blob: 9dda54f6d379c8eecbbbe59370d170522682c068 [file] [log] [blame]
Michael Walsh70369fd2016-11-22 11:25:57 -06001#!/usr/bin/env python
2
3r"""
4This module contains functions having to do with machine state: get_state,
5check_state, wait_state, etc.
6
7The 'State' is a composite of many pieces of data. Therefore, the functions
8in this module define state as an ordered dictionary. Here is an example of
9some test output showing machine state:
10
Michael Walsh341c21e2017-01-17 16:25:20 -060011default_state:
Michael Walsh65b12542017-02-03 15:34:38 -060012 default_state[chassis]: On
Michael Walsh01975fa2017-08-20 20:51:36 -050013 default_state[boot_progress]: OSStart
Michael Walsh56749222017-09-29 15:26:07 -050014 default_state[operating_system]: BootComplete
Michael Walsh65b12542017-02-03 15:34:38 -060015 default_state[host]: Running
Michael Walsh341c21e2017-01-17 16:25:20 -060016 default_state[os_ping]: 1
17 default_state[os_login]: 1
18 default_state[os_run_cmd]: 1
Michael Walsh70369fd2016-11-22 11:25:57 -060019
20Different users may very well have different needs when inquiring about
Michael Walsh8fae6ea2017-02-20 16:14:44 -060021state. Support for new pieces of state information may be added to this
22module as needed.
Michael Walsh70369fd2016-11-22 11:25:57 -060023
24By using the wait_state function, a caller can start a boot and then wait for
25a precisely defined state to indicate that the boot has succeeded. If
26the boot fails, they can see exactly why by looking at the current state as
27compared with the expected state.
28"""
29
30import gen_print as gp
31import gen_robot_print as grp
32import gen_valid as gv
Michael Walsh16cbb7f2017-02-02 15:54:16 -060033import gen_robot_utils as gru
Michael Walsh8fae6ea2017-02-20 16:14:44 -060034import gen_cmd as gc
Michael Walsh70369fd2016-11-22 11:25:57 -060035
36import commands
37from robot.libraries.BuiltIn import BuiltIn
Michael Walsh341c21e2017-01-17 16:25:20 -060038from robot.utils import DotDict
Michael Walsh70369fd2016-11-22 11:25:57 -060039
40import re
Michael Walsh341c21e2017-01-17 16:25:20 -060041import os
Michael Walsh56749222017-09-29 15:26:07 -050042import sys
43import imp
44
Michael Walsh70369fd2016-11-22 11:25:57 -060045
Michael Walsh619aa332017-04-12 15:56:51 -050046# We need utils.robot to get keywords like "Get Chassis Power State".
Michael Walsh16cbb7f2017-02-02 15:54:16 -060047gru.my_import_resource("utils.robot")
48gru.my_import_resource("state_manager.robot")
Michael Walsh70369fd2016-11-22 11:25:57 -060049
Michael Walsh56749222017-09-29 15:26:07 -050050base_path = os.path.dirname(os.path.dirname(
51 imp.find_module("gen_robot_print")[1])) + os.sep
52sys.path.append(base_path + "data/")
Michael Walsh56749222017-09-29 15:26:07 -050053
Michael Walsh940d6912017-10-27 12:32:33 -050054# Previously, I had this coded:
55# import variables as var
56# However, we ran into a problem where a robot program did this...
57# Variables ../../lib/ras/variables.py
58# Prior to doing this...
59# Library ../lib/state.py
60
61# This caused the wrong variables.py file to be selected. Attempts to fix this
62# have failed so far. For the moment, we will hard-code the value we need from
63# the file.
64
65SYSTEM_STATE_URI = "/xyz/openbmc_project/state/"
Michael Walsh56749222017-09-29 15:26:07 -050066
Michael Walsh8fae6ea2017-02-20 16:14:44 -060067# The BMC code has recently been changed as far as what states are defined and
68# what the state values can be. This module now has a means of processing both
69# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e.
Michael Walsh16cbb7f2017-02-02 15:54:16 -060070# OBMC_STATES_VERSION = 1).
Michael Walsh341c21e2017-01-17 16:25:20 -060071# The caller can set environment variable OBMC_STATES_VERSION to dictate
72# whether we're processing old or new style states. If OBMC_STATES_VERSION is
Michael Walsh8fae6ea2017-02-20 16:14:44 -060073# not set it will default to 1.
Michael Walsh341c21e2017-01-17 16:25:20 -060074
Michael Walsh619aa332017-04-12 15:56:51 -050075# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old
76# that it is no longer worthwhile to maintain. The OBMC_STATES_VERSION 0 code
77# is being removed but the OBMC_STATES_VERSION value will stay for now in the
78# event that it is needed in the future.
79
Michael Walsh8fae6ea2017-02-20 16:14:44 -060080OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1))
Michael Walsh341c21e2017-01-17 16:25:20 -060081
Michael Walsh619aa332017-04-12 15:56:51 -050082# When a user calls get_state w/o specifying req_states, default_req_states
83# is used as its value.
84default_req_states = ['rest',
85 'chassis',
86 'bmc',
87 'boot_progress',
Michael Walsh56749222017-09-29 15:26:07 -050088 'operating_system',
Michael Walsh619aa332017-04-12 15:56:51 -050089 'host',
90 'os_ping',
91 'os_login',
92 'os_run_cmd']
Michael Walsh8fae6ea2017-02-20 16:14:44 -060093
Michael Walsh619aa332017-04-12 15:56:51 -050094# valid_req_states is a list of sub states supported by the get_state function.
95# valid_req_states, default_req_states and master_os_up_match are used by the
96# get_state function.
97valid_req_states = ['ping',
98 'packet_loss',
99 'uptime',
100 'epoch_seconds',
101 'rest',
102 'chassis',
Michael Walsh56749222017-09-29 15:26:07 -0500103 'requested_chassis',
Michael Walsh619aa332017-04-12 15:56:51 -0500104 'bmc',
Michael Walsh56749222017-09-29 15:26:07 -0500105 'requested_bmc',
Michael Walsh619aa332017-04-12 15:56:51 -0500106 'boot_progress',
Michael Walsh56749222017-09-29 15:26:07 -0500107 'operating_system',
Michael Walsh619aa332017-04-12 15:56:51 -0500108 'host',
Michael Walsh56749222017-09-29 15:26:07 -0500109 'requested_host',
110 'attempts_left',
Michael Walsh619aa332017-04-12 15:56:51 -0500111 'os_ping',
112 'os_login',
113 'os_run_cmd']
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600114
115# valid_os_req_states and default_os_req_states are used by the os_get_state
116# function.
117# valid_os_req_states is a list of state information supported by the
118# get_os_state function.
119valid_os_req_states = ['os_ping',
120 'os_login',
121 'os_run_cmd']
122# When a user calls get_os_state w/o specifying req_states,
123# default_os_req_states is used as its value.
124default_os_req_states = ['os_ping',
125 'os_login',
126 'os_run_cmd']
127
128# Presently, some BMCs appear to not keep time very well. This environment
129# variable directs the get_state function to use either the BMC's epoch time
130# or the local epoch time.
131USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0))
Michael Walsh341c21e2017-01-17 16:25:20 -0600132
Michael Walsh619aa332017-04-12 15:56:51 -0500133# Useful state constant definition(s).
Michael Walsh619aa332017-04-12 15:56:51 -0500134# default_state is an initial value which may be of use to callers.
135default_state = DotDict([('rest', '1'),
136 ('chassis', 'On'),
137 ('bmc', 'Ready'),
Michael Walsh01975fa2017-08-20 20:51:36 -0500138 ('boot_progress', 'OSStart'),
Michael Walsh56749222017-09-29 15:26:07 -0500139 ('operating_system', 'BootComplete'),
Michael Walsh619aa332017-04-12 15:56:51 -0500140 ('host', 'Running'),
141 ('os_ping', '1'),
142 ('os_login', '1'),
143 ('os_run_cmd', '1')])
144
Michael Walsh7dc885b2018-03-14 17:51:59 -0500145# A match state for checking that the system is at "standby".
146standby_match_state = DotDict([('rest', '^1$'),
147 ('chassis', '^Off$'),
148 ('bmc', '^Ready$'),
149 ('boot_progress', '^$'),
150 ('operating_system', '^$'),
151 ('host', '^$')])
152
153# A match state for checking that the system is at "os running".
154os_running_match_state = DotDict([('chassis', '^On$'),
155 ('bmc', '^Ready$'),
156 ('boot_progress',
157 'FW Progress, Starting OS|OSStart'),
158 ('operating_system', 'BootComplete'),
159 ('host', '^Running$'),
160 ('os_ping', '^1$'),
161 ('os_login', '^1$'),
162 ('os_run_cmd', '^1$')])
163
Michael Walsh619aa332017-04-12 15:56:51 -0500164# A master dictionary to determine whether the os may be up.
165master_os_up_match = DotDict([('chassis', '^On$'),
166 ('bmc', '^Ready$'),
167 ('boot_progress',
Michael Walsh01975fa2017-08-20 20:51:36 -0500168 'FW Progress, Starting OS|OSStart'),
Michael Walsh56749222017-09-29 15:26:07 -0500169 ('operating_system', 'BootComplete'),
Michael Walsh619aa332017-04-12 15:56:51 -0500170 ('host', '^Running$')])
171
Michael Walsh45ca6e42017-09-14 17:29:12 -0500172invalid_state_match = DotDict([('rest', '^$'),
173 ('chassis', '^$'),
174 ('bmc', '^$'),
175 ('boot_progress', '^$'),
Michael Walsh56749222017-09-29 15:26:07 -0500176 ('operating_system', '^$'),
Michael Walsh45ca6e42017-09-14 17:29:12 -0500177 ('host', '^$')])
178
Michael Walsh341c21e2017-01-17 16:25:20 -0600179
Michael Walsh619aa332017-04-12 15:56:51 -0500180def return_state_constant(state_name='default'):
Michael Walsh619aa332017-04-12 15:56:51 -0500181 r"""
Michael Walsh7dc885b2018-03-14 17:51:59 -0500182 Return the named state dictionary constant.
Michael Walsh619aa332017-04-12 15:56:51 -0500183 """
184
Michael Walsh7dc885b2018-03-14 17:51:59 -0500185 cmd_buf = "state = " + state_name
186 exec(cmd_buf)
187 return state
Michael Walsh619aa332017-04-12 15:56:51 -0500188
Michael Walsh619aa332017-04-12 15:56:51 -0500189
Michael Walsh70369fd2016-11-22 11:25:57 -0600190def anchor_state(state):
Michael Walsh70369fd2016-11-22 11:25:57 -0600191 r"""
192 Add regular expression anchors ("^" and "$") to the beginning and end of
193 each item in the state dictionary passed in. Return the resulting
194 dictionary.
195
196 Description of Arguments:
197 state A dictionary such as the one returned by the get_state()
198 function.
199 """
200
Michael Walsh2ce067a2017-02-27 14:24:07 -0600201 anchored_state = state.copy()
Michael Walsh70369fd2016-11-22 11:25:57 -0600202 for key, match_state_value in anchored_state.items():
203 anchored_state[key] = "^" + str(anchored_state[key]) + "$"
204
205 return anchored_state
206
Michael Walsh70369fd2016-11-22 11:25:57 -0600207
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600208def strip_anchor_state(state):
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600209 r"""
210 Strip regular expression anchors ("^" and "$") from the beginning and end
211 of each item in the state dictionary passed in. Return the resulting
212 dictionary.
213
214 Description of Arguments:
215 state A dictionary such as the one returned by the get_state()
216 function.
217 """
218
Michael Walsh2ce067a2017-02-27 14:24:07 -0600219 stripped_state = state.copy()
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600220 for key, match_state_value in stripped_state.items():
221 stripped_state[key] = stripped_state[key].strip("^$")
222
223 return stripped_state
224
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600225
Michael Walsh70369fd2016-11-22 11:25:57 -0600226def compare_states(state,
Michael Walsh45ca6e42017-09-14 17:29:12 -0500227 match_state,
228 match_type='and'):
Michael Walsh70369fd2016-11-22 11:25:57 -0600229 r"""
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600230 Compare 2 state dictionaries. Return True if they match and False if they
Michael Walsh70369fd2016-11-22 11:25:57 -0600231 don't. Note that the match_state dictionary does not need to have an entry
232 corresponding to each entry in the state dictionary. But for each entry
233 that it does have, the corresponding state entry will be checked for a
234 match.
235
236 Description of arguments:
237 state A state dictionary such as the one returned by the
238 get_state function.
239 match_state A dictionary whose key/value pairs are "state field"/
240 "state value". The state value is interpreted as a
241 regular expression. Every value in this dictionary is
Michael Walsh45ca6e42017-09-14 17:29:12 -0500242 considered. When match_type is 'and', if each and every
243 comparison matches, the two dictionaries are considered to
244 be matching. If match_type is 'or', if any two of the
245 elements compared match, the two dictionaries are
246 considered to be matching.
Michael Walsh7dc885b2018-03-14 17:51:59 -0500247 This value may also be any string accepted by
248 return_state_constant (e.g. "standby_match_state").
249 In such a case this function will call
250 return_state_constant to convert it to a proper
251 dictionary as described above.
Michael Walsh45ca6e42017-09-14 17:29:12 -0500252 match_type This may be 'and' or 'or'.
Michael Walsh70369fd2016-11-22 11:25:57 -0600253 """
254
Michael Walsh45ca6e42017-09-14 17:29:12 -0500255 error_message = gv.svalid_value(match_type, var_name="match_type",
256 valid_values=['and', 'or'])
257 if error_message != "":
258 BuiltIn().fail(gp.sprint_error(error_message))
259
Michael Walsh7dc885b2018-03-14 17:51:59 -0500260 if type(match_state) in (str, unicode):
261 match_state = return_state_constant(match_state)
262
Michael Walsh45ca6e42017-09-14 17:29:12 -0500263 default_match = (match_type == 'and')
Michael Walsh70369fd2016-11-22 11:25:57 -0600264 for key, match_state_value in match_state.items():
Michael Walsh97df71c2017-03-27 14:33:24 -0500265 # Blank match_state_value means "don't care".
266 if match_state_value == "":
267 continue
Michael Walsh70369fd2016-11-22 11:25:57 -0600268 try:
Michael Walsh45ca6e42017-09-14 17:29:12 -0500269 match = (re.match(match_state_value, str(state[key])) is not None)
Michael Walsh70369fd2016-11-22 11:25:57 -0600270 except KeyError:
271 match = False
Michael Walsh70369fd2016-11-22 11:25:57 -0600272
Michael Walsh45ca6e42017-09-14 17:29:12 -0500273 if match != default_match:
274 return match
275
276 return default_match
Michael Walsh70369fd2016-11-22 11:25:57 -0600277
Michael Walsh70369fd2016-11-22 11:25:57 -0600278
Michael Walsh70369fd2016-11-22 11:25:57 -0600279def get_os_state(os_host="",
280 os_username="",
281 os_password="",
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600282 req_states=default_os_req_states,
283 os_up=True,
Michael Walsh70369fd2016-11-22 11:25:57 -0600284 quiet=None):
Michael Walsh70369fd2016-11-22 11:25:57 -0600285 r"""
286 Get component states for the operating system such as ping, login,
287 etc, put them into a dictionary and return them to the caller.
288
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600289 Note that all substate values are strings.
290
Michael Walsh70369fd2016-11-22 11:25:57 -0600291 Description of arguments:
292 os_host The DNS name or IP address of the operating system.
293 This defaults to global ${OS_HOST}.
294 os_username The username to be used to login to the OS.
295 This defaults to global ${OS_USERNAME}.
296 os_password The password to be used to login to the OS.
297 This defaults to global ${OS_PASSWORD}.
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600298 req_states This is a list of states whose values are being requested by
299 the caller.
300 os_up If the caller knows that the os can't possibly be up, it can
301 improve performance by passing os_up=False. This function
302 will then simply return default values for all requested os
303 sub states.
Michael Walsh70369fd2016-11-22 11:25:57 -0600304 quiet Indicates whether status details (e.g. curl commands) should
305 be written to the console.
306 Defaults to either global value of ${QUIET} or to 1.
307 """
308
Michael Walsh619aa332017-04-12 15:56:51 -0500309 quiet = int(gp.get_var_value(quiet, 0))
Michael Walsh70369fd2016-11-22 11:25:57 -0600310
311 # Set parm defaults where necessary and validate all parms.
312 if os_host == "":
313 os_host = BuiltIn().get_variable_value("${OS_HOST}")
314 error_message = gv.svalid_value(os_host, var_name="os_host",
315 invalid_values=[None, ""])
316 if error_message != "":
317 BuiltIn().fail(gp.sprint_error(error_message))
318
319 if os_username == "":
320 os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
321 error_message = gv.svalid_value(os_username, var_name="os_username",
322 invalid_values=[None, ""])
323 if error_message != "":
324 BuiltIn().fail(gp.sprint_error(error_message))
325
326 if os_password == "":
327 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
328 error_message = gv.svalid_value(os_password, var_name="os_password",
329 invalid_values=[None, ""])
330 if error_message != "":
331 BuiltIn().fail(gp.sprint_error(error_message))
332
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600333 invalid_req_states = [sub_state for sub_state in req_states
334 if sub_state not in valid_os_req_states]
335 if len(invalid_req_states) > 0:
336 error_message = "The following req_states are not supported:\n" +\
337 gp.sprint_var(invalid_req_states)
338 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh70369fd2016-11-22 11:25:57 -0600339
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600340 # Initialize all substate values supported by this function.
341 os_ping = 0
342 os_login = 0
343 os_run_cmd = 0
Michael Walsh70369fd2016-11-22 11:25:57 -0600344
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600345 if os_up:
346 if 'os_ping' in req_states:
347 # See if the OS pings.
348 cmd_buf = "ping -c 1 -w 2 " + os_host
349 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500350 gp.pissuing(cmd_buf)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600351 rc, out_buf = commands.getstatusoutput(cmd_buf)
352 if rc == 0:
353 os_ping = 1
Michael Walsh70369fd2016-11-22 11:25:57 -0600354
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600355 # Programming note: All attributes which do not require an ssh login
356 # should have been processed by this point.
357 master_req_login = ['os_login', 'os_run_cmd']
358 req_login = [sub_state for sub_state in req_states if sub_state in
359 master_req_login]
Michael Walsh97df71c2017-03-27 14:33:24 -0500360 must_login = (len(req_login) > 0)
Michael Walsh70369fd2016-11-22 11:25:57 -0600361
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600362 if must_login:
Michael Walsh3eb50022017-03-21 11:27:30 -0500363 # Open SSH connection to OS. Note that this doesn't fail even when
364 # the OS is not up.
Michael Walshac275512017-03-07 11:39:28 -0600365 cmd_buf = ["SSHLibrary.Open Connection", os_host]
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600366 if not quiet:
367 grp.rpissuing_keyword(cmd_buf)
368 ix = BuiltIn().run_keyword(*cmd_buf)
369
370 # Login to OS.
371 cmd_buf = ["Login", os_username, os_password]
372 if not quiet:
373 grp.rpissuing_keyword(cmd_buf)
Michael Walsh3eb50022017-03-21 11:27:30 -0500374 status, ret_values = \
375 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600376 if status == "PASS":
377 os_login = 1
Michael Walsh3eb50022017-03-21 11:27:30 -0500378 else:
379 gp.dprint_var(status)
380 gp.dprint_var(ret_values)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600381
382 if os_login:
383 if 'os_run_cmd' in req_states:
Michael Walsh3eb50022017-03-21 11:27:30 -0500384 # Try running a simple command (uptime) on the OS.
385 cmd_buf = ["Execute Command", "uptime",
386 "return_stderr=True", "return_rc=True"]
387 if not quiet:
388 grp.rpissuing_keyword(cmd_buf)
389 # Note that in spite of its name, there are occasions
390 # where run_keyword_and_ignore_error can fail.
391 status, ret_values = \
392 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
393 if status == "PASS":
394 stdout, stderr, rc = ret_values
395 if rc == 0 and stderr == "":
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600396 os_run_cmd = 1
Michael Walsh3eb50022017-03-21 11:27:30 -0500397 else:
398 gp.dprint_var(status)
399 gp.dprint_var(stdout)
400 gp.dprint_var(stderr)
401 gp.dprint_var(rc)
402 else:
403 gp.dprint_var(status)
404 gp.dprint_var(ret_values)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600405
406 os_state = DotDict()
407 for sub_state in req_states:
408 cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")"
409 exec(cmd_buf)
Michael Walsh70369fd2016-11-22 11:25:57 -0600410
411 return os_state
412
Michael Walsh70369fd2016-11-22 11:25:57 -0600413
Michael Walsh70369fd2016-11-22 11:25:57 -0600414def get_state(openbmc_host="",
415 openbmc_username="",
416 openbmc_password="",
417 os_host="",
418 os_username="",
419 os_password="",
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600420 req_states=default_req_states,
Michael Walsh70369fd2016-11-22 11:25:57 -0600421 quiet=None):
Michael Walsh70369fd2016-11-22 11:25:57 -0600422 r"""
Michael Walsh619aa332017-04-12 15:56:51 -0500423 Get component states such as chassis state, bmc state, etc, put them into a
Michael Walsh70369fd2016-11-22 11:25:57 -0600424 dictionary and return them to the caller.
425
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600426 Note that all substate values are strings.
427
Michael Walsh70369fd2016-11-22 11:25:57 -0600428 Description of arguments:
429 openbmc_host The DNS name or IP address of the BMC.
430 This defaults to global ${OPENBMC_HOST}.
431 openbmc_username The username to be used to login to the BMC.
432 This defaults to global ${OPENBMC_USERNAME}.
433 openbmc_password The password to be used to login to the BMC.
434 This defaults to global ${OPENBMC_PASSWORD}.
435 os_host The DNS name or IP address of the operating system.
436 This defaults to global ${OS_HOST}.
437 os_username The username to be used to login to the OS.
438 This defaults to global ${OS_USERNAME}.
439 os_password The password to be used to login to the OS.
440 This defaults to global ${OS_PASSWORD}.
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600441 req_states This is a list of states whose values are being requested
442 by the caller.
Michael Walsh70369fd2016-11-22 11:25:57 -0600443 quiet Indicates whether status details (e.g. curl commands)
444 should be written to the console.
445 Defaults to either global value of ${QUIET} or to 1.
446 """
447
Michael Walsh619aa332017-04-12 15:56:51 -0500448 quiet = int(gp.get_var_value(quiet, 0))
Michael Walsh70369fd2016-11-22 11:25:57 -0600449
450 # Set parm defaults where necessary and validate all parms.
451 if openbmc_host == "":
452 openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
453 error_message = gv.svalid_value(openbmc_host,
454 var_name="openbmc_host",
455 invalid_values=[None, ""])
456 if error_message != "":
457 BuiltIn().fail(gp.sprint_error(error_message))
458
459 if openbmc_username == "":
460 openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}")
461 error_message = gv.svalid_value(openbmc_username,
462 var_name="openbmc_username",
463 invalid_values=[None, ""])
464 if error_message != "":
465 BuiltIn().fail(gp.sprint_error(error_message))
466
467 if openbmc_password == "":
468 openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}")
469 error_message = gv.svalid_value(openbmc_password,
470 var_name="openbmc_password",
471 invalid_values=[None, ""])
472 if error_message != "":
473 BuiltIn().fail(gp.sprint_error(error_message))
474
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600475 # NOTE: OS parms are optional.
Michael Walsh70369fd2016-11-22 11:25:57 -0600476 if os_host == "":
477 os_host = BuiltIn().get_variable_value("${OS_HOST}")
478 if os_host is None:
479 os_host = ""
480
481 if os_username is "":
482 os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
483 if os_username is None:
484 os_username = ""
485
486 if os_password is "":
487 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
488 if os_password is None:
489 os_password = ""
490
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600491 invalid_req_states = [sub_state for sub_state in req_states
492 if sub_state not in valid_req_states]
493 if len(invalid_req_states) > 0:
494 error_message = "The following req_states are not supported:\n" +\
495 gp.sprint_var(invalid_req_states)
496 BuiltIn().fail(gp.sprint_error(error_message))
497
498 # Initialize all substate values supported by this function.
499 ping = 0
500 packet_loss = ''
501 uptime = ''
502 epoch_seconds = ''
Michael Walsh2b269de2017-10-09 11:18:11 -0500503 rest = ''
504 chassis = ''
505 requested_chassis = ''
506 bmc = ''
507 requested_bmc = ''
508 boot_progress = ''
509 operating_system = ''
510 host = ''
511 requested_host = ''
512 attempts_left = ''
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600513
Michael Walsh70369fd2016-11-22 11:25:57 -0600514 # Get the component states.
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600515 if 'ping' in req_states:
516 # See if the OS pings.
517 cmd_buf = "ping -c 1 -w 2 " + openbmc_host
518 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500519 gp.pissuing(cmd_buf)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600520 rc, out_buf = commands.getstatusoutput(cmd_buf)
521 if rc == 0:
522 ping = 1
523
524 if 'packet_loss' in req_states:
525 # See if the OS pings.
526 cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\
527 " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'"
528 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500529 gp.pissuing(cmd_buf)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600530 rc, out_buf = commands.getstatusoutput(cmd_buf)
531 if rc == 0:
532 packet_loss = out_buf.rstrip("\n")
533
Michael Walshe53e47a2017-06-30 17:03:24 -0500534 if 'uptime' in req_states:
Michael Walshfa765932017-10-13 14:07:22 -0500535 # Sometimes reading uptime results in a blank value. Call with
536 # wait_until_keyword_succeeds to ensure a non-blank value is obtained.
537 remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\
538 " && [ ! -z \"${uptime}\" ] && echo ${uptime}"
539 cmd_buf = ["BMC Execute Command", re.sub(r'\$', '\$', remote_cmd_buf),
540 'quiet=1']
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600541 if not quiet:
542 grp.rpissuing_keyword(cmd_buf)
Michael Walshfa765932017-10-13 14:07:22 -0500543 grp.rpissuing(remote_cmd_buf)
544 try:
545 stdout, stderr, rc =\
Michael Walshcaccd852017-11-01 17:58:41 -0500546 BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec",
Michael Walshfa765932017-10-13 14:07:22 -0500547 *cmd_buf)
Michael Walsh97df71c2017-03-27 14:33:24 -0500548 if rc == 0 and stderr == "":
549 uptime = stdout
Michael Walshfa765932017-10-13 14:07:22 -0500550 except AssertionError as my_assertion_error:
551 pass
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600552
Michael Walshe53e47a2017-06-30 17:03:24 -0500553 if 'epoch_seconds' in req_states:
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600554 date_cmd_buf = "date -u +%s"
555 if USE_BMC_EPOCH_TIME:
Michael Walshe53e47a2017-06-30 17:03:24 -0500556 cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}']
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600557 if not quiet:
558 grp.rpissuing_keyword(cmd_buf)
Michael Walsh97df71c2017-03-27 14:33:24 -0500559 status, ret_values = \
560 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
561 if status == "PASS":
562 stdout, stderr, rc = ret_values
563 if rc == 0 and stderr == "":
564 epoch_seconds = stdout.rstrip("\n")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600565 else:
566 shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf,
Michael Walshfa765932017-10-13 14:07:22 -0500567 quiet=quiet,
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600568 print_output=0)
569 if shell_rc == 0:
570 epoch_seconds = out_buf.rstrip("\n")
571
Michael Walsh56749222017-09-29 15:26:07 -0500572 master_req_rest = ['rest', 'host', 'requested_host', 'operating_system',
573 'attempts_left', 'boot_progress', 'chassis',
574 'requested_chassis' 'bmc' 'requested_bmc']
575
Michael Walshb95eb542017-03-31 09:39:20 -0500576 req_rest = [sub_state for sub_state in req_states if sub_state in
577 master_req_rest]
578 need_rest = (len(req_rest) > 0)
Michael Walsh56749222017-09-29 15:26:07 -0500579 state = DotDict()
580 if need_rest:
Michael Walsh940d6912017-10-27 12:32:33 -0500581 cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate",
Michael Walsh56749222017-09-29 15:26:07 -0500582 "quiet=${" + str(quiet) + "}"]
Michael Walsh341c21e2017-01-17 16:25:20 -0600583 grp.rdpissuing_keyword(cmd_buf)
Michael Walsh97df71c2017-03-27 14:33:24 -0500584 status, ret_values = \
585 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
586 if status == "PASS":
Michael Walsh56749222017-09-29 15:26:07 -0500587 state['rest'] = '1'
Michael Walsh341c21e2017-01-17 16:25:20 -0600588 else:
Michael Walsh56749222017-09-29 15:26:07 -0500589 state['rest'] = '0'
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600590
Michael Walsh2b269de2017-10-09 11:18:11 -0500591 if int(state['rest']):
592 for url_path in ret_values:
593 for attr_name in ret_values[url_path]:
594 # Create a state key value based on the attr_name.
595 if type(ret_values[url_path][attr_name]) is unicode:
596 ret_values[url_path][attr_name] = \
597 re.sub(r'.*\.', "",
598 ret_values[url_path][attr_name])
599 # Do some key name manipulations.
600 new_attr_name = re.sub(r'^Current|(State|Transition)$',
601 "", attr_name)
602 new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name)
603 new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1',
604 new_attr_name)
605 new_attr_name = new_attr_name.lower().lstrip("_")
606 new_attr_name = re.sub(r'power', r'chassis', new_attr_name)
607 if new_attr_name in req_states:
608 state[new_attr_name] = ret_values[url_path][attr_name]
Michael Walshb95eb542017-03-31 09:39:20 -0500609
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600610 for sub_state in req_states:
Michael Walsh56749222017-09-29 15:26:07 -0500611 if sub_state in state:
612 continue
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600613 if sub_state.startswith("os_"):
614 # We pass "os_" requests on to get_os_state.
615 continue
616 cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")"
617 exec(cmd_buf)
618
619 if os_host == "":
620 # The caller has not specified an os_host so as far as we're concerned,
621 # it doesn't exist.
622 return state
623
624 os_req_states = [sub_state for sub_state in req_states
625 if sub_state.startswith('os_')]
626
627 if len(os_req_states) > 0:
628 # The caller has specified an os_host and they have requested
629 # information on os substates.
630
631 # Based on the information gathered on bmc, we'll try to make a
632 # determination of whether the os is even up. We'll pass the result
633 # of that assessment to get_os_state to enhance performance.
634 os_up_match = DotDict()
635 for sub_state in master_os_up_match:
636 if sub_state in req_states:
637 os_up_match[sub_state] = master_os_up_match[sub_state]
Michael Walsh70369fd2016-11-22 11:25:57 -0600638 os_up = compare_states(state, os_up_match)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600639 os_state = get_os_state(os_host=os_host,
640 os_username=os_username,
641 os_password=os_password,
642 req_states=os_req_states,
643 os_up=os_up,
644 quiet=quiet)
645 # Append os_state dictionary to ours.
646 state.update(os_state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600647
648 return state
649
Michael Walsh70369fd2016-11-22 11:25:57 -0600650
Michael Walsh70369fd2016-11-22 11:25:57 -0600651def check_state(match_state,
652 invert=0,
653 print_string="",
654 openbmc_host="",
655 openbmc_username="",
656 openbmc_password="",
657 os_host="",
658 os_username="",
659 os_password="",
660 quiet=None):
Michael Walsh70369fd2016-11-22 11:25:57 -0600661 r"""
662 Check that the Open BMC machine's composite state matches the specified
663 state. On success, this keyword returns the machine's composite state as a
664 dictionary.
665
666 Description of arguments:
667 match_state A dictionary whose key/value pairs are "state field"/
668 "state value". The state value is interpreted as a
669 regular expression. Example call from robot:
Michael Walsh341c21e2017-01-17 16:25:20 -0600670 ${match_state}= Create Dictionary chassis=^On$
671 ... bmc=^Ready$
Michael Walsh01975fa2017-08-20 20:51:36 -0500672 ... boot_progress=^OSStart$
Michael Walsh70369fd2016-11-22 11:25:57 -0600673 ${state}= Check State &{match_state}
674 invert If this flag is set, this function will succeed if the
675 states do NOT match.
676 print_string This function will print this string to the console prior
677 to getting the state.
678 openbmc_host The DNS name or IP address of the BMC.
679 This defaults to global ${OPENBMC_HOST}.
680 openbmc_username The username to be used to login to the BMC.
681 This defaults to global ${OPENBMC_USERNAME}.
682 openbmc_password The password to be used to login to the BMC.
683 This defaults to global ${OPENBMC_PASSWORD}.
684 os_host The DNS name or IP address of the operating system.
685 This defaults to global ${OS_HOST}.
686 os_username The username to be used to login to the OS.
687 This defaults to global ${OS_USERNAME}.
688 os_password The password to be used to login to the OS.
689 This defaults to global ${OS_PASSWORD}.
690 quiet Indicates whether status details should be written to the
691 console. Defaults to either global value of ${QUIET} or
692 to 1.
693 """
694
Michael Walsh619aa332017-04-12 15:56:51 -0500695 quiet = int(gp.get_var_value(quiet, 0))
Michael Walsh70369fd2016-11-22 11:25:57 -0600696
697 grp.rprint(print_string)
698
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600699 req_states = match_state.keys()
Michael Walsh70369fd2016-11-22 11:25:57 -0600700 # Initialize state.
701 state = get_state(openbmc_host=openbmc_host,
702 openbmc_username=openbmc_username,
703 openbmc_password=openbmc_password,
704 os_host=os_host,
705 os_username=os_username,
706 os_password=os_password,
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600707 req_states=req_states,
Michael Walsh70369fd2016-11-22 11:25:57 -0600708 quiet=quiet)
709 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500710 gp.print_var(state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600711
712 match = compare_states(state, match_state)
713
714 if invert and match:
715 fail_msg = "The current state of the machine matches the match" +\
716 " state:\n" + gp.sprint_varx("state", state)
717 BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
718 elif not invert and not match:
719 fail_msg = "The current state of the machine does NOT match the" +\
720 " match state:\n" +\
721 gp.sprint_varx("state", state)
722 BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
723
724 return state
725
Michael Walsh70369fd2016-11-22 11:25:57 -0600726
Michael Walshf893ba02017-01-10 10:28:05 -0600727def wait_state(match_state=(),
Michael Walsh70369fd2016-11-22 11:25:57 -0600728 wait_time="1 min",
729 interval="1 second",
730 invert=0,
731 openbmc_host="",
732 openbmc_username="",
733 openbmc_password="",
734 os_host="",
735 os_username="",
736 os_password="",
737 quiet=None):
Michael Walsh70369fd2016-11-22 11:25:57 -0600738 r"""
739 Wait for the Open BMC machine's composite state to match the specified
740 state. On success, this keyword returns the machine's composite state as
741 a dictionary.
742
743 Description of arguments:
744 match_state A dictionary whose key/value pairs are "state field"/
745 "state value". See check_state (above) for details.
Michael Walsh619aa332017-04-12 15:56:51 -0500746 This value may also be any string accepted by
747 return_state_constant (e.g. "standby_match_state").
748 In such a case this function will call
749 return_state_constant to convert it to a proper
750 dictionary as described above.
Michael Walsh70369fd2016-11-22 11:25:57 -0600751 wait_time The total amount of time to wait for the desired state.
752 This value may be expressed in Robot Framework's time
753 format (e.g. 1 minute, 2 min 3 s, 4.5).
754 interval The amount of time between state checks.
755 This value may be expressed in Robot Framework's time
756 format (e.g. 1 minute, 2 min 3 s, 4.5).
757 invert If this flag is set, this function will for the state of
758 the machine to cease to match the match state.
759 openbmc_host The DNS name or IP address of the BMC.
760 This defaults to global ${OPENBMC_HOST}.
761 openbmc_username The username to be used to login to the BMC.
762 This defaults to global ${OPENBMC_USERNAME}.
763 openbmc_password The password to be used to login to the BMC.
764 This defaults to global ${OPENBMC_PASSWORD}.
765 os_host The DNS name or IP address of the operating system.
766 This defaults to global ${OS_HOST}.
767 os_username The username to be used to login to the OS.
768 This defaults to global ${OS_USERNAME}.
769 os_password The password to be used to login to the OS.
770 This defaults to global ${OS_PASSWORD}.
771 quiet Indicates whether status details should be written to the
772 console. Defaults to either global value of ${QUIET} or
773 to 1.
774 """
775
Michael Walsh619aa332017-04-12 15:56:51 -0500776 quiet = int(gp.get_var_value(quiet, 0))
777
778 if type(match_state) in (str, unicode):
779 match_state = return_state_constant(match_state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600780
781 if not quiet:
782 if invert:
783 alt_text = "cease to "
784 else:
785 alt_text = ""
Michael Walsh3eb50022017-03-21 11:27:30 -0500786 gp.print_timen("Checking every " + str(interval) + " for up to " +
787 str(wait_time) + " for the state of the machine to " +
788 alt_text + "match the state shown below.")
789 gp.print_var(match_state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600790
Michael Walshf893ba02017-01-10 10:28:05 -0600791 if quiet:
Michael Walsh341c21e2017-01-17 16:25:20 -0600792 print_string = ""
Michael Walshf893ba02017-01-10 10:28:05 -0600793 else:
Michael Walsh341c21e2017-01-17 16:25:20 -0600794 print_string = "#"
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600795
796 debug = int(BuiltIn().get_variable_value("${debug}", "0"))
797 if debug:
798 # In debug we print state so no need to print the "#".
799 print_string = ""
800 check_state_quiet = 1 - debug
Michael Walsh70369fd2016-11-22 11:25:57 -0600801 cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}",
Michael Walshf893ba02017-01-10 10:28:05 -0600802 "print_string=" + print_string, "openbmc_host=" + openbmc_host,
Michael Walsh70369fd2016-11-22 11:25:57 -0600803 "openbmc_username=" + openbmc_username,
804 "openbmc_password=" + openbmc_password, "os_host=" + os_host,
805 "os_username=" + os_username, "os_password=" + os_password,
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600806 "quiet=${" + str(check_state_quiet) + "}"]
Michael Walsh70369fd2016-11-22 11:25:57 -0600807 grp.rdpissuing_keyword(cmd_buf)
Michael Walsh619aa332017-04-12 15:56:51 -0500808 try:
809 state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval,
810 *cmd_buf)
811 except AssertionError as my_assertion_error:
812 gp.printn()
813 message = my_assertion_error.args[0]
814 BuiltIn().fail(message)
815
Michael Walsh70369fd2016-11-22 11:25:57 -0600816 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500817 gp.printn()
Michael Walsh70369fd2016-11-22 11:25:57 -0600818 if invert:
Michael Walsh3eb50022017-03-21 11:27:30 -0500819 gp.print_timen("The states no longer match:")
Michael Walsh70369fd2016-11-22 11:25:57 -0600820 else:
Michael Walsh3eb50022017-03-21 11:27:30 -0500821 gp.print_timen("The states match:")
822 gp.print_var(state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600823
824 return state
825
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600826
Michael Walsh619aa332017-04-12 15:56:51 -0500827def wait_for_comm_cycle(start_boot_seconds,
828 quiet=None):
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600829 r"""
830 Wait for communications to the BMC to stop working and then resume working.
831 This function is useful when you have initiated some kind of reboot.
832
833 Description of arguments:
834 start_boot_seconds The time that the boot test started. The format is the
835 epoch time in seconds, i.e. the number of seconds since
836 1970-01-01 00:00:00 UTC. This value should be obtained
837 from the BMC so that it is not dependent on any kind of
838 synchronization between this machine and the target BMC
839 This will allow this program to work correctly even in
840 a simulated environment. This value should be obtained
841 by the caller prior to initiating a reboot. It can be
842 obtained as follows:
843 state = st.get_state(req_states=['epoch_seconds'])
844 """
845
Michael Walsh619aa332017-04-12 15:56:51 -0500846 quiet = int(gp.get_var_value(quiet, 0))
847
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600848 # Validate parms.
849 error_message = gv.svalid_integer(start_boot_seconds,
850 var_name="start_boot_seconds")
851 if error_message != "":
852 BuiltIn().fail(gp.sprint_error(error_message))
853
854 match_state = anchor_state(DotDict([('packet_loss', '100')]))
855 # Wait for 100% packet loss trying to ping machine.
Michael Walsh619aa332017-04-12 15:56:51 -0500856 wait_state(match_state, wait_time="8 mins", interval="0 seconds")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600857
858 match_state['packet_loss'] = '^0$'
859 # Wait for 0% packet loss trying to ping machine.
Michael Walsh619aa332017-04-12 15:56:51 -0500860 wait_state(match_state, wait_time="8 mins", interval="0 seconds")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600861
862 # Get the uptime and epoch seconds for comparisons. We want to be sure
863 # that the uptime is less than the elapsed boot time. Further proof that
864 # a reboot has indeed occurred (vs random network instability giving a
865 # false positive.
Michael Walsh01975fa2017-08-20 20:51:36 -0500866 state = get_state(req_states=['uptime', 'epoch_seconds'], quiet=quiet)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600867
868 elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds
Michael Walsh619aa332017-04-12 15:56:51 -0500869 gp.qprint_var(elapsed_boot_time)
Michael Walshe77585a2017-12-14 11:02:28 -0600870 if state['uptime'] == "":
871 error_message = "Unable to obtain uptime from the BMC. BMC is not" +\
872 " communicating."
873 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600874 if int(float(state['uptime'])) < elapsed_boot_time:
875 uptime = state['uptime']
Michael Walsh619aa332017-04-12 15:56:51 -0500876 gp.qprint_var(uptime)
877 gp.qprint_timen("The uptime is less than the elapsed boot time," +
878 " as expected.")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600879 else:
880 error_message = "The uptime is greater than the elapsed boot time," +\
881 " which is unexpected:\n" +\
882 gp.sprint_var(start_boot_seconds) +\
883 gp.sprint_var(state)
884 BuiltIn().fail(gp.sprint_error(error_message))
885
Michael Walsh619aa332017-04-12 15:56:51 -0500886 gp.qprint_timen("Verifying that REST API interface is working.")
Michael Walshb95eb542017-03-31 09:39:20 -0500887 match_state = DotDict([('rest', '^1$')])
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600888 state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")