blob: 4c36ef4e5b9bae330e848446719d484f9de21ee6 [file] [log] [blame]
Michael Walsh70369fd2016-11-22 11:25:57 -06001#!/usr/bin/env python
2
3r"""
4This module contains functions having to do with machine state: get_state,
5check_state, wait_state, etc.
6
7The 'State' is a composite of many pieces of data. Therefore, the functions
8in this module define state as an ordered dictionary. Here is an example of
9some test output showing machine state:
10
Michael Walsh341c21e2017-01-17 16:25:20 -060011default_state:
Michael Walsh65b12542017-02-03 15:34:38 -060012 default_state[chassis]: On
Michael Walsh01975fa2017-08-20 20:51:36 -050013 default_state[boot_progress]: OSStart
Michael Walsh56749222017-09-29 15:26:07 -050014 default_state[operating_system]: BootComplete
Michael Walsh65b12542017-02-03 15:34:38 -060015 default_state[host]: Running
Michael Walsh341c21e2017-01-17 16:25:20 -060016 default_state[os_ping]: 1
17 default_state[os_login]: 1
18 default_state[os_run_cmd]: 1
Michael Walsh70369fd2016-11-22 11:25:57 -060019
20Different users may very well have different needs when inquiring about
Michael Walsh8fae6ea2017-02-20 16:14:44 -060021state. Support for new pieces of state information may be added to this
22module as needed.
Michael Walsh70369fd2016-11-22 11:25:57 -060023
24By using the wait_state function, a caller can start a boot and then wait for
25a precisely defined state to indicate that the boot has succeeded. If
26the boot fails, they can see exactly why by looking at the current state as
27compared with the expected state.
28"""
29
30import gen_print as gp
31import gen_robot_print as grp
32import gen_valid as gv
Michael Walsh16cbb7f2017-02-02 15:54:16 -060033import gen_robot_utils as gru
Michael Walsh8fae6ea2017-02-20 16:14:44 -060034import gen_cmd as gc
Michael Walsh70369fd2016-11-22 11:25:57 -060035
36import commands
37from robot.libraries.BuiltIn import BuiltIn
Michael Walsh341c21e2017-01-17 16:25:20 -060038from robot.utils import DotDict
Michael Walsh70369fd2016-11-22 11:25:57 -060039
40import re
Michael Walsh341c21e2017-01-17 16:25:20 -060041import os
Michael Walsh56749222017-09-29 15:26:07 -050042import sys
43import imp
44
Michael Walsh70369fd2016-11-22 11:25:57 -060045
Michael Walsh619aa332017-04-12 15:56:51 -050046# We need utils.robot to get keywords like "Get Chassis Power State".
Michael Walsh16cbb7f2017-02-02 15:54:16 -060047gru.my_import_resource("utils.robot")
48gru.my_import_resource("state_manager.robot")
Michael Walsh70369fd2016-11-22 11:25:57 -060049
Michael Walsh56749222017-09-29 15:26:07 -050050base_path = os.path.dirname(os.path.dirname(
51 imp.find_module("gen_robot_print")[1])) + os.sep
52sys.path.append(base_path + "data/")
Michael Walsh56749222017-09-29 15:26:07 -050053
Michael Walsh940d6912017-10-27 12:32:33 -050054# Previously, I had this coded:
55# import variables as var
56# However, we ran into a problem where a robot program did this...
57# Variables ../../lib/ras/variables.py
58# Prior to doing this...
59# Library ../lib/state.py
60
61# This caused the wrong variables.py file to be selected. Attempts to fix this
62# have failed so far. For the moment, we will hard-code the value we need from
63# the file.
64
65SYSTEM_STATE_URI = "/xyz/openbmc_project/state/"
Michael Walsh56749222017-09-29 15:26:07 -050066
Michael Walsh8fae6ea2017-02-20 16:14:44 -060067# The BMC code has recently been changed as far as what states are defined and
68# what the state values can be. This module now has a means of processing both
69# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e.
Michael Walsh16cbb7f2017-02-02 15:54:16 -060070# OBMC_STATES_VERSION = 1).
Michael Walsh341c21e2017-01-17 16:25:20 -060071# The caller can set environment variable OBMC_STATES_VERSION to dictate
72# whether we're processing old or new style states. If OBMC_STATES_VERSION is
Michael Walsh8fae6ea2017-02-20 16:14:44 -060073# not set it will default to 1.
Michael Walsh341c21e2017-01-17 16:25:20 -060074
Michael Walsh619aa332017-04-12 15:56:51 -050075# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old
76# that it is no longer worthwhile to maintain. The OBMC_STATES_VERSION 0 code
77# is being removed but the OBMC_STATES_VERSION value will stay for now in the
78# event that it is needed in the future.
79
Michael Walsh8fae6ea2017-02-20 16:14:44 -060080OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1))
Michael Walsh341c21e2017-01-17 16:25:20 -060081
Michael Walsh619aa332017-04-12 15:56:51 -050082# When a user calls get_state w/o specifying req_states, default_req_states
83# is used as its value.
84default_req_states = ['rest',
85 'chassis',
86 'bmc',
87 'boot_progress',
Michael Walsh56749222017-09-29 15:26:07 -050088 'operating_system',
Michael Walsh619aa332017-04-12 15:56:51 -050089 'host',
90 'os_ping',
91 'os_login',
92 'os_run_cmd']
Michael Walsh8fae6ea2017-02-20 16:14:44 -060093
Michael Walsh619aa332017-04-12 15:56:51 -050094# valid_req_states is a list of sub states supported by the get_state function.
95# valid_req_states, default_req_states and master_os_up_match are used by the
96# get_state function.
97valid_req_states = ['ping',
98 'packet_loss',
99 'uptime',
100 'epoch_seconds',
101 'rest',
102 'chassis',
Michael Walsh56749222017-09-29 15:26:07 -0500103 'requested_chassis',
Michael Walsh619aa332017-04-12 15:56:51 -0500104 'bmc',
Michael Walsh56749222017-09-29 15:26:07 -0500105 'requested_bmc',
Michael Walsh619aa332017-04-12 15:56:51 -0500106 'boot_progress',
Michael Walsh56749222017-09-29 15:26:07 -0500107 'operating_system',
Michael Walsh619aa332017-04-12 15:56:51 -0500108 'host',
Michael Walsh56749222017-09-29 15:26:07 -0500109 'requested_host',
110 'attempts_left',
Michael Walsh619aa332017-04-12 15:56:51 -0500111 'os_ping',
112 'os_login',
113 'os_run_cmd']
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600114
115# valid_os_req_states and default_os_req_states are used by the os_get_state
116# function.
117# valid_os_req_states is a list of state information supported by the
118# get_os_state function.
119valid_os_req_states = ['os_ping',
120 'os_login',
121 'os_run_cmd']
122# When a user calls get_os_state w/o specifying req_states,
123# default_os_req_states is used as its value.
124default_os_req_states = ['os_ping',
125 'os_login',
126 'os_run_cmd']
127
128# Presently, some BMCs appear to not keep time very well. This environment
129# variable directs the get_state function to use either the BMC's epoch time
130# or the local epoch time.
131USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0))
Michael Walsh341c21e2017-01-17 16:25:20 -0600132
Michael Walsh619aa332017-04-12 15:56:51 -0500133# Useful state constant definition(s).
134# A match state for checking that the system is at "standby".
135standby_match_state = DotDict([('rest', '^1$'),
136 ('chassis', '^Off$'),
137 ('bmc', '^Ready$'),
138 ('boot_progress', ''),
Michael Walsh56749222017-09-29 15:26:07 -0500139 ('operating_system', ''),
Michael Walsh619aa332017-04-12 15:56:51 -0500140 ('host', '')])
141
142# default_state is an initial value which may be of use to callers.
143default_state = DotDict([('rest', '1'),
144 ('chassis', 'On'),
145 ('bmc', 'Ready'),
Michael Walsh01975fa2017-08-20 20:51:36 -0500146 ('boot_progress', 'OSStart'),
Michael Walsh56749222017-09-29 15:26:07 -0500147 ('operating_system', 'BootComplete'),
Michael Walsh619aa332017-04-12 15:56:51 -0500148 ('host', 'Running'),
149 ('os_ping', '1'),
150 ('os_login', '1'),
151 ('os_run_cmd', '1')])
152
153# A master dictionary to determine whether the os may be up.
154master_os_up_match = DotDict([('chassis', '^On$'),
155 ('bmc', '^Ready$'),
156 ('boot_progress',
Michael Walsh01975fa2017-08-20 20:51:36 -0500157 'FW Progress, Starting OS|OSStart'),
Michael Walsh56749222017-09-29 15:26:07 -0500158 ('operating_system', 'BootComplete'),
Michael Walsh619aa332017-04-12 15:56:51 -0500159 ('host', '^Running$')])
160
Michael Walsh45ca6e42017-09-14 17:29:12 -0500161invalid_state_match = DotDict([('rest', '^$'),
162 ('chassis', '^$'),
163 ('bmc', '^$'),
164 ('boot_progress', '^$'),
Michael Walsh56749222017-09-29 15:26:07 -0500165 ('operating_system', '^$'),
Michael Walsh45ca6e42017-09-14 17:29:12 -0500166 ('host', '^$')])
167
Michael Walsh341c21e2017-01-17 16:25:20 -0600168
Michael Walsh341c21e2017-01-17 16:25:20 -0600169def return_default_state():
170
171 r"""
172 Return default state dictionary.
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600173
174 default_state is an initial value which may be of use to callers.
Michael Walsh341c21e2017-01-17 16:25:20 -0600175 """
176
177 return default_state
178
Michael Walsh70369fd2016-11-22 11:25:57 -0600179
Michael Walsh619aa332017-04-12 15:56:51 -0500180valid_state_constants = ['default', 'standby_match_state']
181
182
Michael Walsh619aa332017-04-12 15:56:51 -0500183def return_state_constant(state_name='default'):
184
185 r"""
186 Return default state dictionary.
187
188 default_state is an initial value which may be of use to callers.
189 """
190
191 error_message = gv.svalid_value(state_name, var_name='state_name',
192 valid_values=valid_state_constants)
193 if error_message != "":
194 BuiltIn().fail(gp.sprint_error(error_message))
195
196 if state_name == 'default':
197 return default_state
198 elif state_name == 'standby_match_state':
199 return standby_match_state
200
Michael Walsh619aa332017-04-12 15:56:51 -0500201
Michael Walsh70369fd2016-11-22 11:25:57 -0600202def anchor_state(state):
203
204 r"""
205 Add regular expression anchors ("^" and "$") to the beginning and end of
206 each item in the state dictionary passed in. Return the resulting
207 dictionary.
208
209 Description of Arguments:
210 state A dictionary such as the one returned by the get_state()
211 function.
212 """
213
Michael Walsh2ce067a2017-02-27 14:24:07 -0600214 anchored_state = state.copy()
Michael Walsh70369fd2016-11-22 11:25:57 -0600215 for key, match_state_value in anchored_state.items():
216 anchored_state[key] = "^" + str(anchored_state[key]) + "$"
217
218 return anchored_state
219
Michael Walsh70369fd2016-11-22 11:25:57 -0600220
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600221def strip_anchor_state(state):
222
223 r"""
224 Strip regular expression anchors ("^" and "$") from the beginning and end
225 of each item in the state dictionary passed in. Return the resulting
226 dictionary.
227
228 Description of Arguments:
229 state A dictionary such as the one returned by the get_state()
230 function.
231 """
232
Michael Walsh2ce067a2017-02-27 14:24:07 -0600233 stripped_state = state.copy()
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600234 for key, match_state_value in stripped_state.items():
235 stripped_state[key] = stripped_state[key].strip("^$")
236
237 return stripped_state
238
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600239
Michael Walsh70369fd2016-11-22 11:25:57 -0600240def compare_states(state,
Michael Walsh45ca6e42017-09-14 17:29:12 -0500241 match_state,
242 match_type='and'):
Michael Walsh70369fd2016-11-22 11:25:57 -0600243
244 r"""
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600245 Compare 2 state dictionaries. Return True if they match and False if they
Michael Walsh70369fd2016-11-22 11:25:57 -0600246 don't. Note that the match_state dictionary does not need to have an entry
247 corresponding to each entry in the state dictionary. But for each entry
248 that it does have, the corresponding state entry will be checked for a
249 match.
250
251 Description of arguments:
252 state A state dictionary such as the one returned by the
253 get_state function.
254 match_state A dictionary whose key/value pairs are "state field"/
255 "state value". The state value is interpreted as a
256 regular expression. Every value in this dictionary is
Michael Walsh45ca6e42017-09-14 17:29:12 -0500257 considered. When match_type is 'and', if each and every
258 comparison matches, the two dictionaries are considered to
259 be matching. If match_type is 'or', if any two of the
260 elements compared match, the two dictionaries are
261 considered to be matching.
262 match_type This may be 'and' or 'or'.
Michael Walsh70369fd2016-11-22 11:25:57 -0600263 """
264
Michael Walsh45ca6e42017-09-14 17:29:12 -0500265 error_message = gv.svalid_value(match_type, var_name="match_type",
266 valid_values=['and', 'or'])
267 if error_message != "":
268 BuiltIn().fail(gp.sprint_error(error_message))
269
270 default_match = (match_type == 'and')
Michael Walsh70369fd2016-11-22 11:25:57 -0600271 for key, match_state_value in match_state.items():
Michael Walsh97df71c2017-03-27 14:33:24 -0500272 # Blank match_state_value means "don't care".
273 if match_state_value == "":
274 continue
Michael Walsh70369fd2016-11-22 11:25:57 -0600275 try:
Michael Walsh45ca6e42017-09-14 17:29:12 -0500276 match = (re.match(match_state_value, str(state[key])) is not None)
Michael Walsh70369fd2016-11-22 11:25:57 -0600277 except KeyError:
278 match = False
Michael Walsh70369fd2016-11-22 11:25:57 -0600279
Michael Walsh45ca6e42017-09-14 17:29:12 -0500280 if match != default_match:
281 return match
282
283 return default_match
Michael Walsh70369fd2016-11-22 11:25:57 -0600284
Michael Walsh70369fd2016-11-22 11:25:57 -0600285
Michael Walsh70369fd2016-11-22 11:25:57 -0600286def get_os_state(os_host="",
287 os_username="",
288 os_password="",
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600289 req_states=default_os_req_states,
290 os_up=True,
Michael Walsh70369fd2016-11-22 11:25:57 -0600291 quiet=None):
292
293 r"""
294 Get component states for the operating system such as ping, login,
295 etc, put them into a dictionary and return them to the caller.
296
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600297 Note that all substate values are strings.
298
Michael Walsh70369fd2016-11-22 11:25:57 -0600299 Description of arguments:
300 os_host The DNS name or IP address of the operating system.
301 This defaults to global ${OS_HOST}.
302 os_username The username to be used to login to the OS.
303 This defaults to global ${OS_USERNAME}.
304 os_password The password to be used to login to the OS.
305 This defaults to global ${OS_PASSWORD}.
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600306 req_states This is a list of states whose values are being requested by
307 the caller.
308 os_up If the caller knows that the os can't possibly be up, it can
309 improve performance by passing os_up=False. This function
310 will then simply return default values for all requested os
311 sub states.
Michael Walsh70369fd2016-11-22 11:25:57 -0600312 quiet Indicates whether status details (e.g. curl commands) should
313 be written to the console.
314 Defaults to either global value of ${QUIET} or to 1.
315 """
316
Michael Walsh619aa332017-04-12 15:56:51 -0500317 quiet = int(gp.get_var_value(quiet, 0))
Michael Walsh70369fd2016-11-22 11:25:57 -0600318
319 # Set parm defaults where necessary and validate all parms.
320 if os_host == "":
321 os_host = BuiltIn().get_variable_value("${OS_HOST}")
322 error_message = gv.svalid_value(os_host, var_name="os_host",
323 invalid_values=[None, ""])
324 if error_message != "":
325 BuiltIn().fail(gp.sprint_error(error_message))
326
327 if os_username == "":
328 os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
329 error_message = gv.svalid_value(os_username, var_name="os_username",
330 invalid_values=[None, ""])
331 if error_message != "":
332 BuiltIn().fail(gp.sprint_error(error_message))
333
334 if os_password == "":
335 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
336 error_message = gv.svalid_value(os_password, var_name="os_password",
337 invalid_values=[None, ""])
338 if error_message != "":
339 BuiltIn().fail(gp.sprint_error(error_message))
340
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600341 invalid_req_states = [sub_state for sub_state in req_states
342 if sub_state not in valid_os_req_states]
343 if len(invalid_req_states) > 0:
344 error_message = "The following req_states are not supported:\n" +\
345 gp.sprint_var(invalid_req_states)
346 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh70369fd2016-11-22 11:25:57 -0600347
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600348 # Initialize all substate values supported by this function.
349 os_ping = 0
350 os_login = 0
351 os_run_cmd = 0
Michael Walsh70369fd2016-11-22 11:25:57 -0600352
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600353 if os_up:
354 if 'os_ping' in req_states:
355 # See if the OS pings.
356 cmd_buf = "ping -c 1 -w 2 " + os_host
357 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500358 gp.pissuing(cmd_buf)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600359 rc, out_buf = commands.getstatusoutput(cmd_buf)
360 if rc == 0:
361 os_ping = 1
Michael Walsh70369fd2016-11-22 11:25:57 -0600362
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600363 # Programming note: All attributes which do not require an ssh login
364 # should have been processed by this point.
365 master_req_login = ['os_login', 'os_run_cmd']
366 req_login = [sub_state for sub_state in req_states if sub_state in
367 master_req_login]
Michael Walsh97df71c2017-03-27 14:33:24 -0500368 must_login = (len(req_login) > 0)
Michael Walsh70369fd2016-11-22 11:25:57 -0600369
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600370 if must_login:
Michael Walsh3eb50022017-03-21 11:27:30 -0500371 # Open SSH connection to OS. Note that this doesn't fail even when
372 # the OS is not up.
Michael Walshac275512017-03-07 11:39:28 -0600373 cmd_buf = ["SSHLibrary.Open Connection", os_host]
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600374 if not quiet:
375 grp.rpissuing_keyword(cmd_buf)
376 ix = BuiltIn().run_keyword(*cmd_buf)
377
378 # Login to OS.
379 cmd_buf = ["Login", os_username, os_password]
380 if not quiet:
381 grp.rpissuing_keyword(cmd_buf)
Michael Walsh3eb50022017-03-21 11:27:30 -0500382 status, ret_values = \
383 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600384 if status == "PASS":
385 os_login = 1
Michael Walsh3eb50022017-03-21 11:27:30 -0500386 else:
387 gp.dprint_var(status)
388 gp.dprint_var(ret_values)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600389
390 if os_login:
391 if 'os_run_cmd' in req_states:
Michael Walsh3eb50022017-03-21 11:27:30 -0500392 # Try running a simple command (uptime) on the OS.
393 cmd_buf = ["Execute Command", "uptime",
394 "return_stderr=True", "return_rc=True"]
395 if not quiet:
396 grp.rpissuing_keyword(cmd_buf)
397 # Note that in spite of its name, there are occasions
398 # where run_keyword_and_ignore_error can fail.
399 status, ret_values = \
400 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
401 if status == "PASS":
402 stdout, stderr, rc = ret_values
403 if rc == 0 and stderr == "":
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600404 os_run_cmd = 1
Michael Walsh3eb50022017-03-21 11:27:30 -0500405 else:
406 gp.dprint_var(status)
407 gp.dprint_var(stdout)
408 gp.dprint_var(stderr)
409 gp.dprint_var(rc)
410 else:
411 gp.dprint_var(status)
412 gp.dprint_var(ret_values)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600413
414 os_state = DotDict()
415 for sub_state in req_states:
416 cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")"
417 exec(cmd_buf)
Michael Walsh70369fd2016-11-22 11:25:57 -0600418
419 return os_state
420
Michael Walsh70369fd2016-11-22 11:25:57 -0600421
Michael Walsh70369fd2016-11-22 11:25:57 -0600422def get_state(openbmc_host="",
423 openbmc_username="",
424 openbmc_password="",
425 os_host="",
426 os_username="",
427 os_password="",
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600428 req_states=default_req_states,
Michael Walsh70369fd2016-11-22 11:25:57 -0600429 quiet=None):
430
431 r"""
Michael Walsh619aa332017-04-12 15:56:51 -0500432 Get component states such as chassis state, bmc state, etc, put them into a
Michael Walsh70369fd2016-11-22 11:25:57 -0600433 dictionary and return them to the caller.
434
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600435 Note that all substate values are strings.
436
Michael Walsh70369fd2016-11-22 11:25:57 -0600437 Description of arguments:
438 openbmc_host The DNS name or IP address of the BMC.
439 This defaults to global ${OPENBMC_HOST}.
440 openbmc_username The username to be used to login to the BMC.
441 This defaults to global ${OPENBMC_USERNAME}.
442 openbmc_password The password to be used to login to the BMC.
443 This defaults to global ${OPENBMC_PASSWORD}.
444 os_host The DNS name or IP address of the operating system.
445 This defaults to global ${OS_HOST}.
446 os_username The username to be used to login to the OS.
447 This defaults to global ${OS_USERNAME}.
448 os_password The password to be used to login to the OS.
449 This defaults to global ${OS_PASSWORD}.
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600450 req_states This is a list of states whose values are being requested
451 by the caller.
Michael Walsh70369fd2016-11-22 11:25:57 -0600452 quiet Indicates whether status details (e.g. curl commands)
453 should be written to the console.
454 Defaults to either global value of ${QUIET} or to 1.
455 """
456
Michael Walsh619aa332017-04-12 15:56:51 -0500457 quiet = int(gp.get_var_value(quiet, 0))
Michael Walsh70369fd2016-11-22 11:25:57 -0600458
459 # Set parm defaults where necessary and validate all parms.
460 if openbmc_host == "":
461 openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
462 error_message = gv.svalid_value(openbmc_host,
463 var_name="openbmc_host",
464 invalid_values=[None, ""])
465 if error_message != "":
466 BuiltIn().fail(gp.sprint_error(error_message))
467
468 if openbmc_username == "":
469 openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}")
470 error_message = gv.svalid_value(openbmc_username,
471 var_name="openbmc_username",
472 invalid_values=[None, ""])
473 if error_message != "":
474 BuiltIn().fail(gp.sprint_error(error_message))
475
476 if openbmc_password == "":
477 openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}")
478 error_message = gv.svalid_value(openbmc_password,
479 var_name="openbmc_password",
480 invalid_values=[None, ""])
481 if error_message != "":
482 BuiltIn().fail(gp.sprint_error(error_message))
483
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600484 # NOTE: OS parms are optional.
Michael Walsh70369fd2016-11-22 11:25:57 -0600485 if os_host == "":
486 os_host = BuiltIn().get_variable_value("${OS_HOST}")
487 if os_host is None:
488 os_host = ""
489
490 if os_username is "":
491 os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
492 if os_username is None:
493 os_username = ""
494
495 if os_password is "":
496 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
497 if os_password is None:
498 os_password = ""
499
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600500 invalid_req_states = [sub_state for sub_state in req_states
501 if sub_state not in valid_req_states]
502 if len(invalid_req_states) > 0:
503 error_message = "The following req_states are not supported:\n" +\
504 gp.sprint_var(invalid_req_states)
505 BuiltIn().fail(gp.sprint_error(error_message))
506
507 # Initialize all substate values supported by this function.
508 ping = 0
509 packet_loss = ''
510 uptime = ''
511 epoch_seconds = ''
Michael Walsh2b269de2017-10-09 11:18:11 -0500512 rest = ''
513 chassis = ''
514 requested_chassis = ''
515 bmc = ''
516 requested_bmc = ''
517 boot_progress = ''
518 operating_system = ''
519 host = ''
520 requested_host = ''
521 attempts_left = ''
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600522
Michael Walsh70369fd2016-11-22 11:25:57 -0600523 # Get the component states.
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600524 if 'ping' in req_states:
525 # See if the OS pings.
526 cmd_buf = "ping -c 1 -w 2 " + openbmc_host
527 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500528 gp.pissuing(cmd_buf)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600529 rc, out_buf = commands.getstatusoutput(cmd_buf)
530 if rc == 0:
531 ping = 1
532
533 if 'packet_loss' in req_states:
534 # See if the OS pings.
535 cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\
536 " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'"
537 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500538 gp.pissuing(cmd_buf)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600539 rc, out_buf = commands.getstatusoutput(cmd_buf)
540 if rc == 0:
541 packet_loss = out_buf.rstrip("\n")
542
Michael Walshe53e47a2017-06-30 17:03:24 -0500543 if 'uptime' in req_states:
Michael Walshfa765932017-10-13 14:07:22 -0500544 # Sometimes reading uptime results in a blank value. Call with
545 # wait_until_keyword_succeeds to ensure a non-blank value is obtained.
546 remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\
547 " && [ ! -z \"${uptime}\" ] && echo ${uptime}"
548 cmd_buf = ["BMC Execute Command", re.sub(r'\$', '\$', remote_cmd_buf),
549 'quiet=1']
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600550 if not quiet:
551 grp.rpissuing_keyword(cmd_buf)
Michael Walshfa765932017-10-13 14:07:22 -0500552 grp.rpissuing(remote_cmd_buf)
553 try:
554 stdout, stderr, rc =\
Michael Walshcaccd852017-11-01 17:58:41 -0500555 BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec",
Michael Walshfa765932017-10-13 14:07:22 -0500556 *cmd_buf)
Michael Walsh97df71c2017-03-27 14:33:24 -0500557 if rc == 0 and stderr == "":
558 uptime = stdout
Michael Walshfa765932017-10-13 14:07:22 -0500559 except AssertionError as my_assertion_error:
560 pass
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600561
Michael Walshe53e47a2017-06-30 17:03:24 -0500562 if 'epoch_seconds' in req_states:
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600563 date_cmd_buf = "date -u +%s"
564 if USE_BMC_EPOCH_TIME:
Michael Walshe53e47a2017-06-30 17:03:24 -0500565 cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}']
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600566 if not quiet:
567 grp.rpissuing_keyword(cmd_buf)
Michael Walsh97df71c2017-03-27 14:33:24 -0500568 status, ret_values = \
569 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
570 if status == "PASS":
571 stdout, stderr, rc = ret_values
572 if rc == 0 and stderr == "":
573 epoch_seconds = stdout.rstrip("\n")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600574 else:
575 shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf,
Michael Walshfa765932017-10-13 14:07:22 -0500576 quiet=quiet,
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600577 print_output=0)
578 if shell_rc == 0:
579 epoch_seconds = out_buf.rstrip("\n")
580
Michael Walsh56749222017-09-29 15:26:07 -0500581 master_req_rest = ['rest', 'host', 'requested_host', 'operating_system',
582 'attempts_left', 'boot_progress', 'chassis',
583 'requested_chassis' 'bmc' 'requested_bmc']
584
Michael Walshb95eb542017-03-31 09:39:20 -0500585 req_rest = [sub_state for sub_state in req_states if sub_state in
586 master_req_rest]
587 need_rest = (len(req_rest) > 0)
Michael Walsh56749222017-09-29 15:26:07 -0500588 state = DotDict()
589 if need_rest:
Michael Walsh940d6912017-10-27 12:32:33 -0500590 cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate",
Michael Walsh56749222017-09-29 15:26:07 -0500591 "quiet=${" + str(quiet) + "}"]
Michael Walsh341c21e2017-01-17 16:25:20 -0600592 grp.rdpissuing_keyword(cmd_buf)
Michael Walsh97df71c2017-03-27 14:33:24 -0500593 status, ret_values = \
594 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
595 if status == "PASS":
Michael Walsh56749222017-09-29 15:26:07 -0500596 state['rest'] = '1'
Michael Walsh341c21e2017-01-17 16:25:20 -0600597 else:
Michael Walsh56749222017-09-29 15:26:07 -0500598 state['rest'] = '0'
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600599
Michael Walsh2b269de2017-10-09 11:18:11 -0500600 if int(state['rest']):
601 for url_path in ret_values:
602 for attr_name in ret_values[url_path]:
603 # Create a state key value based on the attr_name.
604 if type(ret_values[url_path][attr_name]) is unicode:
605 ret_values[url_path][attr_name] = \
606 re.sub(r'.*\.', "",
607 ret_values[url_path][attr_name])
608 # Do some key name manipulations.
609 new_attr_name = re.sub(r'^Current|(State|Transition)$',
610 "", attr_name)
611 new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name)
612 new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1',
613 new_attr_name)
614 new_attr_name = new_attr_name.lower().lstrip("_")
615 new_attr_name = re.sub(r'power', r'chassis', new_attr_name)
616 if new_attr_name in req_states:
617 state[new_attr_name] = ret_values[url_path][attr_name]
Michael Walshb95eb542017-03-31 09:39:20 -0500618
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600619 for sub_state in req_states:
Michael Walsh56749222017-09-29 15:26:07 -0500620 if sub_state in state:
621 continue
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600622 if sub_state.startswith("os_"):
623 # We pass "os_" requests on to get_os_state.
624 continue
625 cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")"
626 exec(cmd_buf)
627
628 if os_host == "":
629 # The caller has not specified an os_host so as far as we're concerned,
630 # it doesn't exist.
631 return state
632
633 os_req_states = [sub_state for sub_state in req_states
634 if sub_state.startswith('os_')]
635
636 if len(os_req_states) > 0:
637 # The caller has specified an os_host and they have requested
638 # information on os substates.
639
640 # Based on the information gathered on bmc, we'll try to make a
641 # determination of whether the os is even up. We'll pass the result
642 # of that assessment to get_os_state to enhance performance.
643 os_up_match = DotDict()
644 for sub_state in master_os_up_match:
645 if sub_state in req_states:
646 os_up_match[sub_state] = master_os_up_match[sub_state]
Michael Walsh70369fd2016-11-22 11:25:57 -0600647 os_up = compare_states(state, os_up_match)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600648 os_state = get_os_state(os_host=os_host,
649 os_username=os_username,
650 os_password=os_password,
651 req_states=os_req_states,
652 os_up=os_up,
653 quiet=quiet)
654 # Append os_state dictionary to ours.
655 state.update(os_state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600656
657 return state
658
Michael Walsh70369fd2016-11-22 11:25:57 -0600659
Michael Walsh70369fd2016-11-22 11:25:57 -0600660def check_state(match_state,
661 invert=0,
662 print_string="",
663 openbmc_host="",
664 openbmc_username="",
665 openbmc_password="",
666 os_host="",
667 os_username="",
668 os_password="",
669 quiet=None):
670
671 r"""
672 Check that the Open BMC machine's composite state matches the specified
673 state. On success, this keyword returns the machine's composite state as a
674 dictionary.
675
676 Description of arguments:
677 match_state A dictionary whose key/value pairs are "state field"/
678 "state value". The state value is interpreted as a
679 regular expression. Example call from robot:
Michael Walsh341c21e2017-01-17 16:25:20 -0600680 ${match_state}= Create Dictionary chassis=^On$
681 ... bmc=^Ready$
Michael Walsh01975fa2017-08-20 20:51:36 -0500682 ... boot_progress=^OSStart$
Michael Walsh70369fd2016-11-22 11:25:57 -0600683 ${state}= Check State &{match_state}
684 invert If this flag is set, this function will succeed if the
685 states do NOT match.
686 print_string This function will print this string to the console prior
687 to getting the state.
688 openbmc_host The DNS name or IP address of the BMC.
689 This defaults to global ${OPENBMC_HOST}.
690 openbmc_username The username to be used to login to the BMC.
691 This defaults to global ${OPENBMC_USERNAME}.
692 openbmc_password The password to be used to login to the BMC.
693 This defaults to global ${OPENBMC_PASSWORD}.
694 os_host The DNS name or IP address of the operating system.
695 This defaults to global ${OS_HOST}.
696 os_username The username to be used to login to the OS.
697 This defaults to global ${OS_USERNAME}.
698 os_password The password to be used to login to the OS.
699 This defaults to global ${OS_PASSWORD}.
700 quiet Indicates whether status details should be written to the
701 console. Defaults to either global value of ${QUIET} or
702 to 1.
703 """
704
Michael Walsh619aa332017-04-12 15:56:51 -0500705 quiet = int(gp.get_var_value(quiet, 0))
Michael Walsh70369fd2016-11-22 11:25:57 -0600706
707 grp.rprint(print_string)
708
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600709 req_states = match_state.keys()
Michael Walsh70369fd2016-11-22 11:25:57 -0600710 # Initialize state.
711 state = get_state(openbmc_host=openbmc_host,
712 openbmc_username=openbmc_username,
713 openbmc_password=openbmc_password,
714 os_host=os_host,
715 os_username=os_username,
716 os_password=os_password,
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600717 req_states=req_states,
Michael Walsh70369fd2016-11-22 11:25:57 -0600718 quiet=quiet)
719 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500720 gp.print_var(state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600721
722 match = compare_states(state, match_state)
723
724 if invert and match:
725 fail_msg = "The current state of the machine matches the match" +\
726 " state:\n" + gp.sprint_varx("state", state)
727 BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
728 elif not invert and not match:
729 fail_msg = "The current state of the machine does NOT match the" +\
730 " match state:\n" +\
731 gp.sprint_varx("state", state)
732 BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
733
734 return state
735
Michael Walsh70369fd2016-11-22 11:25:57 -0600736
Michael Walshf893ba02017-01-10 10:28:05 -0600737def wait_state(match_state=(),
Michael Walsh70369fd2016-11-22 11:25:57 -0600738 wait_time="1 min",
739 interval="1 second",
740 invert=0,
741 openbmc_host="",
742 openbmc_username="",
743 openbmc_password="",
744 os_host="",
745 os_username="",
746 os_password="",
747 quiet=None):
748
749 r"""
750 Wait for the Open BMC machine's composite state to match the specified
751 state. On success, this keyword returns the machine's composite state as
752 a dictionary.
753
754 Description of arguments:
755 match_state A dictionary whose key/value pairs are "state field"/
756 "state value". See check_state (above) for details.
Michael Walsh619aa332017-04-12 15:56:51 -0500757 This value may also be any string accepted by
758 return_state_constant (e.g. "standby_match_state").
759 In such a case this function will call
760 return_state_constant to convert it to a proper
761 dictionary as described above.
Michael Walsh70369fd2016-11-22 11:25:57 -0600762 wait_time The total amount of time to wait for the desired state.
763 This value may be expressed in Robot Framework's time
764 format (e.g. 1 minute, 2 min 3 s, 4.5).
765 interval The amount of time between state checks.
766 This value may be expressed in Robot Framework's time
767 format (e.g. 1 minute, 2 min 3 s, 4.5).
768 invert If this flag is set, this function will for the state of
769 the machine to cease to match the match state.
770 openbmc_host The DNS name or IP address of the BMC.
771 This defaults to global ${OPENBMC_HOST}.
772 openbmc_username The username to be used to login to the BMC.
773 This defaults to global ${OPENBMC_USERNAME}.
774 openbmc_password The password to be used to login to the BMC.
775 This defaults to global ${OPENBMC_PASSWORD}.
776 os_host The DNS name or IP address of the operating system.
777 This defaults to global ${OS_HOST}.
778 os_username The username to be used to login to the OS.
779 This defaults to global ${OS_USERNAME}.
780 os_password The password to be used to login to the OS.
781 This defaults to global ${OS_PASSWORD}.
782 quiet Indicates whether status details should be written to the
783 console. Defaults to either global value of ${QUIET} or
784 to 1.
785 """
786
Michael Walsh619aa332017-04-12 15:56:51 -0500787 quiet = int(gp.get_var_value(quiet, 0))
788
789 if type(match_state) in (str, unicode):
790 match_state = return_state_constant(match_state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600791
792 if not quiet:
793 if invert:
794 alt_text = "cease to "
795 else:
796 alt_text = ""
Michael Walsh3eb50022017-03-21 11:27:30 -0500797 gp.print_timen("Checking every " + str(interval) + " for up to " +
798 str(wait_time) + " for the state of the machine to " +
799 alt_text + "match the state shown below.")
800 gp.print_var(match_state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600801
Michael Walshf893ba02017-01-10 10:28:05 -0600802 if quiet:
Michael Walsh341c21e2017-01-17 16:25:20 -0600803 print_string = ""
Michael Walshf893ba02017-01-10 10:28:05 -0600804 else:
Michael Walsh341c21e2017-01-17 16:25:20 -0600805 print_string = "#"
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600806
807 debug = int(BuiltIn().get_variable_value("${debug}", "0"))
808 if debug:
809 # In debug we print state so no need to print the "#".
810 print_string = ""
811 check_state_quiet = 1 - debug
Michael Walsh70369fd2016-11-22 11:25:57 -0600812 cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}",
Michael Walshf893ba02017-01-10 10:28:05 -0600813 "print_string=" + print_string, "openbmc_host=" + openbmc_host,
Michael Walsh70369fd2016-11-22 11:25:57 -0600814 "openbmc_username=" + openbmc_username,
815 "openbmc_password=" + openbmc_password, "os_host=" + os_host,
816 "os_username=" + os_username, "os_password=" + os_password,
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600817 "quiet=${" + str(check_state_quiet) + "}"]
Michael Walsh70369fd2016-11-22 11:25:57 -0600818 grp.rdpissuing_keyword(cmd_buf)
Michael Walsh619aa332017-04-12 15:56:51 -0500819 try:
820 state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval,
821 *cmd_buf)
822 except AssertionError as my_assertion_error:
823 gp.printn()
824 message = my_assertion_error.args[0]
825 BuiltIn().fail(message)
826
Michael Walsh70369fd2016-11-22 11:25:57 -0600827 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500828 gp.printn()
Michael Walsh70369fd2016-11-22 11:25:57 -0600829 if invert:
Michael Walsh3eb50022017-03-21 11:27:30 -0500830 gp.print_timen("The states no longer match:")
Michael Walsh70369fd2016-11-22 11:25:57 -0600831 else:
Michael Walsh3eb50022017-03-21 11:27:30 -0500832 gp.print_timen("The states match:")
833 gp.print_var(state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600834
835 return state
836
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600837
Michael Walsh619aa332017-04-12 15:56:51 -0500838def wait_for_comm_cycle(start_boot_seconds,
839 quiet=None):
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600840
841 r"""
842 Wait for communications to the BMC to stop working and then resume working.
843 This function is useful when you have initiated some kind of reboot.
844
845 Description of arguments:
846 start_boot_seconds The time that the boot test started. The format is the
847 epoch time in seconds, i.e. the number of seconds since
848 1970-01-01 00:00:00 UTC. This value should be obtained
849 from the BMC so that it is not dependent on any kind of
850 synchronization between this machine and the target BMC
851 This will allow this program to work correctly even in
852 a simulated environment. This value should be obtained
853 by the caller prior to initiating a reboot. It can be
854 obtained as follows:
855 state = st.get_state(req_states=['epoch_seconds'])
856 """
857
Michael Walsh619aa332017-04-12 15:56:51 -0500858 quiet = int(gp.get_var_value(quiet, 0))
859
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600860 # Validate parms.
861 error_message = gv.svalid_integer(start_boot_seconds,
862 var_name="start_boot_seconds")
863 if error_message != "":
864 BuiltIn().fail(gp.sprint_error(error_message))
865
866 match_state = anchor_state(DotDict([('packet_loss', '100')]))
867 # Wait for 100% packet loss trying to ping machine.
Michael Walsh619aa332017-04-12 15:56:51 -0500868 wait_state(match_state, wait_time="8 mins", interval="0 seconds")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600869
870 match_state['packet_loss'] = '^0$'
871 # Wait for 0% packet loss trying to ping machine.
Michael Walsh619aa332017-04-12 15:56:51 -0500872 wait_state(match_state, wait_time="8 mins", interval="0 seconds")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600873
874 # Get the uptime and epoch seconds for comparisons. We want to be sure
875 # that the uptime is less than the elapsed boot time. Further proof that
876 # a reboot has indeed occurred (vs random network instability giving a
877 # false positive.
Michael Walsh01975fa2017-08-20 20:51:36 -0500878 state = get_state(req_states=['uptime', 'epoch_seconds'], quiet=quiet)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600879
880 elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds
Michael Walsh619aa332017-04-12 15:56:51 -0500881 gp.qprint_var(elapsed_boot_time)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600882 if int(float(state['uptime'])) < elapsed_boot_time:
883 uptime = state['uptime']
Michael Walsh619aa332017-04-12 15:56:51 -0500884 gp.qprint_var(uptime)
885 gp.qprint_timen("The uptime is less than the elapsed boot time," +
886 " as expected.")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600887 else:
888 error_message = "The uptime is greater than the elapsed boot time," +\
889 " which is unexpected:\n" +\
890 gp.sprint_var(start_boot_seconds) +\
891 gp.sprint_var(state)
892 BuiltIn().fail(gp.sprint_error(error_message))
893
Michael Walsh619aa332017-04-12 15:56:51 -0500894 gp.qprint_timen("Verifying that REST API interface is working.")
Michael Walshb95eb542017-03-31 09:39:20 -0500895 match_state = DotDict([('rest', '^1$')])
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600896 state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")
897