blob: 6c1c6d68bb745e8d4ff6b7ae7a2b63ccbd27de2b [file] [log] [blame]
Michael Walsh70369fd2016-11-22 11:25:57 -06001#!/usr/bin/env python
2
3r"""
4This module contains functions having to do with machine state: get_state,
5check_state, wait_state, etc.
6
7The 'State' is a composite of many pieces of data. Therefore, the functions
8in this module define state as an ordered dictionary. Here is an example of
9some test output showing machine state:
10
Michael Walsh341c21e2017-01-17 16:25:20 -060011default_state:
Michael Walsh65b12542017-02-03 15:34:38 -060012 default_state[chassis]: On
Michael Walsh01975fa2017-08-20 20:51:36 -050013 default_state[boot_progress]: OSStart
Michael Walsh56749222017-09-29 15:26:07 -050014 default_state[operating_system]: BootComplete
Michael Walsh65b12542017-02-03 15:34:38 -060015 default_state[host]: Running
Michael Walsh341c21e2017-01-17 16:25:20 -060016 default_state[os_ping]: 1
17 default_state[os_login]: 1
18 default_state[os_run_cmd]: 1
Michael Walsh70369fd2016-11-22 11:25:57 -060019
20Different users may very well have different needs when inquiring about
Michael Walsh8fae6ea2017-02-20 16:14:44 -060021state. Support for new pieces of state information may be added to this
22module as needed.
Michael Walsh70369fd2016-11-22 11:25:57 -060023
24By using the wait_state function, a caller can start a boot and then wait for
25a precisely defined state to indicate that the boot has succeeded. If
26the boot fails, they can see exactly why by looking at the current state as
27compared with the expected state.
28"""
29
30import gen_print as gp
Michael Walsh70369fd2016-11-22 11:25:57 -060031import gen_valid as gv
Michael Walsh16cbb7f2017-02-02 15:54:16 -060032import gen_robot_utils as gru
Michael Walsh8fae6ea2017-02-20 16:14:44 -060033import gen_cmd as gc
Michael Walsh6a9bd142018-07-24 16:10:29 -050034import bmc_ssh_utils as bsu
Michael Walsh70369fd2016-11-22 11:25:57 -060035
Michael Walsh70369fd2016-11-22 11:25:57 -060036from robot.libraries.BuiltIn import BuiltIn
Michael Walsh341c21e2017-01-17 16:25:20 -060037from robot.utils import DotDict
Michael Walsh70369fd2016-11-22 11:25:57 -060038
39import re
Michael Walsh341c21e2017-01-17 16:25:20 -060040import os
Michael Walsh56749222017-09-29 15:26:07 -050041import sys
42import imp
43
Michael Walsh70369fd2016-11-22 11:25:57 -060044
Michael Walsh5a5868a2018-10-31 15:20:04 -050045# NOTE: Avoid importing utils.robot because utils.robot imports state.py
46# (indirectly) which will cause failures.
47gru.my_import_resource("rest_client.robot")
Michael Walsh70369fd2016-11-22 11:25:57 -060048
Michael Walsh56749222017-09-29 15:26:07 -050049base_path = os.path.dirname(os.path.dirname(
50 imp.find_module("gen_robot_print")[1])) + os.sep
51sys.path.append(base_path + "data/")
Michael Walsh56749222017-09-29 15:26:07 -050052
Michael Walsh940d6912017-10-27 12:32:33 -050053# Previously, I had this coded:
54# import variables as var
55# However, we ran into a problem where a robot program did this...
56# Variables ../../lib/ras/variables.py
57# Prior to doing this...
58# Library ../lib/state.py
59
60# This caused the wrong variables.py file to be selected. Attempts to fix this
61# have failed so far. For the moment, we will hard-code the value we need from
62# the file.
63
64SYSTEM_STATE_URI = "/xyz/openbmc_project/state/"
Michael Walsh56749222017-09-29 15:26:07 -050065
Michael Walsh8fae6ea2017-02-20 16:14:44 -060066# The BMC code has recently been changed as far as what states are defined and
67# what the state values can be. This module now has a means of processing both
68# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e.
Michael Walsh16cbb7f2017-02-02 15:54:16 -060069# OBMC_STATES_VERSION = 1).
Michael Walsh341c21e2017-01-17 16:25:20 -060070# The caller can set environment variable OBMC_STATES_VERSION to dictate
71# whether we're processing old or new style states. If OBMC_STATES_VERSION is
Michael Walsh8fae6ea2017-02-20 16:14:44 -060072# not set it will default to 1.
Michael Walsh341c21e2017-01-17 16:25:20 -060073
Michael Walsh619aa332017-04-12 15:56:51 -050074# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old
75# that it is no longer worthwhile to maintain. The OBMC_STATES_VERSION 0 code
76# is being removed but the OBMC_STATES_VERSION value will stay for now in the
77# event that it is needed in the future.
78
Michael Walsh8fae6ea2017-02-20 16:14:44 -060079OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1))
Michael Walsh341c21e2017-01-17 16:25:20 -060080
Michael Walsh619aa332017-04-12 15:56:51 -050081# When a user calls get_state w/o specifying req_states, default_req_states
82# is used as its value.
83default_req_states = ['rest',
84 'chassis',
85 'bmc',
86 'boot_progress',
Michael Walsh56749222017-09-29 15:26:07 -050087 'operating_system',
Michael Walsh619aa332017-04-12 15:56:51 -050088 'host',
89 'os_ping',
90 'os_login',
91 'os_run_cmd']
Michael Walsh8fae6ea2017-02-20 16:14:44 -060092
Michael Walsh619aa332017-04-12 15:56:51 -050093# valid_req_states is a list of sub states supported by the get_state function.
94# valid_req_states, default_req_states and master_os_up_match are used by the
95# get_state function.
96valid_req_states = ['ping',
97 'packet_loss',
98 'uptime',
99 'epoch_seconds',
100 'rest',
101 'chassis',
Michael Walsh56749222017-09-29 15:26:07 -0500102 'requested_chassis',
Michael Walsh619aa332017-04-12 15:56:51 -0500103 'bmc',
Michael Walsh56749222017-09-29 15:26:07 -0500104 'requested_bmc',
Michael Walsh619aa332017-04-12 15:56:51 -0500105 'boot_progress',
Michael Walsh56749222017-09-29 15:26:07 -0500106 'operating_system',
Michael Walsh619aa332017-04-12 15:56:51 -0500107 'host',
Michael Walsh56749222017-09-29 15:26:07 -0500108 'requested_host',
109 'attempts_left',
Michael Walsh619aa332017-04-12 15:56:51 -0500110 'os_ping',
111 'os_login',
112 'os_run_cmd']
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600113
114# valid_os_req_states and default_os_req_states are used by the os_get_state
115# function.
116# valid_os_req_states is a list of state information supported by the
117# get_os_state function.
118valid_os_req_states = ['os_ping',
119 'os_login',
120 'os_run_cmd']
121# When a user calls get_os_state w/o specifying req_states,
122# default_os_req_states is used as its value.
123default_os_req_states = ['os_ping',
124 'os_login',
125 'os_run_cmd']
126
127# Presently, some BMCs appear to not keep time very well. This environment
128# variable directs the get_state function to use either the BMC's epoch time
129# or the local epoch time.
130USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0))
Michael Walsh341c21e2017-01-17 16:25:20 -0600131
Michael Walsh619aa332017-04-12 15:56:51 -0500132# Useful state constant definition(s).
Michael Walsh619aa332017-04-12 15:56:51 -0500133# default_state is an initial value which may be of use to callers.
134default_state = DotDict([('rest', '1'),
135 ('chassis', 'On'),
136 ('bmc', 'Ready'),
Michael Walsh01975fa2017-08-20 20:51:36 -0500137 ('boot_progress', 'OSStart'),
Michael Walsh56749222017-09-29 15:26:07 -0500138 ('operating_system', 'BootComplete'),
Michael Walsh619aa332017-04-12 15:56:51 -0500139 ('host', 'Running'),
140 ('os_ping', '1'),
141 ('os_login', '1'),
142 ('os_run_cmd', '1')])
143
Michael Walsh7dc885b2018-03-14 17:51:59 -0500144# A match state for checking that the system is at "standby".
145standby_match_state = DotDict([('rest', '^1$'),
146 ('chassis', '^Off$'),
147 ('bmc', '^Ready$'),
George Keishing6a69d262019-04-03 03:45:27 -0500148 ('boot_progress', '^Off|Unspecified$'),
149 ('operating_system', '^Inactive$'),
150 ('host', '^Off$')])
Michael Walsh7dc885b2018-03-14 17:51:59 -0500151
152# A match state for checking that the system is at "os running".
153os_running_match_state = DotDict([('chassis', '^On$'),
154 ('bmc', '^Ready$'),
155 ('boot_progress',
156 'FW Progress, Starting OS|OSStart'),
157 ('operating_system', 'BootComplete'),
158 ('host', '^Running$'),
159 ('os_ping', '^1$'),
160 ('os_login', '^1$'),
161 ('os_run_cmd', '^1$')])
162
Michael Walsh619aa332017-04-12 15:56:51 -0500163# A master dictionary to determine whether the os may be up.
164master_os_up_match = DotDict([('chassis', '^On$'),
165 ('bmc', '^Ready$'),
166 ('boot_progress',
Michael Walsh01975fa2017-08-20 20:51:36 -0500167 'FW Progress, Starting OS|OSStart'),
Michael Walsh56749222017-09-29 15:26:07 -0500168 ('operating_system', 'BootComplete'),
Michael Walsh192d5e72018-11-01 14:09:11 -0500169 ('host', '^Running|Quiesced$')])
Michael Walsh619aa332017-04-12 15:56:51 -0500170
Michael Walsh45ca6e42017-09-14 17:29:12 -0500171invalid_state_match = DotDict([('rest', '^$'),
172 ('chassis', '^$'),
173 ('bmc', '^$'),
174 ('boot_progress', '^$'),
Michael Walsh56749222017-09-29 15:26:07 -0500175 ('operating_system', '^$'),
Michael Walsh45ca6e42017-09-14 17:29:12 -0500176 ('host', '^$')])
177
Michael Walsh341c21e2017-01-17 16:25:20 -0600178
George Keishing36efbc02018-12-12 10:18:23 -0600179def return_state_constant(state_name='default_state'):
Michael Walsh619aa332017-04-12 15:56:51 -0500180 r"""
Michael Walsh7dc885b2018-03-14 17:51:59 -0500181 Return the named state dictionary constant.
Michael Walsh619aa332017-04-12 15:56:51 -0500182 """
183
George Keishing36efbc02018-12-12 10:18:23 -0600184 return eval(state_name)
Michael Walsh619aa332017-04-12 15:56:51 -0500185
Michael Walsh619aa332017-04-12 15:56:51 -0500186
Michael Walsh70369fd2016-11-22 11:25:57 -0600187def anchor_state(state):
Michael Walsh70369fd2016-11-22 11:25:57 -0600188 r"""
189 Add regular expression anchors ("^" and "$") to the beginning and end of
190 each item in the state dictionary passed in. Return the resulting
191 dictionary.
192
193 Description of Arguments:
194 state A dictionary such as the one returned by the get_state()
195 function.
196 """
197
Michael Walsh2ce067a2017-02-27 14:24:07 -0600198 anchored_state = state.copy()
Michael Walsh70369fd2016-11-22 11:25:57 -0600199 for key, match_state_value in anchored_state.items():
200 anchored_state[key] = "^" + str(anchored_state[key]) + "$"
201
202 return anchored_state
203
Michael Walsh70369fd2016-11-22 11:25:57 -0600204
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600205def strip_anchor_state(state):
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600206 r"""
207 Strip regular expression anchors ("^" and "$") from the beginning and end
208 of each item in the state dictionary passed in. Return the resulting
209 dictionary.
210
211 Description of Arguments:
212 state A dictionary such as the one returned by the get_state()
213 function.
214 """
215
Michael Walsh2ce067a2017-02-27 14:24:07 -0600216 stripped_state = state.copy()
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600217 for key, match_state_value in stripped_state.items():
218 stripped_state[key] = stripped_state[key].strip("^$")
219
220 return stripped_state
221
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600222
Michael Walsh70369fd2016-11-22 11:25:57 -0600223def compare_states(state,
Michael Walsh45ca6e42017-09-14 17:29:12 -0500224 match_state,
225 match_type='and'):
Michael Walsh70369fd2016-11-22 11:25:57 -0600226 r"""
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600227 Compare 2 state dictionaries. Return True if they match and False if they
Michael Walsh70369fd2016-11-22 11:25:57 -0600228 don't. Note that the match_state dictionary does not need to have an entry
229 corresponding to each entry in the state dictionary. But for each entry
230 that it does have, the corresponding state entry will be checked for a
231 match.
232
233 Description of arguments:
234 state A state dictionary such as the one returned by the
235 get_state function.
236 match_state A dictionary whose key/value pairs are "state field"/
237 "state value". The state value is interpreted as a
238 regular expression. Every value in this dictionary is
Michael Walsh45ca6e42017-09-14 17:29:12 -0500239 considered. When match_type is 'and', if each and every
240 comparison matches, the two dictionaries are considered to
241 be matching. If match_type is 'or', if any two of the
242 elements compared match, the two dictionaries are
243 considered to be matching.
Michael Walsh7dc885b2018-03-14 17:51:59 -0500244 This value may also be any string accepted by
245 return_state_constant (e.g. "standby_match_state").
246 In such a case this function will call
247 return_state_constant to convert it to a proper
248 dictionary as described above.
Michael Walsh45ca6e42017-09-14 17:29:12 -0500249 match_type This may be 'and' or 'or'.
Michael Walsh70369fd2016-11-22 11:25:57 -0600250 """
251
Michael Walsh45ca6e42017-09-14 17:29:12 -0500252 error_message = gv.svalid_value(match_type, var_name="match_type",
253 valid_values=['and', 'or'])
254 if error_message != "":
255 BuiltIn().fail(gp.sprint_error(error_message))
256
George Keishing36efbc02018-12-12 10:18:23 -0600257 try:
Michael Walsh7dc885b2018-03-14 17:51:59 -0500258 match_state = return_state_constant(match_state)
George Keishing36efbc02018-12-12 10:18:23 -0600259 except TypeError:
260 pass
Michael Walsh7dc885b2018-03-14 17:51:59 -0500261
Michael Walsh45ca6e42017-09-14 17:29:12 -0500262 default_match = (match_type == 'and')
Michael Walsh70369fd2016-11-22 11:25:57 -0600263 for key, match_state_value in match_state.items():
Michael Walsh97df71c2017-03-27 14:33:24 -0500264 # Blank match_state_value means "don't care".
265 if match_state_value == "":
266 continue
Michael Walsh70369fd2016-11-22 11:25:57 -0600267 try:
Michael Walsh45ca6e42017-09-14 17:29:12 -0500268 match = (re.match(match_state_value, str(state[key])) is not None)
Michael Walsh70369fd2016-11-22 11:25:57 -0600269 except KeyError:
270 match = False
Michael Walsh70369fd2016-11-22 11:25:57 -0600271
Michael Walsh45ca6e42017-09-14 17:29:12 -0500272 if match != default_match:
273 return match
274
275 return default_match
Michael Walsh70369fd2016-11-22 11:25:57 -0600276
Michael Walsh70369fd2016-11-22 11:25:57 -0600277
Michael Walsh70369fd2016-11-22 11:25:57 -0600278def get_os_state(os_host="",
279 os_username="",
280 os_password="",
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600281 req_states=default_os_req_states,
282 os_up=True,
Michael Walsh70369fd2016-11-22 11:25:57 -0600283 quiet=None):
Michael Walsh70369fd2016-11-22 11:25:57 -0600284 r"""
285 Get component states for the operating system such as ping, login,
286 etc, put them into a dictionary and return them to the caller.
287
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600288 Note that all substate values are strings.
289
Michael Walsh70369fd2016-11-22 11:25:57 -0600290 Description of arguments:
291 os_host The DNS name or IP address of the operating system.
292 This defaults to global ${OS_HOST}.
293 os_username The username to be used to login to the OS.
294 This defaults to global ${OS_USERNAME}.
295 os_password The password to be used to login to the OS.
296 This defaults to global ${OS_PASSWORD}.
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600297 req_states This is a list of states whose values are being requested by
298 the caller.
299 os_up If the caller knows that the os can't possibly be up, it can
300 improve performance by passing os_up=False. This function
301 will then simply return default values for all requested os
302 sub states.
Michael Walsh70369fd2016-11-22 11:25:57 -0600303 quiet Indicates whether status details (e.g. curl commands) should
304 be written to the console.
305 Defaults to either global value of ${QUIET} or to 1.
306 """
307
Michael Walsh619aa332017-04-12 15:56:51 -0500308 quiet = int(gp.get_var_value(quiet, 0))
Michael Walsh70369fd2016-11-22 11:25:57 -0600309
310 # Set parm defaults where necessary and validate all parms.
311 if os_host == "":
312 os_host = BuiltIn().get_variable_value("${OS_HOST}")
313 error_message = gv.svalid_value(os_host, var_name="os_host",
314 invalid_values=[None, ""])
315 if error_message != "":
316 BuiltIn().fail(gp.sprint_error(error_message))
317
318 if os_username == "":
319 os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
320 error_message = gv.svalid_value(os_username, var_name="os_username",
321 invalid_values=[None, ""])
322 if error_message != "":
323 BuiltIn().fail(gp.sprint_error(error_message))
324
325 if os_password == "":
326 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
327 error_message = gv.svalid_value(os_password, var_name="os_password",
328 invalid_values=[None, ""])
329 if error_message != "":
330 BuiltIn().fail(gp.sprint_error(error_message))
331
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600332 invalid_req_states = [sub_state for sub_state in req_states
333 if sub_state not in valid_os_req_states]
334 if len(invalid_req_states) > 0:
335 error_message = "The following req_states are not supported:\n" +\
336 gp.sprint_var(invalid_req_states)
337 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh70369fd2016-11-22 11:25:57 -0600338
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600339 # Initialize all substate values supported by this function.
340 os_ping = 0
341 os_login = 0
342 os_run_cmd = 0
Michael Walsh70369fd2016-11-22 11:25:57 -0600343
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600344 if os_up:
345 if 'os_ping' in req_states:
346 # See if the OS pings.
George Keishing36efbc02018-12-12 10:18:23 -0600347 rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + os_host,
348 print_output=0, show_err=0,
349 ignore_err=1)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600350 if rc == 0:
351 os_ping = 1
Michael Walsh70369fd2016-11-22 11:25:57 -0600352
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600353 # Programming note: All attributes which do not require an ssh login
354 # should have been processed by this point.
355 master_req_login = ['os_login', 'os_run_cmd']
356 req_login = [sub_state for sub_state in req_states if sub_state in
357 master_req_login]
Michael Walsh97df71c2017-03-27 14:33:24 -0500358 must_login = (len(req_login) > 0)
Michael Walsh70369fd2016-11-22 11:25:57 -0600359
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600360 if must_login:
Michael Walsh6a9bd142018-07-24 16:10:29 -0500361 output, stderr, rc = bsu.os_execute_command("uptime", quiet=quiet,
Michael Walsh7fc33972018-08-07 14:55:03 -0500362 ignore_err=1,
363 time_out=20)
Michael Walsh6a9bd142018-07-24 16:10:29 -0500364 if rc == 0:
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600365 os_login = 1
Michael Walsh6a9bd142018-07-24 16:10:29 -0500366 os_run_cmd = 1
Michael Walsh3eb50022017-03-21 11:27:30 -0500367 else:
Michael Walsh6a9bd142018-07-24 16:10:29 -0500368 gp.dprint_vars(output, stderr)
369 gp.dprint_vars(rc, 1)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600370
371 os_state = DotDict()
372 for sub_state in req_states:
373 cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")"
374 exec(cmd_buf)
Michael Walsh70369fd2016-11-22 11:25:57 -0600375
376 return os_state
377
Michael Walsh70369fd2016-11-22 11:25:57 -0600378
Michael Walsh70369fd2016-11-22 11:25:57 -0600379def get_state(openbmc_host="",
380 openbmc_username="",
381 openbmc_password="",
382 os_host="",
383 os_username="",
384 os_password="",
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600385 req_states=default_req_states,
Michael Walsh70369fd2016-11-22 11:25:57 -0600386 quiet=None):
Michael Walsh70369fd2016-11-22 11:25:57 -0600387 r"""
Michael Walsh619aa332017-04-12 15:56:51 -0500388 Get component states such as chassis state, bmc state, etc, put them into a
Michael Walsh70369fd2016-11-22 11:25:57 -0600389 dictionary and return them to the caller.
390
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600391 Note that all substate values are strings.
392
Michael Walsh70369fd2016-11-22 11:25:57 -0600393 Description of arguments:
394 openbmc_host The DNS name or IP address of the BMC.
395 This defaults to global ${OPENBMC_HOST}.
396 openbmc_username The username to be used to login to the BMC.
397 This defaults to global ${OPENBMC_USERNAME}.
398 openbmc_password The password to be used to login to the BMC.
399 This defaults to global ${OPENBMC_PASSWORD}.
400 os_host The DNS name or IP address of the operating system.
401 This defaults to global ${OS_HOST}.
402 os_username The username to be used to login to the OS.
403 This defaults to global ${OS_USERNAME}.
404 os_password The password to be used to login to the OS.
405 This defaults to global ${OS_PASSWORD}.
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600406 req_states This is a list of states whose values are being requested
407 by the caller.
Michael Walsh70369fd2016-11-22 11:25:57 -0600408 quiet Indicates whether status details (e.g. curl commands)
409 should be written to the console.
410 Defaults to either global value of ${QUIET} or to 1.
411 """
412
Michael Walsh619aa332017-04-12 15:56:51 -0500413 quiet = int(gp.get_var_value(quiet, 0))
Michael Walsh70369fd2016-11-22 11:25:57 -0600414
415 # Set parm defaults where necessary and validate all parms.
416 if openbmc_host == "":
417 openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
418 error_message = gv.svalid_value(openbmc_host,
419 var_name="openbmc_host",
420 invalid_values=[None, ""])
421 if error_message != "":
422 BuiltIn().fail(gp.sprint_error(error_message))
423
424 if openbmc_username == "":
425 openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}")
426 error_message = gv.svalid_value(openbmc_username,
427 var_name="openbmc_username",
428 invalid_values=[None, ""])
429 if error_message != "":
430 BuiltIn().fail(gp.sprint_error(error_message))
431
432 if openbmc_password == "":
433 openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}")
434 error_message = gv.svalid_value(openbmc_password,
435 var_name="openbmc_password",
436 invalid_values=[None, ""])
437 if error_message != "":
438 BuiltIn().fail(gp.sprint_error(error_message))
439
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600440 # NOTE: OS parms are optional.
Michael Walsh70369fd2016-11-22 11:25:57 -0600441 if os_host == "":
442 os_host = BuiltIn().get_variable_value("${OS_HOST}")
443 if os_host is None:
444 os_host = ""
445
446 if os_username is "":
447 os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
448 if os_username is None:
449 os_username = ""
450
451 if os_password is "":
452 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
453 if os_password is None:
454 os_password = ""
455
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600456 invalid_req_states = [sub_state for sub_state in req_states
457 if sub_state not in valid_req_states]
458 if len(invalid_req_states) > 0:
459 error_message = "The following req_states are not supported:\n" +\
460 gp.sprint_var(invalid_req_states)
461 BuiltIn().fail(gp.sprint_error(error_message))
462
463 # Initialize all substate values supported by this function.
464 ping = 0
465 packet_loss = ''
466 uptime = ''
467 epoch_seconds = ''
Michael Walsh2b269de2017-10-09 11:18:11 -0500468 rest = ''
469 chassis = ''
470 requested_chassis = ''
471 bmc = ''
472 requested_bmc = ''
473 boot_progress = ''
474 operating_system = ''
475 host = ''
476 requested_host = ''
477 attempts_left = ''
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600478
Michael Walsh70369fd2016-11-22 11:25:57 -0600479 # Get the component states.
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600480 if 'ping' in req_states:
481 # See if the OS pings.
George Keishing36efbc02018-12-12 10:18:23 -0600482 rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + openbmc_host,
483 print_output=0, show_err=0,
484 ignore_err=1)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600485 if rc == 0:
486 ping = 1
487
488 if 'packet_loss' in req_states:
489 # See if the OS pings.
490 cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\
491 " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'"
George Keishing36efbc02018-12-12 10:18:23 -0600492 rc, out_buf = gc.shell_cmd(cmd_buf,
493 print_output=0, show_err=0,
494 ignore_err=1)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600495 if rc == 0:
496 packet_loss = out_buf.rstrip("\n")
497
Michael Walshe53e47a2017-06-30 17:03:24 -0500498 if 'uptime' in req_states:
Michael Walshfa765932017-10-13 14:07:22 -0500499 # Sometimes reading uptime results in a blank value. Call with
500 # wait_until_keyword_succeeds to ensure a non-blank value is obtained.
501 remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\
502 " && [ ! -z \"${uptime}\" ] && echo ${uptime}"
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500503 cmd_buf = ["BMC Execute Command",
Michael Walsh888d85a2019-04-18 11:03:28 -0500504 re.sub('\\$', '\\$', remote_cmd_buf), 'quiet=1',
505 'test_mode=0']
506 gp.print_issuing(cmd_buf, 0)
507 gp.print_issuing(remote_cmd_buf, 0)
Michael Walshfa765932017-10-13 14:07:22 -0500508 try:
509 stdout, stderr, rc =\
Michael Walshcaccd852017-11-01 17:58:41 -0500510 BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec",
Michael Walshfa765932017-10-13 14:07:22 -0500511 *cmd_buf)
Michael Walsh97df71c2017-03-27 14:33:24 -0500512 if rc == 0 and stderr == "":
513 uptime = stdout
Michael Walshfa765932017-10-13 14:07:22 -0500514 except AssertionError as my_assertion_error:
515 pass
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600516
Michael Walshe53e47a2017-06-30 17:03:24 -0500517 if 'epoch_seconds' in req_states:
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600518 date_cmd_buf = "date -u +%s"
519 if USE_BMC_EPOCH_TIME:
Michael Walshe53e47a2017-06-30 17:03:24 -0500520 cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}']
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600521 if not quiet:
Michael Walshedb5c942019-03-28 12:40:50 -0500522 gp.print_issuing(cmd_buf)
Michael Walsh97df71c2017-03-27 14:33:24 -0500523 status, ret_values = \
524 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
525 if status == "PASS":
526 stdout, stderr, rc = ret_values
527 if rc == 0 and stderr == "":
528 epoch_seconds = stdout.rstrip("\n")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600529 else:
530 shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf,
Michael Walshfa765932017-10-13 14:07:22 -0500531 quiet=quiet,
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600532 print_output=0)
533 if shell_rc == 0:
534 epoch_seconds = out_buf.rstrip("\n")
535
Michael Walsh56749222017-09-29 15:26:07 -0500536 master_req_rest = ['rest', 'host', 'requested_host', 'operating_system',
537 'attempts_left', 'boot_progress', 'chassis',
538 'requested_chassis' 'bmc' 'requested_bmc']
539
Michael Walshb95eb542017-03-31 09:39:20 -0500540 req_rest = [sub_state for sub_state in req_states if sub_state in
541 master_req_rest]
542 need_rest = (len(req_rest) > 0)
Michael Walsh56749222017-09-29 15:26:07 -0500543 state = DotDict()
544 if need_rest:
Michael Walsh940d6912017-10-27 12:32:33 -0500545 cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate",
Michael Walsh56749222017-09-29 15:26:07 -0500546 "quiet=${" + str(quiet) + "}"]
Michael Walshedb5c942019-03-28 12:40:50 -0500547 gp.dprint_issuing(cmd_buf)
Michael Walsh97df71c2017-03-27 14:33:24 -0500548 status, ret_values = \
549 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
550 if status == "PASS":
Michael Walsh56749222017-09-29 15:26:07 -0500551 state['rest'] = '1'
Michael Walsh341c21e2017-01-17 16:25:20 -0600552 else:
Michael Walsh56749222017-09-29 15:26:07 -0500553 state['rest'] = '0'
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600554
Michael Walsh2b269de2017-10-09 11:18:11 -0500555 if int(state['rest']):
556 for url_path in ret_values:
557 for attr_name in ret_values[url_path]:
558 # Create a state key value based on the attr_name.
George Keishing36efbc02018-12-12 10:18:23 -0600559 try:
Michael Walsh2b269de2017-10-09 11:18:11 -0500560 ret_values[url_path][attr_name] = \
561 re.sub(r'.*\.', "",
562 ret_values[url_path][attr_name])
George Keishing36efbc02018-12-12 10:18:23 -0600563 except TypeError:
564 pass
Michael Walsh2b269de2017-10-09 11:18:11 -0500565 # Do some key name manipulations.
566 new_attr_name = re.sub(r'^Current|(State|Transition)$',
567 "", attr_name)
568 new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name)
569 new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1',
570 new_attr_name)
571 new_attr_name = new_attr_name.lower().lstrip("_")
572 new_attr_name = re.sub(r'power', r'chassis', new_attr_name)
573 if new_attr_name in req_states:
574 state[new_attr_name] = ret_values[url_path][attr_name]
Michael Walshb95eb542017-03-31 09:39:20 -0500575
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600576 for sub_state in req_states:
Michael Walsh56749222017-09-29 15:26:07 -0500577 if sub_state in state:
578 continue
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600579 if sub_state.startswith("os_"):
580 # We pass "os_" requests on to get_os_state.
581 continue
582 cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")"
583 exec(cmd_buf)
584
585 if os_host == "":
586 # The caller has not specified an os_host so as far as we're concerned,
587 # it doesn't exist.
588 return state
589
590 os_req_states = [sub_state for sub_state in req_states
591 if sub_state.startswith('os_')]
592
593 if len(os_req_states) > 0:
594 # The caller has specified an os_host and they have requested
595 # information on os substates.
596
597 # Based on the information gathered on bmc, we'll try to make a
598 # determination of whether the os is even up. We'll pass the result
599 # of that assessment to get_os_state to enhance performance.
600 os_up_match = DotDict()
601 for sub_state in master_os_up_match:
602 if sub_state in req_states:
603 os_up_match[sub_state] = master_os_up_match[sub_state]
Michael Walsh70369fd2016-11-22 11:25:57 -0600604 os_up = compare_states(state, os_up_match)
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600605 os_state = get_os_state(os_host=os_host,
606 os_username=os_username,
607 os_password=os_password,
608 req_states=os_req_states,
609 os_up=os_up,
610 quiet=quiet)
611 # Append os_state dictionary to ours.
612 state.update(os_state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600613
614 return state
615
Michael Walsh70369fd2016-11-22 11:25:57 -0600616
Michael Walshfd5a8682019-02-01 14:28:42 -0600617exit_wait_early_message = ""
618
619
620def set_exit_wait_early_message(value):
621 r"""
622 Set global exit_wait_early_message to the indicated value.
623
624 This is a mechanism by which the programmer can do an early exit from
625 wait_until_keyword_succeeds() based on some special condition.
626
627 Description of argument(s):
628 value The value to assign to the global
629 exit_wait_early_message.
630 """
631
632 global exit_wait_early_message
633 exit_wait_early_message = value
634
635
Michael Walsh70369fd2016-11-22 11:25:57 -0600636def check_state(match_state,
637 invert=0,
638 print_string="",
639 openbmc_host="",
640 openbmc_username="",
641 openbmc_password="",
642 os_host="",
643 os_username="",
644 os_password="",
645 quiet=None):
Michael Walsh70369fd2016-11-22 11:25:57 -0600646 r"""
647 Check that the Open BMC machine's composite state matches the specified
648 state. On success, this keyword returns the machine's composite state as a
649 dictionary.
650
651 Description of arguments:
652 match_state A dictionary whose key/value pairs are "state field"/
653 "state value". The state value is interpreted as a
654 regular expression. Example call from robot:
Michael Walsh341c21e2017-01-17 16:25:20 -0600655 ${match_state}= Create Dictionary chassis=^On$
656 ... bmc=^Ready$
Michael Walsh01975fa2017-08-20 20:51:36 -0500657 ... boot_progress=^OSStart$
Michael Walsh70369fd2016-11-22 11:25:57 -0600658 ${state}= Check State &{match_state}
659 invert If this flag is set, this function will succeed if the
660 states do NOT match.
661 print_string This function will print this string to the console prior
662 to getting the state.
663 openbmc_host The DNS name or IP address of the BMC.
664 This defaults to global ${OPENBMC_HOST}.
665 openbmc_username The username to be used to login to the BMC.
666 This defaults to global ${OPENBMC_USERNAME}.
667 openbmc_password The password to be used to login to the BMC.
668 This defaults to global ${OPENBMC_PASSWORD}.
669 os_host The DNS name or IP address of the operating system.
670 This defaults to global ${OS_HOST}.
671 os_username The username to be used to login to the OS.
672 This defaults to global ${OS_USERNAME}.
673 os_password The password to be used to login to the OS.
674 This defaults to global ${OS_PASSWORD}.
675 quiet Indicates whether status details should be written to the
676 console. Defaults to either global value of ${QUIET} or
677 to 1.
678 """
679
Michael Walsh619aa332017-04-12 15:56:51 -0500680 quiet = int(gp.get_var_value(quiet, 0))
Michael Walsh70369fd2016-11-22 11:25:57 -0600681
Michael Walshedb5c942019-03-28 12:40:50 -0500682 gp.gp_print(print_string)
683
684 try:
685 match_state = return_state_constant(match_state)
686 except TypeError:
687 pass
Michael Walsh70369fd2016-11-22 11:25:57 -0600688
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600689 req_states = match_state.keys()
Michael Walsh70369fd2016-11-22 11:25:57 -0600690 # Initialize state.
691 state = get_state(openbmc_host=openbmc_host,
692 openbmc_username=openbmc_username,
693 openbmc_password=openbmc_password,
694 os_host=os_host,
695 os_username=os_username,
696 os_password=os_password,
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600697 req_states=req_states,
Michael Walsh70369fd2016-11-22 11:25:57 -0600698 quiet=quiet)
699 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500700 gp.print_var(state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600701
Michael Walshfd5a8682019-02-01 14:28:42 -0600702 if exit_wait_early_message != "":
703 # The exit_wait_early_message has been set by a signal handler so we
704 # will exit "successfully". It is incumbent upon the calling function
705 # (e.g. wait_state) to check/clear this variable and to fail
706 # appropriately.
707 return state
708
Michael Walsh70369fd2016-11-22 11:25:57 -0600709 match = compare_states(state, match_state)
710
711 if invert and match:
712 fail_msg = "The current state of the machine matches the match" +\
713 " state:\n" + gp.sprint_varx("state", state)
714 BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
715 elif not invert and not match:
716 fail_msg = "The current state of the machine does NOT match the" +\
717 " match state:\n" +\
718 gp.sprint_varx("state", state)
719 BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
720
721 return state
722
Michael Walsh70369fd2016-11-22 11:25:57 -0600723
Michael Walshf893ba02017-01-10 10:28:05 -0600724def wait_state(match_state=(),
Michael Walsh70369fd2016-11-22 11:25:57 -0600725 wait_time="1 min",
726 interval="1 second",
727 invert=0,
728 openbmc_host="",
729 openbmc_username="",
730 openbmc_password="",
731 os_host="",
732 os_username="",
733 os_password="",
734 quiet=None):
Michael Walsh70369fd2016-11-22 11:25:57 -0600735 r"""
736 Wait for the Open BMC machine's composite state to match the specified
737 state. On success, this keyword returns the machine's composite state as
738 a dictionary.
739
740 Description of arguments:
741 match_state A dictionary whose key/value pairs are "state field"/
742 "state value". See check_state (above) for details.
Michael Walsh619aa332017-04-12 15:56:51 -0500743 This value may also be any string accepted by
744 return_state_constant (e.g. "standby_match_state").
745 In such a case this function will call
746 return_state_constant to convert it to a proper
747 dictionary as described above.
Michael Walsh70369fd2016-11-22 11:25:57 -0600748 wait_time The total amount of time to wait for the desired state.
749 This value may be expressed in Robot Framework's time
750 format (e.g. 1 minute, 2 min 3 s, 4.5).
751 interval The amount of time between state checks.
752 This value may be expressed in Robot Framework's time
753 format (e.g. 1 minute, 2 min 3 s, 4.5).
754 invert If this flag is set, this function will for the state of
755 the machine to cease to match the match state.
756 openbmc_host The DNS name or IP address of the BMC.
757 This defaults to global ${OPENBMC_HOST}.
758 openbmc_username The username to be used to login to the BMC.
759 This defaults to global ${OPENBMC_USERNAME}.
760 openbmc_password The password to be used to login to the BMC.
761 This defaults to global ${OPENBMC_PASSWORD}.
762 os_host The DNS name or IP address of the operating system.
763 This defaults to global ${OS_HOST}.
764 os_username The username to be used to login to the OS.
765 This defaults to global ${OS_USERNAME}.
766 os_password The password to be used to login to the OS.
767 This defaults to global ${OS_PASSWORD}.
768 quiet Indicates whether status details should be written to the
769 console. Defaults to either global value of ${QUIET} or
770 to 1.
771 """
772
Michael Walsh619aa332017-04-12 15:56:51 -0500773 quiet = int(gp.get_var_value(quiet, 0))
774
George Keishing36efbc02018-12-12 10:18:23 -0600775 try:
Michael Walsh619aa332017-04-12 15:56:51 -0500776 match_state = return_state_constant(match_state)
George Keishing36efbc02018-12-12 10:18:23 -0600777 except TypeError:
778 pass
Michael Walsh70369fd2016-11-22 11:25:57 -0600779
780 if not quiet:
781 if invert:
782 alt_text = "cease to "
783 else:
784 alt_text = ""
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500785 gp.print_timen("Checking every " + str(interval) + " for up to "
786 + str(wait_time) + " for the state of the machine to "
787 + alt_text + "match the state shown below.")
Michael Walsh3eb50022017-03-21 11:27:30 -0500788 gp.print_var(match_state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600789
Michael Walshf893ba02017-01-10 10:28:05 -0600790 if quiet:
Michael Walsh341c21e2017-01-17 16:25:20 -0600791 print_string = ""
Michael Walshf893ba02017-01-10 10:28:05 -0600792 else:
Michael Walsh341c21e2017-01-17 16:25:20 -0600793 print_string = "#"
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600794
795 debug = int(BuiltIn().get_variable_value("${debug}", "0"))
796 if debug:
797 # In debug we print state so no need to print the "#".
798 print_string = ""
799 check_state_quiet = 1 - debug
Michael Walsh70369fd2016-11-22 11:25:57 -0600800 cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}",
Michael Walshf893ba02017-01-10 10:28:05 -0600801 "print_string=" + print_string, "openbmc_host=" + openbmc_host,
Michael Walsh70369fd2016-11-22 11:25:57 -0600802 "openbmc_username=" + openbmc_username,
803 "openbmc_password=" + openbmc_password, "os_host=" + os_host,
804 "os_username=" + os_username, "os_password=" + os_password,
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600805 "quiet=${" + str(check_state_quiet) + "}"]
Michael Walshedb5c942019-03-28 12:40:50 -0500806 gp.dprint_issuing(cmd_buf)
Michael Walsh619aa332017-04-12 15:56:51 -0500807 try:
808 state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval,
809 *cmd_buf)
810 except AssertionError as my_assertion_error:
811 gp.printn()
812 message = my_assertion_error.args[0]
813 BuiltIn().fail(message)
814
Michael Walshfd5a8682019-02-01 14:28:42 -0600815 if exit_wait_early_message:
816 # The global exit_wait_early_message was set by a signal handler
817 # indicating that we should fail.
818 message = exit_wait_early_message
819 # Clear the exit_wait_early_message variable for future use.
820 set_exit_wait_early_message("")
821 BuiltIn().fail(gp.sprint_error(message))
822
Michael Walsh70369fd2016-11-22 11:25:57 -0600823 if not quiet:
Michael Walsh3eb50022017-03-21 11:27:30 -0500824 gp.printn()
Michael Walsh70369fd2016-11-22 11:25:57 -0600825 if invert:
Michael Walsh3eb50022017-03-21 11:27:30 -0500826 gp.print_timen("The states no longer match:")
Michael Walsh70369fd2016-11-22 11:25:57 -0600827 else:
Michael Walsh3eb50022017-03-21 11:27:30 -0500828 gp.print_timen("The states match:")
829 gp.print_var(state)
Michael Walsh70369fd2016-11-22 11:25:57 -0600830
831 return state
832
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600833
Michael Walsh619aa332017-04-12 15:56:51 -0500834def wait_for_comm_cycle(start_boot_seconds,
835 quiet=None):
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600836 r"""
837 Wait for communications to the BMC to stop working and then resume working.
838 This function is useful when you have initiated some kind of reboot.
839
840 Description of arguments:
841 start_boot_seconds The time that the boot test started. The format is the
842 epoch time in seconds, i.e. the number of seconds since
843 1970-01-01 00:00:00 UTC. This value should be obtained
844 from the BMC so that it is not dependent on any kind of
845 synchronization between this machine and the target BMC
846 This will allow this program to work correctly even in
847 a simulated environment. This value should be obtained
848 by the caller prior to initiating a reboot. It can be
849 obtained as follows:
850 state = st.get_state(req_states=['epoch_seconds'])
851 """
852
Michael Walsh619aa332017-04-12 15:56:51 -0500853 quiet = int(gp.get_var_value(quiet, 0))
854
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600855 # Validate parms.
856 error_message = gv.svalid_integer(start_boot_seconds,
857 var_name="start_boot_seconds")
858 if error_message != "":
859 BuiltIn().fail(gp.sprint_error(error_message))
860
861 match_state = anchor_state(DotDict([('packet_loss', '100')]))
862 # Wait for 100% packet loss trying to ping machine.
Michael Walsh619aa332017-04-12 15:56:51 -0500863 wait_state(match_state, wait_time="8 mins", interval="0 seconds")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600864
865 match_state['packet_loss'] = '^0$'
866 # Wait for 0% packet loss trying to ping machine.
Michael Walsh619aa332017-04-12 15:56:51 -0500867 wait_state(match_state, wait_time="8 mins", interval="0 seconds")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600868
869 # Get the uptime and epoch seconds for comparisons. We want to be sure
870 # that the uptime is less than the elapsed boot time. Further proof that
871 # a reboot has indeed occurred (vs random network instability giving a
Michael Walshc4c05d32018-05-29 11:39:39 -0500872 # false positive. We also use wait_state because the BMC may take a short
873 # while to be ready to process SSH requests.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500874 match_state = DotDict([('uptime', '^[0-9\\.]+$'),
Michael Walshc4c05d32018-05-29 11:39:39 -0500875 ('epoch_seconds', '^[0-9]+$')])
876 state = wait_state(match_state, wait_time="2 mins", interval="1 second")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600877
878 elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds
Michael Walsh619aa332017-04-12 15:56:51 -0500879 gp.qprint_var(elapsed_boot_time)
Michael Walshe77585a2017-12-14 11:02:28 -0600880 if state['uptime'] == "":
881 error_message = "Unable to obtain uptime from the BMC. BMC is not" +\
882 " communicating."
883 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600884 if int(float(state['uptime'])) < elapsed_boot_time:
885 uptime = state['uptime']
Michael Walsh619aa332017-04-12 15:56:51 -0500886 gp.qprint_var(uptime)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500887 gp.qprint_timen("The uptime is less than the elapsed boot time,"
888 + " as expected.")
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600889 else:
890 error_message = "The uptime is greater than the elapsed boot time," +\
891 " which is unexpected:\n" +\
892 gp.sprint_var(start_boot_seconds) +\
893 gp.sprint_var(state)
894 BuiltIn().fail(gp.sprint_error(error_message))
895
Michael Walsh619aa332017-04-12 15:56:51 -0500896 gp.qprint_timen("Verifying that REST API interface is working.")
Michael Walshb95eb542017-03-31 09:39:20 -0500897 match_state = DotDict([('rest', '^1$')])
Michael Walsh8fae6ea2017-02-20 16:14:44 -0600898 state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")