blob: f407569eadbbdcd5acab3f4f0df12d504d53a486 [file] [log] [blame]
Michael Walsh0bbd8602016-11-22 11:31:49 -06001#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
Michael Walsh0b93fbf2017-03-02 14:42:41 -06007import os
8import imp
9import time
10import glob
11import random
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050012import re
Michael Walsh0b93fbf2017-03-02 14:42:41 -060013import cPickle as pickle
Michael Walshdc80d672017-05-09 12:58:32 -050014import socket
Michael Walsh0b93fbf2017-03-02 14:42:41 -060015
16from robot.utils import DotDict
17from robot.libraries.BuiltIn import BuiltIn
18
Michael Walsh6741f742017-02-20 16:16:38 -060019from boot_data import *
Michael Walshc9116812017-03-10 14:23:06 -060020import gen_print as gp
Michael Walsh0bbd8602016-11-22 11:31:49 -060021import gen_robot_print as grp
Michael Walsh55302292017-01-10 11:43:02 -060022import gen_robot_plug_in as grpi
Michael Walsh6741f742017-02-20 16:16:38 -060023import gen_robot_valid as grv
24import gen_misc as gm
25import gen_cmd as gc
Michael Walshb5839d02017-04-12 16:11:20 -050026import gen_robot_keyword as grk
Michael Walsh55302292017-01-10 11:43:02 -060027import state as st
Michael Walsh0bbd8602016-11-22 11:31:49 -060028
Michael Walsh0b93fbf2017-03-02 14:42:41 -060029base_path = os.path.dirname(os.path.dirname(
30 imp.find_module("gen_robot_print")[1])) +\
Michael Walshc9116812017-03-10 14:23:06 -060031 os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060032sys.path.append(base_path + "extended/")
33import run_keyword as rk
Michael Walsh0bbd8602016-11-22 11:31:49 -060034
Michael Walshe1e26442017-03-06 17:50:07 -060035# Setting master_pid correctly influences the behavior of plug-ins like
36# DB_Logging
37program_pid = os.getpid()
38master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
39
Michael Walshb5839d02017-04-12 16:11:20 -050040# Set up boot data structures.
41boot_table = create_boot_table()
42valid_boot_types = create_valid_boot_list(boot_table)
Michael Walsh0b93fbf2017-03-02 14:42:41 -060043
Michael Walsh6741f742017-02-20 16:16:38 -060044boot_lists = read_boot_lists()
45last_ten = []
Michael Walsh6741f742017-02-20 16:16:38 -060046
47state = st.return_default_state()
48cp_setup_called = 0
49next_boot = ""
50base_tool_dir_path = os.path.normpath(os.environ.get(
51 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
Michael Walshb5839d02017-04-12 16:11:20 -050052
Michael Walsh6741f742017-02-20 16:16:38 -060053ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
Michael Walsh6741f742017-02-20 16:16:38 -060054boot_success = 0
Michael Walsh6741f742017-02-20 16:16:38 -060055status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
56if status_dir_path != "":
57 status_dir_path = os.path.normpath(status_dir_path) + os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060058default_power_on = "REST Power On"
59default_power_off = "REST Power Off"
Michael Walsh6741f742017-02-20 16:16:38 -060060boot_count = 0
Michael Walsh0bbd8602016-11-22 11:31:49 -060061
Michael Walsh85678942017-03-27 14:34:22 -050062LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
63
64
65###############################################################################
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050066def process_host(host,
67 host_var_name=""):
68
69 r"""
70 Process a host by getting the associated host name and IP address and
71 setting them in global variables.
72
73 If the caller does not pass the host_var_name, this function will try to
74 figure out the name of the variable used by the caller for the host parm.
75 Callers are advised to explicitly specify the host_var_name when calling
76 with an exec command. In such cases, the get_arg_name cannot figure out
77 the host variable name.
78
79 This function will then create similar global variable names by
80 removing "_host" and appending "_host_name" or "_ip" to the host variable
81 name.
82
83 Example:
84
85 If a call is made like this:
86 process_host(openbmc_host)
87
88 Global variables openbmc_host_name and openbmc_ip will be set.
89
90 Description of argument(s):
91 host A host name or IP. The name of the variable used should
92 have a suffix of "_host".
93 host_var_name The name of the variable being used as the host parm.
94 """
95
96 if host_var_name == "":
97 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
98
99 host_name_var_name = re.sub("host", "host_name", host_var_name)
100 ip_var_name = re.sub("host", "ip", host_var_name)
101 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
102 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
103 host + "')"
104 exec(cmd_buf)
105
106###############################################################################
107
108
109###############################################################################
Michael Walshb5839d02017-04-12 16:11:20 -0500110def process_pgm_parms():
111
112 r"""
113 Process the program parameters by assigning them all to corresponding
114 globals. Also, set some global values that depend on program parameters.
115 """
116
117 # Program parameter processing.
118 # Assign all program parms to python variables which are global to this
119 # module.
120
121 global parm_list
122 parm_list = BuiltIn().get_variable_value("${parm_list}")
123 # The following subset of parms should be processed as integers.
124 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
125 'boot_fail_threshold', 'quiet', 'test_mode', 'debug']
126 for parm in parm_list:
127 if parm in int_list:
128 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
129 "}\", \"0\"))"
130 else:
131 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
132 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
133 exec(cmd_buf)
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500134 if re.match(r".*_host$", parm):
135 cmd_buf = "process_host(" + parm + ", '" + parm + "')"
136 exec(cmd_buf)
137 if re.match(r".*_password$", parm):
138 # Register the value of any parm whose name ends in _password.
139 # This will cause the print functions to replace passwords with
140 # asterisks in the output.
141 cmd_buf = "gp.register_passwords(" + parm + ")"
142 exec(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500143
144 global ffdc_dir_path_style
145 global boot_list
146 global boot_stack
147 global boot_results_file_path
148 global boot_results
149 global ffdc_list_file_path
Michael Walshe0cf8d72017-05-17 13:20:46 -0500150 global ffdc_report_list_path
Michael Walsh600876d2017-05-30 17:58:58 -0500151 global ffdc_summary_list_path
Michael Walshb5839d02017-04-12 16:11:20 -0500152
153 if ffdc_dir_path_style == "":
154 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
155
156 # Convert these program parms to lists for easier processing..
157 boot_list = filter(None, boot_list.split(":"))
158 boot_stack = filter(None, boot_stack.split(":"))
159
160 boot_results_file_path = "/tmp/" + openbmc_nickname + ":pid_" +\
161 str(master_pid) + ":boot_results"
162
163 if os.path.isfile(boot_results_file_path):
164 # We've been called before in this run so we'll load the saved
165 # boot_results object.
166 boot_results = pickle.load(open(boot_results_file_path, 'rb'))
167 else:
168 boot_results = boot_results(boot_table, boot_pass, boot_fail)
169
170 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
171 "/FFDC_FILE_LIST"
Michael Walshe0cf8d72017-05-17 13:20:46 -0500172 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
173 "/FFDC_REPORT_FILE_LIST"
Michael Walshb5839d02017-04-12 16:11:20 -0500174
Michael Walsh600876d2017-05-30 17:58:58 -0500175 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
176 "/FFDC_SUMMARY_FILE_LIST"
177
Michael Walshb5839d02017-04-12 16:11:20 -0500178###############################################################################
179
180
181###############################################################################
Michael Walsh85678942017-03-27 14:34:22 -0500182def initial_plug_in_setup():
183
184 r"""
185 Initialize all plug-in environment variables which do not change for the
186 duration of the program.
187
188 """
189
190 global LOG_LEVEL
191 BuiltIn().set_log_level("NONE")
192
193 BuiltIn().set_global_variable("${master_pid}", master_pid)
194 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
195 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
196 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
197 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
198 ffdc_list_file_path)
Michael Walshe0cf8d72017-05-17 13:20:46 -0500199 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
200 ffdc_report_list_path)
Michael Walsh600876d2017-05-30 17:58:58 -0500201 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
202 ffdc_summary_list_path)
Michael Walsh85678942017-03-27 14:34:22 -0500203
204 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
205 ffdc_dir_path_style)
206 BuiltIn().set_global_variable("${FFDC_CHECK}",
207 ffdc_check)
208
209 # For each program parameter, set the corresponding AUTOBOOT_ environment
210 # variable value. Also, set an AUTOBOOT_ environment variable for every
211 # element in additional_values.
212 additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
213 "status_dir_path", "base_tool_dir_path",
Michael Walsh600876d2017-05-30 17:58:58 -0500214 "ffdc_list_file_path", "ffdc_report_list_path",
215 "ffdc_summary_list_path"]
Michael Walsh85678942017-03-27 14:34:22 -0500216
217 plug_in_vars = parm_list + additional_values
218
219 for var_name in plug_in_vars:
220 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
221 var_name = var_name.upper()
222 if var_value is None:
223 var_value = ""
224 os.environ["AUTOBOOT_" + var_name] = str(var_value)
225
226 BuiltIn().set_log_level(LOG_LEVEL)
227
Michael Walsh68a61162017-04-25 11:54:06 -0500228 # Make sure the ffdc list directory exists.
229 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
230 if not os.path.exists(ffdc_list_dir_path):
231 os.makedirs(ffdc_list_dir_path)
Michael Walsh85678942017-03-27 14:34:22 -0500232
233###############################################################################
234
Michael Walsh0bbd8602016-11-22 11:31:49 -0600235
236###############################################################################
Michael Walsh0bbd8602016-11-22 11:31:49 -0600237def plug_in_setup():
238
239 r"""
Michael Walsh85678942017-03-27 14:34:22 -0500240 Initialize all changing plug-in environment variables for use by the
241 plug-in programs.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600242 """
243
Michael Walsh85678942017-03-27 14:34:22 -0500244 global LOG_LEVEL
245 global test_really_running
246
247 BuiltIn().set_log_level("NONE")
248
Michael Walsh6741f742017-02-20 16:16:38 -0600249 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600250 if boot_pass > 1:
251 test_really_running = 1
252 else:
253 test_really_running = 0
254
Michael Walsh0bbd8602016-11-22 11:31:49 -0600255 seconds = time.time()
256 loc_time = time.localtime(seconds)
257 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
258
Michael Walsh6741f742017-02-20 16:16:38 -0600259 ffdc_prefix = openbmc_nickname + "." + time_string
Michael Walsh0bbd8602016-11-22 11:31:49 -0600260
Michael Walsh6741f742017-02-20 16:16:38 -0600261 BuiltIn().set_global_variable("${test_really_running}",
262 test_really_running)
263 BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
Michael Walsh6741f742017-02-20 16:16:38 -0600264 BuiltIn().set_global_variable("${boot_pass}", boot_pass)
265 BuiltIn().set_global_variable("${boot_fail}", boot_fail)
266 BuiltIn().set_global_variable("${boot_success}", boot_success)
267 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
Michael Walsh4c9a6452016-12-13 16:03:11 -0600268
Michael Walsh0bbd8602016-11-22 11:31:49 -0600269 # For each program parameter, set the corresponding AUTOBOOT_ environment
270 # variable value. Also, set an AUTOBOOT_ environment variable for every
271 # element in additional_values.
272 additional_values = ["boot_type_desc", "boot_success", "boot_pass",
Michael Walsh85678942017-03-27 14:34:22 -0500273 "boot_fail", "test_really_running", "ffdc_prefix"]
Michael Walsh0bbd8602016-11-22 11:31:49 -0600274
Michael Walsh85678942017-03-27 14:34:22 -0500275 plug_in_vars = additional_values
Michael Walsh0bbd8602016-11-22 11:31:49 -0600276
277 for var_name in plug_in_vars:
278 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
279 var_name = var_name.upper()
280 if var_value is None:
281 var_value = ""
Michael Walsh6741f742017-02-20 16:16:38 -0600282 os.environ["AUTOBOOT_" + var_name] = str(var_value)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600283
Michael Walsh0bbd8602016-11-22 11:31:49 -0600284 if debug:
Michael Walsh6741f742017-02-20 16:16:38 -0600285 shell_rc, out_buf = \
286 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600287
Michael Walsh85678942017-03-27 14:34:22 -0500288 BuiltIn().set_log_level(LOG_LEVEL)
289
Michael Walsh0bbd8602016-11-22 11:31:49 -0600290###############################################################################
291
292
293###############################################################################
Michael Walshe0cf8d72017-05-17 13:20:46 -0500294def pre_boot_plug_in_setup():
295
296 # Clear the ffdc_list_file_path file. Plug-ins may now write to it.
297 try:
298 os.remove(ffdc_list_file_path)
299 except OSError:
300 pass
301
302 # Clear the ffdc_report_list_path file. Plug-ins may now write to it.
303 try:
304 os.remove(ffdc_report_list_path)
305 except OSError:
306 pass
307
Michael Walsh600876d2017-05-30 17:58:58 -0500308 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it.
309 try:
310 os.remove(ffdc_summary_list_path)
311 except OSError:
312 pass
313
Michael Walshe0cf8d72017-05-17 13:20:46 -0500314###############################################################################
315
316
317###############################################################################
Michael Walsh6741f742017-02-20 16:16:38 -0600318def setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600319
320 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600321 Do general program setup tasks.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600322 """
323
Michael Walsh6741f742017-02-20 16:16:38 -0600324 global cp_setup_called
Michael Walsh0bbd8602016-11-22 11:31:49 -0600325
Michael Walshb5839d02017-04-12 16:11:20 -0500326 gp.qprintn()
327
Michael Walsh83f4bc72017-04-20 16:49:43 -0500328 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
329 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
Michael Walshd061c042017-05-23 14:46:57 -0500330 # If we can't find process_plug_in_packages.py, ssh_pw or
331 # validate_plug_ins.py, then we don't have our repo bin in PATH.
332 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" +
333 " ssh_pw validate_plug_ins.py", quiet=1,
334 print_output=0, show_err=0)
Michael Walshb5839d02017-04-12 16:11:20 -0500335 if shell_rc != 0:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500336 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
337 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
338 if robot_pgm_dir_path not in sys.path:
339 sys.path.append(robot_pgm_dir_path)
340 PYTHONPATH = os.environ.get("PYTHONPATH", "")
341 if PYTHONPATH == "":
342 os.environ['PYTHONPATH'] = robot_pgm_dir_path
343 else:
344 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
Michael Walsh6741f742017-02-20 16:16:38 -0600345
346 validate_parms()
347
348 grp.rqprint_pgm_header()
349
Michael Walshb5839d02017-04-12 16:11:20 -0500350 grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE")
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500351
Michael Walsh85678942017-03-27 14:34:22 -0500352 initial_plug_in_setup()
353
Michael Walsh6741f742017-02-20 16:16:38 -0600354 plug_in_setup()
355 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
356 call_point='setup')
357 if rc != 0:
358 error_message = "Plug-in setup failed.\n"
359 grp.rprint_error_report(error_message)
360 BuiltIn().fail(error_message)
361 # Setting cp_setup_called lets our Teardown know that it needs to call
362 # the cleanup plug-in call point.
363 cp_setup_called = 1
364
365 # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
366 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
Michael Walsh85678942017-03-27 14:34:22 -0500367 # FFDC_LOG_PATH is used by "FFDC" keyword.
368 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
Michael Walsh6741f742017-02-20 16:16:38 -0600369
Michael Walshdc80d672017-05-09 12:58:32 -0500370 # Also printed by FFDC.
371 global host_name
372 global host_ip
373 host = socket.gethostname()
374 host_name, host_ip = gm.get_host_name_ip(host)
375
Michael Walshb5839d02017-04-12 16:11:20 -0500376 gp.dprint_var(boot_table, 1)
377 gp.dprint_var(boot_lists)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600378
379###############################################################################
380
381
382###############################################################################
Michael Walsh6741f742017-02-20 16:16:38 -0600383def validate_parms():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600384
385 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600386 Validate all program parameters.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600387 """
388
Michael Walshb5839d02017-04-12 16:11:20 -0500389 process_pgm_parms()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600390
Michael Walshb5839d02017-04-12 16:11:20 -0500391 gp.qprintn()
392
393 global openbmc_model
Michael Walsh6741f742017-02-20 16:16:38 -0600394 grv.rvalid_value("openbmc_host")
395 grv.rvalid_value("openbmc_username")
396 grv.rvalid_value("openbmc_password")
397 if os_host != "":
398 grv.rvalid_value("os_username")
399 grv.rvalid_value("os_password")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600400
Michael Walsh6741f742017-02-20 16:16:38 -0600401 if pdu_host != "":
402 grv.rvalid_value("pdu_username")
403 grv.rvalid_value("pdu_password")
Michael Walsh85678942017-03-27 14:34:22 -0500404 grv.rvalid_integer("pdu_slot_no")
Michael Walsh6741f742017-02-20 16:16:38 -0600405 if openbmc_serial_host != "":
406 grv.rvalid_integer("openbmc_serial_port")
Michael Walshb5839d02017-04-12 16:11:20 -0500407 if openbmc_model == "":
408 status, ret_values =\
409 grk.run_key_u("Get BMC System Model")
410 openbmc_model = ret_values
411 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
Michael Walsh6741f742017-02-20 16:16:38 -0600412 grv.rvalid_value("openbmc_model")
Michael Walshb5839d02017-04-12 16:11:20 -0500413 grv.rvalid_integer("max_num_tests")
Michael Walsh6741f742017-02-20 16:16:38 -0600414 grv.rvalid_integer("boot_pass")
415 grv.rvalid_integer("boot_fail")
416
417 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
418 BuiltIn().set_global_variable("${plug_in_packages_list}",
419 plug_in_packages_list)
420
Michael Walshb5839d02017-04-12 16:11:20 -0500421 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip'])
Michael Walsha20da402017-03-31 16:27:45 -0500422 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
Michael Walsh6741f742017-02-20 16:16:38 -0600423 error_message = "You must provide either a value for either the" +\
424 " boot_list or the boot_stack parm.\n"
425 BuiltIn().fail(gp.sprint_error(error_message))
426
427 valid_boot_list(boot_list, valid_boot_types)
428 valid_boot_list(boot_stack, valid_boot_types)
429
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500430 selected_PDU_boots = list(set(boot_list + boot_stack) &
431 set(boot_lists['PDU_reboot']))
432
433 if len(selected_PDU_boots) > 0 and pdu_host == "":
434 error_message = "You have selected the following boots which" +\
435 " require a PDU host but no value for pdu_host:\n"
436 error_message += gp.sprint_var(selected_PDU_boots)
437 error_message += gp.sprint_var(pdu_host, 2)
438 BuiltIn().fail(gp.sprint_error(error_message))
439
Michael Walsh6741f742017-02-20 16:16:38 -0600440 return
Michael Walsh0bbd8602016-11-22 11:31:49 -0600441
442###############################################################################
443
444
445###############################################################################
Michael Walsh6741f742017-02-20 16:16:38 -0600446def my_get_state():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600447
448 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600449 Get the system state plus a little bit of wrapping.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600450 """
451
Michael Walsh6741f742017-02-20 16:16:38 -0600452 global state
453
454 req_states = ['epoch_seconds'] + st.default_req_states
455
Michael Walshb5839d02017-04-12 16:11:20 -0500456 gp.qprint_timen("Getting system state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600457 if test_mode:
458 state['epoch_seconds'] = int(time.time())
459 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500460 state = st.get_state(req_states=req_states, quiet=quiet)
461 gp.qprint_var(state)
Michael Walsh341c21e2017-01-17 16:25:20 -0600462
463###############################################################################
464
465
466###############################################################################
Michael Walsh6741f742017-02-20 16:16:38 -0600467def select_boot():
Michael Walsh341c21e2017-01-17 16:25:20 -0600468
469 r"""
470 Select a boot test to be run based on our current state and return the
471 chosen boot type.
472
473 Description of arguments:
Michael Walsh6741f742017-02-20 16:16:38 -0600474 state The state of the machine.
Michael Walsh341c21e2017-01-17 16:25:20 -0600475 """
476
Michael Walsh30dadae2017-02-27 14:25:52 -0600477 global boot_stack
478
Michael Walshb5839d02017-04-12 16:11:20 -0500479 gp.qprint_timen("Selecting a boot test.")
Michael Walsh6741f742017-02-20 16:16:38 -0600480
481 my_get_state()
482
483 stack_popped = 0
484 if len(boot_stack) > 0:
485 stack_popped = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500486 gp.qprint_dashes()
487 gp.qprint_var(boot_stack)
488 gp.qprint_dashes()
489 skip_boot_printed = 0
490 while len(boot_stack) > 0:
491 boot_candidate = boot_stack.pop()
492 if stack_mode == 'normal':
493 break
494 else:
495 if st.compare_states(state, boot_table[boot_candidate]['end']):
496 if not skip_boot_printed:
497 gp.print_var(stack_mode)
498 gp.printn()
499 gp.print_timen("Skipping the following boot tests" +
500 " which are unnecessary since their" +
501 " required end states match the" +
502 " current machine state:")
503 skip_boot_printed = 1
504 gp.print_var(boot_candidate)
505 boot_candidate = ""
506 if boot_candidate == "":
507 gp.qprint_dashes()
508 gp.qprint_var(boot_stack)
509 gp.qprint_dashes()
510 return boot_candidate
Michael Walsh6741f742017-02-20 16:16:38 -0600511 if st.compare_states(state, boot_table[boot_candidate]['start']):
Michael Walshb5839d02017-04-12 16:11:20 -0500512 gp.qprint_timen("The machine state is valid for a '" +
513 boot_candidate + "' boot test.")
514 gp.qprint_dashes()
515 gp.qprint_var(boot_stack)
516 gp.qprint_dashes()
Michael Walsh6741f742017-02-20 16:16:38 -0600517 return boot_candidate
Michael Walsh341c21e2017-01-17 16:25:20 -0600518 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500519 gp.qprint_timen("The machine state does not match the required" +
520 " starting state for a '" + boot_candidate +
521 "' boot test:")
522 gp.print_varx("boot_table[" + boot_candidate + "][start]",
523 boot_table[boot_candidate]['start'], 1)
Michael Walsh6741f742017-02-20 16:16:38 -0600524 boot_stack.append(boot_candidate)
525 popped_boot = boot_candidate
526
527 # Loop through your list selecting a boot_candidates
528 boot_candidates = []
529 for boot_candidate in boot_list:
530 if st.compare_states(state, boot_table[boot_candidate]['start']):
531 if stack_popped:
532 if st.compare_states(boot_table[boot_candidate]['end'],
533 boot_table[popped_boot]['start']):
534 boot_candidates.append(boot_candidate)
535 else:
536 boot_candidates.append(boot_candidate)
537
538 if len(boot_candidates) == 0:
Michael Walshb5839d02017-04-12 16:11:20 -0500539 gp.qprint_timen("The user's boot list contained no boot tests" +
540 " which are valid for the current machine state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600541 boot_candidate = default_power_on
542 if not st.compare_states(state, boot_table[default_power_on]['start']):
543 boot_candidate = default_power_off
544 boot_candidates.append(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500545 gp.qprint_timen("Using default '" + boot_candidate +
546 "' boot type to transition to valid state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600547
Michael Walshb5839d02017-04-12 16:11:20 -0500548 gp.dprint_var(boot_candidates)
Michael Walsh6741f742017-02-20 16:16:38 -0600549
550 # Randomly select a boot from the candidate list.
551 boot = random.choice(boot_candidates)
Michael Walsh341c21e2017-01-17 16:25:20 -0600552
553 return boot
Michael Walsh0bbd8602016-11-22 11:31:49 -0600554
555###############################################################################
Michael Walsh55302292017-01-10 11:43:02 -0600556
557
558###############################################################################
Michael Walsh341c21e2017-01-17 16:25:20 -0600559def print_last_boots():
560
561 r"""
562 Print the last ten boots done with their time stamps.
563 """
564
565 # indent 0, 90 chars wide, linefeed, char is "="
Michael Walshb5839d02017-04-12 16:11:20 -0500566 gp.qprint_dashes(0, 90)
567 gp.qprintn("Last 10 boots:\n")
Michael Walsh341c21e2017-01-17 16:25:20 -0600568
569 for boot_entry in last_ten:
570 grp.rqprint(boot_entry)
Michael Walshb5839d02017-04-12 16:11:20 -0500571 gp.qprint_dashes(0, 90)
Michael Walsh341c21e2017-01-17 16:25:20 -0600572
573###############################################################################
574
575
576###############################################################################
Michael Walsh341c21e2017-01-17 16:25:20 -0600577def print_defect_report():
578
579 r"""
580 Print a defect report.
581 """
582
Michael Walsh600876d2017-05-30 17:58:58 -0500583 # Making deliberate choice to NOT run plug_in_setup(). We don't want
584 # ffdc_prefix updated.
585 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
586 call_point='ffdc_report', stop_on_plug_in_failure=0)
587
Michael Walsh341c21e2017-01-17 16:25:20 -0600588 # At some point I'd like to have the 'Call FFDC Methods' return a list
589 # of files it has collected. In that case, the following "ls" command
590 # would no longer be needed. For now, however, glob shows the files
591 # named in FFDC_LIST_FILE_PATH so I will refrain from printing those
592 # out (so we don't see duplicates in the list).
593
Michael Walshe0cf8d72017-05-17 13:20:46 -0500594 # Get additional header data which may have been created by ffdc plug-ins.
595 # Also, delete the individual header files to cleanup.
596 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
597 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
598 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
599 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
600 show_err=0)
601
Michael Walsh600876d2017-05-30 17:58:58 -0500602 # Get additional header data which may have been created by ffdc plug-ins.
603 # Also, delete the individual header files to cleanup.
604 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
605 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
606 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
607 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
608 show_err=0)
609
Michael Walsh341c21e2017-01-17 16:25:20 -0600610 LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}")
611
Michael Walshe0cf8d72017-05-17 13:20:46 -0500612 output = '\n'.join(sorted(glob.glob(LOG_PREFIX + '*')))
Michael Walsh341c21e2017-01-17 16:25:20 -0600613 try:
Michael Walsh6741f742017-02-20 16:16:38 -0600614 ffdc_list = open(ffdc_list_file_path, 'r')
Michael Walsh341c21e2017-01-17 16:25:20 -0600615 except IOError:
616 ffdc_list = ""
617
Michael Walsh68a61162017-04-25 11:54:06 -0500618 # Open ffdc_file_list for writing. We will write a complete list of
619 # FFDC files to it for possible use by plug-ins like cp_stop_check.
620 ffdc_list_file = open(ffdc_list_file_path, 'w')
621
622 gp.qprintn()
623 # indent=0, width=90, linefeed=1, char="="
624 gp.qprint_dashes(0, 90, 1, "=")
625 gp.qprintn("Copy this data to the defect:\n")
626
Michael Walshe0cf8d72017-05-17 13:20:46 -0500627 if len(more_header_info) > 0:
628 gp.printn(more_header_info)
Michael Walshdc80d672017-05-09 12:58:32 -0500629 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
630 openbmc_host_name, openbmc_ip, openbmc_username,
631 openbmc_password, os_host, os_host_name, os_ip, os_username,
632 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
633 pdu_password, pdu_slot_no, openbmc_serial_host,
634 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
Michael Walsh68a61162017-04-25 11:54:06 -0500635
636 gp.qprintn()
637
638 print_last_boots()
639 gp.qprintn()
640 gp.qprint_var(state)
641
Michael Walshb5839d02017-04-12 16:11:20 -0500642 gp.qprintn()
643 gp.qprintn("FFDC data files:")
Michael Walsh341c21e2017-01-17 16:25:20 -0600644 if status_file_path != "":
Michael Walshb5839d02017-04-12 16:11:20 -0500645 gp.qprintn(status_file_path)
Michael Walsh68a61162017-04-25 11:54:06 -0500646 ffdc_list_file.write(status_file_path + "\n")
Michael Walsh341c21e2017-01-17 16:25:20 -0600647
Michael Walshb5839d02017-04-12 16:11:20 -0500648 gp.qprintn(output)
649 # gp.qprintn(ffdc_list)
650 gp.qprintn()
Michael Walsh341c21e2017-01-17 16:25:20 -0600651
Michael Walsh600876d2017-05-30 17:58:58 -0500652 if len(ffdc_summary_info) > 0:
653 gp.printn(ffdc_summary_info)
654
Michael Walshb5839d02017-04-12 16:11:20 -0500655 gp.qprint_dashes(0, 90, 1, "=")
Michael Walsh341c21e2017-01-17 16:25:20 -0600656
Michael Walsh68a61162017-04-25 11:54:06 -0500657 ffdc_list_file.write(output + "\n")
658 ffdc_list_file.close()
659
Michael Walsh341c21e2017-01-17 16:25:20 -0600660###############################################################################
Michael Walsh6741f742017-02-20 16:16:38 -0600661
662
663###############################################################################
664def my_ffdc():
665
666 r"""
667 Collect FFDC data.
668 """
669
670 global state
671
672 plug_in_setup()
673 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500674 call_point='ffdc', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600675
676 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
Michael Walsh83f4bc72017-04-20 16:49:43 -0500677 status, ret_values = grk.run_key_u("FFDC ffdc_prefix=" +
678 AUTOBOOT_FFDC_PREFIX +
679 " ffdc_function_list=" +
680 ffdc_function_list, ignore=1)
681 if status != 'PASS':
Michael Walsh3328caf2017-03-21 17:04:08 -0500682 gp.print_error("Call to ffdc failed.\n")
Michael Walsh6741f742017-02-20 16:16:38 -0600683
684 my_get_state()
685
686 print_defect_report()
687
688###############################################################################
689
690
691###############################################################################
692def print_test_start_message(boot_keyword):
693
694 r"""
695 Print a message indicating what boot test is about to run.
696
697 Description of arguments:
698 boot_keyword The name of the boot which is to be run
699 (e.g. "BMC Power On").
700 """
701
702 global last_ten
703
704 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
Michael Walshb5839d02017-04-12 16:11:20 -0500705 gp.qprint(doing_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600706
707 last_ten.append(doing_msg)
708
709 if len(last_ten) > 10:
710 del last_ten[0]
711
712###############################################################################
713
714
715###############################################################################
716def run_boot(boot):
717
718 r"""
719 Run the specified boot.
720
721 Description of arguments:
722 boot The name of the boot test to be performed.
723 """
724
725 global state
726
727 print_test_start_message(boot)
728
729 plug_in_setup()
730 rc, shell_rc, failed_plug_in_name = \
731 grpi.rprocess_plug_in_packages(call_point="pre_boot")
732 if rc != 0:
733 error_message = "Plug-in failed with non-zero return code.\n" +\
734 gp.sprint_var(rc, 1)
735 BuiltIn().fail(gp.sprint_error(error_message))
736
737 if test_mode:
738 # In test mode, we'll pretend the boot worked by assigning its
739 # required end state to the default state value.
Michael Walsh30dadae2017-02-27 14:25:52 -0600740 state = st.strip_anchor_state(boot_table[boot]['end'])
Michael Walsh6741f742017-02-20 16:16:38 -0600741 else:
742 # Assertion: We trust that the state data was made fresh by the
743 # caller.
744
Michael Walshb5839d02017-04-12 16:11:20 -0500745 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600746
747 if boot_table[boot]['method_type'] == "keyword":
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600748 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
Michael Walshb5839d02017-04-12 16:11:20 -0500749 boot_table[boot]['method'],
750 quiet=quiet)
Michael Walsh6741f742017-02-20 16:16:38 -0600751
752 if boot_table[boot]['bmc_reboot']:
753 st.wait_for_comm_cycle(int(state['epoch_seconds']))
Michael Walsh30dadae2017-02-27 14:25:52 -0600754 plug_in_setup()
755 rc, shell_rc, failed_plug_in_name = \
756 grpi.rprocess_plug_in_packages(call_point="post_reboot")
757 if rc != 0:
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600758 error_message = "Plug-in failed with non-zero return code.\n"
759 error_message += gp.sprint_var(rc, 1)
Michael Walsh30dadae2017-02-27 14:25:52 -0600760 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600761 else:
762 match_state = st.anchor_state(state)
763 del match_state['epoch_seconds']
764 # Wait for the state to change in any way.
765 st.wait_state(match_state, wait_time=state_change_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500766 interval="10 seconds", invert=1)
Michael Walsh6741f742017-02-20 16:16:38 -0600767
Michael Walshb5839d02017-04-12 16:11:20 -0500768 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600769 if boot_table[boot]['end']['chassis'] == "Off":
770 boot_timeout = power_off_timeout
771 else:
772 boot_timeout = power_on_timeout
773 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500774 interval="10 seconds")
Michael Walsh6741f742017-02-20 16:16:38 -0600775
776 plug_in_setup()
777 rc, shell_rc, failed_plug_in_name = \
778 grpi.rprocess_plug_in_packages(call_point="post_boot")
779 if rc != 0:
780 error_message = "Plug-in failed with non-zero return code.\n" +\
781 gp.sprint_var(rc, 1)
782 BuiltIn().fail(gp.sprint_error(error_message))
783
784###############################################################################
785
786
787###############################################################################
788def test_loop_body():
789
790 r"""
791 The main loop body for the loop in main_py.
792
793 Description of arguments:
794 boot_count The iteration number (starts at 1).
795 """
796
797 global boot_count
798 global state
799 global next_boot
800 global boot_success
801
Michael Walshb5839d02017-04-12 16:11:20 -0500802 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600803
804 next_boot = select_boot()
Michael Walshb5839d02017-04-12 16:11:20 -0500805 if next_boot == "":
806 return True
Michael Walsh6741f742017-02-20 16:16:38 -0600807
Michael Walshb5839d02017-04-12 16:11:20 -0500808 boot_count += 1
809 gp.qprint_timen("Starting boot " + str(boot_count) + ".")
Michael Walsh6741f742017-02-20 16:16:38 -0600810
Michael Walshe0cf8d72017-05-17 13:20:46 -0500811 pre_boot_plug_in_setup()
Michael Walsh6741f742017-02-20 16:16:38 -0600812
813 cmd_buf = ["run_boot", next_boot]
814 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
815 if boot_status == "FAIL":
Michael Walshb5839d02017-04-12 16:11:20 -0500816 gp.qprint(msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600817
Michael Walshb5839d02017-04-12 16:11:20 -0500818 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600819 if boot_status == "PASS":
820 boot_success = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500821 gp.qprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.")
Michael Walsh6741f742017-02-20 16:16:38 -0600822 else:
823 boot_success = 0
Michael Walshb5839d02017-04-12 16:11:20 -0500824 gp.qprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.")
Michael Walsh6741f742017-02-20 16:16:38 -0600825
826 boot_results.update(next_boot, boot_status)
827
828 plug_in_setup()
829 # NOTE: A post_test_case call point failure is NOT counted as a boot
830 # failure.
831 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500832 call_point='post_test_case', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600833
834 plug_in_setup()
835 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
836 call_point='ffdc_check', shell_rc=0x00000200,
837 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
838 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500839 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
840 if status != 'PASS':
Michael Walsh3328caf2017-03-21 17:04:08 -0500841 gp.print_error("Call to my_ffdc failed.\n")
Michael Walsh6741f742017-02-20 16:16:38 -0600842
Michael Walshd139f282017-04-04 18:00:23 -0500843 # We need to purge error logs between boots or they build up.
Michael Walshb5839d02017-04-12 16:11:20 -0500844 grk.run_key("Delete Error logs", ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -0500845
Michael Walsh952f9b02017-03-09 13:11:14 -0600846 boot_results.print_report()
Michael Walshb5839d02017-04-12 16:11:20 -0500847 gp.qprint_timen("Finished boot " + str(boot_count) + ".")
Michael Walsh952f9b02017-03-09 13:11:14 -0600848
Michael Walsh6741f742017-02-20 16:16:38 -0600849 plug_in_setup()
850 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
851 call_point='stop_check')
852 if rc != 0:
853 error_message = "Stopping as requested by user.\n"
854 grp.rprint_error_report(error_message)
855 BuiltIn().fail(error_message)
856
Michael Walshd139f282017-04-04 18:00:23 -0500857 # This should help prevent ConnectionErrors.
Michael Walsh0960b382017-06-22 16:23:37 -0500858 grk.run_key_u("Close All Connections")
Michael Walshd139f282017-04-04 18:00:23 -0500859
Michael Walsh6741f742017-02-20 16:16:38 -0600860 return True
861
862###############################################################################
863
864
865###############################################################################
Michael Walsh83f4bc72017-04-20 16:49:43 -0500866def obmc_boot_test_teardown():
Michael Walsh6741f742017-02-20 16:16:38 -0600867
868 r"""
Michael Walshc9116812017-03-10 14:23:06 -0600869 Clean up after the Main keyword.
Michael Walsh6741f742017-02-20 16:16:38 -0600870 """
871
872 if cp_setup_called:
873 plug_in_setup()
874 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500875 call_point='cleanup', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600876
Michael Walsh600876d2017-05-30 17:58:58 -0500877 if 'boot_results_file_path' in globals():
878 # Save boot_results object to a file in case it is needed again.
879 gp.qprint_timen("Saving boot_results to the following path.")
880 gp.qprint_var(boot_results_file_path)
881 pickle.dump(boot_results, open(boot_results_file_path, 'wb'),
882 pickle.HIGHEST_PROTOCOL)
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600883
Michael Walsh6741f742017-02-20 16:16:38 -0600884###############################################################################
885
886
887###############################################################################
Michael Walshc9116812017-03-10 14:23:06 -0600888def test_teardown():
889
890 r"""
891 Clean up after this test case.
892 """
893
894 gp.qprintn()
895 cmd_buf = ["Print Error",
896 "A keyword timeout occurred ending this program.\n"]
897 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
898
Michael Walshb5839d02017-04-12 16:11:20 -0500899 grp.rqprint_pgm_footer()
900
Michael Walshc9116812017-03-10 14:23:06 -0600901###############################################################################
902
903
904###############################################################################
Michael Walsh83f4bc72017-04-20 16:49:43 -0500905def obmc_boot_test_py(alt_boot_stack=None):
Michael Walsh6741f742017-02-20 16:16:38 -0600906
907 r"""
908 Do main program processing.
909 """
910
Michael Walshb5839d02017-04-12 16:11:20 -0500911 if alt_boot_stack is not None:
912 BuiltIn().set_global_variable("${boot_stack}", alt_boot_stack)
913
Michael Walsh6741f742017-02-20 16:16:38 -0600914 setup()
915
Michael Walsha20da402017-03-31 16:27:45 -0500916 if ffdc_only:
917 gp.qprint_timen("Caller requested ffdc_only.")
Michael Walshe0cf8d72017-05-17 13:20:46 -0500918 pre_boot_plug_in_setup()
Michael Walsh83f4bc72017-04-20 16:49:43 -0500919 grk.run_key_u("my_ffdc")
Michael Walsh764d2f82017-04-27 16:01:08 -0500920 return
Michael Walsha20da402017-03-31 16:27:45 -0500921
Michael Walsh6741f742017-02-20 16:16:38 -0600922 # Process caller's boot_stack.
923 while (len(boot_stack) > 0):
924 test_loop_body()
925
Michael Walshb5839d02017-04-12 16:11:20 -0500926 gp.qprint_timen("Finished processing stack.")
Michael Walsh30dadae2017-02-27 14:25:52 -0600927
Michael Walsh6741f742017-02-20 16:16:38 -0600928 # Process caller's boot_list.
929 if len(boot_list) > 0:
930 for ix in range(1, max_num_tests + 1):
931 test_loop_body()
932
Michael Walshb5839d02017-04-12 16:11:20 -0500933 gp.qprint_timen("Completed all requested boot tests.")
934
935 boot_pass, boot_fail = boot_results.return_total_pass_fail()
936 if boot_fail > boot_fail_threshold:
937 error_message = "Boot failures exceed the boot failure" +\
938 " threshold:\n" +\
939 gp.sprint_var(boot_fail) +\
940 gp.sprint_var(boot_fail_threshold)
941 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600942
943###############################################################################