blob: ccc81c4cb700c4aa62de0c3e3200e8c0505609e1 [file] [log] [blame]
Michael Walsh0bbd8602016-11-22 11:31:49 -06001#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
Michael Walsh0b93fbf2017-03-02 14:42:41 -06007import os
8import imp
9import time
10import glob
11import random
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050012import re
Michael Walshf566fb12019-02-01 14:35:09 -060013import signal
George Keishingd54bbc22018-08-03 08:24:58 -050014try:
15 import cPickle as pickle
16except ImportError:
17 import pickle
Michael Walshdc80d672017-05-09 12:58:32 -050018import socket
Michael Walsh0b93fbf2017-03-02 14:42:41 -060019
20from robot.utils import DotDict
21from robot.libraries.BuiltIn import BuiltIn
22
Michael Walsh6741f742017-02-20 16:16:38 -060023from boot_data import *
Michael Walshc9116812017-03-10 14:23:06 -060024import gen_print as gp
Michael Walsh55302292017-01-10 11:43:02 -060025import gen_robot_plug_in as grpi
Michael Walshf75d4352019-12-05 17:01:20 -060026import gen_arg as ga
Michael Walsh44cef252019-08-01 12:38:56 -050027import gen_valid as gv
Michael Walsh6741f742017-02-20 16:16:38 -060028import gen_misc as gm
29import gen_cmd as gc
Michael Walshb5839d02017-04-12 16:11:20 -050030import gen_robot_keyword as grk
Michael Walsh55302292017-01-10 11:43:02 -060031import state as st
Michael Walshff340002017-08-29 11:18:27 -050032import var_stack as vs
Michael Walshc9bd2e82019-04-18 11:06:52 -050033import gen_plug_in_utils as gpu
Michael Shepos1a67b082020-08-28 16:01:58 -050034import pel_utils as pel
Michael Shepos0e5f1132020-09-30 16:24:25 -050035import logging_utils as log
Michael Walsh0bbd8602016-11-22 11:31:49 -060036
Michael Walsh0b93fbf2017-03-02 14:42:41 -060037base_path = os.path.dirname(os.path.dirname(
38 imp.find_module("gen_robot_print")[1])) +\
Michael Walshc9116812017-03-10 14:23:06 -060039 os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060040sys.path.append(base_path + "extended/")
41import run_keyword as rk
Michael Walsh0bbd8602016-11-22 11:31:49 -060042
Michael Walshe1e26442017-03-06 17:50:07 -060043# Setting master_pid correctly influences the behavior of plug-ins like
44# DB_Logging
45program_pid = os.getpid()
46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -050047pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
Michael Walshe1e26442017-03-06 17:50:07 -060048
Michael Walshb5839d02017-04-12 16:11:20 -050049# Set up boot data structures.
Michael Walsh986d8ae2019-07-17 10:02:23 -050050os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
Michael Walsh0b93fbf2017-03-02 14:42:41 -060051
Michael Walsh6741f742017-02-20 16:16:38 -060052boot_lists = read_boot_lists()
Michael Walsh986d8ae2019-07-17 10:02:23 -050053
54# The maximum number of entries that can be in the boot_history global variable.
Michael Walsh815b1d52018-10-30 13:32:26 -050055max_boot_history = 10
Michael Walsh986d8ae2019-07-17 10:02:23 -050056boot_history = []
Michael Walsh6741f742017-02-20 16:16:38 -060057
Michael Walsh7dc885b2018-03-14 17:51:59 -050058state = st.return_state_constant('default_state')
Michael Walsh6741f742017-02-20 16:16:38 -060059cp_setup_called = 0
60next_boot = ""
61base_tool_dir_path = os.path.normpath(os.environ.get(
62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
Michael Walshb5839d02017-04-12 16:11:20 -050063
Michael Walsh6741f742017-02-20 16:16:38 -060064ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
Michael Walsh6741f742017-02-20 16:16:38 -060065boot_success = 0
Michael Walsh6741f742017-02-20 16:16:38 -060066status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
67if status_dir_path != "":
68 status_dir_path = os.path.normpath(status_dir_path) + os.sep
Michael Walshe58df1c2019-08-07 09:57:43 -050069redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False)
George Keishingeb1fe352020-06-19 03:02:22 -050070redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False)
Michael Walshe58df1c2019-08-07 09:57:43 -050071if redfish_supported:
George Keishing89537a82020-06-17 00:37:25 -050072 redfish = BuiltIn().get_library_instance('redfish')
Michael Walshe58df1c2019-08-07 09:57:43 -050073 default_power_on = "Redfish Power On"
74 default_power_off = "Redfish Power Off"
George Keishingeb1fe352020-06-19 03:02:22 -050075 if redfish_rest_supported:
Michael Sheposcc490b42020-08-26 12:53:01 -050076 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}"
Michael Shepos92a54bf2020-11-11 11:48:55 -060077 delete_bmcdump_cmd = "Delete All BMC Dump"
George Keishingeb1fe352020-06-19 03:02:22 -050078 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF"
79 else:
80 delete_errlogs_cmd = "Redfish Purge Event Log"
Michael Shepos92a54bf2020-11-11 11:48:55 -060081 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps"
George Keishingeb1fe352020-06-19 03:02:22 -050082 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff"
Michael Walshe58df1c2019-08-07 09:57:43 -050083else:
84 default_power_on = "REST Power On"
85 default_power_off = "REST Power Off"
Michael Sheposcc490b42020-08-26 12:53:01 -050086 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}"
Michael Shepos92a54bf2020-11-11 11:48:55 -060087 delete_bmcdump_cmd = "Delete All BMC Dump"
George Keishinga54e06f2020-06-12 10:42:41 -050088 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF"
Michael Walsh6741f742017-02-20 16:16:38 -060089boot_count = 0
Michael Walsh0bbd8602016-11-22 11:31:49 -060090
Michael Walsh85678942017-03-27 14:34:22 -050091LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
Michael Walsh986d8ae2019-07-17 10:02:23 -050092AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '')
93ffdc_prefix = AUTOBOOT_FFDC_PREFIX
Sunil M325eb542017-08-10 07:09:43 -050094boot_start_time = ""
95boot_end_time = ""
Michael Walshff340002017-08-29 11:18:27 -050096save_stack = vs.var_stack('save_stack')
97main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
Michael Walsh85678942017-03-27 14:34:22 -050098
99
Michael Walsh89de14a2018-10-01 16:51:37 -0500100def dump_ffdc_rc():
101 r"""
102 Return the constant dump ffdc test return code value.
103
104 When a plug-in call point program returns this value, it indicates that
105 this program should collect FFDC.
106 """
107
108 return 0x00000200
109
110
111def stop_test_rc():
112 r"""
113 Return the constant stop test return code value.
114
115 When a plug-in call point program returns this value, it indicates that
116 this program should stop running.
117 """
118
119 return 0x00000200
120
121
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500122def process_host(host,
123 host_var_name=""):
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500124 r"""
125 Process a host by getting the associated host name and IP address and
126 setting them in global variables.
127
128 If the caller does not pass the host_var_name, this function will try to
129 figure out the name of the variable used by the caller for the host parm.
130 Callers are advised to explicitly specify the host_var_name when calling
131 with an exec command. In such cases, the get_arg_name cannot figure out
132 the host variable name.
133
134 This function will then create similar global variable names by
135 removing "_host" and appending "_host_name" or "_ip" to the host variable
136 name.
137
138 Example:
139
140 If a call is made like this:
141 process_host(openbmc_host)
142
143 Global variables openbmc_host_name and openbmc_ip will be set.
144
145 Description of argument(s):
146 host A host name or IP. The name of the variable used should
147 have a suffix of "_host".
148 host_var_name The name of the variable being used as the host parm.
149 """
150
151 if host_var_name == "":
152 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
153
154 host_name_var_name = re.sub("host", "host_name", host_var_name)
155 ip_var_name = re.sub("host", "ip", host_var_name)
156 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
157 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
158 host + "')"
159 exec(cmd_buf)
160
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500161
Michael Walshb5839d02017-04-12 16:11:20 -0500162def process_pgm_parms():
Michael Walshb5839d02017-04-12 16:11:20 -0500163 r"""
164 Process the program parameters by assigning them all to corresponding
165 globals. Also, set some global values that depend on program parameters.
166 """
167
168 # Program parameter processing.
169 # Assign all program parms to python variables which are global to this
170 # module.
171
172 global parm_list
173 parm_list = BuiltIn().get_variable_value("${parm_list}")
174 # The following subset of parms should be processed as integers.
175 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
Michael Walsh89de14a2018-10-01 16:51:37 -0500176 'boot_fail_threshold', 'delete_errlogs',
Michael Walsh986d8ae2019-07-17 10:02:23 -0500177 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet',
178 'test_mode', 'debug']
Michael Walshb5839d02017-04-12 16:11:20 -0500179 for parm in parm_list:
180 if parm in int_list:
181 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
182 "}\", \"0\"))"
183 else:
184 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
185 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
Michael Walshff340002017-08-29 11:18:27 -0500186 gp.dpissuing(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500187 exec(cmd_buf)
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500188 if re.match(r".*_host$", parm):
189 cmd_buf = "process_host(" + parm + ", '" + parm + "')"
190 exec(cmd_buf)
191 if re.match(r".*_password$", parm):
192 # Register the value of any parm whose name ends in _password.
193 # This will cause the print functions to replace passwords with
194 # asterisks in the output.
195 cmd_buf = "gp.register_passwords(" + parm + ")"
196 exec(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500197
198 global ffdc_dir_path_style
199 global boot_list
200 global boot_stack
201 global boot_results_file_path
202 global boot_results
Michael Walsh986d8ae2019-07-17 10:02:23 -0500203 global boot_history
Michael Walshb5839d02017-04-12 16:11:20 -0500204 global ffdc_list_file_path
Michael Walshe0cf8d72017-05-17 13:20:46 -0500205 global ffdc_report_list_path
Michael Walsh600876d2017-05-30 17:58:58 -0500206 global ffdc_summary_list_path
Michael Walsha3e7b222020-02-03 15:32:16 -0600207 global boot_table
208 global valid_boot_types
Michael Walshb5839d02017-04-12 16:11:20 -0500209
210 if ffdc_dir_path_style == "":
211 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
212
213 # Convert these program parms to lists for easier processing..
George Keishing36efbc02018-12-12 10:18:23 -0600214 boot_list = list(filter(None, boot_list.split(":")))
215 boot_stack = list(filter(None, boot_stack.split(":")))
Michael Walshb5839d02017-04-12 16:11:20 -0500216
Michael Walsha3e7b222020-02-03 15:32:16 -0600217 boot_table = create_boot_table(boot_table_path, os_host=os_host)
218 valid_boot_types = create_valid_boot_list(boot_table)
219
Michael Walsh903e0b22017-09-19 17:00:33 -0500220 cleanup_boot_results_file()
221 boot_results_file_path = create_boot_results_file_path(pgm_name,
222 openbmc_nickname,
223 master_pid)
Michael Walshb5839d02017-04-12 16:11:20 -0500224
225 if os.path.isfile(boot_results_file_path):
226 # We've been called before in this run so we'll load the saved
Michael Walsh986d8ae2019-07-17 10:02:23 -0500227 # boot_results and boot_history objects.
228 boot_results, boot_history =\
Michael Walsh6c645742018-08-17 15:02:17 -0500229 pickle.load(open(boot_results_file_path, 'rb'))
Michael Walshb5839d02017-04-12 16:11:20 -0500230 else:
231 boot_results = boot_results(boot_table, boot_pass, boot_fail)
232
233 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
234 "/FFDC_FILE_LIST"
Michael Walshe0cf8d72017-05-17 13:20:46 -0500235 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
236 "/FFDC_REPORT_FILE_LIST"
Michael Walshb5839d02017-04-12 16:11:20 -0500237
Michael Walsh600876d2017-05-30 17:58:58 -0500238 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
239 "/FFDC_SUMMARY_FILE_LIST"
240
Michael Walshb5839d02017-04-12 16:11:20 -0500241
Michael Walsh85678942017-03-27 14:34:22 -0500242def initial_plug_in_setup():
Michael Walsh85678942017-03-27 14:34:22 -0500243 r"""
244 Initialize all plug-in environment variables which do not change for the
245 duration of the program.
246
247 """
248
249 global LOG_LEVEL
250 BuiltIn().set_log_level("NONE")
251
252 BuiltIn().set_global_variable("${master_pid}", master_pid)
253 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
254 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
255 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
256 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
257 ffdc_list_file_path)
Michael Walshe0cf8d72017-05-17 13:20:46 -0500258 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
259 ffdc_report_list_path)
Michael Walsh600876d2017-05-30 17:58:58 -0500260 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
261 ffdc_summary_list_path)
Michael Walsh85678942017-03-27 14:34:22 -0500262
263 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
264 ffdc_dir_path_style)
265 BuiltIn().set_global_variable("${FFDC_CHECK}",
266 ffdc_check)
267
268 # For each program parameter, set the corresponding AUTOBOOT_ environment
269 # variable value. Also, set an AUTOBOOT_ environment variable for every
270 # element in additional_values.
271 additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
272 "status_dir_path", "base_tool_dir_path",
Michael Walsh600876d2017-05-30 17:58:58 -0500273 "ffdc_list_file_path", "ffdc_report_list_path",
Michael Shepos7fe83b32020-09-21 15:46:01 -0500274 "ffdc_summary_list_path", "execdir", "redfish_supported",
George Keishing343cc782021-03-18 15:51:45 +0000275 "redfish_rest_supported"]
Michael Walsh85678942017-03-27 14:34:22 -0500276
277 plug_in_vars = parm_list + additional_values
278
279 for var_name in plug_in_vars:
280 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
281 var_name = var_name.upper()
282 if var_value is None:
283 var_value = ""
284 os.environ["AUTOBOOT_" + var_name] = str(var_value)
285
286 BuiltIn().set_log_level(LOG_LEVEL)
287
Michael Walsh68a61162017-04-25 11:54:06 -0500288 # Make sure the ffdc list directory exists.
289 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
290 if not os.path.exists(ffdc_list_dir_path):
291 os.makedirs(ffdc_list_dir_path)
Michael Walsh85678942017-03-27 14:34:22 -0500292
Michael Walsh85678942017-03-27 14:34:22 -0500293
Michael Walsh0bbd8602016-11-22 11:31:49 -0600294def plug_in_setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600295 r"""
Michael Walsh85678942017-03-27 14:34:22 -0500296 Initialize all changing plug-in environment variables for use by the
297 plug-in programs.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600298 """
299
Michael Walsh85678942017-03-27 14:34:22 -0500300 global LOG_LEVEL
301 global test_really_running
302
303 BuiltIn().set_log_level("NONE")
304
Michael Walsh6741f742017-02-20 16:16:38 -0600305 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600306 if boot_pass > 1:
307 test_really_running = 1
308 else:
309 test_really_running = 0
310
Michael Walsh6741f742017-02-20 16:16:38 -0600311 BuiltIn().set_global_variable("${test_really_running}",
312 test_really_running)
313 BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
Michael Walsh6741f742017-02-20 16:16:38 -0600314 BuiltIn().set_global_variable("${boot_pass}", boot_pass)
315 BuiltIn().set_global_variable("${boot_fail}", boot_fail)
316 BuiltIn().set_global_variable("${boot_success}", boot_success)
317 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
Sunil M325eb542017-08-10 07:09:43 -0500318 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
319 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
Michael Walsh4c9a6452016-12-13 16:03:11 -0600320
Michael Walsh0bbd8602016-11-22 11:31:49 -0600321 # For each program parameter, set the corresponding AUTOBOOT_ environment
322 # variable value. Also, set an AUTOBOOT_ environment variable for every
323 # element in additional_values.
324 additional_values = ["boot_type_desc", "boot_success", "boot_pass",
Sunil M325eb542017-08-10 07:09:43 -0500325 "boot_fail", "test_really_running", "ffdc_prefix",
326 "boot_start_time", "boot_end_time"]
Michael Walsh0bbd8602016-11-22 11:31:49 -0600327
Michael Walsh85678942017-03-27 14:34:22 -0500328 plug_in_vars = additional_values
Michael Walsh0bbd8602016-11-22 11:31:49 -0600329
330 for var_name in plug_in_vars:
331 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
332 var_name = var_name.upper()
333 if var_value is None:
334 var_value = ""
Michael Walsh6741f742017-02-20 16:16:38 -0600335 os.environ["AUTOBOOT_" + var_name] = str(var_value)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600336
Michael Walsh0bbd8602016-11-22 11:31:49 -0600337 if debug:
Michael Walsh6741f742017-02-20 16:16:38 -0600338 shell_rc, out_buf = \
339 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600340
Michael Walsh85678942017-03-27 14:34:22 -0500341 BuiltIn().set_log_level(LOG_LEVEL)
342
Michael Walsh0bbd8602016-11-22 11:31:49 -0600343
Michael Walshe0cf8d72017-05-17 13:20:46 -0500344def pre_boot_plug_in_setup():
345
346 # Clear the ffdc_list_file_path file. Plug-ins may now write to it.
347 try:
348 os.remove(ffdc_list_file_path)
349 except OSError:
350 pass
351
352 # Clear the ffdc_report_list_path file. Plug-ins may now write to it.
353 try:
354 os.remove(ffdc_report_list_path)
355 except OSError:
356 pass
357
Michael Walsh600876d2017-05-30 17:58:58 -0500358 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it.
359 try:
360 os.remove(ffdc_summary_list_path)
361 except OSError:
362 pass
363
Michael Walshe1974b92017-08-03 13:39:51 -0500364 global ffdc_prefix
365
366 seconds = time.time()
367 loc_time = time.localtime(seconds)
368 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
369
370 ffdc_prefix = openbmc_nickname + "." + time_string
371
Michael Walshe0cf8d72017-05-17 13:20:46 -0500372
Michael Walshf566fb12019-02-01 14:35:09 -0600373def default_sigusr1(signal_number=0,
374 frame=None):
375 r"""
376 Handle SIGUSR1 by doing nothing.
377
378 This function assists in debugging SIGUSR1 processing by printing messages
379 to stdout and to the log.html file.
380
381 Description of argument(s):
382 signal_number The signal number (should always be 10 for SIGUSR1).
383 frame The frame data.
384 """
385
Michael Walsh80dddde2019-10-22 13:54:38 -0500386 gp.qprintn()
387 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600388 gp.lprint_executing()
389
390
391def set_default_siguser1():
392 r"""
393 Set the default_sigusr1 function to be the SIGUSR1 handler.
394 """
395
Michael Walsh80dddde2019-10-22 13:54:38 -0500396 gp.qprintn()
397 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600398 gp.lprint_executing()
399 signal.signal(signal.SIGUSR1, default_sigusr1)
400
401
Michael Walsh6741f742017-02-20 16:16:38 -0600402def setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600403 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600404 Do general program setup tasks.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600405 """
406
Michael Walsh6741f742017-02-20 16:16:38 -0600407 global cp_setup_called
Michael Walsh81816742017-09-27 11:02:29 -0500408 global transitional_boot_selected
Michael Walsh0bbd8602016-11-22 11:31:49 -0600409
Michael Walshb5839d02017-04-12 16:11:20 -0500410 gp.qprintn()
411
George Keishinga54e06f2020-06-12 10:42:41 -0500412 if redfish_supported:
413 redfish.login()
414
Michael Walshf566fb12019-02-01 14:35:09 -0600415 set_default_siguser1()
Michael Walsh81816742017-09-27 11:02:29 -0500416 transitional_boot_selected = False
417
Michael Walsh83f4bc72017-04-20 16:49:43 -0500418 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
419 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
Michael Walshd061c042017-05-23 14:46:57 -0500420 # If we can't find process_plug_in_packages.py, ssh_pw or
421 # validate_plug_ins.py, then we don't have our repo bin in PATH.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500422 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
423 + " ssh_pw validate_plug_ins.py", quiet=1,
Michael Walshd061c042017-05-23 14:46:57 -0500424 print_output=0, show_err=0)
Michael Walshb5839d02017-04-12 16:11:20 -0500425 if shell_rc != 0:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500426 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
427 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
428 if robot_pgm_dir_path not in sys.path:
429 sys.path.append(robot_pgm_dir_path)
430 PYTHONPATH = os.environ.get("PYTHONPATH", "")
431 if PYTHONPATH == "":
432 os.environ['PYTHONPATH'] = robot_pgm_dir_path
433 else:
434 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
Michael Walsh6741f742017-02-20 16:16:38 -0600435
436 validate_parms()
437
Michael Walshc108e422019-03-28 12:27:18 -0500438 gp.qprint_pgm_header()
Michael Walsh6741f742017-02-20 16:16:38 -0600439
George Keishinga54e06f2020-06-12 10:42:41 -0500440 grk.run_key_u(default_set_power_policy)
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500441
Michael Walsh85678942017-03-27 14:34:22 -0500442 initial_plug_in_setup()
443
Michael Walsh6741f742017-02-20 16:16:38 -0600444 plug_in_setup()
445 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
446 call_point='setup')
447 if rc != 0:
448 error_message = "Plug-in setup failed.\n"
Michael Walshc108e422019-03-28 12:27:18 -0500449 gp.print_error_report(error_message)
Michael Walsh6741f742017-02-20 16:16:38 -0600450 BuiltIn().fail(error_message)
451 # Setting cp_setup_called lets our Teardown know that it needs to call
452 # the cleanup plug-in call point.
453 cp_setup_called = 1
454
455 # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
456 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
Michael Walsh85678942017-03-27 14:34:22 -0500457 # FFDC_LOG_PATH is used by "FFDC" keyword.
458 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
Michael Walsh6741f742017-02-20 16:16:38 -0600459
Michael Walshdc80d672017-05-09 12:58:32 -0500460 # Also printed by FFDC.
461 global host_name
462 global host_ip
463 host = socket.gethostname()
464 host_name, host_ip = gm.get_host_name_ip(host)
465
Michael Walsh986d8ae2019-07-17 10:02:23 -0500466 gp.dprint_var(boot_table)
Michael Walshb5839d02017-04-12 16:11:20 -0500467 gp.dprint_var(boot_lists)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600468
Michael Walsh0bbd8602016-11-22 11:31:49 -0600469
Michael Walsh6741f742017-02-20 16:16:38 -0600470def validate_parms():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600471 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600472 Validate all program parameters.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600473 """
474
Michael Walshb5839d02017-04-12 16:11:20 -0500475 process_pgm_parms()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600476
Michael Walshb5839d02017-04-12 16:11:20 -0500477 gp.qprintn()
478
479 global openbmc_model
Michael Walshf5ce38c2020-02-27 12:46:20 -0600480 if openbmc_model == "":
481 status, ret_values =\
482 grk.run_key_u("Get BMC System Model")
483 openbmc_model = ret_values
484 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
485 gv.set_exit_on_error(True)
Michael Walsh44cef252019-08-01 12:38:56 -0500486 gv.valid_value(openbmc_host)
487 gv.valid_value(openbmc_username)
488 gv.valid_value(openbmc_password)
489 gv.valid_value(rest_username)
490 gv.valid_value(rest_password)
491 gv.valid_value(ipmi_username)
492 gv.valid_value(ipmi_password)
Michael Walsh6741f742017-02-20 16:16:38 -0600493 if os_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500494 gv.valid_value(os_username)
495 gv.valid_value(os_password)
Michael Walsh6741f742017-02-20 16:16:38 -0600496 if pdu_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500497 gv.valid_value(pdu_username)
498 gv.valid_value(pdu_password)
499 gv.valid_integer(pdu_slot_no)
Michael Walsh6741f742017-02-20 16:16:38 -0600500 if openbmc_serial_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500501 gv.valid_integer(openbmc_serial_port)
Michael Walsh44cef252019-08-01 12:38:56 -0500502 gv.valid_value(openbmc_model)
503 gv.valid_integer(max_num_tests)
504 gv.valid_integer(boot_pass)
505 gv.valid_integer(boot_fail)
Michael Walsh6741f742017-02-20 16:16:38 -0600506 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
507 BuiltIn().set_global_variable("${plug_in_packages_list}",
508 plug_in_packages_list)
Michael Walsh44cef252019-08-01 12:38:56 -0500509 gv.valid_value(stack_mode, valid_values=['normal', 'skip'])
Michael Walshf5ce38c2020-02-27 12:46:20 -0600510 gv.set_exit_on_error(False)
Michael Walsha20da402017-03-31 16:27:45 -0500511 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
Michael Walsh6741f742017-02-20 16:16:38 -0600512 error_message = "You must provide either a value for either the" +\
513 " boot_list or the boot_stack parm.\n"
514 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600515 valid_boot_list(boot_list, valid_boot_types)
516 valid_boot_list(boot_stack, valid_boot_types)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500517 selected_PDU_boots = list(set(boot_list + boot_stack)
518 & set(boot_lists['PDU_reboot']))
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500519 if len(selected_PDU_boots) > 0 and pdu_host == "":
520 error_message = "You have selected the following boots which" +\
521 " require a PDU host but no value for pdu_host:\n"
522 error_message += gp.sprint_var(selected_PDU_boots)
Michael Walsh986d8ae2019-07-17 10:02:23 -0500523 error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500524 BuiltIn().fail(gp.sprint_error(error_message))
525
Michael Walsh6741f742017-02-20 16:16:38 -0600526 return
Michael Walsh0bbd8602016-11-22 11:31:49 -0600527
Michael Walsh0bbd8602016-11-22 11:31:49 -0600528
Michael Walsh6741f742017-02-20 16:16:38 -0600529def my_get_state():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600530 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600531 Get the system state plus a little bit of wrapping.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600532 """
533
Michael Walsh6741f742017-02-20 16:16:38 -0600534 global state
535
536 req_states = ['epoch_seconds'] + st.default_req_states
537
Michael Walshb5839d02017-04-12 16:11:20 -0500538 gp.qprint_timen("Getting system state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600539 if test_mode:
540 state['epoch_seconds'] = int(time.time())
541 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500542 state = st.get_state(req_states=req_states, quiet=quiet)
543 gp.qprint_var(state)
Michael Walsh341c21e2017-01-17 16:25:20 -0600544
Michael Walsh341c21e2017-01-17 16:25:20 -0600545
Michael Walsh45ca6e42017-09-14 17:29:12 -0500546def valid_state():
Michael Walsh45ca6e42017-09-14 17:29:12 -0500547 r"""
548 Verify that our state dictionary contains no blank values. If we don't get
549 valid state data, we cannot continue to work.
550 """
551
552 if st.compare_states(state, st.invalid_state_match, 'or'):
553 error_message = "The state dictionary contains blank fields which" +\
554 " is illegal.\n" + gp.sprint_var(state)
555 BuiltIn().fail(gp.sprint_error(error_message))
556
Michael Walsh45ca6e42017-09-14 17:29:12 -0500557
Michael Walsh6741f742017-02-20 16:16:38 -0600558def select_boot():
Michael Walsh341c21e2017-01-17 16:25:20 -0600559 r"""
560 Select a boot test to be run based on our current state and return the
561 chosen boot type.
562
563 Description of arguments:
Michael Walsh6741f742017-02-20 16:16:38 -0600564 state The state of the machine.
Michael Walsh341c21e2017-01-17 16:25:20 -0600565 """
566
Michael Walsh81816742017-09-27 11:02:29 -0500567 global transitional_boot_selected
Michael Walsh30dadae2017-02-27 14:25:52 -0600568 global boot_stack
569
Michael Walshb5839d02017-04-12 16:11:20 -0500570 gp.qprint_timen("Selecting a boot test.")
Michael Walsh6741f742017-02-20 16:16:38 -0600571
Michael Walsh81816742017-09-27 11:02:29 -0500572 if transitional_boot_selected and not boot_success:
573 prior_boot = next_boot
574 boot_candidate = boot_stack.pop()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500575 gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
576 + " transition to a valid state for '" + boot_candidate
577 + "' which was at the top of the boot_stack. Since"
578 + " the '" + next_boot + "' failed, the '"
579 + boot_candidate + "' has been removed from the stack"
580 + " to avoid and endless failure loop.")
Michael Walsh81816742017-09-27 11:02:29 -0500581 if len(boot_stack) == 0:
582 return ""
583
Michael Walsh6741f742017-02-20 16:16:38 -0600584 my_get_state()
Michael Walsh45ca6e42017-09-14 17:29:12 -0500585 valid_state()
Michael Walsh6741f742017-02-20 16:16:38 -0600586
Michael Walsh81816742017-09-27 11:02:29 -0500587 transitional_boot_selected = False
Michael Walsh6741f742017-02-20 16:16:38 -0600588 stack_popped = 0
589 if len(boot_stack) > 0:
590 stack_popped = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500591 gp.qprint_dashes()
592 gp.qprint_var(boot_stack)
593 gp.qprint_dashes()
594 skip_boot_printed = 0
595 while len(boot_stack) > 0:
596 boot_candidate = boot_stack.pop()
597 if stack_mode == 'normal':
598 break
599 else:
600 if st.compare_states(state, boot_table[boot_candidate]['end']):
601 if not skip_boot_printed:
Michael Walshff340002017-08-29 11:18:27 -0500602 gp.qprint_var(stack_mode)
603 gp.qprintn()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500604 gp.qprint_timen("Skipping the following boot tests"
605 + " which are unnecessary since their"
606 + " required end states match the"
607 + " current machine state:")
Michael Walshb5839d02017-04-12 16:11:20 -0500608 skip_boot_printed = 1
Michael Walshff340002017-08-29 11:18:27 -0500609 gp.qprint_var(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500610 boot_candidate = ""
611 if boot_candidate == "":
612 gp.qprint_dashes()
613 gp.qprint_var(boot_stack)
614 gp.qprint_dashes()
615 return boot_candidate
Michael Walsh6741f742017-02-20 16:16:38 -0600616 if st.compare_states(state, boot_table[boot_candidate]['start']):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500617 gp.qprint_timen("The machine state is valid for a '"
618 + boot_candidate + "' boot test.")
Michael Walshb5839d02017-04-12 16:11:20 -0500619 gp.qprint_dashes()
620 gp.qprint_var(boot_stack)
621 gp.qprint_dashes()
Michael Walsh6741f742017-02-20 16:16:38 -0600622 return boot_candidate
Michael Walsh341c21e2017-01-17 16:25:20 -0600623 else:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500624 gp.qprint_timen("The machine state does not match the required"
625 + " starting state for a '" + boot_candidate
626 + "' boot test:")
Michael Walsh986d8ae2019-07-17 10:02:23 -0500627 gp.qprint_varx("boot_table_start_entry",
628 boot_table[boot_candidate]['start'])
Michael Walsh6741f742017-02-20 16:16:38 -0600629 boot_stack.append(boot_candidate)
Michael Walsh81816742017-09-27 11:02:29 -0500630 transitional_boot_selected = True
Michael Walsh6741f742017-02-20 16:16:38 -0600631 popped_boot = boot_candidate
632
633 # Loop through your list selecting a boot_candidates
634 boot_candidates = []
635 for boot_candidate in boot_list:
636 if st.compare_states(state, boot_table[boot_candidate]['start']):
637 if stack_popped:
638 if st.compare_states(boot_table[boot_candidate]['end'],
Gunnar Mills096cd562018-03-26 10:19:12 -0500639 boot_table[popped_boot]['start']):
Michael Walsh6741f742017-02-20 16:16:38 -0600640 boot_candidates.append(boot_candidate)
641 else:
642 boot_candidates.append(boot_candidate)
643
644 if len(boot_candidates) == 0:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500645 gp.qprint_timen("The user's boot list contained no boot tests"
646 + " which are valid for the current machine state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600647 boot_candidate = default_power_on
648 if not st.compare_states(state, boot_table[default_power_on]['start']):
649 boot_candidate = default_power_off
650 boot_candidates.append(boot_candidate)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500651 gp.qprint_timen("Using default '" + boot_candidate
652 + "' boot type to transition to valid state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600653
Michael Walshb5839d02017-04-12 16:11:20 -0500654 gp.dprint_var(boot_candidates)
Michael Walsh6741f742017-02-20 16:16:38 -0600655
656 # Randomly select a boot from the candidate list.
657 boot = random.choice(boot_candidates)
Michael Walsh341c21e2017-01-17 16:25:20 -0600658
659 return boot
Michael Walsh0bbd8602016-11-22 11:31:49 -0600660
Michael Walsh55302292017-01-10 11:43:02 -0600661
Michael Walshb2e53ec2017-10-30 15:04:36 -0500662def print_defect_report(ffdc_file_list):
Michael Walsh341c21e2017-01-17 16:25:20 -0600663 r"""
664 Print a defect report.
Michael Walshb2e53ec2017-10-30 15:04:36 -0500665
666 Description of argument(s):
667 ffdc_file_list A list of files which were collected by our ffdc functions.
Michael Walsh341c21e2017-01-17 16:25:20 -0600668 """
669
Michael Walsh600876d2017-05-30 17:58:58 -0500670 # Making deliberate choice to NOT run plug_in_setup(). We don't want
671 # ffdc_prefix updated.
672 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
673 call_point='ffdc_report', stop_on_plug_in_failure=0)
674
Michael Walshe0cf8d72017-05-17 13:20:46 -0500675 # Get additional header data which may have been created by ffdc plug-ins.
676 # Also, delete the individual header files to cleanup.
677 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
678 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
679 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
680 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
681 show_err=0)
682
Michael Walshb2e53ec2017-10-30 15:04:36 -0500683 # Get additional summary data which may have been created by ffdc plug-ins.
Michael Walsh600876d2017-05-30 17:58:58 -0500684 # Also, delete the individual header files to cleanup.
685 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
686 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
687 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
688 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
689 show_err=0)
690
Michael Walshb2e53ec2017-10-30 15:04:36 -0500691 # ffdc_list_file_path contains a list of any ffdc files created by plug-
692 # ins, etc. Read that data into a list.
Michael Walsh341c21e2017-01-17 16:25:20 -0600693 try:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500694 plug_in_ffdc_list = \
695 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
George Keishing36efbc02018-12-12 10:18:23 -0600696 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
Michael Walsh341c21e2017-01-17 16:25:20 -0600697 except IOError:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500698 plug_in_ffdc_list = []
699
700 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
701 # in. Eliminate duplicates and sort the list.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500702 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
Michael Walshb2e53ec2017-10-30 15:04:36 -0500703
704 if status_file_path != "":
705 ffdc_file_list.insert(0, status_file_path)
706
707 # Convert the list to a printable list.
708 printable_ffdc_file_list = "\n".join(ffdc_file_list)
Michael Walsh341c21e2017-01-17 16:25:20 -0600709
Michael Walsh68a61162017-04-25 11:54:06 -0500710 # Open ffdc_file_list for writing. We will write a complete list of
711 # FFDC files to it for possible use by plug-ins like cp_stop_check.
712 ffdc_list_file = open(ffdc_list_file_path, 'w')
Michael Walshb2e53ec2017-10-30 15:04:36 -0500713 ffdc_list_file.write(printable_ffdc_file_list + "\n")
714 ffdc_list_file.close()
715
716 indent = 0
717 width = 90
718 linefeed = 1
719 char = "="
Michael Walsh68a61162017-04-25 11:54:06 -0500720
721 gp.qprintn()
Michael Walshb2e53ec2017-10-30 15:04:36 -0500722 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500723 gp.qprintn("Copy this data to the defect:\n")
724
Michael Walshe0cf8d72017-05-17 13:20:46 -0500725 if len(more_header_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500726 gp.qprintn(more_header_info)
Michael Walshdc80d672017-05-09 12:58:32 -0500727 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
728 openbmc_host_name, openbmc_ip, openbmc_username,
Michael Walsh0a3bdb42019-01-31 16:21:44 +0000729 openbmc_password, rest_username, rest_password, ipmi_username,
730 ipmi_password, os_host, os_host_name, os_ip, os_username,
Michael Walshdc80d672017-05-09 12:58:32 -0500731 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
732 pdu_password, pdu_slot_no, openbmc_serial_host,
733 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
Michael Walsh68a61162017-04-25 11:54:06 -0500734
735 gp.qprintn()
Michael Walsh986d8ae2019-07-17 10:02:23 -0500736 print_boot_history(boot_history)
Michael Walsh68a61162017-04-25 11:54:06 -0500737 gp.qprintn()
738 gp.qprint_var(state)
Michael Walshb5839d02017-04-12 16:11:20 -0500739 gp.qprintn()
740 gp.qprintn("FFDC data files:")
Michael Walshb2e53ec2017-10-30 15:04:36 -0500741 gp.qprintn(printable_ffdc_file_list)
Michael Walshb5839d02017-04-12 16:11:20 -0500742 gp.qprintn()
Michael Walsh341c21e2017-01-17 16:25:20 -0600743
Michael Walsh600876d2017-05-30 17:58:58 -0500744 if len(ffdc_summary_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500745 gp.qprintn(ffdc_summary_info)
Michael Walsh600876d2017-05-30 17:58:58 -0500746
Michael Walshb2e53ec2017-10-30 15:04:36 -0500747 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500748
Michael Walsh6741f742017-02-20 16:16:38 -0600749
Michael Walsh6741f742017-02-20 16:16:38 -0600750def my_ffdc():
Michael Walsh6741f742017-02-20 16:16:38 -0600751 r"""
752 Collect FFDC data.
753 """
754
755 global state
756
757 plug_in_setup()
758 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500759 call_point='ffdc', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600760
761 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500762 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix="
763 + AUTOBOOT_FFDC_PREFIX
764 + " ffdc_function_list="
765 + ffdc_function_list, ignore=1)
Michael Walsh83f4bc72017-04-20 16:49:43 -0500766 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500767 gp.qprint_error("Call to ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500768 if type(ffdc_file_list) is not list:
769 ffdc_file_list = []
770 # Leave a record for caller that "soft" errors occurred.
771 soft_errors = 1
772 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600773
774 my_get_state()
775
Michael Walshb2e53ec2017-10-30 15:04:36 -0500776 print_defect_report(ffdc_file_list)
Michael Walsh6741f742017-02-20 16:16:38 -0600777
Michael Walsh6741f742017-02-20 16:16:38 -0600778
Michael Walsh6741f742017-02-20 16:16:38 -0600779def print_test_start_message(boot_keyword):
Michael Walsh6741f742017-02-20 16:16:38 -0600780 r"""
781 Print a message indicating what boot test is about to run.
782
783 Description of arguments:
784 boot_keyword The name of the boot which is to be run
785 (e.g. "BMC Power On").
786 """
787
Michael Walsh986d8ae2019-07-17 10:02:23 -0500788 global boot_history
Sunil M325eb542017-08-10 07:09:43 -0500789 global boot_start_time
Michael Walsh6741f742017-02-20 16:16:38 -0600790
791 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
Sunil M325eb542017-08-10 07:09:43 -0500792
793 # Set boot_start_time for use by plug-ins.
794 boot_start_time = doing_msg[1:33]
795 gp.qprint_var(boot_start_time)
796
Michael Walshb5839d02017-04-12 16:11:20 -0500797 gp.qprint(doing_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600798
Michael Walsh986d8ae2019-07-17 10:02:23 -0500799 update_boot_history(boot_history, doing_msg, max_boot_history)
Michael Walsh6741f742017-02-20 16:16:38 -0600800
Michael Walsh6741f742017-02-20 16:16:38 -0600801
Michael Walshf566fb12019-02-01 14:35:09 -0600802def stop_boot_test(signal_number=0,
803 frame=None):
804 r"""
805 Handle SIGUSR1 by aborting the boot test that is running.
806
807 Description of argument(s):
808 signal_number The signal number (should always be 10 for SIGUSR1).
809 frame The frame data.
810 """
811
Michael Walsh80dddde2019-10-22 13:54:38 -0500812 gp.qprintn()
813 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600814 gp.lprint_executing()
815
816 # Restore original sigusr1 handler.
817 set_default_siguser1()
818
819 message = "The caller has asked that the boot test be stopped and marked"
820 message += " as a failure."
821
822 function_stack = gm.get_function_stack()
823 if "wait_state" in function_stack:
Michael Walshc44aa532019-06-14 13:33:29 -0500824 st.set_exit_wait_early_message(message)
Michael Walshf566fb12019-02-01 14:35:09 -0600825 else:
826 BuiltIn().fail(gp.sprint_error(message))
827
828
Michael Walsh6741f742017-02-20 16:16:38 -0600829def run_boot(boot):
Michael Walsh6741f742017-02-20 16:16:38 -0600830 r"""
831 Run the specified boot.
832
833 Description of arguments:
834 boot The name of the boot test to be performed.
835 """
836
837 global state
838
Michael Walshf566fb12019-02-01 14:35:09 -0600839 signal.signal(signal.SIGUSR1, stop_boot_test)
840 gp.qprint_timen("stop_boot_test is armed.")
841
Michael Walsh6741f742017-02-20 16:16:38 -0600842 print_test_start_message(boot)
843
844 plug_in_setup()
845 rc, shell_rc, failed_plug_in_name = \
846 grpi.rprocess_plug_in_packages(call_point="pre_boot")
847 if rc != 0:
848 error_message = "Plug-in failed with non-zero return code.\n" +\
Michael Walsh986d8ae2019-07-17 10:02:23 -0500849 gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600850 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600851 BuiltIn().fail(gp.sprint_error(error_message))
852
853 if test_mode:
854 # In test mode, we'll pretend the boot worked by assigning its
855 # required end state to the default state value.
Michael Walsh30dadae2017-02-27 14:25:52 -0600856 state = st.strip_anchor_state(boot_table[boot]['end'])
Michael Walsh6741f742017-02-20 16:16:38 -0600857 else:
858 # Assertion: We trust that the state data was made fresh by the
859 # caller.
860
Michael Walshb5839d02017-04-12 16:11:20 -0500861 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600862
863 if boot_table[boot]['method_type'] == "keyword":
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600864 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
Michael Walshb5839d02017-04-12 16:11:20 -0500865 boot_table[boot]['method'],
866 quiet=quiet)
Michael Walsh6741f742017-02-20 16:16:38 -0600867
868 if boot_table[boot]['bmc_reboot']:
869 st.wait_for_comm_cycle(int(state['epoch_seconds']))
Michael Walsh30dadae2017-02-27 14:25:52 -0600870 plug_in_setup()
871 rc, shell_rc, failed_plug_in_name = \
872 grpi.rprocess_plug_in_packages(call_point="post_reboot")
873 if rc != 0:
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600874 error_message = "Plug-in failed with non-zero return code.\n"
Michael Walsh986d8ae2019-07-17 10:02:23 -0500875 error_message += gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600876 set_default_siguser1()
Michael Walsh30dadae2017-02-27 14:25:52 -0600877 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600878 else:
879 match_state = st.anchor_state(state)
880 del match_state['epoch_seconds']
881 # Wait for the state to change in any way.
882 st.wait_state(match_state, wait_time=state_change_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500883 interval="10 seconds", invert=1)
Michael Walsh6741f742017-02-20 16:16:38 -0600884
Michael Walshb5839d02017-04-12 16:11:20 -0500885 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600886 if boot_table[boot]['end']['chassis'] == "Off":
887 boot_timeout = power_off_timeout
888 else:
889 boot_timeout = power_on_timeout
890 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500891 interval="10 seconds")
Michael Walsh6741f742017-02-20 16:16:38 -0600892
893 plug_in_setup()
894 rc, shell_rc, failed_plug_in_name = \
895 grpi.rprocess_plug_in_packages(call_point="post_boot")
896 if rc != 0:
897 error_message = "Plug-in failed with non-zero return code.\n" +\
Michael Walsh986d8ae2019-07-17 10:02:23 -0500898 gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600899 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600900 BuiltIn().fail(gp.sprint_error(error_message))
901
Michael Walshf566fb12019-02-01 14:35:09 -0600902 # Restore original sigusr1 handler.
903 set_default_siguser1()
904
Michael Walsh6741f742017-02-20 16:16:38 -0600905
Michael Walsh6741f742017-02-20 16:16:38 -0600906def test_loop_body():
Michael Walsh6741f742017-02-20 16:16:38 -0600907 r"""
908 The main loop body for the loop in main_py.
909
910 Description of arguments:
911 boot_count The iteration number (starts at 1).
912 """
913
914 global boot_count
915 global state
916 global next_boot
917 global boot_success
Sunil M325eb542017-08-10 07:09:43 -0500918 global boot_end_time
Michael Walsh6741f742017-02-20 16:16:38 -0600919
Michael Walshb5839d02017-04-12 16:11:20 -0500920 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600921
922 next_boot = select_boot()
Michael Walshb5839d02017-04-12 16:11:20 -0500923 if next_boot == "":
924 return True
Michael Walsh6741f742017-02-20 16:16:38 -0600925
Michael Walshb5839d02017-04-12 16:11:20 -0500926 boot_count += 1
927 gp.qprint_timen("Starting boot " + str(boot_count) + ".")
Michael Walsh6741f742017-02-20 16:16:38 -0600928
Michael Walshe0cf8d72017-05-17 13:20:46 -0500929 pre_boot_plug_in_setup()
Michael Walsh6741f742017-02-20 16:16:38 -0600930
931 cmd_buf = ["run_boot", next_boot]
932 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
933 if boot_status == "FAIL":
Michael Walshb5839d02017-04-12 16:11:20 -0500934 gp.qprint(msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600935
Michael Walshb5839d02017-04-12 16:11:20 -0500936 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600937 if boot_status == "PASS":
938 boot_success = 1
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500939 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
940 + "\" succeeded.")
Michael Walsh6741f742017-02-20 16:16:38 -0600941 else:
942 boot_success = 0
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500943 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
944 + "\" failed.")
Sunil M325eb542017-08-10 07:09:43 -0500945
946 # Set boot_end_time for use by plug-ins.
947 boot_end_time = completion_msg[1:33]
948 gp.qprint_var(boot_end_time)
949
950 gp.qprint(completion_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600951
952 boot_results.update(next_boot, boot_status)
953
954 plug_in_setup()
955 # NOTE: A post_test_case call point failure is NOT counted as a boot
956 # failure.
957 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500958 call_point='post_test_case', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600959
960 plug_in_setup()
961 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500962 call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
Michael Walsh6741f742017-02-20 16:16:38 -0600963 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
Michael Walsh12059e22019-03-21 11:03:45 -0500964 if ffdc_check == "All" or\
Michael Walsh89de14a2018-10-01 16:51:37 -0500965 shell_rc == dump_ffdc_rc():
Michael Walsh83f4bc72017-04-20 16:49:43 -0500966 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
967 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500968 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500969 # Leave a record for caller that "soft" errors occurred.
970 soft_errors = 1
971 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600972
Michael Walshaabef1e2017-09-20 15:16:17 -0500973 if delete_errlogs:
Michael Shepos1a67b082020-08-28 16:01:58 -0500974 # print error logs before delete
975 status, error_logs = grk.run_key_u("Get Error Logs")
976 pels = pel.peltool("-l", ignore_err=1)
Michael Shepos0e5f1132020-09-30 16:24:25 -0500977 log.print_error_logs(error_logs, "AdditionalData Message Severity")
978 gp.qprint_var(pels)
Michael Shepos1a67b082020-08-28 16:01:58 -0500979
Michael Walshaabef1e2017-09-20 15:16:17 -0500980 # We need to purge error logs between boots or they build up.
Michael Walsh409ad352020-02-06 11:46:35 -0600981 grk.run_key(delete_errlogs_cmd, ignore=1)
Michael Shepos92a54bf2020-11-11 11:48:55 -0600982 grk.run_key(delete_bmcdump_cmd, ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -0500983
Michael Walsh952f9b02017-03-09 13:11:14 -0600984 boot_results.print_report()
Michael Walshb5839d02017-04-12 16:11:20 -0500985 gp.qprint_timen("Finished boot " + str(boot_count) + ".")
Michael Walsh952f9b02017-03-09 13:11:14 -0600986
Michael Walsh6741f742017-02-20 16:16:38 -0600987 plug_in_setup()
988 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500989 call_point='stop_check', shell_rc=stop_test_rc(),
990 stop_on_non_zero_rc=1)
991 if shell_rc == stop_test_rc():
Michael Walsh3ba8ecd2018-04-24 11:33:25 -0500992 message = "Stopping as requested by user.\n"
Michael Walsh80dddde2019-10-22 13:54:38 -0500993 gp.qprint_time(message)
Michael Walsh3ba8ecd2018-04-24 11:33:25 -0500994 BuiltIn().fail(message)
Michael Walsh6741f742017-02-20 16:16:38 -0600995
Michael Walshd139f282017-04-04 18:00:23 -0500996 # This should help prevent ConnectionErrors.
George Keishing4d65c862020-12-03 06:52:11 -0600997 # Purge all redfish and REST connection sessions.
998 grk.run_key_u("Close All Connections", ignore=1)
999 grk.run_key_u("Delete All Redfish Sessions", ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -05001000
Michael Walsh6741f742017-02-20 16:16:38 -06001001 return True
1002
Michael Walsh6741f742017-02-20 16:16:38 -06001003
Michael Walsh83f4bc72017-04-20 16:49:43 -05001004def obmc_boot_test_teardown():
Michael Walsh6741f742017-02-20 16:16:38 -06001005 r"""
Michael Walshf75d4352019-12-05 17:01:20 -06001006 Clean up after the main keyword.
Michael Walsh6741f742017-02-20 16:16:38 -06001007 """
Michael Walshf75d4352019-12-05 17:01:20 -06001008 gp.qprint_executing()
1009
1010 if ga.psutil_imported:
1011 ga.terminate_descendants()
Michael Walsh6741f742017-02-20 16:16:38 -06001012
1013 if cp_setup_called:
1014 plug_in_setup()
1015 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -05001016 call_point='cleanup', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -06001017
Michael Walsh600876d2017-05-30 17:58:58 -05001018 if 'boot_results_file_path' in globals():
Michael Walsh986d8ae2019-07-17 10:02:23 -05001019 # Save boot_results and boot_history objects to a file in case they are
Michael Walsh6c645742018-08-17 15:02:17 -05001020 # needed again.
Michael Walsh600876d2017-05-30 17:58:58 -05001021 gp.qprint_timen("Saving boot_results to the following path.")
1022 gp.qprint_var(boot_results_file_path)
Michael Walsh986d8ae2019-07-17 10:02:23 -05001023 pickle.dump((boot_results, boot_history),
Michael Walsh6c645742018-08-17 15:02:17 -05001024 open(boot_results_file_path, 'wb'),
Michael Walsh600876d2017-05-30 17:58:58 -05001025 pickle.HIGHEST_PROTOCOL)
Michael Walsh0b93fbf2017-03-02 14:42:41 -06001026
Michael Walshff340002017-08-29 11:18:27 -05001027 global save_stack
1028 # Restore any global values saved on the save_stack.
1029 for parm_name in main_func_parm_list:
1030 # Get the parm_value if it was saved on the stack.
1031 try:
1032 parm_value = save_stack.pop(parm_name)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -05001033 except BaseException:
Michael Walshff340002017-08-29 11:18:27 -05001034 # If it was not saved, no further action is required.
1035 continue
1036
1037 # Restore the saved value.
1038 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1039 "}\", parm_value)"
1040 gp.dpissuing(cmd_buf)
1041 exec(cmd_buf)
1042
1043 gp.dprintn(save_stack.sprint_obj())
1044
Michael Walsh6741f742017-02-20 16:16:38 -06001045
Michael Walshc9116812017-03-10 14:23:06 -06001046def test_teardown():
Michael Walshc9116812017-03-10 14:23:06 -06001047 r"""
1048 Clean up after this test case.
1049 """
1050
1051 gp.qprintn()
Michael Walshf75d4352019-12-05 17:01:20 -06001052 gp.qprint_executing()
1053
1054 if ga.psutil_imported:
1055 ga.terminate_descendants()
1056
Michael Walshc9116812017-03-10 14:23:06 -06001057 cmd_buf = ["Print Error",
1058 "A keyword timeout occurred ending this program.\n"]
1059 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1060
George Keishinga54e06f2020-06-12 10:42:41 -05001061 if redfish_supported:
1062 redfish.logout()
1063
Michael Walshc108e422019-03-28 12:27:18 -05001064 gp.qprint_pgm_footer()
Michael Walshb5839d02017-04-12 16:11:20 -05001065
Michael Walshc9116812017-03-10 14:23:06 -06001066
Michael Walsh89de14a2018-10-01 16:51:37 -05001067def post_stack():
1068 r"""
1069 Process post_stack plug-in programs.
1070 """
1071
1072 if not call_post_stack_plug:
1073 # The caller does not wish to have post_stack plug-in processing done.
1074 return
1075
1076 global boot_success
1077
1078 # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1079 pre_boot_plug_in_setup()
1080 # For the purposes of the following plug-ins, mark the "boot" as a success.
1081 boot_success = 1
1082 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001083 rc, shell_rc, failed_plug_in_name, history =\
1084 grpi.rprocess_plug_in_packages(call_point='post_stack',
1085 stop_on_plug_in_failure=0,
1086 return_history=True)
Michael Walsh986d8ae2019-07-17 10:02:23 -05001087 for doing_msg in history:
1088 update_boot_history(boot_history, doing_msg, max_boot_history)
Michael Walsh815b1d52018-10-30 13:32:26 -05001089 if rc != 0:
1090 boot_success = 0
Michael Walsh89de14a2018-10-01 16:51:37 -05001091
1092 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001093 rc, shell_rc, failed_plug_in_name =\
1094 grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1095 shell_rc=dump_ffdc_rc(),
1096 stop_on_plug_in_failure=1,
1097 stop_on_non_zero_rc=1)
1098 if shell_rc == dump_ffdc_rc():
Michael Walsh89de14a2018-10-01 16:51:37 -05001099 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1100 if status != 'PASS':
1101 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -05001102 # Leave a record for caller that "soft" errors occurred.
1103 soft_errors = 1
1104 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh89de14a2018-10-01 16:51:37 -05001105
1106 plug_in_setup()
1107 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1108 call_point='stop_check', shell_rc=stop_test_rc(),
1109 stop_on_non_zero_rc=1)
1110 if shell_rc == stop_test_rc():
1111 message = "Stopping as requested by user.\n"
Michael Walsh80dddde2019-10-22 13:54:38 -05001112 gp.qprint_time(message)
Michael Walsh89de14a2018-10-01 16:51:37 -05001113 BuiltIn().fail(message)
1114
1115
Michael Walshff340002017-08-29 11:18:27 -05001116def obmc_boot_test_py(loc_boot_stack=None,
1117 loc_stack_mode=None,
1118 loc_quiet=None):
Michael Walsh6741f742017-02-20 16:16:38 -06001119 r"""
1120 Do main program processing.
1121 """
1122
Michael Walshff340002017-08-29 11:18:27 -05001123 global save_stack
1124
Michael Walshf75d4352019-12-05 17:01:20 -06001125 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']})
1126
George Keishing36efbc02018-12-12 10:18:23 -06001127 gp.dprintn()
Michael Walshff340002017-08-29 11:18:27 -05001128 # Process function parms.
1129 for parm_name in main_func_parm_list:
1130 # Get parm's value.
George Keishing36efbc02018-12-12 10:18:23 -06001131 parm_value = eval("loc_" + parm_name)
1132 gp.dpvars(parm_name, parm_value)
Michael Walshff340002017-08-29 11:18:27 -05001133
George Keishing36efbc02018-12-12 10:18:23 -06001134 if parm_value is not None:
Michael Walshff340002017-08-29 11:18:27 -05001135 # Save the global value on a stack.
1136 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1137 parm_name + "}\"), \"" + parm_name + "\")"
1138 gp.dpissuing(cmd_buf)
1139 exec(cmd_buf)
1140
1141 # Set the global value to the passed value.
1142 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1143 "}\", loc_" + parm_name + ")"
1144 gp.dpissuing(cmd_buf)
1145 exec(cmd_buf)
1146
1147 gp.dprintn(save_stack.sprint_obj())
Michael Walshb5839d02017-04-12 16:11:20 -05001148
Michael Walsh6741f742017-02-20 16:16:38 -06001149 setup()
1150
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001151 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1152
Michael Walsha20da402017-03-31 16:27:45 -05001153 if ffdc_only:
1154 gp.qprint_timen("Caller requested ffdc_only.")
Michael Walsh986d8ae2019-07-17 10:02:23 -05001155 if do_pre_boot_plug_in_setup:
1156 pre_boot_plug_in_setup()
Michael Walsh83f4bc72017-04-20 16:49:43 -05001157 grk.run_key_u("my_ffdc")
Michael Walsh764d2f82017-04-27 16:01:08 -05001158 return
Michael Walsha20da402017-03-31 16:27:45 -05001159
Michael Walsh409ad352020-02-06 11:46:35 -06001160 if delete_errlogs:
Michael Shepos1a67b082020-08-28 16:01:58 -05001161 # print error logs before delete
1162 status, error_logs = grk.run_key_u("Get Error Logs")
1163 pels = pel.peltool("-l", ignore_err=1)
Michael Shepos0e5f1132020-09-30 16:24:25 -05001164 log.print_error_logs(error_logs, "AdditionalData Message Severity")
1165 gp.qprint_var(pels)
Michael Shepos1a67b082020-08-28 16:01:58 -05001166
Michael Walsh409ad352020-02-06 11:46:35 -06001167 # Delete errlogs prior to doing any boot tests.
1168 grk.run_key(delete_errlogs_cmd, ignore=1)
Michael Shepos92a54bf2020-11-11 11:48:55 -06001169 grk.run_key(delete_bmcdump_cmd, ignore=1)
Michael Walsh409ad352020-02-06 11:46:35 -06001170
Michael Walsh6741f742017-02-20 16:16:38 -06001171 # Process caller's boot_stack.
1172 while (len(boot_stack) > 0):
1173 test_loop_body()
1174
Michael Walshb5839d02017-04-12 16:11:20 -05001175 gp.qprint_timen("Finished processing stack.")
Michael Walsh30dadae2017-02-27 14:25:52 -06001176
Michael Walsh89de14a2018-10-01 16:51:37 -05001177 post_stack()
1178
Michael Walsh6741f742017-02-20 16:16:38 -06001179 # Process caller's boot_list.
1180 if len(boot_list) > 0:
1181 for ix in range(1, max_num_tests + 1):
1182 test_loop_body()
1183
Michael Walshb5839d02017-04-12 16:11:20 -05001184 gp.qprint_timen("Completed all requested boot tests.")
1185
1186 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001187 new_fail = boot_fail - init_boot_fail
1188 if new_fail > boot_fail_threshold:
Michael Walshb5839d02017-04-12 16:11:20 -05001189 error_message = "Boot failures exceed the boot failure" +\
1190 " threshold:\n" +\
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001191 gp.sprint_var(new_fail) +\
Michael Walshb5839d02017-04-12 16:11:20 -05001192 gp.sprint_var(boot_fail_threshold)
1193 BuiltIn().fail(gp.sprint_error(error_message))