blob: 1729ed11578aaccff5a84123edd613fb1ca09eff [file] [log] [blame]
Michael Walsh0bbd8602016-11-22 11:31:49 -06001#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
Michael Walsh0b93fbf2017-03-02 14:42:41 -06007import os
8import imp
9import time
10import glob
11import random
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050012import re
Michael Walshf566fb12019-02-01 14:35:09 -060013import signal
George Keishingd54bbc22018-08-03 08:24:58 -050014try:
15 import cPickle as pickle
16except ImportError:
17 import pickle
Michael Walshdc80d672017-05-09 12:58:32 -050018import socket
Michael Walsh0b93fbf2017-03-02 14:42:41 -060019
20from robot.utils import DotDict
21from robot.libraries.BuiltIn import BuiltIn
22
Michael Walsh6741f742017-02-20 16:16:38 -060023from boot_data import *
Michael Walshc9116812017-03-10 14:23:06 -060024import gen_print as gp
Michael Walsh55302292017-01-10 11:43:02 -060025import gen_robot_plug_in as grpi
Michael Walshf75d4352019-12-05 17:01:20 -060026import gen_arg as ga
Michael Walsh44cef252019-08-01 12:38:56 -050027import gen_valid as gv
Michael Walsh6741f742017-02-20 16:16:38 -060028import gen_misc as gm
29import gen_cmd as gc
Michael Walshb5839d02017-04-12 16:11:20 -050030import gen_robot_keyword as grk
Michael Walsh55302292017-01-10 11:43:02 -060031import state as st
Michael Walshff340002017-08-29 11:18:27 -050032import var_stack as vs
Michael Walshc9bd2e82019-04-18 11:06:52 -050033import gen_plug_in_utils as gpu
Michael Shepos1a67b082020-08-28 16:01:58 -050034import pel_utils as pel
Michael Shepos0e5f1132020-09-30 16:24:25 -050035import logging_utils as log
Michael Walsh0bbd8602016-11-22 11:31:49 -060036
Michael Walsh0b93fbf2017-03-02 14:42:41 -060037base_path = os.path.dirname(os.path.dirname(
38 imp.find_module("gen_robot_print")[1])) +\
Michael Walshc9116812017-03-10 14:23:06 -060039 os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060040sys.path.append(base_path + "extended/")
41import run_keyword as rk
Michael Walsh0bbd8602016-11-22 11:31:49 -060042
Michael Walshe1e26442017-03-06 17:50:07 -060043# Setting master_pid correctly influences the behavior of plug-ins like
44# DB_Logging
45program_pid = os.getpid()
46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -050047pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
Michael Walshe1e26442017-03-06 17:50:07 -060048
Michael Walshb5839d02017-04-12 16:11:20 -050049# Set up boot data structures.
Michael Walsh986d8ae2019-07-17 10:02:23 -050050os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
Michael Walsh0b93fbf2017-03-02 14:42:41 -060051
Michael Walsh6741f742017-02-20 16:16:38 -060052boot_lists = read_boot_lists()
Michael Walsh986d8ae2019-07-17 10:02:23 -050053
54# The maximum number of entries that can be in the boot_history global variable.
Michael Walsh815b1d52018-10-30 13:32:26 -050055max_boot_history = 10
Michael Walsh986d8ae2019-07-17 10:02:23 -050056boot_history = []
Michael Walsh6741f742017-02-20 16:16:38 -060057
Michael Walsh7dc885b2018-03-14 17:51:59 -050058state = st.return_state_constant('default_state')
Michael Walsh6741f742017-02-20 16:16:38 -060059cp_setup_called = 0
60next_boot = ""
61base_tool_dir_path = os.path.normpath(os.environ.get(
62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
Michael Walshb5839d02017-04-12 16:11:20 -050063
Michael Walsh6741f742017-02-20 16:16:38 -060064ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
Michael Walsh6741f742017-02-20 16:16:38 -060065boot_success = 0
Michael Walsh6741f742017-02-20 16:16:38 -060066status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
67if status_dir_path != "":
68 status_dir_path = os.path.normpath(status_dir_path) + os.sep
Michael Shepos34c79562021-03-18 18:49:44 -050069redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \
70 int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0))
Michael Walshe58df1c2019-08-07 09:57:43 -050071redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False)
George Keishingeb1fe352020-06-19 03:02:22 -050072redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False)
George Keishingd86e45c2021-03-19 07:38:14 -050073redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1))
Michael Walshe58df1c2019-08-07 09:57:43 -050074if redfish_supported:
George Keishing89537a82020-06-17 00:37:25 -050075 redfish = BuiltIn().get_library_instance('redfish')
Michael Walshe58df1c2019-08-07 09:57:43 -050076 default_power_on = "Redfish Power On"
77 default_power_off = "Redfish Power Off"
George Keishing870999a2021-03-31 23:43:57 -050078 if not redfish_support_trans_state:
Michael Sheposcc490b42020-08-26 12:53:01 -050079 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}"
Michael Shepos92a54bf2020-11-11 11:48:55 -060080 delete_bmcdump_cmd = "Delete All BMC Dump"
George Keishingeb1fe352020-06-19 03:02:22 -050081 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF"
82 else:
83 delete_errlogs_cmd = "Redfish Purge Event Log"
Michael Shepos92a54bf2020-11-11 11:48:55 -060084 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps"
George Keishing2ef6a7d2021-05-19 09:05:32 -050085 delete_sysdump_cmd = "Redfish Delete All System Dumps"
George Keishingeb1fe352020-06-19 03:02:22 -050086 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff"
Michael Walshe58df1c2019-08-07 09:57:43 -050087else:
88 default_power_on = "REST Power On"
89 default_power_off = "REST Power Off"
Michael Sheposcc490b42020-08-26 12:53:01 -050090 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}"
Michael Shepos92a54bf2020-11-11 11:48:55 -060091 delete_bmcdump_cmd = "Delete All BMC Dump"
George Keishinga54e06f2020-06-12 10:42:41 -050092 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF"
Michael Walsh6741f742017-02-20 16:16:38 -060093boot_count = 0
Michael Walsh0bbd8602016-11-22 11:31:49 -060094
Michael Walsh85678942017-03-27 14:34:22 -050095LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
Michael Walsh986d8ae2019-07-17 10:02:23 -050096AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '')
97ffdc_prefix = AUTOBOOT_FFDC_PREFIX
Sunil M325eb542017-08-10 07:09:43 -050098boot_start_time = ""
99boot_end_time = ""
Michael Walshff340002017-08-29 11:18:27 -0500100save_stack = vs.var_stack('save_stack')
101main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
Michael Walsh85678942017-03-27 14:34:22 -0500102
103
Michael Walsh89de14a2018-10-01 16:51:37 -0500104def dump_ffdc_rc():
105 r"""
106 Return the constant dump ffdc test return code value.
107
108 When a plug-in call point program returns this value, it indicates that
109 this program should collect FFDC.
110 """
111
112 return 0x00000200
113
114
115def stop_test_rc():
116 r"""
117 Return the constant stop test return code value.
118
119 When a plug-in call point program returns this value, it indicates that
120 this program should stop running.
121 """
122
123 return 0x00000200
124
125
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500126def process_host(host,
127 host_var_name=""):
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500128 r"""
129 Process a host by getting the associated host name and IP address and
130 setting them in global variables.
131
132 If the caller does not pass the host_var_name, this function will try to
133 figure out the name of the variable used by the caller for the host parm.
134 Callers are advised to explicitly specify the host_var_name when calling
135 with an exec command. In such cases, the get_arg_name cannot figure out
136 the host variable name.
137
138 This function will then create similar global variable names by
139 removing "_host" and appending "_host_name" or "_ip" to the host variable
140 name.
141
142 Example:
143
144 If a call is made like this:
145 process_host(openbmc_host)
146
147 Global variables openbmc_host_name and openbmc_ip will be set.
148
149 Description of argument(s):
150 host A host name or IP. The name of the variable used should
151 have a suffix of "_host".
152 host_var_name The name of the variable being used as the host parm.
153 """
154
155 if host_var_name == "":
156 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
157
158 host_name_var_name = re.sub("host", "host_name", host_var_name)
159 ip_var_name = re.sub("host", "ip", host_var_name)
160 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
161 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
162 host + "')"
163 exec(cmd_buf)
164
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500165
Michael Walshb5839d02017-04-12 16:11:20 -0500166def process_pgm_parms():
Michael Walshb5839d02017-04-12 16:11:20 -0500167 r"""
168 Process the program parameters by assigning them all to corresponding
169 globals. Also, set some global values that depend on program parameters.
170 """
171
172 # Program parameter processing.
173 # Assign all program parms to python variables which are global to this
174 # module.
175
176 global parm_list
177 parm_list = BuiltIn().get_variable_value("${parm_list}")
178 # The following subset of parms should be processed as integers.
179 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
Michael Walsh89de14a2018-10-01 16:51:37 -0500180 'boot_fail_threshold', 'delete_errlogs',
Michael Walsh986d8ae2019-07-17 10:02:23 -0500181 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet',
182 'test_mode', 'debug']
Michael Walshb5839d02017-04-12 16:11:20 -0500183 for parm in parm_list:
184 if parm in int_list:
185 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
186 "}\", \"0\"))"
187 else:
188 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
189 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
Michael Walshff340002017-08-29 11:18:27 -0500190 gp.dpissuing(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500191 exec(cmd_buf)
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500192 if re.match(r".*_host$", parm):
193 cmd_buf = "process_host(" + parm + ", '" + parm + "')"
194 exec(cmd_buf)
195 if re.match(r".*_password$", parm):
196 # Register the value of any parm whose name ends in _password.
197 # This will cause the print functions to replace passwords with
198 # asterisks in the output.
199 cmd_buf = "gp.register_passwords(" + parm + ")"
200 exec(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500201
202 global ffdc_dir_path_style
203 global boot_list
204 global boot_stack
205 global boot_results_file_path
206 global boot_results
Michael Walsh986d8ae2019-07-17 10:02:23 -0500207 global boot_history
Michael Walshb5839d02017-04-12 16:11:20 -0500208 global ffdc_list_file_path
Michael Walshe0cf8d72017-05-17 13:20:46 -0500209 global ffdc_report_list_path
Michael Walsh600876d2017-05-30 17:58:58 -0500210 global ffdc_summary_list_path
Michael Walsha3e7b222020-02-03 15:32:16 -0600211 global boot_table
212 global valid_boot_types
Michael Walshb5839d02017-04-12 16:11:20 -0500213
214 if ffdc_dir_path_style == "":
215 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
216
217 # Convert these program parms to lists for easier processing..
George Keishing36efbc02018-12-12 10:18:23 -0600218 boot_list = list(filter(None, boot_list.split(":")))
219 boot_stack = list(filter(None, boot_stack.split(":")))
Michael Walshb5839d02017-04-12 16:11:20 -0500220
Michael Walsha3e7b222020-02-03 15:32:16 -0600221 boot_table = create_boot_table(boot_table_path, os_host=os_host)
222 valid_boot_types = create_valid_boot_list(boot_table)
223
Michael Walsh903e0b22017-09-19 17:00:33 -0500224 cleanup_boot_results_file()
225 boot_results_file_path = create_boot_results_file_path(pgm_name,
226 openbmc_nickname,
227 master_pid)
Michael Walshb5839d02017-04-12 16:11:20 -0500228
229 if os.path.isfile(boot_results_file_path):
230 # We've been called before in this run so we'll load the saved
Michael Walsh986d8ae2019-07-17 10:02:23 -0500231 # boot_results and boot_history objects.
232 boot_results, boot_history =\
Michael Walsh6c645742018-08-17 15:02:17 -0500233 pickle.load(open(boot_results_file_path, 'rb'))
Michael Walshb5839d02017-04-12 16:11:20 -0500234 else:
235 boot_results = boot_results(boot_table, boot_pass, boot_fail)
236
237 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
238 "/FFDC_FILE_LIST"
Michael Walshe0cf8d72017-05-17 13:20:46 -0500239 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
240 "/FFDC_REPORT_FILE_LIST"
Michael Walshb5839d02017-04-12 16:11:20 -0500241
Michael Walsh600876d2017-05-30 17:58:58 -0500242 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
243 "/FFDC_SUMMARY_FILE_LIST"
244
Michael Walshb5839d02017-04-12 16:11:20 -0500245
Michael Walsh85678942017-03-27 14:34:22 -0500246def initial_plug_in_setup():
Michael Walsh85678942017-03-27 14:34:22 -0500247 r"""
248 Initialize all plug-in environment variables which do not change for the
249 duration of the program.
250
251 """
252
253 global LOG_LEVEL
254 BuiltIn().set_log_level("NONE")
255
256 BuiltIn().set_global_variable("${master_pid}", master_pid)
257 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
258 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
259 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
260 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
261 ffdc_list_file_path)
Michael Walshe0cf8d72017-05-17 13:20:46 -0500262 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
263 ffdc_report_list_path)
Michael Walsh600876d2017-05-30 17:58:58 -0500264 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
265 ffdc_summary_list_path)
Michael Walsh85678942017-03-27 14:34:22 -0500266
267 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
268 ffdc_dir_path_style)
269 BuiltIn().set_global_variable("${FFDC_CHECK}",
270 ffdc_check)
271
272 # For each program parameter, set the corresponding AUTOBOOT_ environment
273 # variable value. Also, set an AUTOBOOT_ environment variable for every
274 # element in additional_values.
275 additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
276 "status_dir_path", "base_tool_dir_path",
Michael Walsh600876d2017-05-30 17:58:58 -0500277 "ffdc_list_file_path", "ffdc_report_list_path",
Michael Shepos7fe83b32020-09-21 15:46:01 -0500278 "ffdc_summary_list_path", "execdir", "redfish_supported",
Michael Shepos34c79562021-03-18 18:49:44 -0500279 "redfish_rest_supported", "redfish_support_trans_state"]
Michael Walsh85678942017-03-27 14:34:22 -0500280
281 plug_in_vars = parm_list + additional_values
282
283 for var_name in plug_in_vars:
284 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
285 var_name = var_name.upper()
286 if var_value is None:
287 var_value = ""
288 os.environ["AUTOBOOT_" + var_name] = str(var_value)
289
290 BuiltIn().set_log_level(LOG_LEVEL)
291
Michael Walsh68a61162017-04-25 11:54:06 -0500292 # Make sure the ffdc list directory exists.
293 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
294 if not os.path.exists(ffdc_list_dir_path):
295 os.makedirs(ffdc_list_dir_path)
Michael Walsh85678942017-03-27 14:34:22 -0500296
Michael Walsh85678942017-03-27 14:34:22 -0500297
Michael Walsh0bbd8602016-11-22 11:31:49 -0600298def plug_in_setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600299 r"""
Michael Walsh85678942017-03-27 14:34:22 -0500300 Initialize all changing plug-in environment variables for use by the
301 plug-in programs.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600302 """
303
Michael Walsh85678942017-03-27 14:34:22 -0500304 global LOG_LEVEL
305 global test_really_running
306
307 BuiltIn().set_log_level("NONE")
308
Michael Walsh6741f742017-02-20 16:16:38 -0600309 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600310 if boot_pass > 1:
311 test_really_running = 1
312 else:
313 test_really_running = 0
314
Michael Walsh6741f742017-02-20 16:16:38 -0600315 BuiltIn().set_global_variable("${test_really_running}",
316 test_really_running)
317 BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
Michael Walsh6741f742017-02-20 16:16:38 -0600318 BuiltIn().set_global_variable("${boot_pass}", boot_pass)
319 BuiltIn().set_global_variable("${boot_fail}", boot_fail)
320 BuiltIn().set_global_variable("${boot_success}", boot_success)
321 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
Sunil M325eb542017-08-10 07:09:43 -0500322 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
323 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
Michael Walsh4c9a6452016-12-13 16:03:11 -0600324
Michael Walsh0bbd8602016-11-22 11:31:49 -0600325 # For each program parameter, set the corresponding AUTOBOOT_ environment
326 # variable value. Also, set an AUTOBOOT_ environment variable for every
327 # element in additional_values.
328 additional_values = ["boot_type_desc", "boot_success", "boot_pass",
Sunil M325eb542017-08-10 07:09:43 -0500329 "boot_fail", "test_really_running", "ffdc_prefix",
330 "boot_start_time", "boot_end_time"]
Michael Walsh0bbd8602016-11-22 11:31:49 -0600331
Michael Walsh85678942017-03-27 14:34:22 -0500332 plug_in_vars = additional_values
Michael Walsh0bbd8602016-11-22 11:31:49 -0600333
334 for var_name in plug_in_vars:
335 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
336 var_name = var_name.upper()
337 if var_value is None:
338 var_value = ""
Michael Walsh6741f742017-02-20 16:16:38 -0600339 os.environ["AUTOBOOT_" + var_name] = str(var_value)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600340
Michael Walsh0bbd8602016-11-22 11:31:49 -0600341 if debug:
Michael Walsh6741f742017-02-20 16:16:38 -0600342 shell_rc, out_buf = \
343 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600344
Michael Walsh85678942017-03-27 14:34:22 -0500345 BuiltIn().set_log_level(LOG_LEVEL)
346
Michael Walsh0bbd8602016-11-22 11:31:49 -0600347
Michael Walshe0cf8d72017-05-17 13:20:46 -0500348def pre_boot_plug_in_setup():
349
350 # Clear the ffdc_list_file_path file. Plug-ins may now write to it.
351 try:
352 os.remove(ffdc_list_file_path)
353 except OSError:
354 pass
355
356 # Clear the ffdc_report_list_path file. Plug-ins may now write to it.
357 try:
358 os.remove(ffdc_report_list_path)
359 except OSError:
360 pass
361
Michael Walsh600876d2017-05-30 17:58:58 -0500362 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it.
363 try:
364 os.remove(ffdc_summary_list_path)
365 except OSError:
366 pass
367
Michael Walshe1974b92017-08-03 13:39:51 -0500368 global ffdc_prefix
369
370 seconds = time.time()
371 loc_time = time.localtime(seconds)
372 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
373
374 ffdc_prefix = openbmc_nickname + "." + time_string
375
Michael Walshe0cf8d72017-05-17 13:20:46 -0500376
Michael Walshf566fb12019-02-01 14:35:09 -0600377def default_sigusr1(signal_number=0,
378 frame=None):
379 r"""
380 Handle SIGUSR1 by doing nothing.
381
382 This function assists in debugging SIGUSR1 processing by printing messages
383 to stdout and to the log.html file.
384
385 Description of argument(s):
386 signal_number The signal number (should always be 10 for SIGUSR1).
387 frame The frame data.
388 """
389
Michael Walsh80dddde2019-10-22 13:54:38 -0500390 gp.qprintn()
391 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600392 gp.lprint_executing()
393
394
395def set_default_siguser1():
396 r"""
397 Set the default_sigusr1 function to be the SIGUSR1 handler.
398 """
399
Michael Walsh80dddde2019-10-22 13:54:38 -0500400 gp.qprintn()
401 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600402 gp.lprint_executing()
403 signal.signal(signal.SIGUSR1, default_sigusr1)
404
405
Michael Walsh6741f742017-02-20 16:16:38 -0600406def setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600407 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600408 Do general program setup tasks.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600409 """
410
Michael Walsh6741f742017-02-20 16:16:38 -0600411 global cp_setup_called
Michael Walsh81816742017-09-27 11:02:29 -0500412 global transitional_boot_selected
Michael Walsh0bbd8602016-11-22 11:31:49 -0600413
Michael Walshb5839d02017-04-12 16:11:20 -0500414 gp.qprintn()
415
George Keishinga54e06f2020-06-12 10:42:41 -0500416 if redfish_supported:
417 redfish.login()
418
Michael Walshf566fb12019-02-01 14:35:09 -0600419 set_default_siguser1()
Michael Walsh81816742017-09-27 11:02:29 -0500420 transitional_boot_selected = False
421
Michael Walsh83f4bc72017-04-20 16:49:43 -0500422 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
423 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
Michael Walshd061c042017-05-23 14:46:57 -0500424 # If we can't find process_plug_in_packages.py, ssh_pw or
425 # validate_plug_ins.py, then we don't have our repo bin in PATH.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500426 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
427 + " ssh_pw validate_plug_ins.py", quiet=1,
Michael Walshd061c042017-05-23 14:46:57 -0500428 print_output=0, show_err=0)
Michael Walshb5839d02017-04-12 16:11:20 -0500429 if shell_rc != 0:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500430 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
431 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
432 if robot_pgm_dir_path not in sys.path:
433 sys.path.append(robot_pgm_dir_path)
434 PYTHONPATH = os.environ.get("PYTHONPATH", "")
435 if PYTHONPATH == "":
436 os.environ['PYTHONPATH'] = robot_pgm_dir_path
437 else:
438 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
Michael Walsh6741f742017-02-20 16:16:38 -0600439
440 validate_parms()
441
Michael Walshc108e422019-03-28 12:27:18 -0500442 gp.qprint_pgm_header()
Michael Walsh6741f742017-02-20 16:16:38 -0600443
George Keishinga54e06f2020-06-12 10:42:41 -0500444 grk.run_key_u(default_set_power_policy)
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500445
Michael Walsh85678942017-03-27 14:34:22 -0500446 initial_plug_in_setup()
447
Michael Walsh6741f742017-02-20 16:16:38 -0600448 plug_in_setup()
449 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
450 call_point='setup')
451 if rc != 0:
452 error_message = "Plug-in setup failed.\n"
Michael Walshc108e422019-03-28 12:27:18 -0500453 gp.print_error_report(error_message)
Michael Walsh6741f742017-02-20 16:16:38 -0600454 BuiltIn().fail(error_message)
455 # Setting cp_setup_called lets our Teardown know that it needs to call
456 # the cleanup plug-in call point.
457 cp_setup_called = 1
458
459 # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
460 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
Michael Walsh85678942017-03-27 14:34:22 -0500461 # FFDC_LOG_PATH is used by "FFDC" keyword.
462 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
Michael Walsh6741f742017-02-20 16:16:38 -0600463
Michael Walshdc80d672017-05-09 12:58:32 -0500464 # Also printed by FFDC.
465 global host_name
466 global host_ip
467 host = socket.gethostname()
468 host_name, host_ip = gm.get_host_name_ip(host)
469
Michael Walsh986d8ae2019-07-17 10:02:23 -0500470 gp.dprint_var(boot_table)
Michael Walshb5839d02017-04-12 16:11:20 -0500471 gp.dprint_var(boot_lists)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600472
Michael Walsh0bbd8602016-11-22 11:31:49 -0600473
Michael Walsh6741f742017-02-20 16:16:38 -0600474def validate_parms():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600475 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600476 Validate all program parameters.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600477 """
478
Michael Walshb5839d02017-04-12 16:11:20 -0500479 process_pgm_parms()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600480
Michael Walshb5839d02017-04-12 16:11:20 -0500481 gp.qprintn()
482
483 global openbmc_model
Michael Walshf5ce38c2020-02-27 12:46:20 -0600484 if openbmc_model == "":
485 status, ret_values =\
486 grk.run_key_u("Get BMC System Model")
487 openbmc_model = ret_values
488 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
489 gv.set_exit_on_error(True)
Michael Walsh44cef252019-08-01 12:38:56 -0500490 gv.valid_value(openbmc_host)
491 gv.valid_value(openbmc_username)
492 gv.valid_value(openbmc_password)
493 gv.valid_value(rest_username)
494 gv.valid_value(rest_password)
495 gv.valid_value(ipmi_username)
496 gv.valid_value(ipmi_password)
Michael Walsh6741f742017-02-20 16:16:38 -0600497 if os_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500498 gv.valid_value(os_username)
499 gv.valid_value(os_password)
Michael Walsh6741f742017-02-20 16:16:38 -0600500 if pdu_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500501 gv.valid_value(pdu_username)
502 gv.valid_value(pdu_password)
503 gv.valid_integer(pdu_slot_no)
Michael Walsh6741f742017-02-20 16:16:38 -0600504 if openbmc_serial_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500505 gv.valid_integer(openbmc_serial_port)
Michael Walsh44cef252019-08-01 12:38:56 -0500506 gv.valid_value(openbmc_model)
507 gv.valid_integer(max_num_tests)
508 gv.valid_integer(boot_pass)
509 gv.valid_integer(boot_fail)
Michael Walsh6741f742017-02-20 16:16:38 -0600510 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
511 BuiltIn().set_global_variable("${plug_in_packages_list}",
512 plug_in_packages_list)
Michael Walsh44cef252019-08-01 12:38:56 -0500513 gv.valid_value(stack_mode, valid_values=['normal', 'skip'])
Michael Walshf5ce38c2020-02-27 12:46:20 -0600514 gv.set_exit_on_error(False)
Michael Walsha20da402017-03-31 16:27:45 -0500515 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
Michael Walsh6741f742017-02-20 16:16:38 -0600516 error_message = "You must provide either a value for either the" +\
517 " boot_list or the boot_stack parm.\n"
518 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600519 valid_boot_list(boot_list, valid_boot_types)
520 valid_boot_list(boot_stack, valid_boot_types)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500521 selected_PDU_boots = list(set(boot_list + boot_stack)
522 & set(boot_lists['PDU_reboot']))
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500523 if len(selected_PDU_boots) > 0 and pdu_host == "":
524 error_message = "You have selected the following boots which" +\
525 " require a PDU host but no value for pdu_host:\n"
526 error_message += gp.sprint_var(selected_PDU_boots)
Michael Walsh986d8ae2019-07-17 10:02:23 -0500527 error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500528 BuiltIn().fail(gp.sprint_error(error_message))
529
Michael Walsh6741f742017-02-20 16:16:38 -0600530 return
Michael Walsh0bbd8602016-11-22 11:31:49 -0600531
Michael Walsh0bbd8602016-11-22 11:31:49 -0600532
Michael Walsh6741f742017-02-20 16:16:38 -0600533def my_get_state():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600534 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600535 Get the system state plus a little bit of wrapping.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600536 """
537
Michael Walsh6741f742017-02-20 16:16:38 -0600538 global state
539
540 req_states = ['epoch_seconds'] + st.default_req_states
541
Michael Walshb5839d02017-04-12 16:11:20 -0500542 gp.qprint_timen("Getting system state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600543 if test_mode:
544 state['epoch_seconds'] = int(time.time())
545 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500546 state = st.get_state(req_states=req_states, quiet=quiet)
547 gp.qprint_var(state)
Michael Walsh341c21e2017-01-17 16:25:20 -0600548
Michael Walsh341c21e2017-01-17 16:25:20 -0600549
Michael Walsh45ca6e42017-09-14 17:29:12 -0500550def valid_state():
Michael Walsh45ca6e42017-09-14 17:29:12 -0500551 r"""
552 Verify that our state dictionary contains no blank values. If we don't get
553 valid state data, we cannot continue to work.
554 """
555
556 if st.compare_states(state, st.invalid_state_match, 'or'):
557 error_message = "The state dictionary contains blank fields which" +\
558 " is illegal.\n" + gp.sprint_var(state)
559 BuiltIn().fail(gp.sprint_error(error_message))
560
Michael Walsh45ca6e42017-09-14 17:29:12 -0500561
Michael Walsh6741f742017-02-20 16:16:38 -0600562def select_boot():
Michael Walsh341c21e2017-01-17 16:25:20 -0600563 r"""
564 Select a boot test to be run based on our current state and return the
565 chosen boot type.
566
567 Description of arguments:
Michael Walsh6741f742017-02-20 16:16:38 -0600568 state The state of the machine.
Michael Walsh341c21e2017-01-17 16:25:20 -0600569 """
570
Michael Walsh81816742017-09-27 11:02:29 -0500571 global transitional_boot_selected
Michael Walsh30dadae2017-02-27 14:25:52 -0600572 global boot_stack
573
Michael Walshb5839d02017-04-12 16:11:20 -0500574 gp.qprint_timen("Selecting a boot test.")
Michael Walsh6741f742017-02-20 16:16:38 -0600575
Michael Walsh81816742017-09-27 11:02:29 -0500576 if transitional_boot_selected and not boot_success:
577 prior_boot = next_boot
578 boot_candidate = boot_stack.pop()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500579 gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
580 + " transition to a valid state for '" + boot_candidate
581 + "' which was at the top of the boot_stack. Since"
582 + " the '" + next_boot + "' failed, the '"
583 + boot_candidate + "' has been removed from the stack"
584 + " to avoid and endless failure loop.")
Michael Walsh81816742017-09-27 11:02:29 -0500585 if len(boot_stack) == 0:
586 return ""
587
Michael Walsh6741f742017-02-20 16:16:38 -0600588 my_get_state()
Michael Walsh45ca6e42017-09-14 17:29:12 -0500589 valid_state()
Michael Walsh6741f742017-02-20 16:16:38 -0600590
Michael Walsh81816742017-09-27 11:02:29 -0500591 transitional_boot_selected = False
Michael Walsh6741f742017-02-20 16:16:38 -0600592 stack_popped = 0
593 if len(boot_stack) > 0:
594 stack_popped = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500595 gp.qprint_dashes()
596 gp.qprint_var(boot_stack)
597 gp.qprint_dashes()
598 skip_boot_printed = 0
599 while len(boot_stack) > 0:
600 boot_candidate = boot_stack.pop()
601 if stack_mode == 'normal':
602 break
603 else:
604 if st.compare_states(state, boot_table[boot_candidate]['end']):
605 if not skip_boot_printed:
Michael Walshff340002017-08-29 11:18:27 -0500606 gp.qprint_var(stack_mode)
607 gp.qprintn()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500608 gp.qprint_timen("Skipping the following boot tests"
609 + " which are unnecessary since their"
610 + " required end states match the"
611 + " current machine state:")
Michael Walshb5839d02017-04-12 16:11:20 -0500612 skip_boot_printed = 1
Michael Walshff340002017-08-29 11:18:27 -0500613 gp.qprint_var(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500614 boot_candidate = ""
615 if boot_candidate == "":
616 gp.qprint_dashes()
617 gp.qprint_var(boot_stack)
618 gp.qprint_dashes()
619 return boot_candidate
Michael Walsh6741f742017-02-20 16:16:38 -0600620 if st.compare_states(state, boot_table[boot_candidate]['start']):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500621 gp.qprint_timen("The machine state is valid for a '"
622 + boot_candidate + "' boot test.")
Michael Walshb5839d02017-04-12 16:11:20 -0500623 gp.qprint_dashes()
624 gp.qprint_var(boot_stack)
625 gp.qprint_dashes()
Michael Walsh6741f742017-02-20 16:16:38 -0600626 return boot_candidate
Michael Walsh341c21e2017-01-17 16:25:20 -0600627 else:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500628 gp.qprint_timen("The machine state does not match the required"
629 + " starting state for a '" + boot_candidate
630 + "' boot test:")
Michael Walsh986d8ae2019-07-17 10:02:23 -0500631 gp.qprint_varx("boot_table_start_entry",
632 boot_table[boot_candidate]['start'])
Michael Walsh6741f742017-02-20 16:16:38 -0600633 boot_stack.append(boot_candidate)
Michael Walsh81816742017-09-27 11:02:29 -0500634 transitional_boot_selected = True
Michael Walsh6741f742017-02-20 16:16:38 -0600635 popped_boot = boot_candidate
636
637 # Loop through your list selecting a boot_candidates
638 boot_candidates = []
639 for boot_candidate in boot_list:
640 if st.compare_states(state, boot_table[boot_candidate]['start']):
641 if stack_popped:
642 if st.compare_states(boot_table[boot_candidate]['end'],
Gunnar Mills096cd562018-03-26 10:19:12 -0500643 boot_table[popped_boot]['start']):
Michael Walsh6741f742017-02-20 16:16:38 -0600644 boot_candidates.append(boot_candidate)
645 else:
646 boot_candidates.append(boot_candidate)
647
648 if len(boot_candidates) == 0:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500649 gp.qprint_timen("The user's boot list contained no boot tests"
650 + " which are valid for the current machine state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600651 boot_candidate = default_power_on
652 if not st.compare_states(state, boot_table[default_power_on]['start']):
653 boot_candidate = default_power_off
654 boot_candidates.append(boot_candidate)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500655 gp.qprint_timen("Using default '" + boot_candidate
656 + "' boot type to transition to valid state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600657
Michael Walshb5839d02017-04-12 16:11:20 -0500658 gp.dprint_var(boot_candidates)
Michael Walsh6741f742017-02-20 16:16:38 -0600659
660 # Randomly select a boot from the candidate list.
661 boot = random.choice(boot_candidates)
Michael Walsh341c21e2017-01-17 16:25:20 -0600662
663 return boot
Michael Walsh0bbd8602016-11-22 11:31:49 -0600664
Michael Walsh55302292017-01-10 11:43:02 -0600665
Michael Walshb2e53ec2017-10-30 15:04:36 -0500666def print_defect_report(ffdc_file_list):
Michael Walsh341c21e2017-01-17 16:25:20 -0600667 r"""
668 Print a defect report.
Michael Walshb2e53ec2017-10-30 15:04:36 -0500669
670 Description of argument(s):
671 ffdc_file_list A list of files which were collected by our ffdc functions.
Michael Walsh341c21e2017-01-17 16:25:20 -0600672 """
673
Michael Walsh600876d2017-05-30 17:58:58 -0500674 # Making deliberate choice to NOT run plug_in_setup(). We don't want
675 # ffdc_prefix updated.
676 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
677 call_point='ffdc_report', stop_on_plug_in_failure=0)
678
Michael Walshe0cf8d72017-05-17 13:20:46 -0500679 # Get additional header data which may have been created by ffdc plug-ins.
680 # Also, delete the individual header files to cleanup.
681 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
682 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
683 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
684 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
685 show_err=0)
686
Michael Walshb2e53ec2017-10-30 15:04:36 -0500687 # Get additional summary data which may have been created by ffdc plug-ins.
Michael Walsh600876d2017-05-30 17:58:58 -0500688 # Also, delete the individual header files to cleanup.
689 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
690 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
691 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
692 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
693 show_err=0)
694
Michael Walshb2e53ec2017-10-30 15:04:36 -0500695 # ffdc_list_file_path contains a list of any ffdc files created by plug-
696 # ins, etc. Read that data into a list.
Michael Walsh341c21e2017-01-17 16:25:20 -0600697 try:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500698 plug_in_ffdc_list = \
699 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
George Keishing36efbc02018-12-12 10:18:23 -0600700 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
Michael Walsh341c21e2017-01-17 16:25:20 -0600701 except IOError:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500702 plug_in_ffdc_list = []
703
704 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
705 # in. Eliminate duplicates and sort the list.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500706 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
Michael Walshb2e53ec2017-10-30 15:04:36 -0500707
708 if status_file_path != "":
709 ffdc_file_list.insert(0, status_file_path)
710
711 # Convert the list to a printable list.
712 printable_ffdc_file_list = "\n".join(ffdc_file_list)
Michael Walsh341c21e2017-01-17 16:25:20 -0600713
Michael Walsh68a61162017-04-25 11:54:06 -0500714 # Open ffdc_file_list for writing. We will write a complete list of
715 # FFDC files to it for possible use by plug-ins like cp_stop_check.
716 ffdc_list_file = open(ffdc_list_file_path, 'w')
Michael Walshb2e53ec2017-10-30 15:04:36 -0500717 ffdc_list_file.write(printable_ffdc_file_list + "\n")
718 ffdc_list_file.close()
719
720 indent = 0
721 width = 90
722 linefeed = 1
723 char = "="
Michael Walsh68a61162017-04-25 11:54:06 -0500724
725 gp.qprintn()
Michael Walshb2e53ec2017-10-30 15:04:36 -0500726 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500727 gp.qprintn("Copy this data to the defect:\n")
728
Michael Walshe0cf8d72017-05-17 13:20:46 -0500729 if len(more_header_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500730 gp.qprintn(more_header_info)
Michael Walshdc80d672017-05-09 12:58:32 -0500731 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
732 openbmc_host_name, openbmc_ip, openbmc_username,
Michael Walsh0a3bdb42019-01-31 16:21:44 +0000733 openbmc_password, rest_username, rest_password, ipmi_username,
734 ipmi_password, os_host, os_host_name, os_ip, os_username,
Michael Walshdc80d672017-05-09 12:58:32 -0500735 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
736 pdu_password, pdu_slot_no, openbmc_serial_host,
737 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
Michael Walsh68a61162017-04-25 11:54:06 -0500738
739 gp.qprintn()
Michael Walsh986d8ae2019-07-17 10:02:23 -0500740 print_boot_history(boot_history)
Michael Walsh68a61162017-04-25 11:54:06 -0500741 gp.qprintn()
742 gp.qprint_var(state)
Michael Walshb5839d02017-04-12 16:11:20 -0500743 gp.qprintn()
744 gp.qprintn("FFDC data files:")
Michael Walshb2e53ec2017-10-30 15:04:36 -0500745 gp.qprintn(printable_ffdc_file_list)
Michael Walshb5839d02017-04-12 16:11:20 -0500746 gp.qprintn()
Michael Walsh341c21e2017-01-17 16:25:20 -0600747
Michael Walsh600876d2017-05-30 17:58:58 -0500748 if len(ffdc_summary_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500749 gp.qprintn(ffdc_summary_info)
Michael Walsh600876d2017-05-30 17:58:58 -0500750
Michael Walshb2e53ec2017-10-30 15:04:36 -0500751 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500752
Michael Walsh6741f742017-02-20 16:16:38 -0600753
Michael Walsh6741f742017-02-20 16:16:38 -0600754def my_ffdc():
Michael Walsh6741f742017-02-20 16:16:38 -0600755 r"""
756 Collect FFDC data.
757 """
758
759 global state
760
761 plug_in_setup()
762 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500763 call_point='ffdc', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600764
765 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500766 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix="
767 + AUTOBOOT_FFDC_PREFIX
768 + " ffdc_function_list="
769 + ffdc_function_list, ignore=1)
Michael Walsh83f4bc72017-04-20 16:49:43 -0500770 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500771 gp.qprint_error("Call to ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500772 if type(ffdc_file_list) is not list:
773 ffdc_file_list = []
774 # Leave a record for caller that "soft" errors occurred.
775 soft_errors = 1
776 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600777
778 my_get_state()
779
Michael Walshb2e53ec2017-10-30 15:04:36 -0500780 print_defect_report(ffdc_file_list)
Michael Walsh6741f742017-02-20 16:16:38 -0600781
Michael Walsh6741f742017-02-20 16:16:38 -0600782
Michael Walsh6741f742017-02-20 16:16:38 -0600783def print_test_start_message(boot_keyword):
Michael Walsh6741f742017-02-20 16:16:38 -0600784 r"""
785 Print a message indicating what boot test is about to run.
786
787 Description of arguments:
788 boot_keyword The name of the boot which is to be run
789 (e.g. "BMC Power On").
790 """
791
Michael Walsh986d8ae2019-07-17 10:02:23 -0500792 global boot_history
Sunil M325eb542017-08-10 07:09:43 -0500793 global boot_start_time
Michael Walsh6741f742017-02-20 16:16:38 -0600794
795 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
Sunil M325eb542017-08-10 07:09:43 -0500796
797 # Set boot_start_time for use by plug-ins.
798 boot_start_time = doing_msg[1:33]
799 gp.qprint_var(boot_start_time)
800
Michael Walshb5839d02017-04-12 16:11:20 -0500801 gp.qprint(doing_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600802
Michael Walsh986d8ae2019-07-17 10:02:23 -0500803 update_boot_history(boot_history, doing_msg, max_boot_history)
Michael Walsh6741f742017-02-20 16:16:38 -0600804
Michael Walsh6741f742017-02-20 16:16:38 -0600805
Michael Walshf566fb12019-02-01 14:35:09 -0600806def stop_boot_test(signal_number=0,
807 frame=None):
808 r"""
809 Handle SIGUSR1 by aborting the boot test that is running.
810
811 Description of argument(s):
812 signal_number The signal number (should always be 10 for SIGUSR1).
813 frame The frame data.
814 """
815
Michael Walsh80dddde2019-10-22 13:54:38 -0500816 gp.qprintn()
817 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600818 gp.lprint_executing()
819
820 # Restore original sigusr1 handler.
821 set_default_siguser1()
822
823 message = "The caller has asked that the boot test be stopped and marked"
824 message += " as a failure."
825
826 function_stack = gm.get_function_stack()
827 if "wait_state" in function_stack:
Michael Walshc44aa532019-06-14 13:33:29 -0500828 st.set_exit_wait_early_message(message)
Michael Walshf566fb12019-02-01 14:35:09 -0600829 else:
830 BuiltIn().fail(gp.sprint_error(message))
831
832
Michael Walsh6741f742017-02-20 16:16:38 -0600833def run_boot(boot):
Michael Walsh6741f742017-02-20 16:16:38 -0600834 r"""
835 Run the specified boot.
836
837 Description of arguments:
838 boot The name of the boot test to be performed.
839 """
840
841 global state
842
Michael Walshf566fb12019-02-01 14:35:09 -0600843 signal.signal(signal.SIGUSR1, stop_boot_test)
844 gp.qprint_timen("stop_boot_test is armed.")
845
Michael Walsh6741f742017-02-20 16:16:38 -0600846 print_test_start_message(boot)
847
848 plug_in_setup()
849 rc, shell_rc, failed_plug_in_name = \
850 grpi.rprocess_plug_in_packages(call_point="pre_boot")
851 if rc != 0:
852 error_message = "Plug-in failed with non-zero return code.\n" +\
Michael Walsh986d8ae2019-07-17 10:02:23 -0500853 gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600854 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600855 BuiltIn().fail(gp.sprint_error(error_message))
856
857 if test_mode:
858 # In test mode, we'll pretend the boot worked by assigning its
859 # required end state to the default state value.
Michael Walsh30dadae2017-02-27 14:25:52 -0600860 state = st.strip_anchor_state(boot_table[boot]['end'])
Michael Walsh6741f742017-02-20 16:16:38 -0600861 else:
862 # Assertion: We trust that the state data was made fresh by the
863 # caller.
864
Michael Walshb5839d02017-04-12 16:11:20 -0500865 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600866
867 if boot_table[boot]['method_type'] == "keyword":
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600868 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
Michael Walshb5839d02017-04-12 16:11:20 -0500869 boot_table[boot]['method'],
870 quiet=quiet)
Michael Walsh6741f742017-02-20 16:16:38 -0600871
872 if boot_table[boot]['bmc_reboot']:
873 st.wait_for_comm_cycle(int(state['epoch_seconds']))
Michael Walsh30dadae2017-02-27 14:25:52 -0600874 plug_in_setup()
875 rc, shell_rc, failed_plug_in_name = \
876 grpi.rprocess_plug_in_packages(call_point="post_reboot")
877 if rc != 0:
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600878 error_message = "Plug-in failed with non-zero return code.\n"
Michael Walsh986d8ae2019-07-17 10:02:23 -0500879 error_message += gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600880 set_default_siguser1()
Michael Walsh30dadae2017-02-27 14:25:52 -0600881 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600882 else:
883 match_state = st.anchor_state(state)
884 del match_state['epoch_seconds']
885 # Wait for the state to change in any way.
886 st.wait_state(match_state, wait_time=state_change_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500887 interval="10 seconds", invert=1)
Michael Walsh6741f742017-02-20 16:16:38 -0600888
Michael Walshb5839d02017-04-12 16:11:20 -0500889 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600890 if boot_table[boot]['end']['chassis'] == "Off":
891 boot_timeout = power_off_timeout
892 else:
893 boot_timeout = power_on_timeout
894 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500895 interval="10 seconds")
Michael Walsh6741f742017-02-20 16:16:38 -0600896
897 plug_in_setup()
898 rc, shell_rc, failed_plug_in_name = \
899 grpi.rprocess_plug_in_packages(call_point="post_boot")
900 if rc != 0:
901 error_message = "Plug-in failed with non-zero return code.\n" +\
Michael Walsh986d8ae2019-07-17 10:02:23 -0500902 gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600903 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600904 BuiltIn().fail(gp.sprint_error(error_message))
905
Michael Walshf566fb12019-02-01 14:35:09 -0600906 # Restore original sigusr1 handler.
907 set_default_siguser1()
908
Michael Walsh6741f742017-02-20 16:16:38 -0600909
Michael Walsh6741f742017-02-20 16:16:38 -0600910def test_loop_body():
Michael Walsh6741f742017-02-20 16:16:38 -0600911 r"""
912 The main loop body for the loop in main_py.
913
914 Description of arguments:
915 boot_count The iteration number (starts at 1).
916 """
917
918 global boot_count
919 global state
920 global next_boot
921 global boot_success
Sunil M325eb542017-08-10 07:09:43 -0500922 global boot_end_time
Michael Walsh6741f742017-02-20 16:16:38 -0600923
Michael Walshb5839d02017-04-12 16:11:20 -0500924 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600925
926 next_boot = select_boot()
Michael Walshb5839d02017-04-12 16:11:20 -0500927 if next_boot == "":
928 return True
Michael Walsh6741f742017-02-20 16:16:38 -0600929
Michael Walshb5839d02017-04-12 16:11:20 -0500930 boot_count += 1
931 gp.qprint_timen("Starting boot " + str(boot_count) + ".")
Michael Walsh6741f742017-02-20 16:16:38 -0600932
Michael Walshe0cf8d72017-05-17 13:20:46 -0500933 pre_boot_plug_in_setup()
Michael Walsh6741f742017-02-20 16:16:38 -0600934
935 cmd_buf = ["run_boot", next_boot]
936 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
937 if boot_status == "FAIL":
Michael Walshb5839d02017-04-12 16:11:20 -0500938 gp.qprint(msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600939
Michael Walshb5839d02017-04-12 16:11:20 -0500940 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600941 if boot_status == "PASS":
942 boot_success = 1
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500943 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
944 + "\" succeeded.")
Michael Walsh6741f742017-02-20 16:16:38 -0600945 else:
946 boot_success = 0
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500947 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
948 + "\" failed.")
Sunil M325eb542017-08-10 07:09:43 -0500949
950 # Set boot_end_time for use by plug-ins.
951 boot_end_time = completion_msg[1:33]
952 gp.qprint_var(boot_end_time)
953
954 gp.qprint(completion_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600955
956 boot_results.update(next_boot, boot_status)
957
958 plug_in_setup()
959 # NOTE: A post_test_case call point failure is NOT counted as a boot
960 # failure.
961 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500962 call_point='post_test_case', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600963
964 plug_in_setup()
965 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500966 call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
Michael Walsh6741f742017-02-20 16:16:38 -0600967 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
Michael Walsh12059e22019-03-21 11:03:45 -0500968 if ffdc_check == "All" or\
Michael Walsh89de14a2018-10-01 16:51:37 -0500969 shell_rc == dump_ffdc_rc():
Michael Walsh83f4bc72017-04-20 16:49:43 -0500970 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
971 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500972 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500973 # Leave a record for caller that "soft" errors occurred.
974 soft_errors = 1
975 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600976
Michael Walshaabef1e2017-09-20 15:16:17 -0500977 if delete_errlogs:
Michael Shepos1a67b082020-08-28 16:01:58 -0500978 # print error logs before delete
979 status, error_logs = grk.run_key_u("Get Error Logs")
980 pels = pel.peltool("-l", ignore_err=1)
Michael Shepos0e5f1132020-09-30 16:24:25 -0500981 log.print_error_logs(error_logs, "AdditionalData Message Severity")
982 gp.qprint_var(pels)
Michael Shepos1a67b082020-08-28 16:01:58 -0500983
Michael Walshaabef1e2017-09-20 15:16:17 -0500984 # We need to purge error logs between boots or they build up.
Michael Walsh409ad352020-02-06 11:46:35 -0600985 grk.run_key(delete_errlogs_cmd, ignore=1)
Michael Shepos92a54bf2020-11-11 11:48:55 -0600986 grk.run_key(delete_bmcdump_cmd, ignore=1)
George Keishing2ef6a7d2021-05-19 09:05:32 -0500987 if redfish_support_trans_state:
988 grk.run_key(delete_sysdump_cmd, ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -0500989
Michael Walsh952f9b02017-03-09 13:11:14 -0600990 boot_results.print_report()
Michael Walshb5839d02017-04-12 16:11:20 -0500991 gp.qprint_timen("Finished boot " + str(boot_count) + ".")
Michael Walsh952f9b02017-03-09 13:11:14 -0600992
Michael Walsh6741f742017-02-20 16:16:38 -0600993 plug_in_setup()
994 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500995 call_point='stop_check', shell_rc=stop_test_rc(),
996 stop_on_non_zero_rc=1)
997 if shell_rc == stop_test_rc():
Michael Walsh3ba8ecd2018-04-24 11:33:25 -0500998 message = "Stopping as requested by user.\n"
Michael Walsh80dddde2019-10-22 13:54:38 -0500999 gp.qprint_time(message)
Michael Walsh3ba8ecd2018-04-24 11:33:25 -05001000 BuiltIn().fail(message)
Michael Walsh6741f742017-02-20 16:16:38 -06001001
Michael Walshd139f282017-04-04 18:00:23 -05001002 # This should help prevent ConnectionErrors.
George Keishing4d65c862020-12-03 06:52:11 -06001003 # Purge all redfish and REST connection sessions.
George Keishingd86e45c2021-03-19 07:38:14 -05001004 if redfish_delete_sessions:
1005 grk.run_key_u("Close All Connections", ignore=1)
1006 grk.run_key_u("Delete All Redfish Sessions", ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -05001007
Michael Walsh6741f742017-02-20 16:16:38 -06001008 return True
1009
Michael Walsh6741f742017-02-20 16:16:38 -06001010
Michael Walsh83f4bc72017-04-20 16:49:43 -05001011def obmc_boot_test_teardown():
Michael Walsh6741f742017-02-20 16:16:38 -06001012 r"""
Michael Walshf75d4352019-12-05 17:01:20 -06001013 Clean up after the main keyword.
Michael Walsh6741f742017-02-20 16:16:38 -06001014 """
Michael Walshf75d4352019-12-05 17:01:20 -06001015 gp.qprint_executing()
1016
1017 if ga.psutil_imported:
1018 ga.terminate_descendants()
Michael Walsh6741f742017-02-20 16:16:38 -06001019
1020 if cp_setup_called:
1021 plug_in_setup()
1022 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -05001023 call_point='cleanup', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -06001024
Michael Walsh600876d2017-05-30 17:58:58 -05001025 if 'boot_results_file_path' in globals():
Michael Walsh986d8ae2019-07-17 10:02:23 -05001026 # Save boot_results and boot_history objects to a file in case they are
Michael Walsh6c645742018-08-17 15:02:17 -05001027 # needed again.
Michael Walsh600876d2017-05-30 17:58:58 -05001028 gp.qprint_timen("Saving boot_results to the following path.")
1029 gp.qprint_var(boot_results_file_path)
Michael Walsh986d8ae2019-07-17 10:02:23 -05001030 pickle.dump((boot_results, boot_history),
Michael Walsh6c645742018-08-17 15:02:17 -05001031 open(boot_results_file_path, 'wb'),
Michael Walsh600876d2017-05-30 17:58:58 -05001032 pickle.HIGHEST_PROTOCOL)
Michael Walsh0b93fbf2017-03-02 14:42:41 -06001033
Michael Walshff340002017-08-29 11:18:27 -05001034 global save_stack
1035 # Restore any global values saved on the save_stack.
1036 for parm_name in main_func_parm_list:
1037 # Get the parm_value if it was saved on the stack.
1038 try:
1039 parm_value = save_stack.pop(parm_name)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -05001040 except BaseException:
Michael Walshff340002017-08-29 11:18:27 -05001041 # If it was not saved, no further action is required.
1042 continue
1043
1044 # Restore the saved value.
1045 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1046 "}\", parm_value)"
1047 gp.dpissuing(cmd_buf)
1048 exec(cmd_buf)
1049
1050 gp.dprintn(save_stack.sprint_obj())
1051
Michael Walsh6741f742017-02-20 16:16:38 -06001052
Michael Walshc9116812017-03-10 14:23:06 -06001053def test_teardown():
Michael Walshc9116812017-03-10 14:23:06 -06001054 r"""
1055 Clean up after this test case.
1056 """
1057
1058 gp.qprintn()
Michael Walshf75d4352019-12-05 17:01:20 -06001059 gp.qprint_executing()
1060
1061 if ga.psutil_imported:
1062 ga.terminate_descendants()
1063
Michael Walshc9116812017-03-10 14:23:06 -06001064 cmd_buf = ["Print Error",
1065 "A keyword timeout occurred ending this program.\n"]
1066 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1067
George Keishinga54e06f2020-06-12 10:42:41 -05001068 if redfish_supported:
1069 redfish.logout()
1070
Michael Walshc108e422019-03-28 12:27:18 -05001071 gp.qprint_pgm_footer()
Michael Walshb5839d02017-04-12 16:11:20 -05001072
Michael Walshc9116812017-03-10 14:23:06 -06001073
Michael Walsh89de14a2018-10-01 16:51:37 -05001074def post_stack():
1075 r"""
1076 Process post_stack plug-in programs.
1077 """
1078
1079 if not call_post_stack_plug:
1080 # The caller does not wish to have post_stack plug-in processing done.
1081 return
1082
1083 global boot_success
1084
1085 # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1086 pre_boot_plug_in_setup()
1087 # For the purposes of the following plug-ins, mark the "boot" as a success.
1088 boot_success = 1
1089 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001090 rc, shell_rc, failed_plug_in_name, history =\
1091 grpi.rprocess_plug_in_packages(call_point='post_stack',
1092 stop_on_plug_in_failure=0,
1093 return_history=True)
Michael Walsh986d8ae2019-07-17 10:02:23 -05001094 for doing_msg in history:
1095 update_boot_history(boot_history, doing_msg, max_boot_history)
Michael Walsh815b1d52018-10-30 13:32:26 -05001096 if rc != 0:
1097 boot_success = 0
Michael Walsh89de14a2018-10-01 16:51:37 -05001098
1099 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001100 rc, shell_rc, failed_plug_in_name =\
1101 grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1102 shell_rc=dump_ffdc_rc(),
1103 stop_on_plug_in_failure=1,
1104 stop_on_non_zero_rc=1)
1105 if shell_rc == dump_ffdc_rc():
Michael Walsh89de14a2018-10-01 16:51:37 -05001106 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1107 if status != 'PASS':
1108 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -05001109 # Leave a record for caller that "soft" errors occurred.
1110 soft_errors = 1
1111 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh89de14a2018-10-01 16:51:37 -05001112
1113 plug_in_setup()
1114 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1115 call_point='stop_check', shell_rc=stop_test_rc(),
1116 stop_on_non_zero_rc=1)
1117 if shell_rc == stop_test_rc():
1118 message = "Stopping as requested by user.\n"
Michael Walsh80dddde2019-10-22 13:54:38 -05001119 gp.qprint_time(message)
Michael Walsh89de14a2018-10-01 16:51:37 -05001120 BuiltIn().fail(message)
1121
1122
Michael Walshff340002017-08-29 11:18:27 -05001123def obmc_boot_test_py(loc_boot_stack=None,
1124 loc_stack_mode=None,
1125 loc_quiet=None):
Michael Walsh6741f742017-02-20 16:16:38 -06001126 r"""
1127 Do main program processing.
1128 """
1129
Michael Walshff340002017-08-29 11:18:27 -05001130 global save_stack
1131
Michael Walshf75d4352019-12-05 17:01:20 -06001132 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']})
1133
George Keishing36efbc02018-12-12 10:18:23 -06001134 gp.dprintn()
Michael Walshff340002017-08-29 11:18:27 -05001135 # Process function parms.
1136 for parm_name in main_func_parm_list:
1137 # Get parm's value.
George Keishing36efbc02018-12-12 10:18:23 -06001138 parm_value = eval("loc_" + parm_name)
1139 gp.dpvars(parm_name, parm_value)
Michael Walshff340002017-08-29 11:18:27 -05001140
George Keishing36efbc02018-12-12 10:18:23 -06001141 if parm_value is not None:
Michael Walshff340002017-08-29 11:18:27 -05001142 # Save the global value on a stack.
1143 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1144 parm_name + "}\"), \"" + parm_name + "\")"
1145 gp.dpissuing(cmd_buf)
1146 exec(cmd_buf)
1147
1148 # Set the global value to the passed value.
1149 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1150 "}\", loc_" + parm_name + ")"
1151 gp.dpissuing(cmd_buf)
1152 exec(cmd_buf)
1153
1154 gp.dprintn(save_stack.sprint_obj())
Michael Walshb5839d02017-04-12 16:11:20 -05001155
Michael Walsh6741f742017-02-20 16:16:38 -06001156 setup()
1157
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001158 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1159
Michael Walsha20da402017-03-31 16:27:45 -05001160 if ffdc_only:
1161 gp.qprint_timen("Caller requested ffdc_only.")
Michael Walsh986d8ae2019-07-17 10:02:23 -05001162 if do_pre_boot_plug_in_setup:
1163 pre_boot_plug_in_setup()
Michael Walsh83f4bc72017-04-20 16:49:43 -05001164 grk.run_key_u("my_ffdc")
Michael Walsh764d2f82017-04-27 16:01:08 -05001165 return
Michael Walsha20da402017-03-31 16:27:45 -05001166
Michael Walsh409ad352020-02-06 11:46:35 -06001167 if delete_errlogs:
Michael Shepos1a67b082020-08-28 16:01:58 -05001168 # print error logs before delete
1169 status, error_logs = grk.run_key_u("Get Error Logs")
1170 pels = pel.peltool("-l", ignore_err=1)
Michael Shepos0e5f1132020-09-30 16:24:25 -05001171 log.print_error_logs(error_logs, "AdditionalData Message Severity")
1172 gp.qprint_var(pels)
Michael Shepos1a67b082020-08-28 16:01:58 -05001173
Michael Walsh409ad352020-02-06 11:46:35 -06001174 # Delete errlogs prior to doing any boot tests.
1175 grk.run_key(delete_errlogs_cmd, ignore=1)
Michael Shepos92a54bf2020-11-11 11:48:55 -06001176 grk.run_key(delete_bmcdump_cmd, ignore=1)
George Keishing2ef6a7d2021-05-19 09:05:32 -05001177 if redfish_support_trans_state:
1178 grk.run_key(delete_sysdump_cmd, ignore=1)
Michael Walsh409ad352020-02-06 11:46:35 -06001179
Michael Walsh6741f742017-02-20 16:16:38 -06001180 # Process caller's boot_stack.
1181 while (len(boot_stack) > 0):
1182 test_loop_body()
1183
Michael Walshb5839d02017-04-12 16:11:20 -05001184 gp.qprint_timen("Finished processing stack.")
Michael Walsh30dadae2017-02-27 14:25:52 -06001185
Michael Walsh89de14a2018-10-01 16:51:37 -05001186 post_stack()
1187
Michael Walsh6741f742017-02-20 16:16:38 -06001188 # Process caller's boot_list.
1189 if len(boot_list) > 0:
1190 for ix in range(1, max_num_tests + 1):
1191 test_loop_body()
1192
Michael Walshb5839d02017-04-12 16:11:20 -05001193 gp.qprint_timen("Completed all requested boot tests.")
1194
1195 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001196 new_fail = boot_fail - init_boot_fail
1197 if new_fail > boot_fail_threshold:
Michael Walshb5839d02017-04-12 16:11:20 -05001198 error_message = "Boot failures exceed the boot failure" +\
1199 " threshold:\n" +\
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001200 gp.sprint_var(new_fail) +\
Michael Walshb5839d02017-04-12 16:11:20 -05001201 gp.sprint_var(boot_fail_threshold)
1202 BuiltIn().fail(gp.sprint_error(error_message))