blob: e3404b29b710105d337aa07167b9bd83fe34eb3c [file] [log] [blame]
George Keishinge7e91712021-09-03 11:28:44 -05001#!/usr/bin/env python3
Michael Walsh0bbd8602016-11-22 11:31:49 -06002
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
Michael Walsh0b93fbf2017-03-02 14:42:41 -06007import os
8import imp
9import time
10import glob
11import random
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050012import re
Michael Walshf566fb12019-02-01 14:35:09 -060013import signal
George Keishingd54bbc22018-08-03 08:24:58 -050014try:
15 import cPickle as pickle
16except ImportError:
17 import pickle
Michael Walshdc80d672017-05-09 12:58:32 -050018import socket
Michael Walsh0b93fbf2017-03-02 14:42:41 -060019
20from robot.utils import DotDict
21from robot.libraries.BuiltIn import BuiltIn
22
Michael Walsh6741f742017-02-20 16:16:38 -060023from boot_data import *
Michael Walshc9116812017-03-10 14:23:06 -060024import gen_print as gp
Michael Walsh55302292017-01-10 11:43:02 -060025import gen_robot_plug_in as grpi
Michael Walshf75d4352019-12-05 17:01:20 -060026import gen_arg as ga
Michael Walsh44cef252019-08-01 12:38:56 -050027import gen_valid as gv
Michael Walsh6741f742017-02-20 16:16:38 -060028import gen_misc as gm
29import gen_cmd as gc
Michael Walshb5839d02017-04-12 16:11:20 -050030import gen_robot_keyword as grk
Michael Walsh55302292017-01-10 11:43:02 -060031import state as st
Michael Walshff340002017-08-29 11:18:27 -050032import var_stack as vs
Michael Walshc9bd2e82019-04-18 11:06:52 -050033import gen_plug_in_utils as gpu
Michael Shepos1a67b082020-08-28 16:01:58 -050034import pel_utils as pel
Michael Shepos0e5f1132020-09-30 16:24:25 -050035import logging_utils as log
Michael Walsh0bbd8602016-11-22 11:31:49 -060036
Michael Walsh0b93fbf2017-03-02 14:42:41 -060037base_path = os.path.dirname(os.path.dirname(
38 imp.find_module("gen_robot_print")[1])) +\
Michael Walshc9116812017-03-10 14:23:06 -060039 os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060040sys.path.append(base_path + "extended/")
41import run_keyword as rk
Michael Walsh0bbd8602016-11-22 11:31:49 -060042
Michael Walshe1e26442017-03-06 17:50:07 -060043# Setting master_pid correctly influences the behavior of plug-ins like
44# DB_Logging
45program_pid = os.getpid()
46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -050047pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
Michael Walshe1e26442017-03-06 17:50:07 -060048
Michael Walshb5839d02017-04-12 16:11:20 -050049# Set up boot data structures.
Michael Walsh986d8ae2019-07-17 10:02:23 -050050os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
Michael Walsh0b93fbf2017-03-02 14:42:41 -060051
Michael Walsh6741f742017-02-20 16:16:38 -060052boot_lists = read_boot_lists()
Michael Walsh986d8ae2019-07-17 10:02:23 -050053
54# The maximum number of entries that can be in the boot_history global variable.
Michael Walsh815b1d52018-10-30 13:32:26 -050055max_boot_history = 10
Michael Walsh986d8ae2019-07-17 10:02:23 -050056boot_history = []
Michael Walsh6741f742017-02-20 16:16:38 -060057
Michael Walsh7dc885b2018-03-14 17:51:59 -050058state = st.return_state_constant('default_state')
Michael Walsh6741f742017-02-20 16:16:38 -060059cp_setup_called = 0
60next_boot = ""
61base_tool_dir_path = os.path.normpath(os.environ.get(
62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
Michael Walshb5839d02017-04-12 16:11:20 -050063
Michael Walsh6741f742017-02-20 16:16:38 -060064ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
Michael Walsh6741f742017-02-20 16:16:38 -060065boot_success = 0
George Keishingce90a7d2021-10-12 02:38:51 -050066
67status_dir_path = os.environ.get('STATUS_DIR_PATH', "") or \
68 BuiltIn().get_variable_value("${STATUS_DIR_PATH}", default="")
Michael Walsh6741f742017-02-20 16:16:38 -060069if status_dir_path != "":
70 status_dir_path = os.path.normpath(status_dir_path) + os.sep
George Keishingce90a7d2021-10-12 02:38:51 -050071 # For plugin expecting env gen_call_robot.py
72 os.environ['STATUS_DIR_PATH'] = status_dir_path
73
Michael Shepos34c79562021-03-18 18:49:44 -050074redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \
75 int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0))
Michael Walshe58df1c2019-08-07 09:57:43 -050076redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False)
George Keishingeb1fe352020-06-19 03:02:22 -050077redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False)
George Keishingd86e45c2021-03-19 07:38:14 -050078redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1))
Michael Walshe58df1c2019-08-07 09:57:43 -050079if redfish_supported:
George Keishing89537a82020-06-17 00:37:25 -050080 redfish = BuiltIn().get_library_instance('redfish')
Michael Walshe58df1c2019-08-07 09:57:43 -050081 default_power_on = "Redfish Power On"
82 default_power_off = "Redfish Power Off"
George Keishing870999a2021-03-31 23:43:57 -050083 if not redfish_support_trans_state:
Michael Sheposcc490b42020-08-26 12:53:01 -050084 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}"
Michael Shepos92a54bf2020-11-11 11:48:55 -060085 delete_bmcdump_cmd = "Delete All BMC Dump"
George Keishingeb1fe352020-06-19 03:02:22 -050086 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF"
87 else:
88 delete_errlogs_cmd = "Redfish Purge Event Log"
Michael Shepos92a54bf2020-11-11 11:48:55 -060089 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps"
George Keishing2ef6a7d2021-05-19 09:05:32 -050090 delete_sysdump_cmd = "Redfish Delete All System Dumps"
George Keishingeb1fe352020-06-19 03:02:22 -050091 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff"
Michael Walshe58df1c2019-08-07 09:57:43 -050092else:
93 default_power_on = "REST Power On"
94 default_power_off = "REST Power Off"
Michael Sheposcc490b42020-08-26 12:53:01 -050095 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}"
Michael Shepos92a54bf2020-11-11 11:48:55 -060096 delete_bmcdump_cmd = "Delete All BMC Dump"
George Keishinga54e06f2020-06-12 10:42:41 -050097 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF"
Michael Walsh6741f742017-02-20 16:16:38 -060098boot_count = 0
Michael Walsh0bbd8602016-11-22 11:31:49 -060099
Michael Walsh85678942017-03-27 14:34:22 -0500100LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
Michael Walsh986d8ae2019-07-17 10:02:23 -0500101AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '')
102ffdc_prefix = AUTOBOOT_FFDC_PREFIX
Sunil M325eb542017-08-10 07:09:43 -0500103boot_start_time = ""
104boot_end_time = ""
Michael Walshff340002017-08-29 11:18:27 -0500105save_stack = vs.var_stack('save_stack')
106main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
Michael Walsh85678942017-03-27 14:34:22 -0500107
108
Michael Walsh89de14a2018-10-01 16:51:37 -0500109def dump_ffdc_rc():
110 r"""
111 Return the constant dump ffdc test return code value.
112
113 When a plug-in call point program returns this value, it indicates that
114 this program should collect FFDC.
115 """
116
117 return 0x00000200
118
119
120def stop_test_rc():
121 r"""
122 Return the constant stop test return code value.
123
124 When a plug-in call point program returns this value, it indicates that
125 this program should stop running.
126 """
127
128 return 0x00000200
129
130
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500131def process_host(host,
132 host_var_name=""):
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500133 r"""
134 Process a host by getting the associated host name and IP address and
135 setting them in global variables.
136
137 If the caller does not pass the host_var_name, this function will try to
138 figure out the name of the variable used by the caller for the host parm.
139 Callers are advised to explicitly specify the host_var_name when calling
140 with an exec command. In such cases, the get_arg_name cannot figure out
141 the host variable name.
142
143 This function will then create similar global variable names by
144 removing "_host" and appending "_host_name" or "_ip" to the host variable
145 name.
146
147 Example:
148
149 If a call is made like this:
150 process_host(openbmc_host)
151
152 Global variables openbmc_host_name and openbmc_ip will be set.
153
154 Description of argument(s):
155 host A host name or IP. The name of the variable used should
156 have a suffix of "_host".
157 host_var_name The name of the variable being used as the host parm.
158 """
159
160 if host_var_name == "":
161 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
162
163 host_name_var_name = re.sub("host", "host_name", host_var_name)
164 ip_var_name = re.sub("host", "ip", host_var_name)
165 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
166 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
167 host + "')"
168 exec(cmd_buf)
169
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500170
Michael Walshb5839d02017-04-12 16:11:20 -0500171def process_pgm_parms():
Michael Walshb5839d02017-04-12 16:11:20 -0500172 r"""
173 Process the program parameters by assigning them all to corresponding
174 globals. Also, set some global values that depend on program parameters.
175 """
176
177 # Program parameter processing.
178 # Assign all program parms to python variables which are global to this
179 # module.
180
181 global parm_list
182 parm_list = BuiltIn().get_variable_value("${parm_list}")
183 # The following subset of parms should be processed as integers.
184 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
Michael Walsh89de14a2018-10-01 16:51:37 -0500185 'boot_fail_threshold', 'delete_errlogs',
Michael Walsh986d8ae2019-07-17 10:02:23 -0500186 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet',
187 'test_mode', 'debug']
Michael Walshb5839d02017-04-12 16:11:20 -0500188 for parm in parm_list:
189 if parm in int_list:
190 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
191 "}\", \"0\"))"
192 else:
193 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
194 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
Michael Walshff340002017-08-29 11:18:27 -0500195 gp.dpissuing(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500196 exec(cmd_buf)
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500197 if re.match(r".*_host$", parm):
198 cmd_buf = "process_host(" + parm + ", '" + parm + "')"
199 exec(cmd_buf)
200 if re.match(r".*_password$", parm):
201 # Register the value of any parm whose name ends in _password.
202 # This will cause the print functions to replace passwords with
203 # asterisks in the output.
204 cmd_buf = "gp.register_passwords(" + parm + ")"
205 exec(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500206
207 global ffdc_dir_path_style
208 global boot_list
209 global boot_stack
210 global boot_results_file_path
211 global boot_results
Michael Walsh986d8ae2019-07-17 10:02:23 -0500212 global boot_history
Michael Walshb5839d02017-04-12 16:11:20 -0500213 global ffdc_list_file_path
Michael Walshe0cf8d72017-05-17 13:20:46 -0500214 global ffdc_report_list_path
Michael Walsh600876d2017-05-30 17:58:58 -0500215 global ffdc_summary_list_path
Michael Walsha3e7b222020-02-03 15:32:16 -0600216 global boot_table
217 global valid_boot_types
Michael Walshb5839d02017-04-12 16:11:20 -0500218
219 if ffdc_dir_path_style == "":
220 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
221
222 # Convert these program parms to lists for easier processing..
George Keishing36efbc02018-12-12 10:18:23 -0600223 boot_list = list(filter(None, boot_list.split(":")))
224 boot_stack = list(filter(None, boot_stack.split(":")))
Michael Walshb5839d02017-04-12 16:11:20 -0500225
Michael Walsha3e7b222020-02-03 15:32:16 -0600226 boot_table = create_boot_table(boot_table_path, os_host=os_host)
227 valid_boot_types = create_valid_boot_list(boot_table)
228
Michael Walsh903e0b22017-09-19 17:00:33 -0500229 cleanup_boot_results_file()
230 boot_results_file_path = create_boot_results_file_path(pgm_name,
231 openbmc_nickname,
232 master_pid)
Michael Walshb5839d02017-04-12 16:11:20 -0500233
234 if os.path.isfile(boot_results_file_path):
235 # We've been called before in this run so we'll load the saved
Michael Walsh986d8ae2019-07-17 10:02:23 -0500236 # boot_results and boot_history objects.
237 boot_results, boot_history =\
Michael Walsh6c645742018-08-17 15:02:17 -0500238 pickle.load(open(boot_results_file_path, 'rb'))
Michael Walshb5839d02017-04-12 16:11:20 -0500239 else:
240 boot_results = boot_results(boot_table, boot_pass, boot_fail)
241
242 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
243 "/FFDC_FILE_LIST"
Michael Walshe0cf8d72017-05-17 13:20:46 -0500244 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
245 "/FFDC_REPORT_FILE_LIST"
Michael Walshb5839d02017-04-12 16:11:20 -0500246
Michael Walsh600876d2017-05-30 17:58:58 -0500247 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
248 "/FFDC_SUMMARY_FILE_LIST"
249
Michael Walshb5839d02017-04-12 16:11:20 -0500250
Michael Walsh85678942017-03-27 14:34:22 -0500251def initial_plug_in_setup():
Michael Walsh85678942017-03-27 14:34:22 -0500252 r"""
253 Initialize all plug-in environment variables which do not change for the
254 duration of the program.
255
256 """
257
258 global LOG_LEVEL
259 BuiltIn().set_log_level("NONE")
260
261 BuiltIn().set_global_variable("${master_pid}", master_pid)
262 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
263 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
264 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
265 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
266 ffdc_list_file_path)
Michael Walshe0cf8d72017-05-17 13:20:46 -0500267 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
268 ffdc_report_list_path)
Michael Walsh600876d2017-05-30 17:58:58 -0500269 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
270 ffdc_summary_list_path)
Michael Walsh85678942017-03-27 14:34:22 -0500271
272 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
273 ffdc_dir_path_style)
274 BuiltIn().set_global_variable("${FFDC_CHECK}",
275 ffdc_check)
276
277 # For each program parameter, set the corresponding AUTOBOOT_ environment
278 # variable value. Also, set an AUTOBOOT_ environment variable for every
279 # element in additional_values.
280 additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
281 "status_dir_path", "base_tool_dir_path",
Michael Walsh600876d2017-05-30 17:58:58 -0500282 "ffdc_list_file_path", "ffdc_report_list_path",
Michael Shepos7fe83b32020-09-21 15:46:01 -0500283 "ffdc_summary_list_path", "execdir", "redfish_supported",
Michael Shepos34c79562021-03-18 18:49:44 -0500284 "redfish_rest_supported", "redfish_support_trans_state"]
Michael Walsh85678942017-03-27 14:34:22 -0500285
286 plug_in_vars = parm_list + additional_values
287
288 for var_name in plug_in_vars:
289 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
290 var_name = var_name.upper()
291 if var_value is None:
292 var_value = ""
293 os.environ["AUTOBOOT_" + var_name] = str(var_value)
294
295 BuiltIn().set_log_level(LOG_LEVEL)
296
Michael Walsh68a61162017-04-25 11:54:06 -0500297 # Make sure the ffdc list directory exists.
298 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
299 if not os.path.exists(ffdc_list_dir_path):
300 os.makedirs(ffdc_list_dir_path)
Michael Walsh85678942017-03-27 14:34:22 -0500301
Michael Walsh85678942017-03-27 14:34:22 -0500302
Michael Walsh0bbd8602016-11-22 11:31:49 -0600303def plug_in_setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600304 r"""
Michael Walsh85678942017-03-27 14:34:22 -0500305 Initialize all changing plug-in environment variables for use by the
306 plug-in programs.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600307 """
308
Michael Walsh85678942017-03-27 14:34:22 -0500309 global LOG_LEVEL
310 global test_really_running
311
312 BuiltIn().set_log_level("NONE")
313
Michael Walsh6741f742017-02-20 16:16:38 -0600314 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600315 if boot_pass > 1:
316 test_really_running = 1
317 else:
318 test_really_running = 0
319
Michael Walsh6741f742017-02-20 16:16:38 -0600320 BuiltIn().set_global_variable("${test_really_running}",
321 test_really_running)
322 BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
Michael Walsh6741f742017-02-20 16:16:38 -0600323 BuiltIn().set_global_variable("${boot_pass}", boot_pass)
324 BuiltIn().set_global_variable("${boot_fail}", boot_fail)
325 BuiltIn().set_global_variable("${boot_success}", boot_success)
326 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
Sunil M325eb542017-08-10 07:09:43 -0500327 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
328 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
Michael Walsh4c9a6452016-12-13 16:03:11 -0600329
Michael Walsh0bbd8602016-11-22 11:31:49 -0600330 # For each program parameter, set the corresponding AUTOBOOT_ environment
331 # variable value. Also, set an AUTOBOOT_ environment variable for every
332 # element in additional_values.
333 additional_values = ["boot_type_desc", "boot_success", "boot_pass",
Sunil M325eb542017-08-10 07:09:43 -0500334 "boot_fail", "test_really_running", "ffdc_prefix",
335 "boot_start_time", "boot_end_time"]
Michael Walsh0bbd8602016-11-22 11:31:49 -0600336
Michael Walsh85678942017-03-27 14:34:22 -0500337 plug_in_vars = additional_values
Michael Walsh0bbd8602016-11-22 11:31:49 -0600338
339 for var_name in plug_in_vars:
340 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
341 var_name = var_name.upper()
342 if var_value is None:
343 var_value = ""
Michael Walsh6741f742017-02-20 16:16:38 -0600344 os.environ["AUTOBOOT_" + var_name] = str(var_value)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600345
Michael Walsh0bbd8602016-11-22 11:31:49 -0600346 if debug:
Michael Walsh6741f742017-02-20 16:16:38 -0600347 shell_rc, out_buf = \
348 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600349
Michael Walsh85678942017-03-27 14:34:22 -0500350 BuiltIn().set_log_level(LOG_LEVEL)
351
Michael Walsh0bbd8602016-11-22 11:31:49 -0600352
Michael Walshe0cf8d72017-05-17 13:20:46 -0500353def pre_boot_plug_in_setup():
354
355 # Clear the ffdc_list_file_path file. Plug-ins may now write to it.
356 try:
357 os.remove(ffdc_list_file_path)
358 except OSError:
359 pass
360
361 # Clear the ffdc_report_list_path file. Plug-ins may now write to it.
362 try:
363 os.remove(ffdc_report_list_path)
364 except OSError:
365 pass
366
Michael Walsh600876d2017-05-30 17:58:58 -0500367 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it.
368 try:
369 os.remove(ffdc_summary_list_path)
370 except OSError:
371 pass
372
Michael Walshe1974b92017-08-03 13:39:51 -0500373 global ffdc_prefix
374
375 seconds = time.time()
376 loc_time = time.localtime(seconds)
377 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
378
379 ffdc_prefix = openbmc_nickname + "." + time_string
380
Michael Walshe0cf8d72017-05-17 13:20:46 -0500381
Michael Walshf566fb12019-02-01 14:35:09 -0600382def default_sigusr1(signal_number=0,
383 frame=None):
384 r"""
385 Handle SIGUSR1 by doing nothing.
386
387 This function assists in debugging SIGUSR1 processing by printing messages
388 to stdout and to the log.html file.
389
390 Description of argument(s):
391 signal_number The signal number (should always be 10 for SIGUSR1).
392 frame The frame data.
393 """
394
Michael Walsh80dddde2019-10-22 13:54:38 -0500395 gp.qprintn()
396 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600397 gp.lprint_executing()
398
399
400def set_default_siguser1():
401 r"""
402 Set the default_sigusr1 function to be the SIGUSR1 handler.
403 """
404
Michael Walsh80dddde2019-10-22 13:54:38 -0500405 gp.qprintn()
406 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600407 gp.lprint_executing()
408 signal.signal(signal.SIGUSR1, default_sigusr1)
409
410
Michael Walsh6741f742017-02-20 16:16:38 -0600411def setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600412 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600413 Do general program setup tasks.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600414 """
415
Michael Walsh6741f742017-02-20 16:16:38 -0600416 global cp_setup_called
Michael Walsh81816742017-09-27 11:02:29 -0500417 global transitional_boot_selected
Michael Walsh0bbd8602016-11-22 11:31:49 -0600418
Michael Walshb5839d02017-04-12 16:11:20 -0500419 gp.qprintn()
420
George Keishinga54e06f2020-06-12 10:42:41 -0500421 if redfish_supported:
422 redfish.login()
423
Michael Walshf566fb12019-02-01 14:35:09 -0600424 set_default_siguser1()
Michael Walsh81816742017-09-27 11:02:29 -0500425 transitional_boot_selected = False
426
Michael Walsh83f4bc72017-04-20 16:49:43 -0500427 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
428 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
Michael Walshd061c042017-05-23 14:46:57 -0500429 # If we can't find process_plug_in_packages.py, ssh_pw or
430 # validate_plug_ins.py, then we don't have our repo bin in PATH.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500431 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
432 + " ssh_pw validate_plug_ins.py", quiet=1,
Michael Walshd061c042017-05-23 14:46:57 -0500433 print_output=0, show_err=0)
Michael Walshb5839d02017-04-12 16:11:20 -0500434 if shell_rc != 0:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500435 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
436 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
437 if robot_pgm_dir_path not in sys.path:
438 sys.path.append(robot_pgm_dir_path)
439 PYTHONPATH = os.environ.get("PYTHONPATH", "")
440 if PYTHONPATH == "":
441 os.environ['PYTHONPATH'] = robot_pgm_dir_path
442 else:
443 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
Michael Walsh6741f742017-02-20 16:16:38 -0600444
445 validate_parms()
446
Michael Walshc108e422019-03-28 12:27:18 -0500447 gp.qprint_pgm_header()
Michael Walsh6741f742017-02-20 16:16:38 -0600448
George Keishinga54e06f2020-06-12 10:42:41 -0500449 grk.run_key_u(default_set_power_policy)
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500450
Michael Walsh85678942017-03-27 14:34:22 -0500451 initial_plug_in_setup()
452
Michael Walsh6741f742017-02-20 16:16:38 -0600453 plug_in_setup()
454 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
455 call_point='setup')
456 if rc != 0:
457 error_message = "Plug-in setup failed.\n"
Michael Walshc108e422019-03-28 12:27:18 -0500458 gp.print_error_report(error_message)
Michael Walsh6741f742017-02-20 16:16:38 -0600459 BuiltIn().fail(error_message)
460 # Setting cp_setup_called lets our Teardown know that it needs to call
461 # the cleanup plug-in call point.
462 cp_setup_called = 1
463
464 # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
465 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
Michael Walsh85678942017-03-27 14:34:22 -0500466 # FFDC_LOG_PATH is used by "FFDC" keyword.
467 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
Michael Walsh6741f742017-02-20 16:16:38 -0600468
Michael Walshdc80d672017-05-09 12:58:32 -0500469 # Also printed by FFDC.
470 global host_name
471 global host_ip
472 host = socket.gethostname()
473 host_name, host_ip = gm.get_host_name_ip(host)
474
Michael Walsh986d8ae2019-07-17 10:02:23 -0500475 gp.dprint_var(boot_table)
Michael Walshb5839d02017-04-12 16:11:20 -0500476 gp.dprint_var(boot_lists)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600477
Michael Walsh0bbd8602016-11-22 11:31:49 -0600478
Michael Walsh6741f742017-02-20 16:16:38 -0600479def validate_parms():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600480 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600481 Validate all program parameters.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600482 """
483
Michael Walshb5839d02017-04-12 16:11:20 -0500484 process_pgm_parms()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600485
Michael Walshb5839d02017-04-12 16:11:20 -0500486 gp.qprintn()
487
488 global openbmc_model
Michael Walshf5ce38c2020-02-27 12:46:20 -0600489 if openbmc_model == "":
490 status, ret_values =\
491 grk.run_key_u("Get BMC System Model")
492 openbmc_model = ret_values
493 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
494 gv.set_exit_on_error(True)
Michael Walsh44cef252019-08-01 12:38:56 -0500495 gv.valid_value(openbmc_host)
496 gv.valid_value(openbmc_username)
497 gv.valid_value(openbmc_password)
498 gv.valid_value(rest_username)
499 gv.valid_value(rest_password)
500 gv.valid_value(ipmi_username)
501 gv.valid_value(ipmi_password)
Michael Walsh6741f742017-02-20 16:16:38 -0600502 if os_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500503 gv.valid_value(os_username)
504 gv.valid_value(os_password)
Michael Walsh6741f742017-02-20 16:16:38 -0600505 if pdu_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500506 gv.valid_value(pdu_username)
507 gv.valid_value(pdu_password)
508 gv.valid_integer(pdu_slot_no)
Michael Walsh6741f742017-02-20 16:16:38 -0600509 if openbmc_serial_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500510 gv.valid_integer(openbmc_serial_port)
Michael Walsh44cef252019-08-01 12:38:56 -0500511 gv.valid_value(openbmc_model)
512 gv.valid_integer(max_num_tests)
513 gv.valid_integer(boot_pass)
514 gv.valid_integer(boot_fail)
Michael Walsh6741f742017-02-20 16:16:38 -0600515 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
516 BuiltIn().set_global_variable("${plug_in_packages_list}",
517 plug_in_packages_list)
Michael Walsh44cef252019-08-01 12:38:56 -0500518 gv.valid_value(stack_mode, valid_values=['normal', 'skip'])
Michael Walshf5ce38c2020-02-27 12:46:20 -0600519 gv.set_exit_on_error(False)
Michael Walsha20da402017-03-31 16:27:45 -0500520 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
Michael Walsh6741f742017-02-20 16:16:38 -0600521 error_message = "You must provide either a value for either the" +\
522 " boot_list or the boot_stack parm.\n"
523 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600524 valid_boot_list(boot_list, valid_boot_types)
525 valid_boot_list(boot_stack, valid_boot_types)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500526 selected_PDU_boots = list(set(boot_list + boot_stack)
527 & set(boot_lists['PDU_reboot']))
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500528 if len(selected_PDU_boots) > 0 and pdu_host == "":
529 error_message = "You have selected the following boots which" +\
530 " require a PDU host but no value for pdu_host:\n"
531 error_message += gp.sprint_var(selected_PDU_boots)
Michael Walsh986d8ae2019-07-17 10:02:23 -0500532 error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500533 BuiltIn().fail(gp.sprint_error(error_message))
534
Michael Walsh6741f742017-02-20 16:16:38 -0600535 return
Michael Walsh0bbd8602016-11-22 11:31:49 -0600536
Michael Walsh0bbd8602016-11-22 11:31:49 -0600537
Michael Walsh6741f742017-02-20 16:16:38 -0600538def my_get_state():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600539 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600540 Get the system state plus a little bit of wrapping.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600541 """
542
Michael Walsh6741f742017-02-20 16:16:38 -0600543 global state
544
545 req_states = ['epoch_seconds'] + st.default_req_states
546
Michael Walshb5839d02017-04-12 16:11:20 -0500547 gp.qprint_timen("Getting system state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600548 if test_mode:
549 state['epoch_seconds'] = int(time.time())
550 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500551 state = st.get_state(req_states=req_states, quiet=quiet)
552 gp.qprint_var(state)
Michael Walsh341c21e2017-01-17 16:25:20 -0600553
Michael Walsh341c21e2017-01-17 16:25:20 -0600554
Michael Walsh45ca6e42017-09-14 17:29:12 -0500555def valid_state():
Michael Walsh45ca6e42017-09-14 17:29:12 -0500556 r"""
557 Verify that our state dictionary contains no blank values. If we don't get
558 valid state data, we cannot continue to work.
559 """
560
561 if st.compare_states(state, st.invalid_state_match, 'or'):
562 error_message = "The state dictionary contains blank fields which" +\
563 " is illegal.\n" + gp.sprint_var(state)
564 BuiltIn().fail(gp.sprint_error(error_message))
565
Michael Walsh45ca6e42017-09-14 17:29:12 -0500566
Michael Walsh6741f742017-02-20 16:16:38 -0600567def select_boot():
Michael Walsh341c21e2017-01-17 16:25:20 -0600568 r"""
569 Select a boot test to be run based on our current state and return the
570 chosen boot type.
571
572 Description of arguments:
Michael Walsh6741f742017-02-20 16:16:38 -0600573 state The state of the machine.
Michael Walsh341c21e2017-01-17 16:25:20 -0600574 """
575
Michael Walsh81816742017-09-27 11:02:29 -0500576 global transitional_boot_selected
Michael Walsh30dadae2017-02-27 14:25:52 -0600577 global boot_stack
578
Michael Walshb5839d02017-04-12 16:11:20 -0500579 gp.qprint_timen("Selecting a boot test.")
Michael Walsh6741f742017-02-20 16:16:38 -0600580
Michael Walsh81816742017-09-27 11:02:29 -0500581 if transitional_boot_selected and not boot_success:
582 prior_boot = next_boot
583 boot_candidate = boot_stack.pop()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500584 gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
585 + " transition to a valid state for '" + boot_candidate
586 + "' which was at the top of the boot_stack. Since"
587 + " the '" + next_boot + "' failed, the '"
588 + boot_candidate + "' has been removed from the stack"
589 + " to avoid and endless failure loop.")
Michael Walsh81816742017-09-27 11:02:29 -0500590 if len(boot_stack) == 0:
591 return ""
592
Michael Walsh6741f742017-02-20 16:16:38 -0600593 my_get_state()
Michael Walsh45ca6e42017-09-14 17:29:12 -0500594 valid_state()
Michael Walsh6741f742017-02-20 16:16:38 -0600595
Michael Walsh81816742017-09-27 11:02:29 -0500596 transitional_boot_selected = False
Michael Walsh6741f742017-02-20 16:16:38 -0600597 stack_popped = 0
598 if len(boot_stack) > 0:
599 stack_popped = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500600 gp.qprint_dashes()
601 gp.qprint_var(boot_stack)
602 gp.qprint_dashes()
603 skip_boot_printed = 0
604 while len(boot_stack) > 0:
605 boot_candidate = boot_stack.pop()
606 if stack_mode == 'normal':
607 break
608 else:
609 if st.compare_states(state, boot_table[boot_candidate]['end']):
610 if not skip_boot_printed:
Michael Walshff340002017-08-29 11:18:27 -0500611 gp.qprint_var(stack_mode)
612 gp.qprintn()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500613 gp.qprint_timen("Skipping the following boot tests"
614 + " which are unnecessary since their"
615 + " required end states match the"
616 + " current machine state:")
Michael Walshb5839d02017-04-12 16:11:20 -0500617 skip_boot_printed = 1
Michael Walshff340002017-08-29 11:18:27 -0500618 gp.qprint_var(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500619 boot_candidate = ""
620 if boot_candidate == "":
621 gp.qprint_dashes()
622 gp.qprint_var(boot_stack)
623 gp.qprint_dashes()
624 return boot_candidate
Michael Walsh6741f742017-02-20 16:16:38 -0600625 if st.compare_states(state, boot_table[boot_candidate]['start']):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500626 gp.qprint_timen("The machine state is valid for a '"
627 + boot_candidate + "' boot test.")
Michael Walshb5839d02017-04-12 16:11:20 -0500628 gp.qprint_dashes()
629 gp.qprint_var(boot_stack)
630 gp.qprint_dashes()
Michael Walsh6741f742017-02-20 16:16:38 -0600631 return boot_candidate
Michael Walsh341c21e2017-01-17 16:25:20 -0600632 else:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500633 gp.qprint_timen("The machine state does not match the required"
634 + " starting state for a '" + boot_candidate
635 + "' boot test:")
Michael Walsh986d8ae2019-07-17 10:02:23 -0500636 gp.qprint_varx("boot_table_start_entry",
637 boot_table[boot_candidate]['start'])
Michael Walsh6741f742017-02-20 16:16:38 -0600638 boot_stack.append(boot_candidate)
Michael Walsh81816742017-09-27 11:02:29 -0500639 transitional_boot_selected = True
Michael Walsh6741f742017-02-20 16:16:38 -0600640 popped_boot = boot_candidate
641
642 # Loop through your list selecting a boot_candidates
643 boot_candidates = []
644 for boot_candidate in boot_list:
645 if st.compare_states(state, boot_table[boot_candidate]['start']):
646 if stack_popped:
647 if st.compare_states(boot_table[boot_candidate]['end'],
Gunnar Mills096cd562018-03-26 10:19:12 -0500648 boot_table[popped_boot]['start']):
Michael Walsh6741f742017-02-20 16:16:38 -0600649 boot_candidates.append(boot_candidate)
650 else:
651 boot_candidates.append(boot_candidate)
652
653 if len(boot_candidates) == 0:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500654 gp.qprint_timen("The user's boot list contained no boot tests"
655 + " which are valid for the current machine state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600656 boot_candidate = default_power_on
657 if not st.compare_states(state, boot_table[default_power_on]['start']):
658 boot_candidate = default_power_off
659 boot_candidates.append(boot_candidate)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500660 gp.qprint_timen("Using default '" + boot_candidate
661 + "' boot type to transition to valid state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600662
Michael Walshb5839d02017-04-12 16:11:20 -0500663 gp.dprint_var(boot_candidates)
Michael Walsh6741f742017-02-20 16:16:38 -0600664
665 # Randomly select a boot from the candidate list.
666 boot = random.choice(boot_candidates)
Michael Walsh341c21e2017-01-17 16:25:20 -0600667
668 return boot
Michael Walsh0bbd8602016-11-22 11:31:49 -0600669
Michael Walsh55302292017-01-10 11:43:02 -0600670
Michael Walshb2e53ec2017-10-30 15:04:36 -0500671def print_defect_report(ffdc_file_list):
Michael Walsh341c21e2017-01-17 16:25:20 -0600672 r"""
673 Print a defect report.
Michael Walshb2e53ec2017-10-30 15:04:36 -0500674
675 Description of argument(s):
676 ffdc_file_list A list of files which were collected by our ffdc functions.
Michael Walsh341c21e2017-01-17 16:25:20 -0600677 """
678
Michael Walsh600876d2017-05-30 17:58:58 -0500679 # Making deliberate choice to NOT run plug_in_setup(). We don't want
680 # ffdc_prefix updated.
681 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
682 call_point='ffdc_report', stop_on_plug_in_failure=0)
683
Michael Walshe0cf8d72017-05-17 13:20:46 -0500684 # Get additional header data which may have been created by ffdc plug-ins.
685 # Also, delete the individual header files to cleanup.
686 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
687 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
688 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
689 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
690 show_err=0)
691
Michael Walshb2e53ec2017-10-30 15:04:36 -0500692 # Get additional summary data which may have been created by ffdc plug-ins.
Michael Walsh600876d2017-05-30 17:58:58 -0500693 # Also, delete the individual header files to cleanup.
694 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
695 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
696 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
697 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
698 show_err=0)
699
Michael Walshb2e53ec2017-10-30 15:04:36 -0500700 # ffdc_list_file_path contains a list of any ffdc files created by plug-
701 # ins, etc. Read that data into a list.
Michael Walsh341c21e2017-01-17 16:25:20 -0600702 try:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500703 plug_in_ffdc_list = \
704 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
George Keishing36efbc02018-12-12 10:18:23 -0600705 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
Michael Walsh341c21e2017-01-17 16:25:20 -0600706 except IOError:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500707 plug_in_ffdc_list = []
708
709 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
710 # in. Eliminate duplicates and sort the list.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500711 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
Michael Walshb2e53ec2017-10-30 15:04:36 -0500712
713 if status_file_path != "":
714 ffdc_file_list.insert(0, status_file_path)
715
716 # Convert the list to a printable list.
717 printable_ffdc_file_list = "\n".join(ffdc_file_list)
Michael Walsh341c21e2017-01-17 16:25:20 -0600718
Michael Walsh68a61162017-04-25 11:54:06 -0500719 # Open ffdc_file_list for writing. We will write a complete list of
720 # FFDC files to it for possible use by plug-ins like cp_stop_check.
721 ffdc_list_file = open(ffdc_list_file_path, 'w')
Michael Walshb2e53ec2017-10-30 15:04:36 -0500722 ffdc_list_file.write(printable_ffdc_file_list + "\n")
723 ffdc_list_file.close()
724
725 indent = 0
726 width = 90
727 linefeed = 1
728 char = "="
Michael Walsh68a61162017-04-25 11:54:06 -0500729
730 gp.qprintn()
Michael Walshb2e53ec2017-10-30 15:04:36 -0500731 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500732 gp.qprintn("Copy this data to the defect:\n")
733
Michael Walshe0cf8d72017-05-17 13:20:46 -0500734 if len(more_header_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500735 gp.qprintn(more_header_info)
Michael Walshdc80d672017-05-09 12:58:32 -0500736 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
737 openbmc_host_name, openbmc_ip, openbmc_username,
Michael Walsh0a3bdb42019-01-31 16:21:44 +0000738 openbmc_password, rest_username, rest_password, ipmi_username,
739 ipmi_password, os_host, os_host_name, os_ip, os_username,
Michael Walshdc80d672017-05-09 12:58:32 -0500740 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
741 pdu_password, pdu_slot_no, openbmc_serial_host,
742 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
Michael Walsh68a61162017-04-25 11:54:06 -0500743
744 gp.qprintn()
Michael Walsh986d8ae2019-07-17 10:02:23 -0500745 print_boot_history(boot_history)
Michael Walsh68a61162017-04-25 11:54:06 -0500746 gp.qprintn()
747 gp.qprint_var(state)
Michael Walshb5839d02017-04-12 16:11:20 -0500748 gp.qprintn()
749 gp.qprintn("FFDC data files:")
Michael Walshb2e53ec2017-10-30 15:04:36 -0500750 gp.qprintn(printable_ffdc_file_list)
Michael Walshb5839d02017-04-12 16:11:20 -0500751 gp.qprintn()
Michael Walsh341c21e2017-01-17 16:25:20 -0600752
Michael Walsh600876d2017-05-30 17:58:58 -0500753 if len(ffdc_summary_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500754 gp.qprintn(ffdc_summary_info)
Michael Walsh600876d2017-05-30 17:58:58 -0500755
Michael Walshb2e53ec2017-10-30 15:04:36 -0500756 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500757
Michael Walsh6741f742017-02-20 16:16:38 -0600758
Michael Walsh6741f742017-02-20 16:16:38 -0600759def my_ffdc():
Michael Walsh6741f742017-02-20 16:16:38 -0600760 r"""
761 Collect FFDC data.
762 """
763
764 global state
765
766 plug_in_setup()
767 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500768 call_point='ffdc', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600769
770 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500771 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix="
772 + AUTOBOOT_FFDC_PREFIX
773 + " ffdc_function_list="
774 + ffdc_function_list, ignore=1)
Michael Walsh83f4bc72017-04-20 16:49:43 -0500775 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500776 gp.qprint_error("Call to ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500777 if type(ffdc_file_list) is not list:
778 ffdc_file_list = []
779 # Leave a record for caller that "soft" errors occurred.
780 soft_errors = 1
781 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600782
783 my_get_state()
784
Michael Walshb2e53ec2017-10-30 15:04:36 -0500785 print_defect_report(ffdc_file_list)
Michael Walsh6741f742017-02-20 16:16:38 -0600786
Michael Walsh6741f742017-02-20 16:16:38 -0600787
Michael Walsh6741f742017-02-20 16:16:38 -0600788def print_test_start_message(boot_keyword):
Michael Walsh6741f742017-02-20 16:16:38 -0600789 r"""
790 Print a message indicating what boot test is about to run.
791
792 Description of arguments:
793 boot_keyword The name of the boot which is to be run
794 (e.g. "BMC Power On").
795 """
796
Michael Walsh986d8ae2019-07-17 10:02:23 -0500797 global boot_history
Sunil M325eb542017-08-10 07:09:43 -0500798 global boot_start_time
Michael Walsh6741f742017-02-20 16:16:38 -0600799
800 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
Sunil M325eb542017-08-10 07:09:43 -0500801
802 # Set boot_start_time for use by plug-ins.
803 boot_start_time = doing_msg[1:33]
804 gp.qprint_var(boot_start_time)
805
Michael Walshb5839d02017-04-12 16:11:20 -0500806 gp.qprint(doing_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600807
Michael Walsh986d8ae2019-07-17 10:02:23 -0500808 update_boot_history(boot_history, doing_msg, max_boot_history)
Michael Walsh6741f742017-02-20 16:16:38 -0600809
Michael Walsh6741f742017-02-20 16:16:38 -0600810
Michael Walshf566fb12019-02-01 14:35:09 -0600811def stop_boot_test(signal_number=0,
812 frame=None):
813 r"""
814 Handle SIGUSR1 by aborting the boot test that is running.
815
816 Description of argument(s):
817 signal_number The signal number (should always be 10 for SIGUSR1).
818 frame The frame data.
819 """
820
Michael Walsh80dddde2019-10-22 13:54:38 -0500821 gp.qprintn()
822 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600823 gp.lprint_executing()
824
825 # Restore original sigusr1 handler.
826 set_default_siguser1()
827
828 message = "The caller has asked that the boot test be stopped and marked"
829 message += " as a failure."
830
831 function_stack = gm.get_function_stack()
832 if "wait_state" in function_stack:
Michael Walshc44aa532019-06-14 13:33:29 -0500833 st.set_exit_wait_early_message(message)
Michael Walshf566fb12019-02-01 14:35:09 -0600834 else:
835 BuiltIn().fail(gp.sprint_error(message))
836
837
Michael Walsh6741f742017-02-20 16:16:38 -0600838def run_boot(boot):
Michael Walsh6741f742017-02-20 16:16:38 -0600839 r"""
840 Run the specified boot.
841
842 Description of arguments:
843 boot The name of the boot test to be performed.
844 """
845
846 global state
847
Michael Walshf566fb12019-02-01 14:35:09 -0600848 signal.signal(signal.SIGUSR1, stop_boot_test)
849 gp.qprint_timen("stop_boot_test is armed.")
850
Michael Walsh6741f742017-02-20 16:16:38 -0600851 print_test_start_message(boot)
852
853 plug_in_setup()
854 rc, shell_rc, failed_plug_in_name = \
855 grpi.rprocess_plug_in_packages(call_point="pre_boot")
856 if rc != 0:
857 error_message = "Plug-in failed with non-zero return code.\n" +\
Michael Walsh986d8ae2019-07-17 10:02:23 -0500858 gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600859 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600860 BuiltIn().fail(gp.sprint_error(error_message))
861
862 if test_mode:
863 # In test mode, we'll pretend the boot worked by assigning its
864 # required end state to the default state value.
Michael Walsh30dadae2017-02-27 14:25:52 -0600865 state = st.strip_anchor_state(boot_table[boot]['end'])
Michael Walsh6741f742017-02-20 16:16:38 -0600866 else:
867 # Assertion: We trust that the state data was made fresh by the
868 # caller.
869
Michael Walshb5839d02017-04-12 16:11:20 -0500870 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600871
872 if boot_table[boot]['method_type'] == "keyword":
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600873 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
Michael Walshb5839d02017-04-12 16:11:20 -0500874 boot_table[boot]['method'],
875 quiet=quiet)
Michael Walsh6741f742017-02-20 16:16:38 -0600876
877 if boot_table[boot]['bmc_reboot']:
878 st.wait_for_comm_cycle(int(state['epoch_seconds']))
Michael Walsh30dadae2017-02-27 14:25:52 -0600879 plug_in_setup()
880 rc, shell_rc, failed_plug_in_name = \
881 grpi.rprocess_plug_in_packages(call_point="post_reboot")
882 if rc != 0:
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600883 error_message = "Plug-in failed with non-zero return code.\n"
Michael Walsh986d8ae2019-07-17 10:02:23 -0500884 error_message += gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600885 set_default_siguser1()
Michael Walsh30dadae2017-02-27 14:25:52 -0600886 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600887 else:
888 match_state = st.anchor_state(state)
889 del match_state['epoch_seconds']
890 # Wait for the state to change in any way.
891 st.wait_state(match_state, wait_time=state_change_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500892 interval="10 seconds", invert=1)
Michael Walsh6741f742017-02-20 16:16:38 -0600893
Michael Walshb5839d02017-04-12 16:11:20 -0500894 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600895 if boot_table[boot]['end']['chassis'] == "Off":
896 boot_timeout = power_off_timeout
897 else:
898 boot_timeout = power_on_timeout
899 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500900 interval="10 seconds")
Michael Walsh6741f742017-02-20 16:16:38 -0600901
902 plug_in_setup()
903 rc, shell_rc, failed_plug_in_name = \
904 grpi.rprocess_plug_in_packages(call_point="post_boot")
905 if rc != 0:
906 error_message = "Plug-in failed with non-zero return code.\n" +\
Michael Walsh986d8ae2019-07-17 10:02:23 -0500907 gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600908 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600909 BuiltIn().fail(gp.sprint_error(error_message))
910
Michael Walshf566fb12019-02-01 14:35:09 -0600911 # Restore original sigusr1 handler.
912 set_default_siguser1()
913
Michael Walsh6741f742017-02-20 16:16:38 -0600914
Michael Walsh6741f742017-02-20 16:16:38 -0600915def test_loop_body():
Michael Walsh6741f742017-02-20 16:16:38 -0600916 r"""
917 The main loop body for the loop in main_py.
918
919 Description of arguments:
920 boot_count The iteration number (starts at 1).
921 """
922
923 global boot_count
924 global state
925 global next_boot
926 global boot_success
Sunil M325eb542017-08-10 07:09:43 -0500927 global boot_end_time
Michael Walsh6741f742017-02-20 16:16:38 -0600928
Michael Walshb5839d02017-04-12 16:11:20 -0500929 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600930
931 next_boot = select_boot()
Michael Walshb5839d02017-04-12 16:11:20 -0500932 if next_boot == "":
933 return True
Michael Walsh6741f742017-02-20 16:16:38 -0600934
Michael Walshb5839d02017-04-12 16:11:20 -0500935 boot_count += 1
936 gp.qprint_timen("Starting boot " + str(boot_count) + ".")
Michael Walsh6741f742017-02-20 16:16:38 -0600937
Michael Walshe0cf8d72017-05-17 13:20:46 -0500938 pre_boot_plug_in_setup()
Michael Walsh6741f742017-02-20 16:16:38 -0600939
940 cmd_buf = ["run_boot", next_boot]
941 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
942 if boot_status == "FAIL":
Michael Walshb5839d02017-04-12 16:11:20 -0500943 gp.qprint(msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600944
Michael Walshb5839d02017-04-12 16:11:20 -0500945 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600946 if boot_status == "PASS":
947 boot_success = 1
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500948 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
949 + "\" succeeded.")
Michael Walsh6741f742017-02-20 16:16:38 -0600950 else:
951 boot_success = 0
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500952 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
953 + "\" failed.")
Sunil M325eb542017-08-10 07:09:43 -0500954
955 # Set boot_end_time for use by plug-ins.
956 boot_end_time = completion_msg[1:33]
957 gp.qprint_var(boot_end_time)
958
959 gp.qprint(completion_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600960
961 boot_results.update(next_boot, boot_status)
962
963 plug_in_setup()
964 # NOTE: A post_test_case call point failure is NOT counted as a boot
965 # failure.
966 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500967 call_point='post_test_case', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600968
969 plug_in_setup()
970 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500971 call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
Michael Walsh6741f742017-02-20 16:16:38 -0600972 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
Michael Walsh12059e22019-03-21 11:03:45 -0500973 if ffdc_check == "All" or\
Michael Walsh89de14a2018-10-01 16:51:37 -0500974 shell_rc == dump_ffdc_rc():
Michael Walsh83f4bc72017-04-20 16:49:43 -0500975 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
976 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500977 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500978 # Leave a record for caller that "soft" errors occurred.
979 soft_errors = 1
980 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600981
Michael Walshaabef1e2017-09-20 15:16:17 -0500982 if delete_errlogs:
Michael Shepos1a67b082020-08-28 16:01:58 -0500983 # print error logs before delete
George Keishing438fd3b2021-10-08 02:15:18 -0500984 if redfish_support_trans_state:
985 status, error_logs = grk.run_key_u("Get Redfish Event Logs")
986 log.print_error_logs(error_logs, "AdditionalDataURI Message Severity")
987 else:
988 status, error_logs = grk.run_key_u("Get Error Logs")
989 log.print_error_logs(error_logs, "AdditionalData Message Severity")
Michael Shepos1a67b082020-08-28 16:01:58 -0500990 pels = pel.peltool("-l", ignore_err=1)
Michael Shepos0e5f1132020-09-30 16:24:25 -0500991 gp.qprint_var(pels)
Michael Shepos1a67b082020-08-28 16:01:58 -0500992
Michael Walshaabef1e2017-09-20 15:16:17 -0500993 # We need to purge error logs between boots or they build up.
Michael Walsh409ad352020-02-06 11:46:35 -0600994 grk.run_key(delete_errlogs_cmd, ignore=1)
Michael Shepos92a54bf2020-11-11 11:48:55 -0600995 grk.run_key(delete_bmcdump_cmd, ignore=1)
George Keishing2ef6a7d2021-05-19 09:05:32 -0500996 if redfish_support_trans_state:
997 grk.run_key(delete_sysdump_cmd, ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -0500998
Michael Walsh952f9b02017-03-09 13:11:14 -0600999 boot_results.print_report()
Michael Walshb5839d02017-04-12 16:11:20 -05001000 gp.qprint_timen("Finished boot " + str(boot_count) + ".")
Michael Walsh952f9b02017-03-09 13:11:14 -06001001
Michael Walsh6741f742017-02-20 16:16:38 -06001002 plug_in_setup()
1003 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -05001004 call_point='stop_check', shell_rc=stop_test_rc(),
1005 stop_on_non_zero_rc=1)
1006 if shell_rc == stop_test_rc():
Michael Walsh3ba8ecd2018-04-24 11:33:25 -05001007 message = "Stopping as requested by user.\n"
Michael Walsh80dddde2019-10-22 13:54:38 -05001008 gp.qprint_time(message)
Michael Walsh3ba8ecd2018-04-24 11:33:25 -05001009 BuiltIn().fail(message)
Michael Walsh6741f742017-02-20 16:16:38 -06001010
Michael Walshd139f282017-04-04 18:00:23 -05001011 # This should help prevent ConnectionErrors.
George Keishing4d65c862020-12-03 06:52:11 -06001012 # Purge all redfish and REST connection sessions.
George Keishingd86e45c2021-03-19 07:38:14 -05001013 if redfish_delete_sessions:
1014 grk.run_key_u("Close All Connections", ignore=1)
1015 grk.run_key_u("Delete All Redfish Sessions", ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -05001016
Michael Walsh6741f742017-02-20 16:16:38 -06001017 return True
1018
Michael Walsh6741f742017-02-20 16:16:38 -06001019
Michael Walsh83f4bc72017-04-20 16:49:43 -05001020def obmc_boot_test_teardown():
Michael Walsh6741f742017-02-20 16:16:38 -06001021 r"""
Michael Walshf75d4352019-12-05 17:01:20 -06001022 Clean up after the main keyword.
Michael Walsh6741f742017-02-20 16:16:38 -06001023 """
Michael Walshf75d4352019-12-05 17:01:20 -06001024 gp.qprint_executing()
1025
1026 if ga.psutil_imported:
1027 ga.terminate_descendants()
Michael Walsh6741f742017-02-20 16:16:38 -06001028
1029 if cp_setup_called:
1030 plug_in_setup()
1031 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -05001032 call_point='cleanup', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -06001033
Michael Walsh600876d2017-05-30 17:58:58 -05001034 if 'boot_results_file_path' in globals():
Michael Walsh986d8ae2019-07-17 10:02:23 -05001035 # Save boot_results and boot_history objects to a file in case they are
Michael Walsh6c645742018-08-17 15:02:17 -05001036 # needed again.
Michael Walsh600876d2017-05-30 17:58:58 -05001037 gp.qprint_timen("Saving boot_results to the following path.")
1038 gp.qprint_var(boot_results_file_path)
Michael Walsh986d8ae2019-07-17 10:02:23 -05001039 pickle.dump((boot_results, boot_history),
Michael Walsh6c645742018-08-17 15:02:17 -05001040 open(boot_results_file_path, 'wb'),
Michael Walsh600876d2017-05-30 17:58:58 -05001041 pickle.HIGHEST_PROTOCOL)
Michael Walsh0b93fbf2017-03-02 14:42:41 -06001042
Michael Walshff340002017-08-29 11:18:27 -05001043 global save_stack
1044 # Restore any global values saved on the save_stack.
1045 for parm_name in main_func_parm_list:
1046 # Get the parm_value if it was saved on the stack.
1047 try:
1048 parm_value = save_stack.pop(parm_name)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -05001049 except BaseException:
Michael Walshff340002017-08-29 11:18:27 -05001050 # If it was not saved, no further action is required.
1051 continue
1052
1053 # Restore the saved value.
1054 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1055 "}\", parm_value)"
1056 gp.dpissuing(cmd_buf)
1057 exec(cmd_buf)
1058
1059 gp.dprintn(save_stack.sprint_obj())
1060
Michael Walsh6741f742017-02-20 16:16:38 -06001061
Michael Walshc9116812017-03-10 14:23:06 -06001062def test_teardown():
Michael Walshc9116812017-03-10 14:23:06 -06001063 r"""
1064 Clean up after this test case.
1065 """
1066
1067 gp.qprintn()
Michael Walshf75d4352019-12-05 17:01:20 -06001068 gp.qprint_executing()
1069
1070 if ga.psutil_imported:
1071 ga.terminate_descendants()
1072
Michael Walshc9116812017-03-10 14:23:06 -06001073 cmd_buf = ["Print Error",
1074 "A keyword timeout occurred ending this program.\n"]
1075 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1076
George Keishinga54e06f2020-06-12 10:42:41 -05001077 if redfish_supported:
1078 redfish.logout()
1079
Michael Walshc108e422019-03-28 12:27:18 -05001080 gp.qprint_pgm_footer()
Michael Walshb5839d02017-04-12 16:11:20 -05001081
Michael Walshc9116812017-03-10 14:23:06 -06001082
Michael Walsh89de14a2018-10-01 16:51:37 -05001083def post_stack():
1084 r"""
1085 Process post_stack plug-in programs.
1086 """
1087
1088 if not call_post_stack_plug:
1089 # The caller does not wish to have post_stack plug-in processing done.
1090 return
1091
1092 global boot_success
1093
1094 # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1095 pre_boot_plug_in_setup()
1096 # For the purposes of the following plug-ins, mark the "boot" as a success.
1097 boot_success = 1
1098 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001099 rc, shell_rc, failed_plug_in_name, history =\
1100 grpi.rprocess_plug_in_packages(call_point='post_stack',
1101 stop_on_plug_in_failure=0,
1102 return_history=True)
Michael Walsh986d8ae2019-07-17 10:02:23 -05001103 for doing_msg in history:
1104 update_boot_history(boot_history, doing_msg, max_boot_history)
Michael Walsh815b1d52018-10-30 13:32:26 -05001105 if rc != 0:
1106 boot_success = 0
Michael Walsh89de14a2018-10-01 16:51:37 -05001107
1108 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001109 rc, shell_rc, failed_plug_in_name =\
1110 grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1111 shell_rc=dump_ffdc_rc(),
1112 stop_on_plug_in_failure=1,
1113 stop_on_non_zero_rc=1)
1114 if shell_rc == dump_ffdc_rc():
Michael Walsh89de14a2018-10-01 16:51:37 -05001115 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1116 if status != 'PASS':
1117 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -05001118 # Leave a record for caller that "soft" errors occurred.
1119 soft_errors = 1
1120 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh89de14a2018-10-01 16:51:37 -05001121
1122 plug_in_setup()
1123 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1124 call_point='stop_check', shell_rc=stop_test_rc(),
1125 stop_on_non_zero_rc=1)
1126 if shell_rc == stop_test_rc():
1127 message = "Stopping as requested by user.\n"
Michael Walsh80dddde2019-10-22 13:54:38 -05001128 gp.qprint_time(message)
Michael Walsh89de14a2018-10-01 16:51:37 -05001129 BuiltIn().fail(message)
1130
1131
Michael Walshff340002017-08-29 11:18:27 -05001132def obmc_boot_test_py(loc_boot_stack=None,
1133 loc_stack_mode=None,
1134 loc_quiet=None):
Michael Walsh6741f742017-02-20 16:16:38 -06001135 r"""
1136 Do main program processing.
1137 """
1138
Michael Walshff340002017-08-29 11:18:27 -05001139 global save_stack
1140
Michael Walshf75d4352019-12-05 17:01:20 -06001141 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']})
1142
George Keishing36efbc02018-12-12 10:18:23 -06001143 gp.dprintn()
Michael Walshff340002017-08-29 11:18:27 -05001144 # Process function parms.
1145 for parm_name in main_func_parm_list:
1146 # Get parm's value.
George Keishing36efbc02018-12-12 10:18:23 -06001147 parm_value = eval("loc_" + parm_name)
1148 gp.dpvars(parm_name, parm_value)
Michael Walshff340002017-08-29 11:18:27 -05001149
George Keishing36efbc02018-12-12 10:18:23 -06001150 if parm_value is not None:
Michael Walshff340002017-08-29 11:18:27 -05001151 # Save the global value on a stack.
1152 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1153 parm_name + "}\"), \"" + parm_name + "\")"
1154 gp.dpissuing(cmd_buf)
1155 exec(cmd_buf)
1156
1157 # Set the global value to the passed value.
1158 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1159 "}\", loc_" + parm_name + ")"
1160 gp.dpissuing(cmd_buf)
1161 exec(cmd_buf)
1162
1163 gp.dprintn(save_stack.sprint_obj())
Michael Walshb5839d02017-04-12 16:11:20 -05001164
Michael Walsh6741f742017-02-20 16:16:38 -06001165 setup()
1166
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001167 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1168
Michael Walsha20da402017-03-31 16:27:45 -05001169 if ffdc_only:
1170 gp.qprint_timen("Caller requested ffdc_only.")
Michael Walsh986d8ae2019-07-17 10:02:23 -05001171 if do_pre_boot_plug_in_setup:
1172 pre_boot_plug_in_setup()
Michael Walsh83f4bc72017-04-20 16:49:43 -05001173 grk.run_key_u("my_ffdc")
Michael Walsh764d2f82017-04-27 16:01:08 -05001174 return
Michael Walsha20da402017-03-31 16:27:45 -05001175
Michael Walsh409ad352020-02-06 11:46:35 -06001176 if delete_errlogs:
Michael Shepos1a67b082020-08-28 16:01:58 -05001177 # print error logs before delete
George Keishing438fd3b2021-10-08 02:15:18 -05001178 if redfish_support_trans_state:
1179 status, error_logs = grk.run_key_u("Get Redfish Event Logs")
1180 log.print_error_logs(error_logs, "AdditionalDataURI Message Severity")
1181 else:
1182 status, error_logs = grk.run_key_u("Get Error Logs")
1183 log.print_error_logs(error_logs, "AdditionalData Message Severity")
Michael Shepos1a67b082020-08-28 16:01:58 -05001184 pels = pel.peltool("-l", ignore_err=1)
Michael Shepos0e5f1132020-09-30 16:24:25 -05001185 gp.qprint_var(pels)
Michael Shepos1a67b082020-08-28 16:01:58 -05001186
Michael Walsh409ad352020-02-06 11:46:35 -06001187 # Delete errlogs prior to doing any boot tests.
1188 grk.run_key(delete_errlogs_cmd, ignore=1)
Michael Shepos92a54bf2020-11-11 11:48:55 -06001189 grk.run_key(delete_bmcdump_cmd, ignore=1)
George Keishing2ef6a7d2021-05-19 09:05:32 -05001190 if redfish_support_trans_state:
1191 grk.run_key(delete_sysdump_cmd, ignore=1)
Michael Walsh409ad352020-02-06 11:46:35 -06001192
Michael Walsh6741f742017-02-20 16:16:38 -06001193 # Process caller's boot_stack.
1194 while (len(boot_stack) > 0):
1195 test_loop_body()
1196
Michael Walshb5839d02017-04-12 16:11:20 -05001197 gp.qprint_timen("Finished processing stack.")
Michael Walsh30dadae2017-02-27 14:25:52 -06001198
Michael Walsh89de14a2018-10-01 16:51:37 -05001199 post_stack()
1200
Michael Walsh6741f742017-02-20 16:16:38 -06001201 # Process caller's boot_list.
1202 if len(boot_list) > 0:
1203 for ix in range(1, max_num_tests + 1):
1204 test_loop_body()
1205
Michael Walshb5839d02017-04-12 16:11:20 -05001206 gp.qprint_timen("Completed all requested boot tests.")
1207
1208 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001209 new_fail = boot_fail - init_boot_fail
1210 if new_fail > boot_fail_threshold:
Michael Walshb5839d02017-04-12 16:11:20 -05001211 error_message = "Boot failures exceed the boot failure" +\
1212 " threshold:\n" +\
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001213 gp.sprint_var(new_fail) +\
Michael Walshb5839d02017-04-12 16:11:20 -05001214 gp.sprint_var(boot_fail_threshold)
1215 BuiltIn().fail(gp.sprint_error(error_message))