blob: 5b3d7de56e760de39799906d29c8d5bf81948ee5 [file] [log] [blame]
George Keishinge7e91712021-09-03 11:28:44 -05001#!/usr/bin/env python3
Michael Walsh0bbd8602016-11-22 11:31:49 -06002
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
George Keishinge635ddc2022-12-08 07:38:02 -06007import glob
Patrick Williams20f38712022-12-08 06:18:26 -06008import imp
9import os
Michael Walsh0b93fbf2017-03-02 14:42:41 -060010import random
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050011import re
Michael Walshf566fb12019-02-01 14:35:09 -060012import signal
Patrick Williams20f38712022-12-08 06:18:26 -060013import time
14
George Keishingd54bbc22018-08-03 08:24:58 -050015try:
16 import cPickle as pickle
17except ImportError:
18 import pickle
Patrick Williams20f38712022-12-08 06:18:26 -060019
Michael Walshdc80d672017-05-09 12:58:32 -050020import socket
Michael Walsh0b93fbf2017-03-02 14:42:41 -060021
George Keishinge635ddc2022-12-08 07:38:02 -060022import gen_arg as ga
George Keishinge635ddc2022-12-08 07:38:02 -060023import gen_cmd as gc
Patrick Williams20f38712022-12-08 06:18:26 -060024import gen_misc as gm
25import gen_plug_in_utils as gpu
26import gen_print as gp
George Keishinge635ddc2022-12-08 07:38:02 -060027import gen_robot_keyword as grk
Patrick Williams20f38712022-12-08 06:18:26 -060028import gen_robot_plug_in as grpi
29import gen_valid as gv
30import logging_utils as log
31import pel_utils as pel
Michael Walsh55302292017-01-10 11:43:02 -060032import state as st
Michael Walshff340002017-08-29 11:18:27 -050033import var_stack as vs
Patrick Williams20f38712022-12-08 06:18:26 -060034from boot_data import *
35from robot.libraries.BuiltIn import BuiltIn
36from robot.utils import DotDict
Michael Walsh0bbd8602016-11-22 11:31:49 -060037
Patrick Williams20f38712022-12-08 06:18:26 -060038base_path = (
39 os.path.dirname(os.path.dirname(imp.find_module("gen_robot_print")[1]))
40 + os.sep
41)
Michael Walsh0b93fbf2017-03-02 14:42:41 -060042sys.path.append(base_path + "extended/")
George Keishing09679892022-12-08 08:21:52 -060043import run_keyword as rk # NOQA
Michael Walsh0bbd8602016-11-22 11:31:49 -060044
Michael Walshe1e26442017-03-06 17:50:07 -060045# Setting master_pid correctly influences the behavior of plug-ins like
46# DB_Logging
47program_pid = os.getpid()
Patrick Williams20f38712022-12-08 06:18:26 -060048master_pid = os.environ.get("AUTOBOOT_MASTER_PID", program_pid)
49pgm_name = re.sub("\\.py$", "", os.path.basename(__file__))
Michael Walshe1e26442017-03-06 17:50:07 -060050
Michael Walshb5839d02017-04-12 16:11:20 -050051# Set up boot data structures.
Michael Walsh986d8ae2019-07-17 10:02:23 -050052os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
Michael Walsh0b93fbf2017-03-02 14:42:41 -060053
Michael Walsh6741f742017-02-20 16:16:38 -060054boot_lists = read_boot_lists()
Michael Walsh986d8ae2019-07-17 10:02:23 -050055
56# The maximum number of entries that can be in the boot_history global variable.
Michael Walsh815b1d52018-10-30 13:32:26 -050057max_boot_history = 10
Michael Walsh986d8ae2019-07-17 10:02:23 -050058boot_history = []
Michael Walsh6741f742017-02-20 16:16:38 -060059
Patrick Williams20f38712022-12-08 06:18:26 -060060state = st.return_state_constant("default_state")
Michael Walsh6741f742017-02-20 16:16:38 -060061cp_setup_called = 0
62next_boot = ""
Patrick Williams20f38712022-12-08 06:18:26 -060063base_tool_dir_path = (
64 os.path.normpath(os.environ.get("AUTOBOOT_BASE_TOOL_DIR_PATH", "/tmp"))
65 + os.sep
66)
Michael Walshb5839d02017-04-12 16:11:20 -050067
Patrick Williams20f38712022-12-08 06:18:26 -060068ffdc_dir_path = os.path.normpath(os.environ.get("FFDC_DIR_PATH", "")) + os.sep
Michael Walsh6741f742017-02-20 16:16:38 -060069boot_success = 0
George Keishingce90a7d2021-10-12 02:38:51 -050070
Patrick Williams20f38712022-12-08 06:18:26 -060071status_dir_path = os.environ.get(
72 "STATUS_DIR_PATH", ""
73) or BuiltIn().get_variable_value("${STATUS_DIR_PATH}", default="")
Michael Walsh6741f742017-02-20 16:16:38 -060074if status_dir_path != "":
75 status_dir_path = os.path.normpath(status_dir_path) + os.sep
George Keishingce90a7d2021-10-12 02:38:51 -050076 # For plugin expecting env gen_call_robot.py
Patrick Williams20f38712022-12-08 06:18:26 -060077 os.environ["STATUS_DIR_PATH"] = status_dir_path
George Keishingce90a7d2021-10-12 02:38:51 -050078
Patrick Williams20f38712022-12-08 06:18:26 -060079redfish_support_trans_state = int(
80 os.environ.get("REDFISH_SUPPORT_TRANS_STATE", 0)
81) or int(
82 BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0)
83)
84redfish_supported = BuiltIn().get_variable_value(
85 "${REDFISH_SUPPORTED}", default=False
86)
87redfish_rest_supported = BuiltIn().get_variable_value(
88 "${REDFISH_REST_SUPPORTED}", default=False
89)
90redfish_delete_sessions = int(
91 BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)
92)
Michael Walshe58df1c2019-08-07 09:57:43 -050093if redfish_supported:
Patrick Williams20f38712022-12-08 06:18:26 -060094 redfish = BuiltIn().get_library_instance("redfish")
Michael Walshe58df1c2019-08-07 09:57:43 -050095 default_power_on = "Redfish Power On"
96 default_power_off = "Redfish Power Off"
George Keishing870999a2021-03-31 23:43:57 -050097 if not redfish_support_trans_state:
Michael Sheposcc490b42020-08-26 12:53:01 -050098 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}"
Michael Shepos92a54bf2020-11-11 11:48:55 -060099 delete_bmcdump_cmd = "Delete All BMC Dump"
George Keishingeb1fe352020-06-19 03:02:22 -0500100 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF"
101 else:
102 delete_errlogs_cmd = "Redfish Purge Event Log"
Michael Shepos92a54bf2020-11-11 11:48:55 -0600103 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps"
George Keishing2ef6a7d2021-05-19 09:05:32 -0500104 delete_sysdump_cmd = "Redfish Delete All System Dumps"
Patrick Williams20f38712022-12-08 06:18:26 -0600105 default_set_power_policy = (
106 "Redfish Set Power Restore Policy AlwaysOff"
107 )
Michael Walshe58df1c2019-08-07 09:57:43 -0500108else:
109 default_power_on = "REST Power On"
110 default_power_off = "REST Power Off"
Michael Sheposcc490b42020-08-26 12:53:01 -0500111 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}"
Michael Shepos92a54bf2020-11-11 11:48:55 -0600112 delete_bmcdump_cmd = "Delete All BMC Dump"
George Keishinga54e06f2020-06-12 10:42:41 -0500113 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF"
Michael Walsh6741f742017-02-20 16:16:38 -0600114boot_count = 0
Michael Walsh0bbd8602016-11-22 11:31:49 -0600115
Michael Walsh85678942017-03-27 14:34:22 -0500116LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
Patrick Williams20f38712022-12-08 06:18:26 -0600117AUTOBOOT_FFDC_PREFIX = os.environ.get("AUTOBOOT_FFDC_PREFIX", "")
Michael Walsh986d8ae2019-07-17 10:02:23 -0500118ffdc_prefix = AUTOBOOT_FFDC_PREFIX
Sunil M325eb542017-08-10 07:09:43 -0500119boot_start_time = ""
120boot_end_time = ""
Patrick Williams20f38712022-12-08 06:18:26 -0600121save_stack = vs.var_stack("save_stack")
122main_func_parm_list = ["boot_stack", "stack_mode", "quiet"]
Michael Walsh85678942017-03-27 14:34:22 -0500123
124
Michael Walsh89de14a2018-10-01 16:51:37 -0500125def dump_ffdc_rc():
126 r"""
127 Return the constant dump ffdc test return code value.
128
129 When a plug-in call point program returns this value, it indicates that
130 this program should collect FFDC.
131 """
132
133 return 0x00000200
134
135
136def stop_test_rc():
137 r"""
138 Return the constant stop test return code value.
139
140 When a plug-in call point program returns this value, it indicates that
141 this program should stop running.
142 """
143
144 return 0x00000200
145
146
Patrick Williams20f38712022-12-08 06:18:26 -0600147def process_host(host, host_var_name=""):
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500148 r"""
149 Process a host by getting the associated host name and IP address and
150 setting them in global variables.
151
152 If the caller does not pass the host_var_name, this function will try to
153 figure out the name of the variable used by the caller for the host parm.
154 Callers are advised to explicitly specify the host_var_name when calling
155 with an exec command. In such cases, the get_arg_name cannot figure out
156 the host variable name.
157
158 This function will then create similar global variable names by
159 removing "_host" and appending "_host_name" or "_ip" to the host variable
160 name.
161
162 Example:
163
164 If a call is made like this:
165 process_host(openbmc_host)
166
167 Global variables openbmc_host_name and openbmc_ip will be set.
168
169 Description of argument(s):
170 host A host name or IP. The name of the variable used should
171 have a suffix of "_host".
172 host_var_name The name of the variable being used as the host parm.
173 """
174
175 if host_var_name == "":
176 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
177
178 host_name_var_name = re.sub("host", "host_name", host_var_name)
179 ip_var_name = re.sub("host", "ip", host_var_name)
Patrick Williams20f38712022-12-08 06:18:26 -0600180 cmd_buf = (
181 "global "
182 + host_name_var_name
183 + ", "
184 + ip_var_name
185 + " ; "
186 + host_name_var_name
187 + ", "
188 + ip_var_name
189 + " = gm.get_host_name_ip('"
190 + host
191 + "')"
192 )
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500193 exec(cmd_buf)
194
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500195
Michael Walshb5839d02017-04-12 16:11:20 -0500196def process_pgm_parms():
Michael Walshb5839d02017-04-12 16:11:20 -0500197 r"""
198 Process the program parameters by assigning them all to corresponding
199 globals. Also, set some global values that depend on program parameters.
200 """
201
202 # Program parameter processing.
203 # Assign all program parms to python variables which are global to this
204 # module.
205
206 global parm_list
207 parm_list = BuiltIn().get_variable_value("${parm_list}")
208 # The following subset of parms should be processed as integers.
Patrick Williams20f38712022-12-08 06:18:26 -0600209 int_list = [
210 "max_num_tests",
211 "boot_pass",
212 "boot_fail",
213 "ffdc_only",
214 "boot_fail_threshold",
215 "delete_errlogs",
216 "call_post_stack_plug",
217 "do_pre_boot_plug_in_setup",
218 "quiet",
219 "test_mode",
220 "debug",
221 ]
Michael Walshb5839d02017-04-12 16:11:20 -0500222 for parm in parm_list:
223 if parm in int_list:
Patrick Williams20f38712022-12-08 06:18:26 -0600224 sub_cmd = (
225 'int(BuiltIn().get_variable_value("${' + parm + '}", "0"))'
226 )
Michael Walshb5839d02017-04-12 16:11:20 -0500227 else:
Patrick Williams20f38712022-12-08 06:18:26 -0600228 sub_cmd = 'BuiltIn().get_variable_value("${' + parm + '}")'
Michael Walshb5839d02017-04-12 16:11:20 -0500229 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
Michael Walshff340002017-08-29 11:18:27 -0500230 gp.dpissuing(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500231 exec(cmd_buf)
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500232 if re.match(r".*_host$", parm):
233 cmd_buf = "process_host(" + parm + ", '" + parm + "')"
234 exec(cmd_buf)
235 if re.match(r".*_password$", parm):
236 # Register the value of any parm whose name ends in _password.
237 # This will cause the print functions to replace passwords with
238 # asterisks in the output.
239 cmd_buf = "gp.register_passwords(" + parm + ")"
240 exec(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500241
242 global ffdc_dir_path_style
243 global boot_list
244 global boot_stack
245 global boot_results_file_path
246 global boot_results
Michael Walsh986d8ae2019-07-17 10:02:23 -0500247 global boot_history
Michael Walshb5839d02017-04-12 16:11:20 -0500248 global ffdc_list_file_path
Michael Walshe0cf8d72017-05-17 13:20:46 -0500249 global ffdc_report_list_path
Michael Walsh600876d2017-05-30 17:58:58 -0500250 global ffdc_summary_list_path
Michael Walsha3e7b222020-02-03 15:32:16 -0600251 global boot_table
252 global valid_boot_types
Michael Walshb5839d02017-04-12 16:11:20 -0500253
254 if ffdc_dir_path_style == "":
Patrick Williams20f38712022-12-08 06:18:26 -0600255 ffdc_dir_path_style = int(os.environ.get("FFDC_DIR_PATH_STYLE", "0"))
Michael Walshb5839d02017-04-12 16:11:20 -0500256
257 # Convert these program parms to lists for easier processing..
George Keishing36efbc02018-12-12 10:18:23 -0600258 boot_list = list(filter(None, boot_list.split(":")))
259 boot_stack = list(filter(None, boot_stack.split(":")))
Michael Walshb5839d02017-04-12 16:11:20 -0500260
Michael Walsha3e7b222020-02-03 15:32:16 -0600261 boot_table = create_boot_table(boot_table_path, os_host=os_host)
262 valid_boot_types = create_valid_boot_list(boot_table)
263
Michael Walsh903e0b22017-09-19 17:00:33 -0500264 cleanup_boot_results_file()
Patrick Williams20f38712022-12-08 06:18:26 -0600265 boot_results_file_path = create_boot_results_file_path(
266 pgm_name, openbmc_nickname, master_pid
267 )
Michael Walshb5839d02017-04-12 16:11:20 -0500268
269 if os.path.isfile(boot_results_file_path):
270 # We've been called before in this run so we'll load the saved
Michael Walsh986d8ae2019-07-17 10:02:23 -0500271 # boot_results and boot_history objects.
Patrick Williams20f38712022-12-08 06:18:26 -0600272 boot_results, boot_history = pickle.load(
273 open(boot_results_file_path, "rb")
274 )
Michael Walshb5839d02017-04-12 16:11:20 -0500275 else:
276 boot_results = boot_results(boot_table, boot_pass, boot_fail)
277
Patrick Williams20f38712022-12-08 06:18:26 -0600278 ffdc_list_file_path = (
279 base_tool_dir_path + openbmc_nickname + "/FFDC_FILE_LIST"
280 )
281 ffdc_report_list_path = (
282 base_tool_dir_path + openbmc_nickname + "/FFDC_REPORT_FILE_LIST"
283 )
Michael Walshb5839d02017-04-12 16:11:20 -0500284
Patrick Williams20f38712022-12-08 06:18:26 -0600285 ffdc_summary_list_path = (
286 base_tool_dir_path + openbmc_nickname + "/FFDC_SUMMARY_FILE_LIST"
287 )
Michael Walsh600876d2017-05-30 17:58:58 -0500288
Michael Walshb5839d02017-04-12 16:11:20 -0500289
Michael Walsh85678942017-03-27 14:34:22 -0500290def initial_plug_in_setup():
Michael Walsh85678942017-03-27 14:34:22 -0500291 r"""
292 Initialize all plug-in environment variables which do not change for the
293 duration of the program.
294
295 """
296
297 global LOG_LEVEL
298 BuiltIn().set_log_level("NONE")
299
300 BuiltIn().set_global_variable("${master_pid}", master_pid)
301 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
302 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
303 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
Patrick Williams20f38712022-12-08 06:18:26 -0600304 BuiltIn().set_global_variable(
305 "${FFDC_LIST_FILE_PATH}", ffdc_list_file_path
306 )
307 BuiltIn().set_global_variable(
308 "${FFDC_REPORT_LIST_PATH}", ffdc_report_list_path
309 )
310 BuiltIn().set_global_variable(
311 "${FFDC_SUMMARY_LIST_PATH}", ffdc_summary_list_path
312 )
Michael Walsh85678942017-03-27 14:34:22 -0500313
Patrick Williams20f38712022-12-08 06:18:26 -0600314 BuiltIn().set_global_variable(
315 "${FFDC_DIR_PATH_STYLE}", ffdc_dir_path_style
316 )
317 BuiltIn().set_global_variable("${FFDC_CHECK}", ffdc_check)
Michael Walsh85678942017-03-27 14:34:22 -0500318
319 # For each program parameter, set the corresponding AUTOBOOT_ environment
320 # variable value. Also, set an AUTOBOOT_ environment variable for every
321 # element in additional_values.
Patrick Williams20f38712022-12-08 06:18:26 -0600322 additional_values = [
323 "program_pid",
324 "master_pid",
325 "ffdc_dir_path",
326 "status_dir_path",
327 "base_tool_dir_path",
328 "ffdc_list_file_path",
329 "ffdc_report_list_path",
330 "ffdc_summary_list_path",
331 "execdir",
332 "redfish_supported",
333 "redfish_rest_supported",
334 "redfish_support_trans_state",
335 ]
Michael Walsh85678942017-03-27 14:34:22 -0500336
337 plug_in_vars = parm_list + additional_values
338
339 for var_name in plug_in_vars:
340 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
341 var_name = var_name.upper()
342 if var_value is None:
343 var_value = ""
344 os.environ["AUTOBOOT_" + var_name] = str(var_value)
345
346 BuiltIn().set_log_level(LOG_LEVEL)
347
Michael Walsh68a61162017-04-25 11:54:06 -0500348 # Make sure the ffdc list directory exists.
349 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
350 if not os.path.exists(ffdc_list_dir_path):
351 os.makedirs(ffdc_list_dir_path)
Michael Walsh85678942017-03-27 14:34:22 -0500352
Michael Walsh85678942017-03-27 14:34:22 -0500353
Michael Walsh0bbd8602016-11-22 11:31:49 -0600354def plug_in_setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600355 r"""
Michael Walsh85678942017-03-27 14:34:22 -0500356 Initialize all changing plug-in environment variables for use by the
357 plug-in programs.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600358 """
359
Michael Walsh85678942017-03-27 14:34:22 -0500360 global LOG_LEVEL
361 global test_really_running
362
363 BuiltIn().set_log_level("NONE")
364
Michael Walsh6741f742017-02-20 16:16:38 -0600365 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600366 if boot_pass > 1:
367 test_really_running = 1
368 else:
369 test_really_running = 0
370
Patrick Williams20f38712022-12-08 06:18:26 -0600371 BuiltIn().set_global_variable(
372 "${test_really_running}", test_really_running
373 )
Michael Walsh6741f742017-02-20 16:16:38 -0600374 BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
Michael Walsh6741f742017-02-20 16:16:38 -0600375 BuiltIn().set_global_variable("${boot_pass}", boot_pass)
376 BuiltIn().set_global_variable("${boot_fail}", boot_fail)
377 BuiltIn().set_global_variable("${boot_success}", boot_success)
378 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
Sunil M325eb542017-08-10 07:09:43 -0500379 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
380 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
Michael Walsh4c9a6452016-12-13 16:03:11 -0600381
Michael Walsh0bbd8602016-11-22 11:31:49 -0600382 # For each program parameter, set the corresponding AUTOBOOT_ environment
383 # variable value. Also, set an AUTOBOOT_ environment variable for every
384 # element in additional_values.
Patrick Williams20f38712022-12-08 06:18:26 -0600385 additional_values = [
386 "boot_type_desc",
387 "boot_success",
388 "boot_pass",
389 "boot_fail",
390 "test_really_running",
391 "ffdc_prefix",
392 "boot_start_time",
393 "boot_end_time",
394 ]
Michael Walsh0bbd8602016-11-22 11:31:49 -0600395
Michael Walsh85678942017-03-27 14:34:22 -0500396 plug_in_vars = additional_values
Michael Walsh0bbd8602016-11-22 11:31:49 -0600397
398 for var_name in plug_in_vars:
399 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
400 var_name = var_name.upper()
401 if var_value is None:
402 var_value = ""
Michael Walsh6741f742017-02-20 16:16:38 -0600403 os.environ["AUTOBOOT_" + var_name] = str(var_value)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600404
Michael Walsh0bbd8602016-11-22 11:31:49 -0600405 if debug:
Patrick Williams20f38712022-12-08 06:18:26 -0600406 shell_rc, out_buf = gc.cmd_fnc_u(
407 "printenv | egrep AUTOBOOT_ | sort -u"
408 )
Michael Walsh0bbd8602016-11-22 11:31:49 -0600409
Michael Walsh85678942017-03-27 14:34:22 -0500410 BuiltIn().set_log_level(LOG_LEVEL)
411
Michael Walsh0bbd8602016-11-22 11:31:49 -0600412
Michael Walshe0cf8d72017-05-17 13:20:46 -0500413def pre_boot_plug_in_setup():
Michael Walshe0cf8d72017-05-17 13:20:46 -0500414 # Clear the ffdc_list_file_path file. Plug-ins may now write to it.
415 try:
416 os.remove(ffdc_list_file_path)
417 except OSError:
418 pass
419
420 # Clear the ffdc_report_list_path file. Plug-ins may now write to it.
421 try:
422 os.remove(ffdc_report_list_path)
423 except OSError:
424 pass
425
Michael Walsh600876d2017-05-30 17:58:58 -0500426 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it.
427 try:
428 os.remove(ffdc_summary_list_path)
429 except OSError:
430 pass
431
Michael Walshe1974b92017-08-03 13:39:51 -0500432 global ffdc_prefix
433
434 seconds = time.time()
435 loc_time = time.localtime(seconds)
436 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
437
438 ffdc_prefix = openbmc_nickname + "." + time_string
439
Michael Walshe0cf8d72017-05-17 13:20:46 -0500440
Patrick Williams20f38712022-12-08 06:18:26 -0600441def default_sigusr1(signal_number=0, frame=None):
Michael Walshf566fb12019-02-01 14:35:09 -0600442 r"""
443 Handle SIGUSR1 by doing nothing.
444
445 This function assists in debugging SIGUSR1 processing by printing messages
446 to stdout and to the log.html file.
447
448 Description of argument(s):
449 signal_number The signal number (should always be 10 for SIGUSR1).
450 frame The frame data.
451 """
452
Michael Walsh80dddde2019-10-22 13:54:38 -0500453 gp.qprintn()
454 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600455 gp.lprint_executing()
456
457
458def set_default_siguser1():
459 r"""
460 Set the default_sigusr1 function to be the SIGUSR1 handler.
461 """
462
Michael Walsh80dddde2019-10-22 13:54:38 -0500463 gp.qprintn()
464 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600465 gp.lprint_executing()
466 signal.signal(signal.SIGUSR1, default_sigusr1)
467
468
Michael Walsh6741f742017-02-20 16:16:38 -0600469def setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600470 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600471 Do general program setup tasks.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600472 """
473
Michael Walsh6741f742017-02-20 16:16:38 -0600474 global cp_setup_called
Michael Walsh81816742017-09-27 11:02:29 -0500475 global transitional_boot_selected
Michael Walsh0bbd8602016-11-22 11:31:49 -0600476
Michael Walshb5839d02017-04-12 16:11:20 -0500477 gp.qprintn()
478
George Keishinga54e06f2020-06-12 10:42:41 -0500479 if redfish_supported:
480 redfish.login()
481
Michael Walshf566fb12019-02-01 14:35:09 -0600482 set_default_siguser1()
Michael Walsh81816742017-09-27 11:02:29 -0500483 transitional_boot_selected = False
484
Michael Walsh83f4bc72017-04-20 16:49:43 -0500485 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
486 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
Michael Walshd061c042017-05-23 14:46:57 -0500487 # If we can't find process_plug_in_packages.py, ssh_pw or
488 # validate_plug_ins.py, then we don't have our repo bin in PATH.
Patrick Williams20f38712022-12-08 06:18:26 -0600489 shell_rc, out_buf = gc.cmd_fnc_u(
490 "which process_plug_in_packages.py" + " ssh_pw validate_plug_ins.py",
491 quiet=1,
492 print_output=0,
493 show_err=0,
494 )
Michael Walshb5839d02017-04-12 16:11:20 -0500495 if shell_rc != 0:
Patrick Williams20f38712022-12-08 06:18:26 -0600496 os.environ["PATH"] = repo_bin_path + ":" + os.environ.get("PATH", "")
Michael Walsh83f4bc72017-04-20 16:49:43 -0500497 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
498 if robot_pgm_dir_path not in sys.path:
499 sys.path.append(robot_pgm_dir_path)
500 PYTHONPATH = os.environ.get("PYTHONPATH", "")
501 if PYTHONPATH == "":
Patrick Williams20f38712022-12-08 06:18:26 -0600502 os.environ["PYTHONPATH"] = robot_pgm_dir_path
Michael Walsh83f4bc72017-04-20 16:49:43 -0500503 else:
Patrick Williams20f38712022-12-08 06:18:26 -0600504 os.environ["PYTHONPATH"] = robot_pgm_dir_path + ":" + PYTHONPATH
Michael Walsh6741f742017-02-20 16:16:38 -0600505
506 validate_parms()
507
Michael Walshc108e422019-03-28 12:27:18 -0500508 gp.qprint_pgm_header()
Michael Walsh6741f742017-02-20 16:16:38 -0600509
George Keishingeae945b2021-12-13 11:05:04 -0600510 grk.run_key_u(default_set_power_policy, ignore=1)
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500511
Michael Walsh85678942017-03-27 14:34:22 -0500512 initial_plug_in_setup()
513
Michael Walsh6741f742017-02-20 16:16:38 -0600514 plug_in_setup()
515 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Patrick Williams20f38712022-12-08 06:18:26 -0600516 call_point="setup"
517 )
Michael Walsh6741f742017-02-20 16:16:38 -0600518 if rc != 0:
519 error_message = "Plug-in setup failed.\n"
Michael Walshc108e422019-03-28 12:27:18 -0500520 gp.print_error_report(error_message)
Michael Walsh6741f742017-02-20 16:16:38 -0600521 BuiltIn().fail(error_message)
522 # Setting cp_setup_called lets our Teardown know that it needs to call
523 # the cleanup plug-in call point.
524 cp_setup_called = 1
525
526 # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
527 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
Michael Walsh85678942017-03-27 14:34:22 -0500528 # FFDC_LOG_PATH is used by "FFDC" keyword.
529 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
Michael Walsh6741f742017-02-20 16:16:38 -0600530
Michael Walshdc80d672017-05-09 12:58:32 -0500531 # Also printed by FFDC.
532 global host_name
533 global host_ip
534 host = socket.gethostname()
535 host_name, host_ip = gm.get_host_name_ip(host)
536
Michael Walsh986d8ae2019-07-17 10:02:23 -0500537 gp.dprint_var(boot_table)
Michael Walshb5839d02017-04-12 16:11:20 -0500538 gp.dprint_var(boot_lists)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600539
Michael Walsh0bbd8602016-11-22 11:31:49 -0600540
Michael Walsh6741f742017-02-20 16:16:38 -0600541def validate_parms():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600542 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600543 Validate all program parameters.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600544 """
545
Michael Walshb5839d02017-04-12 16:11:20 -0500546 process_pgm_parms()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600547
Michael Walshb5839d02017-04-12 16:11:20 -0500548 gp.qprintn()
549
550 global openbmc_model
Michael Walshf5ce38c2020-02-27 12:46:20 -0600551 if openbmc_model == "":
Patrick Williams20f38712022-12-08 06:18:26 -0600552 status, ret_values = grk.run_key_u("Get BMC System Model", ignore=1)
George Keishingec2836d2022-01-19 12:05:54 -0600553 # Set the model to default "OPENBMC" if getting it from BMC fails.
Patrick Williams20f38712022-12-08 06:18:26 -0600554 if status == "FAIL":
555 openbmc_model = "OPENBMC"
George Keishingec2836d2022-01-19 12:05:54 -0600556 else:
557 openbmc_model = ret_values
Michael Walshf5ce38c2020-02-27 12:46:20 -0600558 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
559 gv.set_exit_on_error(True)
Michael Walsh44cef252019-08-01 12:38:56 -0500560 gv.valid_value(openbmc_host)
561 gv.valid_value(openbmc_username)
562 gv.valid_value(openbmc_password)
563 gv.valid_value(rest_username)
564 gv.valid_value(rest_password)
565 gv.valid_value(ipmi_username)
566 gv.valid_value(ipmi_password)
Michael Walsh6741f742017-02-20 16:16:38 -0600567 if os_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500568 gv.valid_value(os_username)
569 gv.valid_value(os_password)
Michael Walsh6741f742017-02-20 16:16:38 -0600570 if pdu_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500571 gv.valid_value(pdu_username)
572 gv.valid_value(pdu_password)
573 gv.valid_integer(pdu_slot_no)
Michael Walsh6741f742017-02-20 16:16:38 -0600574 if openbmc_serial_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500575 gv.valid_integer(openbmc_serial_port)
Michael Walsh44cef252019-08-01 12:38:56 -0500576 gv.valid_value(openbmc_model)
577 gv.valid_integer(max_num_tests)
578 gv.valid_integer(boot_pass)
579 gv.valid_integer(boot_fail)
Michael Walsh6741f742017-02-20 16:16:38 -0600580 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
Patrick Williams20f38712022-12-08 06:18:26 -0600581 BuiltIn().set_global_variable(
582 "${plug_in_packages_list}", plug_in_packages_list
583 )
584 gv.valid_value(stack_mode, valid_values=["normal", "skip"])
Michael Walshf5ce38c2020-02-27 12:46:20 -0600585 gv.set_exit_on_error(False)
Michael Walsha20da402017-03-31 16:27:45 -0500586 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
Patrick Williams20f38712022-12-08 06:18:26 -0600587 error_message = (
588 "You must provide either a value for either the"
589 + " boot_list or the boot_stack parm.\n"
590 )
Michael Walsh6741f742017-02-20 16:16:38 -0600591 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600592 valid_boot_list(boot_list, valid_boot_types)
593 valid_boot_list(boot_stack, valid_boot_types)
Patrick Williams20f38712022-12-08 06:18:26 -0600594 selected_PDU_boots = list(
595 set(boot_list + boot_stack) & set(boot_lists["PDU_reboot"])
596 )
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500597 if len(selected_PDU_boots) > 0 and pdu_host == "":
Patrick Williams20f38712022-12-08 06:18:26 -0600598 error_message = (
599 "You have selected the following boots which"
600 + " require a PDU host but no value for pdu_host:\n"
601 )
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500602 error_message += gp.sprint_var(selected_PDU_boots)
Michael Walsh986d8ae2019-07-17 10:02:23 -0500603 error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500604 BuiltIn().fail(gp.sprint_error(error_message))
605
Michael Walsh6741f742017-02-20 16:16:38 -0600606 return
Michael Walsh0bbd8602016-11-22 11:31:49 -0600607
Michael Walsh0bbd8602016-11-22 11:31:49 -0600608
Michael Walsh6741f742017-02-20 16:16:38 -0600609def my_get_state():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600610 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600611 Get the system state plus a little bit of wrapping.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600612 """
613
Michael Walsh6741f742017-02-20 16:16:38 -0600614 global state
615
Patrick Williams20f38712022-12-08 06:18:26 -0600616 req_states = ["epoch_seconds"] + st.default_req_states
Michael Walsh6741f742017-02-20 16:16:38 -0600617
Michael Walshb5839d02017-04-12 16:11:20 -0500618 gp.qprint_timen("Getting system state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600619 if test_mode:
Patrick Williams20f38712022-12-08 06:18:26 -0600620 state["epoch_seconds"] = int(time.time())
Michael Walsh6741f742017-02-20 16:16:38 -0600621 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500622 state = st.get_state(req_states=req_states, quiet=quiet)
623 gp.qprint_var(state)
Michael Walsh341c21e2017-01-17 16:25:20 -0600624
Michael Walsh341c21e2017-01-17 16:25:20 -0600625
Michael Walsh45ca6e42017-09-14 17:29:12 -0500626def valid_state():
Michael Walsh45ca6e42017-09-14 17:29:12 -0500627 r"""
628 Verify that our state dictionary contains no blank values. If we don't get
629 valid state data, we cannot continue to work.
630 """
631
Patrick Williams20f38712022-12-08 06:18:26 -0600632 if st.compare_states(state, st.invalid_state_match, "or"):
633 error_message = (
634 "The state dictionary contains blank fields which"
635 + " is illegal.\n"
636 + gp.sprint_var(state)
637 )
Michael Walsh45ca6e42017-09-14 17:29:12 -0500638 BuiltIn().fail(gp.sprint_error(error_message))
639
Michael Walsh45ca6e42017-09-14 17:29:12 -0500640
Michael Walsh6741f742017-02-20 16:16:38 -0600641def select_boot():
Michael Walsh341c21e2017-01-17 16:25:20 -0600642 r"""
643 Select a boot test to be run based on our current state and return the
644 chosen boot type.
645
646 Description of arguments:
Michael Walsh6741f742017-02-20 16:16:38 -0600647 state The state of the machine.
Michael Walsh341c21e2017-01-17 16:25:20 -0600648 """
649
Michael Walsh81816742017-09-27 11:02:29 -0500650 global transitional_boot_selected
Michael Walsh30dadae2017-02-27 14:25:52 -0600651 global boot_stack
652
Michael Walshb5839d02017-04-12 16:11:20 -0500653 gp.qprint_timen("Selecting a boot test.")
Michael Walsh6741f742017-02-20 16:16:38 -0600654
Michael Walsh81816742017-09-27 11:02:29 -0500655 if transitional_boot_selected and not boot_success:
656 prior_boot = next_boot
657 boot_candidate = boot_stack.pop()
Patrick Williams20f38712022-12-08 06:18:26 -0600658 gp.qprint_timen(
659 "The prior '"
660 + next_boot
661 + "' was chosen to"
662 + " transition to a valid state for '"
663 + boot_candidate
664 + "' which was at the top of the boot_stack. Since"
665 + " the '"
666 + next_boot
667 + "' failed, the '"
668 + boot_candidate
669 + "' has been removed from the stack"
670 + " to avoid and endless failure loop."
671 )
Michael Walsh81816742017-09-27 11:02:29 -0500672 if len(boot_stack) == 0:
673 return ""
674
Michael Walsh6741f742017-02-20 16:16:38 -0600675 my_get_state()
Michael Walsh45ca6e42017-09-14 17:29:12 -0500676 valid_state()
Michael Walsh6741f742017-02-20 16:16:38 -0600677
Michael Walsh81816742017-09-27 11:02:29 -0500678 transitional_boot_selected = False
Michael Walsh6741f742017-02-20 16:16:38 -0600679 stack_popped = 0
680 if len(boot_stack) > 0:
681 stack_popped = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500682 gp.qprint_dashes()
683 gp.qprint_var(boot_stack)
684 gp.qprint_dashes()
685 skip_boot_printed = 0
686 while len(boot_stack) > 0:
687 boot_candidate = boot_stack.pop()
Patrick Williams20f38712022-12-08 06:18:26 -0600688 if stack_mode == "normal":
Michael Walshb5839d02017-04-12 16:11:20 -0500689 break
690 else:
Patrick Williams20f38712022-12-08 06:18:26 -0600691 if st.compare_states(state, boot_table[boot_candidate]["end"]):
Michael Walshb5839d02017-04-12 16:11:20 -0500692 if not skip_boot_printed:
Michael Walshff340002017-08-29 11:18:27 -0500693 gp.qprint_var(stack_mode)
694 gp.qprintn()
Patrick Williams20f38712022-12-08 06:18:26 -0600695 gp.qprint_timen(
696 "Skipping the following boot tests"
697 + " which are unnecessary since their"
698 + " required end states match the"
699 + " current machine state:"
700 )
Michael Walshb5839d02017-04-12 16:11:20 -0500701 skip_boot_printed = 1
Michael Walshff340002017-08-29 11:18:27 -0500702 gp.qprint_var(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500703 boot_candidate = ""
704 if boot_candidate == "":
705 gp.qprint_dashes()
706 gp.qprint_var(boot_stack)
707 gp.qprint_dashes()
708 return boot_candidate
Patrick Williams20f38712022-12-08 06:18:26 -0600709 if st.compare_states(state, boot_table[boot_candidate]["start"]):
710 gp.qprint_timen(
711 "The machine state is valid for a '"
712 + boot_candidate
713 + "' boot test."
714 )
Michael Walshb5839d02017-04-12 16:11:20 -0500715 gp.qprint_dashes()
716 gp.qprint_var(boot_stack)
717 gp.qprint_dashes()
Michael Walsh6741f742017-02-20 16:16:38 -0600718 return boot_candidate
Michael Walsh341c21e2017-01-17 16:25:20 -0600719 else:
Patrick Williams20f38712022-12-08 06:18:26 -0600720 gp.qprint_timen(
721 "The machine state does not match the required"
722 + " starting state for a '"
723 + boot_candidate
724 + "' boot test:"
725 )
726 gp.qprint_varx(
727 "boot_table_start_entry", boot_table[boot_candidate]["start"]
728 )
Michael Walsh6741f742017-02-20 16:16:38 -0600729 boot_stack.append(boot_candidate)
Michael Walsh81816742017-09-27 11:02:29 -0500730 transitional_boot_selected = True
Michael Walsh6741f742017-02-20 16:16:38 -0600731 popped_boot = boot_candidate
732
733 # Loop through your list selecting a boot_candidates
734 boot_candidates = []
735 for boot_candidate in boot_list:
Patrick Williams20f38712022-12-08 06:18:26 -0600736 if st.compare_states(state, boot_table[boot_candidate]["start"]):
Michael Walsh6741f742017-02-20 16:16:38 -0600737 if stack_popped:
Patrick Williams20f38712022-12-08 06:18:26 -0600738 if st.compare_states(
739 boot_table[boot_candidate]["end"],
740 boot_table[popped_boot]["start"],
741 ):
Michael Walsh6741f742017-02-20 16:16:38 -0600742 boot_candidates.append(boot_candidate)
743 else:
744 boot_candidates.append(boot_candidate)
745
746 if len(boot_candidates) == 0:
Patrick Williams20f38712022-12-08 06:18:26 -0600747 gp.qprint_timen(
748 "The user's boot list contained no boot tests"
749 + " which are valid for the current machine state."
750 )
Michael Walsh6741f742017-02-20 16:16:38 -0600751 boot_candidate = default_power_on
Patrick Williams20f38712022-12-08 06:18:26 -0600752 if not st.compare_states(state, boot_table[default_power_on]["start"]):
Michael Walsh6741f742017-02-20 16:16:38 -0600753 boot_candidate = default_power_off
754 boot_candidates.append(boot_candidate)
Patrick Williams20f38712022-12-08 06:18:26 -0600755 gp.qprint_timen(
756 "Using default '"
757 + boot_candidate
758 + "' boot type to transition to valid state."
759 )
Michael Walsh6741f742017-02-20 16:16:38 -0600760
Michael Walshb5839d02017-04-12 16:11:20 -0500761 gp.dprint_var(boot_candidates)
Michael Walsh6741f742017-02-20 16:16:38 -0600762
763 # Randomly select a boot from the candidate list.
764 boot = random.choice(boot_candidates)
Michael Walsh341c21e2017-01-17 16:25:20 -0600765
766 return boot
Michael Walsh0bbd8602016-11-22 11:31:49 -0600767
Michael Walsh55302292017-01-10 11:43:02 -0600768
Michael Walshb2e53ec2017-10-30 15:04:36 -0500769def print_defect_report(ffdc_file_list):
Michael Walsh341c21e2017-01-17 16:25:20 -0600770 r"""
771 Print a defect report.
Michael Walshb2e53ec2017-10-30 15:04:36 -0500772
773 Description of argument(s):
774 ffdc_file_list A list of files which were collected by our ffdc functions.
Michael Walsh341c21e2017-01-17 16:25:20 -0600775 """
776
Michael Walsh600876d2017-05-30 17:58:58 -0500777 # Making deliberate choice to NOT run plug_in_setup(). We don't want
778 # ffdc_prefix updated.
779 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Patrick Williams20f38712022-12-08 06:18:26 -0600780 call_point="ffdc_report", stop_on_plug_in_failure=0
781 )
Michael Walsh600876d2017-05-30 17:58:58 -0500782
Michael Walshe0cf8d72017-05-17 13:20:46 -0500783 # Get additional header data which may have been created by ffdc plug-ins.
784 # Also, delete the individual header files to cleanup.
Patrick Williams20f38712022-12-08 06:18:26 -0600785 cmd_buf = (
786 "file_list=$(cat "
787 + ffdc_report_list_path
788 + " 2>/dev/null)"
789 + ' ; [ ! -z "${file_list}" ] && cat ${file_list}'
790 + " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
791 )
792 shell_rc, more_header_info = gc.cmd_fnc_u(
793 cmd_buf, print_output=0, show_err=0
794 )
Michael Walshe0cf8d72017-05-17 13:20:46 -0500795
Michael Walshb2e53ec2017-10-30 15:04:36 -0500796 # Get additional summary data which may have been created by ffdc plug-ins.
Michael Walsh600876d2017-05-30 17:58:58 -0500797 # Also, delete the individual header files to cleanup.
Patrick Williams20f38712022-12-08 06:18:26 -0600798 cmd_buf = (
799 "file_list=$(cat "
800 + ffdc_summary_list_path
801 + " 2>/dev/null)"
802 + ' ; [ ! -z "${file_list}" ] && cat ${file_list}'
803 + " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
804 )
805 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(
806 cmd_buf, print_output=0, show_err=0
807 )
Michael Walsh600876d2017-05-30 17:58:58 -0500808
Michael Walshb2e53ec2017-10-30 15:04:36 -0500809 # ffdc_list_file_path contains a list of any ffdc files created by plug-
810 # ins, etc. Read that data into a list.
Michael Walsh341c21e2017-01-17 16:25:20 -0600811 try:
Patrick Williams20f38712022-12-08 06:18:26 -0600812 plug_in_ffdc_list = (
813 open(ffdc_list_file_path, "r").read().rstrip("\n").split("\n")
814 )
George Keishing36efbc02018-12-12 10:18:23 -0600815 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
Michael Walsh341c21e2017-01-17 16:25:20 -0600816 except IOError:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500817 plug_in_ffdc_list = []
818
819 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
820 # in. Eliminate duplicates and sort the list.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500821 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
Michael Walshb2e53ec2017-10-30 15:04:36 -0500822
823 if status_file_path != "":
824 ffdc_file_list.insert(0, status_file_path)
825
826 # Convert the list to a printable list.
827 printable_ffdc_file_list = "\n".join(ffdc_file_list)
Michael Walsh341c21e2017-01-17 16:25:20 -0600828
Michael Walsh68a61162017-04-25 11:54:06 -0500829 # Open ffdc_file_list for writing. We will write a complete list of
830 # FFDC files to it for possible use by plug-ins like cp_stop_check.
Patrick Williams20f38712022-12-08 06:18:26 -0600831 ffdc_list_file = open(ffdc_list_file_path, "w")
Michael Walshb2e53ec2017-10-30 15:04:36 -0500832 ffdc_list_file.write(printable_ffdc_file_list + "\n")
833 ffdc_list_file.close()
834
835 indent = 0
836 width = 90
837 linefeed = 1
838 char = "="
Michael Walsh68a61162017-04-25 11:54:06 -0500839
840 gp.qprintn()
Michael Walshb2e53ec2017-10-30 15:04:36 -0500841 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500842 gp.qprintn("Copy this data to the defect:\n")
843
Michael Walshe0cf8d72017-05-17 13:20:46 -0500844 if len(more_header_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500845 gp.qprintn(more_header_info)
Patrick Williams20f38712022-12-08 06:18:26 -0600846 gp.qpvars(
847 host_name,
848 host_ip,
849 openbmc_nickname,
850 openbmc_host,
851 openbmc_host_name,
852 openbmc_ip,
853 openbmc_username,
854 openbmc_password,
855 rest_username,
856 rest_password,
857 ipmi_username,
858 ipmi_password,
859 os_host,
860 os_host_name,
861 os_ip,
862 os_username,
863 os_password,
864 pdu_host,
865 pdu_host_name,
866 pdu_ip,
867 pdu_username,
868 pdu_password,
869 pdu_slot_no,
870 openbmc_serial_host,
871 openbmc_serial_host_name,
872 openbmc_serial_ip,
873 openbmc_serial_port,
874 )
Michael Walsh68a61162017-04-25 11:54:06 -0500875
876 gp.qprintn()
Michael Walsh986d8ae2019-07-17 10:02:23 -0500877 print_boot_history(boot_history)
Michael Walsh68a61162017-04-25 11:54:06 -0500878 gp.qprintn()
879 gp.qprint_var(state)
Michael Walshb5839d02017-04-12 16:11:20 -0500880 gp.qprintn()
881 gp.qprintn("FFDC data files:")
Michael Walshb2e53ec2017-10-30 15:04:36 -0500882 gp.qprintn(printable_ffdc_file_list)
Michael Walshb5839d02017-04-12 16:11:20 -0500883 gp.qprintn()
Michael Walsh341c21e2017-01-17 16:25:20 -0600884
Michael Walsh600876d2017-05-30 17:58:58 -0500885 if len(ffdc_summary_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500886 gp.qprintn(ffdc_summary_info)
Michael Walsh600876d2017-05-30 17:58:58 -0500887
Michael Walshb2e53ec2017-10-30 15:04:36 -0500888 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500889
Michael Walsh6741f742017-02-20 16:16:38 -0600890
Michael Walsh6741f742017-02-20 16:16:38 -0600891def my_ffdc():
Michael Walsh6741f742017-02-20 16:16:38 -0600892 r"""
893 Collect FFDC data.
894 """
895
896 global state
897
898 plug_in_setup()
899 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Patrick Williams20f38712022-12-08 06:18:26 -0600900 call_point="ffdc", stop_on_plug_in_failure=0
901 )
Michael Walsh6741f742017-02-20 16:16:38 -0600902
Patrick Williams20f38712022-12-08 06:18:26 -0600903 AUTOBOOT_FFDC_PREFIX = os.environ["AUTOBOOT_FFDC_PREFIX"]
904 status, ffdc_file_list = grk.run_key_u(
905 "FFDC ffdc_prefix="
906 + AUTOBOOT_FFDC_PREFIX
907 + " ffdc_function_list="
908 + ffdc_function_list,
909 ignore=1,
910 )
911 if status != "PASS":
Michael Walshff340002017-08-29 11:18:27 -0500912 gp.qprint_error("Call to ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500913 if type(ffdc_file_list) is not list:
914 ffdc_file_list = []
915 # Leave a record for caller that "soft" errors occurred.
916 soft_errors = 1
917 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600918
919 my_get_state()
920
Michael Walshb2e53ec2017-10-30 15:04:36 -0500921 print_defect_report(ffdc_file_list)
Michael Walsh6741f742017-02-20 16:16:38 -0600922
Michael Walsh6741f742017-02-20 16:16:38 -0600923
Michael Walsh6741f742017-02-20 16:16:38 -0600924def print_test_start_message(boot_keyword):
Michael Walsh6741f742017-02-20 16:16:38 -0600925 r"""
926 Print a message indicating what boot test is about to run.
927
928 Description of arguments:
929 boot_keyword The name of the boot which is to be run
930 (e.g. "BMC Power On").
931 """
932
Michael Walsh986d8ae2019-07-17 10:02:23 -0500933 global boot_history
Sunil M325eb542017-08-10 07:09:43 -0500934 global boot_start_time
Michael Walsh6741f742017-02-20 16:16:38 -0600935
Patrick Williams20f38712022-12-08 06:18:26 -0600936 doing_msg = gp.sprint_timen('Doing "' + boot_keyword + '".')
Sunil M325eb542017-08-10 07:09:43 -0500937
938 # Set boot_start_time for use by plug-ins.
939 boot_start_time = doing_msg[1:33]
940 gp.qprint_var(boot_start_time)
941
Michael Walshb5839d02017-04-12 16:11:20 -0500942 gp.qprint(doing_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600943
Michael Walsh986d8ae2019-07-17 10:02:23 -0500944 update_boot_history(boot_history, doing_msg, max_boot_history)
Michael Walsh6741f742017-02-20 16:16:38 -0600945
Michael Walsh6741f742017-02-20 16:16:38 -0600946
Patrick Williams20f38712022-12-08 06:18:26 -0600947def stop_boot_test(signal_number=0, frame=None):
Michael Walshf566fb12019-02-01 14:35:09 -0600948 r"""
949 Handle SIGUSR1 by aborting the boot test that is running.
950
951 Description of argument(s):
952 signal_number The signal number (should always be 10 for SIGUSR1).
953 frame The frame data.
954 """
955
Michael Walsh80dddde2019-10-22 13:54:38 -0500956 gp.qprintn()
957 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600958 gp.lprint_executing()
959
960 # Restore original sigusr1 handler.
961 set_default_siguser1()
962
963 message = "The caller has asked that the boot test be stopped and marked"
964 message += " as a failure."
965
966 function_stack = gm.get_function_stack()
967 if "wait_state" in function_stack:
Michael Walshc44aa532019-06-14 13:33:29 -0500968 st.set_exit_wait_early_message(message)
Michael Walshf566fb12019-02-01 14:35:09 -0600969 else:
970 BuiltIn().fail(gp.sprint_error(message))
971
972
Michael Walsh6741f742017-02-20 16:16:38 -0600973def run_boot(boot):
Michael Walsh6741f742017-02-20 16:16:38 -0600974 r"""
975 Run the specified boot.
976
977 Description of arguments:
978 boot The name of the boot test to be performed.
979 """
980
981 global state
982
Michael Walshf566fb12019-02-01 14:35:09 -0600983 signal.signal(signal.SIGUSR1, stop_boot_test)
984 gp.qprint_timen("stop_boot_test is armed.")
985
Michael Walsh6741f742017-02-20 16:16:38 -0600986 print_test_start_message(boot)
987
988 plug_in_setup()
Patrick Williams20f38712022-12-08 06:18:26 -0600989 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
990 call_point="pre_boot"
991 )
Michael Walsh6741f742017-02-20 16:16:38 -0600992 if rc != 0:
Patrick Williams20f38712022-12-08 06:18:26 -0600993 error_message = (
994 "Plug-in failed with non-zero return code.\n"
995 + gp.sprint_var(rc, fmt=gp.hexa())
996 )
Michael Walshf566fb12019-02-01 14:35:09 -0600997 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600998 BuiltIn().fail(gp.sprint_error(error_message))
999
1000 if test_mode:
1001 # In test mode, we'll pretend the boot worked by assigning its
1002 # required end state to the default state value.
Patrick Williams20f38712022-12-08 06:18:26 -06001003 state = st.strip_anchor_state(boot_table[boot]["end"])
Michael Walsh6741f742017-02-20 16:16:38 -06001004 else:
1005 # Assertion: We trust that the state data was made fresh by the
1006 # caller.
1007
Michael Walshb5839d02017-04-12 16:11:20 -05001008 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -06001009
Patrick Williams20f38712022-12-08 06:18:26 -06001010 if boot_table[boot]["method_type"] == "keyword":
1011 rk.my_run_keywords(
1012 boot_table[boot].get("lib_file_path", ""),
1013 boot_table[boot]["method"],
1014 quiet=quiet,
1015 )
Michael Walsh6741f742017-02-20 16:16:38 -06001016
Patrick Williams20f38712022-12-08 06:18:26 -06001017 if boot_table[boot]["bmc_reboot"]:
1018 st.wait_for_comm_cycle(int(state["epoch_seconds"]))
Michael Walsh30dadae2017-02-27 14:25:52 -06001019 plug_in_setup()
Patrick Williams20f38712022-12-08 06:18:26 -06001020 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1021 call_point="post_reboot"
1022 )
Michael Walsh30dadae2017-02-27 14:25:52 -06001023 if rc != 0:
Michael Walsh0b93fbf2017-03-02 14:42:41 -06001024 error_message = "Plug-in failed with non-zero return code.\n"
Michael Walsh986d8ae2019-07-17 10:02:23 -05001025 error_message += gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -06001026 set_default_siguser1()
Michael Walsh30dadae2017-02-27 14:25:52 -06001027 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -06001028 else:
1029 match_state = st.anchor_state(state)
Patrick Williams20f38712022-12-08 06:18:26 -06001030 del match_state["epoch_seconds"]
Michael Walsh6741f742017-02-20 16:16:38 -06001031 # Wait for the state to change in any way.
Patrick Williams20f38712022-12-08 06:18:26 -06001032 st.wait_state(
1033 match_state,
1034 wait_time=state_change_timeout,
1035 interval="10 seconds",
1036 invert=1,
1037 )
Michael Walsh6741f742017-02-20 16:16:38 -06001038
Michael Walshb5839d02017-04-12 16:11:20 -05001039 gp.qprintn()
Patrick Williams20f38712022-12-08 06:18:26 -06001040 if boot_table[boot]["end"]["chassis"] == "Off":
Michael Walsh6741f742017-02-20 16:16:38 -06001041 boot_timeout = power_off_timeout
1042 else:
1043 boot_timeout = power_on_timeout
Patrick Williams20f38712022-12-08 06:18:26 -06001044 st.wait_state(
1045 boot_table[boot]["end"],
1046 wait_time=boot_timeout,
1047 interval="10 seconds",
1048 )
Michael Walsh6741f742017-02-20 16:16:38 -06001049
1050 plug_in_setup()
Patrick Williams20f38712022-12-08 06:18:26 -06001051 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1052 call_point="post_boot"
1053 )
Michael Walsh6741f742017-02-20 16:16:38 -06001054 if rc != 0:
Patrick Williams20f38712022-12-08 06:18:26 -06001055 error_message = (
1056 "Plug-in failed with non-zero return code.\n"
1057 + gp.sprint_var(rc, fmt=gp.hexa())
1058 )
Michael Walshf566fb12019-02-01 14:35:09 -06001059 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -06001060 BuiltIn().fail(gp.sprint_error(error_message))
1061
Michael Walshf566fb12019-02-01 14:35:09 -06001062 # Restore original sigusr1 handler.
1063 set_default_siguser1()
1064
Michael Walsh6741f742017-02-20 16:16:38 -06001065
Michael Walsh6741f742017-02-20 16:16:38 -06001066def test_loop_body():
Michael Walsh6741f742017-02-20 16:16:38 -06001067 r"""
1068 The main loop body for the loop in main_py.
1069
1070 Description of arguments:
1071 boot_count The iteration number (starts at 1).
1072 """
1073
1074 global boot_count
1075 global state
1076 global next_boot
1077 global boot_success
Sunil M325eb542017-08-10 07:09:43 -05001078 global boot_end_time
Michael Walsh6741f742017-02-20 16:16:38 -06001079
Yi Hu02d32762024-03-07 14:34:34 -08001080 # The flag can be enabled or disabled on the go
1081 redfish_delete_sessions = int(
1082 BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)
1083 )
1084
Michael Walshb5839d02017-04-12 16:11:20 -05001085 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -06001086
1087 next_boot = select_boot()
Michael Walshb5839d02017-04-12 16:11:20 -05001088 if next_boot == "":
1089 return True
Michael Walsh6741f742017-02-20 16:16:38 -06001090
Michael Walshb5839d02017-04-12 16:11:20 -05001091 boot_count += 1
1092 gp.qprint_timen("Starting boot " + str(boot_count) + ".")
Michael Walsh6741f742017-02-20 16:16:38 -06001093
Michael Walshe0cf8d72017-05-17 13:20:46 -05001094 pre_boot_plug_in_setup()
Michael Walsh6741f742017-02-20 16:16:38 -06001095
1096 cmd_buf = ["run_boot", next_boot]
1097 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
1098 if boot_status == "FAIL":
Michael Walshb5839d02017-04-12 16:11:20 -05001099 gp.qprint(msg)
Michael Walsh6741f742017-02-20 16:16:38 -06001100
Michael Walshb5839d02017-04-12 16:11:20 -05001101 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -06001102 if boot_status == "PASS":
1103 boot_success = 1
Patrick Williams20f38712022-12-08 06:18:26 -06001104 completion_msg = gp.sprint_timen(
1105 'BOOT_SUCCESS: "' + next_boot + '" succeeded.'
1106 )
Michael Walsh6741f742017-02-20 16:16:38 -06001107 else:
1108 boot_success = 0
Patrick Williams20f38712022-12-08 06:18:26 -06001109 completion_msg = gp.sprint_timen(
1110 'BOOT_FAILED: "' + next_boot + '" failed.'
1111 )
Sunil M325eb542017-08-10 07:09:43 -05001112
1113 # Set boot_end_time for use by plug-ins.
1114 boot_end_time = completion_msg[1:33]
1115 gp.qprint_var(boot_end_time)
1116
1117 gp.qprint(completion_msg)
Michael Walsh6741f742017-02-20 16:16:38 -06001118
1119 boot_results.update(next_boot, boot_status)
1120
1121 plug_in_setup()
1122 # NOTE: A post_test_case call point failure is NOT counted as a boot
1123 # failure.
1124 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Patrick Williams20f38712022-12-08 06:18:26 -06001125 call_point="post_test_case", stop_on_plug_in_failure=0
1126 )
Michael Walsh6741f742017-02-20 16:16:38 -06001127
1128 plug_in_setup()
1129 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Patrick Williams20f38712022-12-08 06:18:26 -06001130 call_point="ffdc_check",
1131 shell_rc=dump_ffdc_rc(),
1132 stop_on_plug_in_failure=1,
1133 stop_on_non_zero_rc=1,
1134 )
1135 if ffdc_check == "All" or shell_rc == dump_ffdc_rc():
Michael Walsh83f4bc72017-04-20 16:49:43 -05001136 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
Patrick Williams20f38712022-12-08 06:18:26 -06001137 if status != "PASS":
Michael Walshff340002017-08-29 11:18:27 -05001138 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -05001139 # Leave a record for caller that "soft" errors occurred.
1140 soft_errors = 1
1141 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -06001142
Michael Walshaabef1e2017-09-20 15:16:17 -05001143 if delete_errlogs:
Michael Shepos1a67b082020-08-28 16:01:58 -05001144 # print error logs before delete
George Keishing438fd3b2021-10-08 02:15:18 -05001145 if redfish_support_trans_state:
1146 status, error_logs = grk.run_key_u("Get Redfish Event Logs")
Patrick Williams20f38712022-12-08 06:18:26 -06001147 log.print_error_logs(
1148 error_logs, "AdditionalDataURI Message Severity"
1149 )
George Keishing438fd3b2021-10-08 02:15:18 -05001150 else:
1151 status, error_logs = grk.run_key_u("Get Error Logs")
1152 log.print_error_logs(error_logs, "AdditionalData Message Severity")
Michael Shepos1a67b082020-08-28 16:01:58 -05001153 pels = pel.peltool("-l", ignore_err=1)
Michael Shepos0e5f1132020-09-30 16:24:25 -05001154 gp.qprint_var(pels)
Michael Shepos1a67b082020-08-28 16:01:58 -05001155
Michael Walshaabef1e2017-09-20 15:16:17 -05001156 # We need to purge error logs between boots or they build up.
Michael Walsh409ad352020-02-06 11:46:35 -06001157 grk.run_key(delete_errlogs_cmd, ignore=1)
Michael Shepos92a54bf2020-11-11 11:48:55 -06001158 grk.run_key(delete_bmcdump_cmd, ignore=1)
George Keishing2ef6a7d2021-05-19 09:05:32 -05001159 if redfish_support_trans_state:
1160 grk.run_key(delete_sysdump_cmd, ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -05001161
Michael Walsh952f9b02017-03-09 13:11:14 -06001162 boot_results.print_report()
Michael Walshb5839d02017-04-12 16:11:20 -05001163 gp.qprint_timen("Finished boot " + str(boot_count) + ".")
Michael Walsh952f9b02017-03-09 13:11:14 -06001164
Michael Walsh6741f742017-02-20 16:16:38 -06001165 plug_in_setup()
1166 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Patrick Williams20f38712022-12-08 06:18:26 -06001167 call_point="stop_check", shell_rc=stop_test_rc(), stop_on_non_zero_rc=1
1168 )
Michael Walsh89de14a2018-10-01 16:51:37 -05001169 if shell_rc == stop_test_rc():
Michael Walsh3ba8ecd2018-04-24 11:33:25 -05001170 message = "Stopping as requested by user.\n"
Michael Walsh80dddde2019-10-22 13:54:38 -05001171 gp.qprint_time(message)
Michael Walsh3ba8ecd2018-04-24 11:33:25 -05001172 BuiltIn().fail(message)
Michael Walsh6741f742017-02-20 16:16:38 -06001173
Michael Walshd139f282017-04-04 18:00:23 -05001174 # This should help prevent ConnectionErrors.
George Keishing4d65c862020-12-03 06:52:11 -06001175 # Purge all redfish and REST connection sessions.
George Keishingd86e45c2021-03-19 07:38:14 -05001176 if redfish_delete_sessions:
1177 grk.run_key_u("Close All Connections", ignore=1)
1178 grk.run_key_u("Delete All Redfish Sessions", ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -05001179
Michael Walsh6741f742017-02-20 16:16:38 -06001180 return True
1181
Michael Walsh6741f742017-02-20 16:16:38 -06001182
Michael Walsh83f4bc72017-04-20 16:49:43 -05001183def obmc_boot_test_teardown():
Michael Walsh6741f742017-02-20 16:16:38 -06001184 r"""
Michael Walshf75d4352019-12-05 17:01:20 -06001185 Clean up after the main keyword.
Michael Walsh6741f742017-02-20 16:16:38 -06001186 """
Michael Walshf75d4352019-12-05 17:01:20 -06001187 gp.qprint_executing()
1188
1189 if ga.psutil_imported:
1190 ga.terminate_descendants()
Michael Walsh6741f742017-02-20 16:16:38 -06001191
1192 if cp_setup_called:
1193 plug_in_setup()
1194 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Patrick Williams20f38712022-12-08 06:18:26 -06001195 call_point="cleanup", stop_on_plug_in_failure=0
1196 )
Michael Walsh6741f742017-02-20 16:16:38 -06001197
Patrick Williams20f38712022-12-08 06:18:26 -06001198 if "boot_results_file_path" in globals():
Michael Walsh986d8ae2019-07-17 10:02:23 -05001199 # Save boot_results and boot_history objects to a file in case they are
Michael Walsh6c645742018-08-17 15:02:17 -05001200 # needed again.
Michael Walsh600876d2017-05-30 17:58:58 -05001201 gp.qprint_timen("Saving boot_results to the following path.")
1202 gp.qprint_var(boot_results_file_path)
Patrick Williams20f38712022-12-08 06:18:26 -06001203 pickle.dump(
1204 (boot_results, boot_history),
1205 open(boot_results_file_path, "wb"),
1206 pickle.HIGHEST_PROTOCOL,
1207 )
Michael Walsh0b93fbf2017-03-02 14:42:41 -06001208
Michael Walshff340002017-08-29 11:18:27 -05001209 global save_stack
1210 # Restore any global values saved on the save_stack.
1211 for parm_name in main_func_parm_list:
1212 # Get the parm_value if it was saved on the stack.
1213 try:
1214 parm_value = save_stack.pop(parm_name)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -05001215 except BaseException:
Michael Walshff340002017-08-29 11:18:27 -05001216 # If it was not saved, no further action is required.
1217 continue
1218
1219 # Restore the saved value.
Patrick Williams20f38712022-12-08 06:18:26 -06001220 cmd_buf = (
1221 'BuiltIn().set_global_variable("${' + parm_name + '}", parm_value)'
1222 )
Michael Walshff340002017-08-29 11:18:27 -05001223 gp.dpissuing(cmd_buf)
1224 exec(cmd_buf)
1225
1226 gp.dprintn(save_stack.sprint_obj())
1227
Michael Walsh6741f742017-02-20 16:16:38 -06001228
Michael Walshc9116812017-03-10 14:23:06 -06001229def test_teardown():
Michael Walshc9116812017-03-10 14:23:06 -06001230 r"""
1231 Clean up after this test case.
1232 """
1233
1234 gp.qprintn()
Michael Walshf75d4352019-12-05 17:01:20 -06001235 gp.qprint_executing()
1236
1237 if ga.psutil_imported:
1238 ga.terminate_descendants()
1239
Patrick Williams20f38712022-12-08 06:18:26 -06001240 cmd_buf = [
1241 "Print Error",
1242 "A keyword timeout occurred ending this program.\n",
1243 ]
Michael Walshc9116812017-03-10 14:23:06 -06001244 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1245
George Keishinga54e06f2020-06-12 10:42:41 -05001246 if redfish_supported:
1247 redfish.logout()
1248
Michael Walshc108e422019-03-28 12:27:18 -05001249 gp.qprint_pgm_footer()
Michael Walshb5839d02017-04-12 16:11:20 -05001250
Michael Walshc9116812017-03-10 14:23:06 -06001251
Michael Walsh89de14a2018-10-01 16:51:37 -05001252def post_stack():
1253 r"""
1254 Process post_stack plug-in programs.
1255 """
1256
1257 if not call_post_stack_plug:
1258 # The caller does not wish to have post_stack plug-in processing done.
1259 return
1260
1261 global boot_success
1262
1263 # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1264 pre_boot_plug_in_setup()
1265 # For the purposes of the following plug-ins, mark the "boot" as a success.
1266 boot_success = 1
1267 plug_in_setup()
Patrick Williams20f38712022-12-08 06:18:26 -06001268 (
1269 rc,
1270 shell_rc,
1271 failed_plug_in_name,
1272 history,
1273 ) = grpi.rprocess_plug_in_packages(
1274 call_point="post_stack", stop_on_plug_in_failure=0, return_history=True
1275 )
Michael Walsh986d8ae2019-07-17 10:02:23 -05001276 for doing_msg in history:
1277 update_boot_history(boot_history, doing_msg, max_boot_history)
Michael Walsh815b1d52018-10-30 13:32:26 -05001278 if rc != 0:
1279 boot_success = 0
Michael Walsh89de14a2018-10-01 16:51:37 -05001280
1281 plug_in_setup()
Patrick Williams20f38712022-12-08 06:18:26 -06001282 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1283 call_point="ffdc_check",
1284 shell_rc=dump_ffdc_rc(),
1285 stop_on_plug_in_failure=1,
1286 stop_on_non_zero_rc=1,
1287 )
Michael Walsh815b1d52018-10-30 13:32:26 -05001288 if shell_rc == dump_ffdc_rc():
Michael Walsh89de14a2018-10-01 16:51:37 -05001289 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
Patrick Williams20f38712022-12-08 06:18:26 -06001290 if status != "PASS":
Michael Walsh89de14a2018-10-01 16:51:37 -05001291 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -05001292 # Leave a record for caller that "soft" errors occurred.
1293 soft_errors = 1
1294 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh89de14a2018-10-01 16:51:37 -05001295
1296 plug_in_setup()
1297 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Patrick Williams20f38712022-12-08 06:18:26 -06001298 call_point="stop_check", shell_rc=stop_test_rc(), stop_on_non_zero_rc=1
1299 )
Michael Walsh89de14a2018-10-01 16:51:37 -05001300 if shell_rc == stop_test_rc():
1301 message = "Stopping as requested by user.\n"
Michael Walsh80dddde2019-10-22 13:54:38 -05001302 gp.qprint_time(message)
Michael Walsh89de14a2018-10-01 16:51:37 -05001303 BuiltIn().fail(message)
1304
1305
Patrick Williams20f38712022-12-08 06:18:26 -06001306def obmc_boot_test_py(
1307 loc_boot_stack=None, loc_stack_mode=None, loc_quiet=None
1308):
Michael Walsh6741f742017-02-20 16:16:38 -06001309 r"""
1310 Do main program processing.
1311 """
1312
Michael Walshff340002017-08-29 11:18:27 -05001313 global save_stack
1314
Patrick Williams20f38712022-12-08 06:18:26 -06001315 ga.set_term_options(
1316 term_requests={"pgm_names": ["process_plug_in_packages.py"]}
1317 )
Michael Walshf75d4352019-12-05 17:01:20 -06001318
George Keishing36efbc02018-12-12 10:18:23 -06001319 gp.dprintn()
Michael Walshff340002017-08-29 11:18:27 -05001320 # Process function parms.
1321 for parm_name in main_func_parm_list:
1322 # Get parm's value.
George Keishing36efbc02018-12-12 10:18:23 -06001323 parm_value = eval("loc_" + parm_name)
1324 gp.dpvars(parm_name, parm_value)
Michael Walshff340002017-08-29 11:18:27 -05001325
George Keishing36efbc02018-12-12 10:18:23 -06001326 if parm_value is not None:
Michael Walshff340002017-08-29 11:18:27 -05001327 # Save the global value on a stack.
Patrick Williams20f38712022-12-08 06:18:26 -06001328 cmd_buf = (
1329 'save_stack.push(BuiltIn().get_variable_value("${'
1330 + parm_name
1331 + '}"), "'
1332 + parm_name
1333 + '")'
1334 )
Michael Walshff340002017-08-29 11:18:27 -05001335 gp.dpissuing(cmd_buf)
1336 exec(cmd_buf)
1337
1338 # Set the global value to the passed value.
Patrick Williams20f38712022-12-08 06:18:26 -06001339 cmd_buf = (
1340 'BuiltIn().set_global_variable("${'
1341 + parm_name
1342 + '}", loc_'
1343 + parm_name
1344 + ")"
1345 )
Michael Walshff340002017-08-29 11:18:27 -05001346 gp.dpissuing(cmd_buf)
1347 exec(cmd_buf)
1348
1349 gp.dprintn(save_stack.sprint_obj())
Michael Walshb5839d02017-04-12 16:11:20 -05001350
Michael Walsh6741f742017-02-20 16:16:38 -06001351 setup()
1352
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001353 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1354
Michael Walsha20da402017-03-31 16:27:45 -05001355 if ffdc_only:
1356 gp.qprint_timen("Caller requested ffdc_only.")
Michael Walsh986d8ae2019-07-17 10:02:23 -05001357 if do_pre_boot_plug_in_setup:
1358 pre_boot_plug_in_setup()
Michael Walsh83f4bc72017-04-20 16:49:43 -05001359 grk.run_key_u("my_ffdc")
Michael Walsh764d2f82017-04-27 16:01:08 -05001360 return
Michael Walsha20da402017-03-31 16:27:45 -05001361
Michael Walsh409ad352020-02-06 11:46:35 -06001362 if delete_errlogs:
Michael Shepos1a67b082020-08-28 16:01:58 -05001363 # print error logs before delete
George Keishing438fd3b2021-10-08 02:15:18 -05001364 if redfish_support_trans_state:
1365 status, error_logs = grk.run_key_u("Get Redfish Event Logs")
Patrick Williams20f38712022-12-08 06:18:26 -06001366 log.print_error_logs(
1367 error_logs, "AdditionalDataURI Message Severity"
1368 )
George Keishing438fd3b2021-10-08 02:15:18 -05001369 else:
1370 status, error_logs = grk.run_key_u("Get Error Logs")
1371 log.print_error_logs(error_logs, "AdditionalData Message Severity")
Michael Shepos1a67b082020-08-28 16:01:58 -05001372 pels = pel.peltool("-l", ignore_err=1)
Michael Shepos0e5f1132020-09-30 16:24:25 -05001373 gp.qprint_var(pels)
Michael Shepos1a67b082020-08-28 16:01:58 -05001374
Michael Walsh409ad352020-02-06 11:46:35 -06001375 # Delete errlogs prior to doing any boot tests.
1376 grk.run_key(delete_errlogs_cmd, ignore=1)
Michael Shepos92a54bf2020-11-11 11:48:55 -06001377 grk.run_key(delete_bmcdump_cmd, ignore=1)
George Keishing2ef6a7d2021-05-19 09:05:32 -05001378 if redfish_support_trans_state:
1379 grk.run_key(delete_sysdump_cmd, ignore=1)
Michael Walsh409ad352020-02-06 11:46:35 -06001380
Michael Walsh6741f742017-02-20 16:16:38 -06001381 # Process caller's boot_stack.
Patrick Williams20f38712022-12-08 06:18:26 -06001382 while len(boot_stack) > 0:
Michael Walsh6741f742017-02-20 16:16:38 -06001383 test_loop_body()
1384
Michael Walshb5839d02017-04-12 16:11:20 -05001385 gp.qprint_timen("Finished processing stack.")
Michael Walsh30dadae2017-02-27 14:25:52 -06001386
Michael Walsh89de14a2018-10-01 16:51:37 -05001387 post_stack()
1388
Michael Walsh6741f742017-02-20 16:16:38 -06001389 # Process caller's boot_list.
1390 if len(boot_list) > 0:
1391 for ix in range(1, max_num_tests + 1):
1392 test_loop_body()
1393
Michael Walshb5839d02017-04-12 16:11:20 -05001394 gp.qprint_timen("Completed all requested boot tests.")
1395
1396 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001397 new_fail = boot_fail - init_boot_fail
1398 if new_fail > boot_fail_threshold:
Patrick Williams20f38712022-12-08 06:18:26 -06001399 error_message = (
1400 "Boot failures exceed the boot failure"
1401 + " threshold:\n"
1402 + gp.sprint_var(new_fail)
1403 + gp.sprint_var(boot_fail_threshold)
1404 )
Michael Walshb5839d02017-04-12 16:11:20 -05001405 BuiltIn().fail(gp.sprint_error(error_message))