blob: 936855d808ee5ab6c4c3d4e8a31b465d22b99609 [file] [log] [blame]
Michael Walsh0bbd8602016-11-22 11:31:49 -06001#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
Michael Walsh0b93fbf2017-03-02 14:42:41 -06007import os
8import imp
9import time
10import glob
11import random
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050012import re
Michael Walshf566fb12019-02-01 14:35:09 -060013import signal
George Keishingd54bbc22018-08-03 08:24:58 -050014try:
15 import cPickle as pickle
16except ImportError:
17 import pickle
Michael Walshdc80d672017-05-09 12:58:32 -050018import socket
Michael Walsh0b93fbf2017-03-02 14:42:41 -060019
20from robot.utils import DotDict
21from robot.libraries.BuiltIn import BuiltIn
22
Michael Walsh6741f742017-02-20 16:16:38 -060023from boot_data import *
Michael Walshc9116812017-03-10 14:23:06 -060024import gen_print as gp
Michael Walsh55302292017-01-10 11:43:02 -060025import gen_robot_plug_in as grpi
Michael Walshf75d4352019-12-05 17:01:20 -060026import gen_arg as ga
Michael Walsh44cef252019-08-01 12:38:56 -050027import gen_valid as gv
Michael Walsh6741f742017-02-20 16:16:38 -060028import gen_misc as gm
29import gen_cmd as gc
Michael Walshb5839d02017-04-12 16:11:20 -050030import gen_robot_keyword as grk
Michael Walsh55302292017-01-10 11:43:02 -060031import state as st
Michael Walshff340002017-08-29 11:18:27 -050032import var_stack as vs
Michael Walshc9bd2e82019-04-18 11:06:52 -050033import gen_plug_in_utils as gpu
Michael Walsh0bbd8602016-11-22 11:31:49 -060034
Michael Walsh0b93fbf2017-03-02 14:42:41 -060035base_path = os.path.dirname(os.path.dirname(
36 imp.find_module("gen_robot_print")[1])) +\
Michael Walshc9116812017-03-10 14:23:06 -060037 os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060038sys.path.append(base_path + "extended/")
39import run_keyword as rk
Michael Walsh0bbd8602016-11-22 11:31:49 -060040
Michael Walshe1e26442017-03-06 17:50:07 -060041# Setting master_pid correctly influences the behavior of plug-ins like
42# DB_Logging
43program_pid = os.getpid()
44master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -050045pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
Michael Walshe1e26442017-03-06 17:50:07 -060046
Michael Walshb5839d02017-04-12 16:11:20 -050047# Set up boot data structures.
Michael Walsh986d8ae2019-07-17 10:02:23 -050048os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
Michael Walsh0b93fbf2017-03-02 14:42:41 -060049
Michael Walsh6741f742017-02-20 16:16:38 -060050boot_lists = read_boot_lists()
Michael Walsh986d8ae2019-07-17 10:02:23 -050051
52# The maximum number of entries that can be in the boot_history global variable.
Michael Walsh815b1d52018-10-30 13:32:26 -050053max_boot_history = 10
Michael Walsh986d8ae2019-07-17 10:02:23 -050054boot_history = []
Michael Walsh6741f742017-02-20 16:16:38 -060055
Michael Walsh7dc885b2018-03-14 17:51:59 -050056state = st.return_state_constant('default_state')
Michael Walsh6741f742017-02-20 16:16:38 -060057cp_setup_called = 0
58next_boot = ""
59base_tool_dir_path = os.path.normpath(os.environ.get(
60 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
Michael Walshb5839d02017-04-12 16:11:20 -050061
Michael Walsh6741f742017-02-20 16:16:38 -060062ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
Michael Walsh6741f742017-02-20 16:16:38 -060063boot_success = 0
Michael Walsh6741f742017-02-20 16:16:38 -060064status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
65if status_dir_path != "":
66 status_dir_path = os.path.normpath(status_dir_path) + os.sep
Michael Walshe58df1c2019-08-07 09:57:43 -050067redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False)
George Keishingeb1fe352020-06-19 03:02:22 -050068redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False)
Michael Walshe58df1c2019-08-07 09:57:43 -050069if redfish_supported:
George Keishing89537a82020-06-17 00:37:25 -050070 redfish = BuiltIn().get_library_instance('redfish')
Michael Walshe58df1c2019-08-07 09:57:43 -050071 default_power_on = "Redfish Power On"
72 default_power_off = "Redfish Power Off"
George Keishingeb1fe352020-06-19 03:02:22 -050073 if redfish_rest_supported:
74 delete_errlogs_cmd = "Delete Error Logs"
75 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF"
76 else:
77 delete_errlogs_cmd = "Redfish Purge Event Log"
78 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff"
Michael Walshe58df1c2019-08-07 09:57:43 -050079else:
80 default_power_on = "REST Power On"
81 default_power_off = "REST Power Off"
Michael Walsh409ad352020-02-06 11:46:35 -060082 delete_errlogs_cmd = "Delete Error Logs"
George Keishinga54e06f2020-06-12 10:42:41 -050083 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF"
Michael Walsh6741f742017-02-20 16:16:38 -060084boot_count = 0
Michael Walsh0bbd8602016-11-22 11:31:49 -060085
Michael Walsh85678942017-03-27 14:34:22 -050086LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
Michael Walsh986d8ae2019-07-17 10:02:23 -050087AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '')
88ffdc_prefix = AUTOBOOT_FFDC_PREFIX
Sunil M325eb542017-08-10 07:09:43 -050089boot_start_time = ""
90boot_end_time = ""
Michael Walshff340002017-08-29 11:18:27 -050091save_stack = vs.var_stack('save_stack')
92main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
Michael Walsh85678942017-03-27 14:34:22 -050093
94
Michael Walsh89de14a2018-10-01 16:51:37 -050095def dump_ffdc_rc():
96 r"""
97 Return the constant dump ffdc test return code value.
98
99 When a plug-in call point program returns this value, it indicates that
100 this program should collect FFDC.
101 """
102
103 return 0x00000200
104
105
106def stop_test_rc():
107 r"""
108 Return the constant stop test return code value.
109
110 When a plug-in call point program returns this value, it indicates that
111 this program should stop running.
112 """
113
114 return 0x00000200
115
116
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500117def process_host(host,
118 host_var_name=""):
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500119 r"""
120 Process a host by getting the associated host name and IP address and
121 setting them in global variables.
122
123 If the caller does not pass the host_var_name, this function will try to
124 figure out the name of the variable used by the caller for the host parm.
125 Callers are advised to explicitly specify the host_var_name when calling
126 with an exec command. In such cases, the get_arg_name cannot figure out
127 the host variable name.
128
129 This function will then create similar global variable names by
130 removing "_host" and appending "_host_name" or "_ip" to the host variable
131 name.
132
133 Example:
134
135 If a call is made like this:
136 process_host(openbmc_host)
137
138 Global variables openbmc_host_name and openbmc_ip will be set.
139
140 Description of argument(s):
141 host A host name or IP. The name of the variable used should
142 have a suffix of "_host".
143 host_var_name The name of the variable being used as the host parm.
144 """
145
146 if host_var_name == "":
147 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
148
149 host_name_var_name = re.sub("host", "host_name", host_var_name)
150 ip_var_name = re.sub("host", "ip", host_var_name)
151 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
152 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
153 host + "')"
154 exec(cmd_buf)
155
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500156
Michael Walshb5839d02017-04-12 16:11:20 -0500157def process_pgm_parms():
Michael Walshb5839d02017-04-12 16:11:20 -0500158 r"""
159 Process the program parameters by assigning them all to corresponding
160 globals. Also, set some global values that depend on program parameters.
161 """
162
163 # Program parameter processing.
164 # Assign all program parms to python variables which are global to this
165 # module.
166
167 global parm_list
168 parm_list = BuiltIn().get_variable_value("${parm_list}")
169 # The following subset of parms should be processed as integers.
170 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
Michael Walsh89de14a2018-10-01 16:51:37 -0500171 'boot_fail_threshold', 'delete_errlogs',
Michael Walsh986d8ae2019-07-17 10:02:23 -0500172 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet',
173 'test_mode', 'debug']
Michael Walshb5839d02017-04-12 16:11:20 -0500174 for parm in parm_list:
175 if parm in int_list:
176 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
177 "}\", \"0\"))"
178 else:
179 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
180 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
Michael Walshff340002017-08-29 11:18:27 -0500181 gp.dpissuing(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500182 exec(cmd_buf)
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500183 if re.match(r".*_host$", parm):
184 cmd_buf = "process_host(" + parm + ", '" + parm + "')"
185 exec(cmd_buf)
186 if re.match(r".*_password$", parm):
187 # Register the value of any parm whose name ends in _password.
188 # This will cause the print functions to replace passwords with
189 # asterisks in the output.
190 cmd_buf = "gp.register_passwords(" + parm + ")"
191 exec(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500192
193 global ffdc_dir_path_style
194 global boot_list
195 global boot_stack
196 global boot_results_file_path
197 global boot_results
Michael Walsh986d8ae2019-07-17 10:02:23 -0500198 global boot_history
Michael Walshb5839d02017-04-12 16:11:20 -0500199 global ffdc_list_file_path
Michael Walshe0cf8d72017-05-17 13:20:46 -0500200 global ffdc_report_list_path
Michael Walsh600876d2017-05-30 17:58:58 -0500201 global ffdc_summary_list_path
Michael Walsha3e7b222020-02-03 15:32:16 -0600202 global boot_table
203 global valid_boot_types
Michael Walshb5839d02017-04-12 16:11:20 -0500204
205 if ffdc_dir_path_style == "":
206 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
207
208 # Convert these program parms to lists for easier processing..
George Keishing36efbc02018-12-12 10:18:23 -0600209 boot_list = list(filter(None, boot_list.split(":")))
210 boot_stack = list(filter(None, boot_stack.split(":")))
Michael Walshb5839d02017-04-12 16:11:20 -0500211
Michael Walsha3e7b222020-02-03 15:32:16 -0600212 boot_table = create_boot_table(boot_table_path, os_host=os_host)
213 valid_boot_types = create_valid_boot_list(boot_table)
214
Michael Walsh903e0b22017-09-19 17:00:33 -0500215 cleanup_boot_results_file()
216 boot_results_file_path = create_boot_results_file_path(pgm_name,
217 openbmc_nickname,
218 master_pid)
Michael Walshb5839d02017-04-12 16:11:20 -0500219
220 if os.path.isfile(boot_results_file_path):
221 # We've been called before in this run so we'll load the saved
Michael Walsh986d8ae2019-07-17 10:02:23 -0500222 # boot_results and boot_history objects.
223 boot_results, boot_history =\
Michael Walsh6c645742018-08-17 15:02:17 -0500224 pickle.load(open(boot_results_file_path, 'rb'))
Michael Walshb5839d02017-04-12 16:11:20 -0500225 else:
226 boot_results = boot_results(boot_table, boot_pass, boot_fail)
227
228 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
229 "/FFDC_FILE_LIST"
Michael Walshe0cf8d72017-05-17 13:20:46 -0500230 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
231 "/FFDC_REPORT_FILE_LIST"
Michael Walshb5839d02017-04-12 16:11:20 -0500232
Michael Walsh600876d2017-05-30 17:58:58 -0500233 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
234 "/FFDC_SUMMARY_FILE_LIST"
235
Michael Walshb5839d02017-04-12 16:11:20 -0500236
Michael Walsh85678942017-03-27 14:34:22 -0500237def initial_plug_in_setup():
Michael Walsh85678942017-03-27 14:34:22 -0500238 r"""
239 Initialize all plug-in environment variables which do not change for the
240 duration of the program.
241
242 """
243
244 global LOG_LEVEL
245 BuiltIn().set_log_level("NONE")
246
247 BuiltIn().set_global_variable("${master_pid}", master_pid)
248 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
249 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
250 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
251 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
252 ffdc_list_file_path)
Michael Walshe0cf8d72017-05-17 13:20:46 -0500253 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
254 ffdc_report_list_path)
Michael Walsh600876d2017-05-30 17:58:58 -0500255 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
256 ffdc_summary_list_path)
Michael Walsh85678942017-03-27 14:34:22 -0500257
258 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
259 ffdc_dir_path_style)
260 BuiltIn().set_global_variable("${FFDC_CHECK}",
261 ffdc_check)
262
263 # For each program parameter, set the corresponding AUTOBOOT_ environment
264 # variable value. Also, set an AUTOBOOT_ environment variable for every
265 # element in additional_values.
266 additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
267 "status_dir_path", "base_tool_dir_path",
Michael Walsh600876d2017-05-30 17:58:58 -0500268 "ffdc_list_file_path", "ffdc_report_list_path",
George Keishinga54e06f2020-06-12 10:42:41 -0500269 "ffdc_summary_list_path", "execdir", "redfish_supported"]
Michael Walsh85678942017-03-27 14:34:22 -0500270
271 plug_in_vars = parm_list + additional_values
272
273 for var_name in plug_in_vars:
274 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
275 var_name = var_name.upper()
276 if var_value is None:
277 var_value = ""
278 os.environ["AUTOBOOT_" + var_name] = str(var_value)
279
280 BuiltIn().set_log_level(LOG_LEVEL)
281
Michael Walsh68a61162017-04-25 11:54:06 -0500282 # Make sure the ffdc list directory exists.
283 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
284 if not os.path.exists(ffdc_list_dir_path):
285 os.makedirs(ffdc_list_dir_path)
Michael Walsh85678942017-03-27 14:34:22 -0500286
Michael Walsh85678942017-03-27 14:34:22 -0500287
Michael Walsh0bbd8602016-11-22 11:31:49 -0600288def plug_in_setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600289 r"""
Michael Walsh85678942017-03-27 14:34:22 -0500290 Initialize all changing plug-in environment variables for use by the
291 plug-in programs.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600292 """
293
Michael Walsh85678942017-03-27 14:34:22 -0500294 global LOG_LEVEL
295 global test_really_running
296
297 BuiltIn().set_log_level("NONE")
298
Michael Walsh6741f742017-02-20 16:16:38 -0600299 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600300 if boot_pass > 1:
301 test_really_running = 1
302 else:
303 test_really_running = 0
304
Michael Walsh6741f742017-02-20 16:16:38 -0600305 BuiltIn().set_global_variable("${test_really_running}",
306 test_really_running)
307 BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
Michael Walsh6741f742017-02-20 16:16:38 -0600308 BuiltIn().set_global_variable("${boot_pass}", boot_pass)
309 BuiltIn().set_global_variable("${boot_fail}", boot_fail)
310 BuiltIn().set_global_variable("${boot_success}", boot_success)
311 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
Sunil M325eb542017-08-10 07:09:43 -0500312 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
313 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
Michael Walsh4c9a6452016-12-13 16:03:11 -0600314
Michael Walsh0bbd8602016-11-22 11:31:49 -0600315 # For each program parameter, set the corresponding AUTOBOOT_ environment
316 # variable value. Also, set an AUTOBOOT_ environment variable for every
317 # element in additional_values.
318 additional_values = ["boot_type_desc", "boot_success", "boot_pass",
Sunil M325eb542017-08-10 07:09:43 -0500319 "boot_fail", "test_really_running", "ffdc_prefix",
320 "boot_start_time", "boot_end_time"]
Michael Walsh0bbd8602016-11-22 11:31:49 -0600321
Michael Walsh85678942017-03-27 14:34:22 -0500322 plug_in_vars = additional_values
Michael Walsh0bbd8602016-11-22 11:31:49 -0600323
324 for var_name in plug_in_vars:
325 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
326 var_name = var_name.upper()
327 if var_value is None:
328 var_value = ""
Michael Walsh6741f742017-02-20 16:16:38 -0600329 os.environ["AUTOBOOT_" + var_name] = str(var_value)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600330
Michael Walsh0bbd8602016-11-22 11:31:49 -0600331 if debug:
Michael Walsh6741f742017-02-20 16:16:38 -0600332 shell_rc, out_buf = \
333 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600334
Michael Walsh85678942017-03-27 14:34:22 -0500335 BuiltIn().set_log_level(LOG_LEVEL)
336
Michael Walsh0bbd8602016-11-22 11:31:49 -0600337
Michael Walshe0cf8d72017-05-17 13:20:46 -0500338def pre_boot_plug_in_setup():
339
340 # Clear the ffdc_list_file_path file. Plug-ins may now write to it.
341 try:
342 os.remove(ffdc_list_file_path)
343 except OSError:
344 pass
345
346 # Clear the ffdc_report_list_path file. Plug-ins may now write to it.
347 try:
348 os.remove(ffdc_report_list_path)
349 except OSError:
350 pass
351
Michael Walsh600876d2017-05-30 17:58:58 -0500352 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it.
353 try:
354 os.remove(ffdc_summary_list_path)
355 except OSError:
356 pass
357
Michael Walshe1974b92017-08-03 13:39:51 -0500358 global ffdc_prefix
359
360 seconds = time.time()
361 loc_time = time.localtime(seconds)
362 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
363
364 ffdc_prefix = openbmc_nickname + "." + time_string
365
Michael Walshe0cf8d72017-05-17 13:20:46 -0500366
Michael Walshf566fb12019-02-01 14:35:09 -0600367def default_sigusr1(signal_number=0,
368 frame=None):
369 r"""
370 Handle SIGUSR1 by doing nothing.
371
372 This function assists in debugging SIGUSR1 processing by printing messages
373 to stdout and to the log.html file.
374
375 Description of argument(s):
376 signal_number The signal number (should always be 10 for SIGUSR1).
377 frame The frame data.
378 """
379
Michael Walsh80dddde2019-10-22 13:54:38 -0500380 gp.qprintn()
381 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600382 gp.lprint_executing()
383
384
385def set_default_siguser1():
386 r"""
387 Set the default_sigusr1 function to be the SIGUSR1 handler.
388 """
389
Michael Walsh80dddde2019-10-22 13:54:38 -0500390 gp.qprintn()
391 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600392 gp.lprint_executing()
393 signal.signal(signal.SIGUSR1, default_sigusr1)
394
395
Michael Walsh6741f742017-02-20 16:16:38 -0600396def setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600397 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600398 Do general program setup tasks.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600399 """
400
Michael Walsh6741f742017-02-20 16:16:38 -0600401 global cp_setup_called
Michael Walsh81816742017-09-27 11:02:29 -0500402 global transitional_boot_selected
Michael Walsh0bbd8602016-11-22 11:31:49 -0600403
Michael Walshb5839d02017-04-12 16:11:20 -0500404 gp.qprintn()
405
George Keishinga54e06f2020-06-12 10:42:41 -0500406 if redfish_supported:
407 redfish.login()
408
Michael Walshf566fb12019-02-01 14:35:09 -0600409 set_default_siguser1()
Michael Walsh81816742017-09-27 11:02:29 -0500410 transitional_boot_selected = False
411
Michael Walsh83f4bc72017-04-20 16:49:43 -0500412 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
413 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
Michael Walshd061c042017-05-23 14:46:57 -0500414 # If we can't find process_plug_in_packages.py, ssh_pw or
415 # validate_plug_ins.py, then we don't have our repo bin in PATH.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500416 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
417 + " ssh_pw validate_plug_ins.py", quiet=1,
Michael Walshd061c042017-05-23 14:46:57 -0500418 print_output=0, show_err=0)
Michael Walshb5839d02017-04-12 16:11:20 -0500419 if shell_rc != 0:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500420 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
421 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
422 if robot_pgm_dir_path not in sys.path:
423 sys.path.append(robot_pgm_dir_path)
424 PYTHONPATH = os.environ.get("PYTHONPATH", "")
425 if PYTHONPATH == "":
426 os.environ['PYTHONPATH'] = robot_pgm_dir_path
427 else:
428 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
Michael Walsh6741f742017-02-20 16:16:38 -0600429
430 validate_parms()
431
Michael Walshc108e422019-03-28 12:27:18 -0500432 gp.qprint_pgm_header()
Michael Walsh6741f742017-02-20 16:16:38 -0600433
George Keishinga54e06f2020-06-12 10:42:41 -0500434 grk.run_key_u(default_set_power_policy)
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500435
Michael Walsh85678942017-03-27 14:34:22 -0500436 initial_plug_in_setup()
437
Michael Walsh6741f742017-02-20 16:16:38 -0600438 plug_in_setup()
439 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
440 call_point='setup')
441 if rc != 0:
442 error_message = "Plug-in setup failed.\n"
Michael Walshc108e422019-03-28 12:27:18 -0500443 gp.print_error_report(error_message)
Michael Walsh6741f742017-02-20 16:16:38 -0600444 BuiltIn().fail(error_message)
445 # Setting cp_setup_called lets our Teardown know that it needs to call
446 # the cleanup plug-in call point.
447 cp_setup_called = 1
448
449 # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
450 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
Michael Walsh85678942017-03-27 14:34:22 -0500451 # FFDC_LOG_PATH is used by "FFDC" keyword.
452 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
Michael Walsh6741f742017-02-20 16:16:38 -0600453
Michael Walshdc80d672017-05-09 12:58:32 -0500454 # Also printed by FFDC.
455 global host_name
456 global host_ip
457 host = socket.gethostname()
458 host_name, host_ip = gm.get_host_name_ip(host)
459
Michael Walsh986d8ae2019-07-17 10:02:23 -0500460 gp.dprint_var(boot_table)
Michael Walshb5839d02017-04-12 16:11:20 -0500461 gp.dprint_var(boot_lists)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600462
Michael Walsh0bbd8602016-11-22 11:31:49 -0600463
Michael Walsh6741f742017-02-20 16:16:38 -0600464def validate_parms():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600465 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600466 Validate all program parameters.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600467 """
468
Michael Walshb5839d02017-04-12 16:11:20 -0500469 process_pgm_parms()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600470
Michael Walshb5839d02017-04-12 16:11:20 -0500471 gp.qprintn()
472
473 global openbmc_model
Michael Walshf5ce38c2020-02-27 12:46:20 -0600474 if openbmc_model == "":
475 status, ret_values =\
476 grk.run_key_u("Get BMC System Model")
477 openbmc_model = ret_values
478 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
479 gv.set_exit_on_error(True)
Michael Walsh44cef252019-08-01 12:38:56 -0500480 gv.valid_value(openbmc_host)
481 gv.valid_value(openbmc_username)
482 gv.valid_value(openbmc_password)
483 gv.valid_value(rest_username)
484 gv.valid_value(rest_password)
485 gv.valid_value(ipmi_username)
486 gv.valid_value(ipmi_password)
Michael Walsh6741f742017-02-20 16:16:38 -0600487 if os_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500488 gv.valid_value(os_username)
489 gv.valid_value(os_password)
Michael Walsh6741f742017-02-20 16:16:38 -0600490 if pdu_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500491 gv.valid_value(pdu_username)
492 gv.valid_value(pdu_password)
493 gv.valid_integer(pdu_slot_no)
Michael Walsh6741f742017-02-20 16:16:38 -0600494 if openbmc_serial_host != "":
Michael Walsh44cef252019-08-01 12:38:56 -0500495 gv.valid_integer(openbmc_serial_port)
Michael Walsh44cef252019-08-01 12:38:56 -0500496 gv.valid_value(openbmc_model)
497 gv.valid_integer(max_num_tests)
498 gv.valid_integer(boot_pass)
499 gv.valid_integer(boot_fail)
Michael Walsh6741f742017-02-20 16:16:38 -0600500 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
501 BuiltIn().set_global_variable("${plug_in_packages_list}",
502 plug_in_packages_list)
Michael Walsh44cef252019-08-01 12:38:56 -0500503 gv.valid_value(stack_mode, valid_values=['normal', 'skip'])
Michael Walshf5ce38c2020-02-27 12:46:20 -0600504 gv.set_exit_on_error(False)
Michael Walsha20da402017-03-31 16:27:45 -0500505 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
Michael Walsh6741f742017-02-20 16:16:38 -0600506 error_message = "You must provide either a value for either the" +\
507 " boot_list or the boot_stack parm.\n"
508 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600509 valid_boot_list(boot_list, valid_boot_types)
510 valid_boot_list(boot_stack, valid_boot_types)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500511 selected_PDU_boots = list(set(boot_list + boot_stack)
512 & set(boot_lists['PDU_reboot']))
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500513 if len(selected_PDU_boots) > 0 and pdu_host == "":
514 error_message = "You have selected the following boots which" +\
515 " require a PDU host but no value for pdu_host:\n"
516 error_message += gp.sprint_var(selected_PDU_boots)
Michael Walsh986d8ae2019-07-17 10:02:23 -0500517 error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500518 BuiltIn().fail(gp.sprint_error(error_message))
519
Michael Walsh6741f742017-02-20 16:16:38 -0600520 return
Michael Walsh0bbd8602016-11-22 11:31:49 -0600521
Michael Walsh0bbd8602016-11-22 11:31:49 -0600522
Michael Walsh6741f742017-02-20 16:16:38 -0600523def my_get_state():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600524 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600525 Get the system state plus a little bit of wrapping.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600526 """
527
Michael Walsh6741f742017-02-20 16:16:38 -0600528 global state
529
530 req_states = ['epoch_seconds'] + st.default_req_states
531
Michael Walshb5839d02017-04-12 16:11:20 -0500532 gp.qprint_timen("Getting system state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600533 if test_mode:
534 state['epoch_seconds'] = int(time.time())
535 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500536 state = st.get_state(req_states=req_states, quiet=quiet)
537 gp.qprint_var(state)
Michael Walsh341c21e2017-01-17 16:25:20 -0600538
Michael Walsh341c21e2017-01-17 16:25:20 -0600539
Michael Walsh45ca6e42017-09-14 17:29:12 -0500540def valid_state():
Michael Walsh45ca6e42017-09-14 17:29:12 -0500541 r"""
542 Verify that our state dictionary contains no blank values. If we don't get
543 valid state data, we cannot continue to work.
544 """
545
546 if st.compare_states(state, st.invalid_state_match, 'or'):
547 error_message = "The state dictionary contains blank fields which" +\
548 " is illegal.\n" + gp.sprint_var(state)
549 BuiltIn().fail(gp.sprint_error(error_message))
550
Michael Walsh45ca6e42017-09-14 17:29:12 -0500551
Michael Walsh6741f742017-02-20 16:16:38 -0600552def select_boot():
Michael Walsh341c21e2017-01-17 16:25:20 -0600553 r"""
554 Select a boot test to be run based on our current state and return the
555 chosen boot type.
556
557 Description of arguments:
Michael Walsh6741f742017-02-20 16:16:38 -0600558 state The state of the machine.
Michael Walsh341c21e2017-01-17 16:25:20 -0600559 """
560
Michael Walsh81816742017-09-27 11:02:29 -0500561 global transitional_boot_selected
Michael Walsh30dadae2017-02-27 14:25:52 -0600562 global boot_stack
563
Michael Walshb5839d02017-04-12 16:11:20 -0500564 gp.qprint_timen("Selecting a boot test.")
Michael Walsh6741f742017-02-20 16:16:38 -0600565
Michael Walsh81816742017-09-27 11:02:29 -0500566 if transitional_boot_selected and not boot_success:
567 prior_boot = next_boot
568 boot_candidate = boot_stack.pop()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500569 gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
570 + " transition to a valid state for '" + boot_candidate
571 + "' which was at the top of the boot_stack. Since"
572 + " the '" + next_boot + "' failed, the '"
573 + boot_candidate + "' has been removed from the stack"
574 + " to avoid and endless failure loop.")
Michael Walsh81816742017-09-27 11:02:29 -0500575 if len(boot_stack) == 0:
576 return ""
577
Michael Walsh6741f742017-02-20 16:16:38 -0600578 my_get_state()
Michael Walsh45ca6e42017-09-14 17:29:12 -0500579 valid_state()
Michael Walsh6741f742017-02-20 16:16:38 -0600580
Michael Walsh81816742017-09-27 11:02:29 -0500581 transitional_boot_selected = False
Michael Walsh6741f742017-02-20 16:16:38 -0600582 stack_popped = 0
583 if len(boot_stack) > 0:
584 stack_popped = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500585 gp.qprint_dashes()
586 gp.qprint_var(boot_stack)
587 gp.qprint_dashes()
588 skip_boot_printed = 0
589 while len(boot_stack) > 0:
590 boot_candidate = boot_stack.pop()
591 if stack_mode == 'normal':
592 break
593 else:
594 if st.compare_states(state, boot_table[boot_candidate]['end']):
595 if not skip_boot_printed:
Michael Walshff340002017-08-29 11:18:27 -0500596 gp.qprint_var(stack_mode)
597 gp.qprintn()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500598 gp.qprint_timen("Skipping the following boot tests"
599 + " which are unnecessary since their"
600 + " required end states match the"
601 + " current machine state:")
Michael Walshb5839d02017-04-12 16:11:20 -0500602 skip_boot_printed = 1
Michael Walshff340002017-08-29 11:18:27 -0500603 gp.qprint_var(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500604 boot_candidate = ""
605 if boot_candidate == "":
606 gp.qprint_dashes()
607 gp.qprint_var(boot_stack)
608 gp.qprint_dashes()
609 return boot_candidate
Michael Walsh6741f742017-02-20 16:16:38 -0600610 if st.compare_states(state, boot_table[boot_candidate]['start']):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500611 gp.qprint_timen("The machine state is valid for a '"
612 + boot_candidate + "' boot test.")
Michael Walshb5839d02017-04-12 16:11:20 -0500613 gp.qprint_dashes()
614 gp.qprint_var(boot_stack)
615 gp.qprint_dashes()
Michael Walsh6741f742017-02-20 16:16:38 -0600616 return boot_candidate
Michael Walsh341c21e2017-01-17 16:25:20 -0600617 else:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500618 gp.qprint_timen("The machine state does not match the required"
619 + " starting state for a '" + boot_candidate
620 + "' boot test:")
Michael Walsh986d8ae2019-07-17 10:02:23 -0500621 gp.qprint_varx("boot_table_start_entry",
622 boot_table[boot_candidate]['start'])
Michael Walsh6741f742017-02-20 16:16:38 -0600623 boot_stack.append(boot_candidate)
Michael Walsh81816742017-09-27 11:02:29 -0500624 transitional_boot_selected = True
Michael Walsh6741f742017-02-20 16:16:38 -0600625 popped_boot = boot_candidate
626
627 # Loop through your list selecting a boot_candidates
628 boot_candidates = []
629 for boot_candidate in boot_list:
630 if st.compare_states(state, boot_table[boot_candidate]['start']):
631 if stack_popped:
632 if st.compare_states(boot_table[boot_candidate]['end'],
Gunnar Mills096cd562018-03-26 10:19:12 -0500633 boot_table[popped_boot]['start']):
Michael Walsh6741f742017-02-20 16:16:38 -0600634 boot_candidates.append(boot_candidate)
635 else:
636 boot_candidates.append(boot_candidate)
637
638 if len(boot_candidates) == 0:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500639 gp.qprint_timen("The user's boot list contained no boot tests"
640 + " which are valid for the current machine state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600641 boot_candidate = default_power_on
642 if not st.compare_states(state, boot_table[default_power_on]['start']):
643 boot_candidate = default_power_off
644 boot_candidates.append(boot_candidate)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500645 gp.qprint_timen("Using default '" + boot_candidate
646 + "' boot type to transition to valid state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600647
Michael Walshb5839d02017-04-12 16:11:20 -0500648 gp.dprint_var(boot_candidates)
Michael Walsh6741f742017-02-20 16:16:38 -0600649
650 # Randomly select a boot from the candidate list.
651 boot = random.choice(boot_candidates)
Michael Walsh341c21e2017-01-17 16:25:20 -0600652
653 return boot
Michael Walsh0bbd8602016-11-22 11:31:49 -0600654
Michael Walsh55302292017-01-10 11:43:02 -0600655
Michael Walshb2e53ec2017-10-30 15:04:36 -0500656def print_defect_report(ffdc_file_list):
Michael Walsh341c21e2017-01-17 16:25:20 -0600657 r"""
658 Print a defect report.
Michael Walshb2e53ec2017-10-30 15:04:36 -0500659
660 Description of argument(s):
661 ffdc_file_list A list of files which were collected by our ffdc functions.
Michael Walsh341c21e2017-01-17 16:25:20 -0600662 """
663
Michael Walsh600876d2017-05-30 17:58:58 -0500664 # Making deliberate choice to NOT run plug_in_setup(). We don't want
665 # ffdc_prefix updated.
666 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
667 call_point='ffdc_report', stop_on_plug_in_failure=0)
668
Michael Walshe0cf8d72017-05-17 13:20:46 -0500669 # Get additional header data which may have been created by ffdc plug-ins.
670 # Also, delete the individual header files to cleanup.
671 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
672 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
673 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
674 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
675 show_err=0)
676
Michael Walshb2e53ec2017-10-30 15:04:36 -0500677 # Get additional summary data which may have been created by ffdc plug-ins.
Michael Walsh600876d2017-05-30 17:58:58 -0500678 # Also, delete the individual header files to cleanup.
679 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
680 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
681 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
682 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
683 show_err=0)
684
Michael Walshb2e53ec2017-10-30 15:04:36 -0500685 # ffdc_list_file_path contains a list of any ffdc files created by plug-
686 # ins, etc. Read that data into a list.
Michael Walsh341c21e2017-01-17 16:25:20 -0600687 try:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500688 plug_in_ffdc_list = \
689 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
George Keishing36efbc02018-12-12 10:18:23 -0600690 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
Michael Walsh341c21e2017-01-17 16:25:20 -0600691 except IOError:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500692 plug_in_ffdc_list = []
693
694 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
695 # in. Eliminate duplicates and sort the list.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500696 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
Michael Walshb2e53ec2017-10-30 15:04:36 -0500697
698 if status_file_path != "":
699 ffdc_file_list.insert(0, status_file_path)
700
701 # Convert the list to a printable list.
702 printable_ffdc_file_list = "\n".join(ffdc_file_list)
Michael Walsh341c21e2017-01-17 16:25:20 -0600703
Michael Walsh68a61162017-04-25 11:54:06 -0500704 # Open ffdc_file_list for writing. We will write a complete list of
705 # FFDC files to it for possible use by plug-ins like cp_stop_check.
706 ffdc_list_file = open(ffdc_list_file_path, 'w')
Michael Walshb2e53ec2017-10-30 15:04:36 -0500707 ffdc_list_file.write(printable_ffdc_file_list + "\n")
708 ffdc_list_file.close()
709
710 indent = 0
711 width = 90
712 linefeed = 1
713 char = "="
Michael Walsh68a61162017-04-25 11:54:06 -0500714
715 gp.qprintn()
Michael Walshb2e53ec2017-10-30 15:04:36 -0500716 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500717 gp.qprintn("Copy this data to the defect:\n")
718
Michael Walshe0cf8d72017-05-17 13:20:46 -0500719 if len(more_header_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500720 gp.qprintn(more_header_info)
Michael Walshdc80d672017-05-09 12:58:32 -0500721 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
722 openbmc_host_name, openbmc_ip, openbmc_username,
Michael Walsh0a3bdb42019-01-31 16:21:44 +0000723 openbmc_password, rest_username, rest_password, ipmi_username,
724 ipmi_password, os_host, os_host_name, os_ip, os_username,
Michael Walshdc80d672017-05-09 12:58:32 -0500725 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
726 pdu_password, pdu_slot_no, openbmc_serial_host,
727 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
Michael Walsh68a61162017-04-25 11:54:06 -0500728
729 gp.qprintn()
Michael Walsh986d8ae2019-07-17 10:02:23 -0500730 print_boot_history(boot_history)
Michael Walsh68a61162017-04-25 11:54:06 -0500731 gp.qprintn()
732 gp.qprint_var(state)
Michael Walshb5839d02017-04-12 16:11:20 -0500733 gp.qprintn()
734 gp.qprintn("FFDC data files:")
Michael Walshb2e53ec2017-10-30 15:04:36 -0500735 gp.qprintn(printable_ffdc_file_list)
Michael Walshb5839d02017-04-12 16:11:20 -0500736 gp.qprintn()
Michael Walsh341c21e2017-01-17 16:25:20 -0600737
Michael Walsh600876d2017-05-30 17:58:58 -0500738 if len(ffdc_summary_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500739 gp.qprintn(ffdc_summary_info)
Michael Walsh600876d2017-05-30 17:58:58 -0500740
Michael Walshb2e53ec2017-10-30 15:04:36 -0500741 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500742
Michael Walsh6741f742017-02-20 16:16:38 -0600743
Michael Walsh6741f742017-02-20 16:16:38 -0600744def my_ffdc():
Michael Walsh6741f742017-02-20 16:16:38 -0600745 r"""
746 Collect FFDC data.
747 """
748
749 global state
750
751 plug_in_setup()
752 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500753 call_point='ffdc', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600754
755 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500756 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix="
757 + AUTOBOOT_FFDC_PREFIX
758 + " ffdc_function_list="
759 + ffdc_function_list, ignore=1)
Michael Walsh83f4bc72017-04-20 16:49:43 -0500760 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500761 gp.qprint_error("Call to ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500762 if type(ffdc_file_list) is not list:
763 ffdc_file_list = []
764 # Leave a record for caller that "soft" errors occurred.
765 soft_errors = 1
766 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600767
768 my_get_state()
769
Michael Walshb2e53ec2017-10-30 15:04:36 -0500770 print_defect_report(ffdc_file_list)
Michael Walsh6741f742017-02-20 16:16:38 -0600771
Michael Walsh6741f742017-02-20 16:16:38 -0600772
Michael Walsh6741f742017-02-20 16:16:38 -0600773def print_test_start_message(boot_keyword):
Michael Walsh6741f742017-02-20 16:16:38 -0600774 r"""
775 Print a message indicating what boot test is about to run.
776
777 Description of arguments:
778 boot_keyword The name of the boot which is to be run
779 (e.g. "BMC Power On").
780 """
781
Michael Walsh986d8ae2019-07-17 10:02:23 -0500782 global boot_history
Sunil M325eb542017-08-10 07:09:43 -0500783 global boot_start_time
Michael Walsh6741f742017-02-20 16:16:38 -0600784
785 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
Sunil M325eb542017-08-10 07:09:43 -0500786
787 # Set boot_start_time for use by plug-ins.
788 boot_start_time = doing_msg[1:33]
789 gp.qprint_var(boot_start_time)
790
Michael Walshb5839d02017-04-12 16:11:20 -0500791 gp.qprint(doing_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600792
Michael Walsh986d8ae2019-07-17 10:02:23 -0500793 update_boot_history(boot_history, doing_msg, max_boot_history)
Michael Walsh6741f742017-02-20 16:16:38 -0600794
Michael Walsh6741f742017-02-20 16:16:38 -0600795
Michael Walshf566fb12019-02-01 14:35:09 -0600796def stop_boot_test(signal_number=0,
797 frame=None):
798 r"""
799 Handle SIGUSR1 by aborting the boot test that is running.
800
801 Description of argument(s):
802 signal_number The signal number (should always be 10 for SIGUSR1).
803 frame The frame data.
804 """
805
Michael Walsh80dddde2019-10-22 13:54:38 -0500806 gp.qprintn()
807 gp.qprint_executing()
Michael Walshf566fb12019-02-01 14:35:09 -0600808 gp.lprint_executing()
809
810 # Restore original sigusr1 handler.
811 set_default_siguser1()
812
813 message = "The caller has asked that the boot test be stopped and marked"
814 message += " as a failure."
815
816 function_stack = gm.get_function_stack()
817 if "wait_state" in function_stack:
Michael Walshc44aa532019-06-14 13:33:29 -0500818 st.set_exit_wait_early_message(message)
Michael Walshf566fb12019-02-01 14:35:09 -0600819 else:
820 BuiltIn().fail(gp.sprint_error(message))
821
822
Michael Walsh6741f742017-02-20 16:16:38 -0600823def run_boot(boot):
Michael Walsh6741f742017-02-20 16:16:38 -0600824 r"""
825 Run the specified boot.
826
827 Description of arguments:
828 boot The name of the boot test to be performed.
829 """
830
831 global state
832
Michael Walshf566fb12019-02-01 14:35:09 -0600833 signal.signal(signal.SIGUSR1, stop_boot_test)
834 gp.qprint_timen("stop_boot_test is armed.")
835
Michael Walsh6741f742017-02-20 16:16:38 -0600836 print_test_start_message(boot)
837
838 plug_in_setup()
839 rc, shell_rc, failed_plug_in_name = \
840 grpi.rprocess_plug_in_packages(call_point="pre_boot")
841 if rc != 0:
842 error_message = "Plug-in failed with non-zero return code.\n" +\
Michael Walsh986d8ae2019-07-17 10:02:23 -0500843 gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600844 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600845 BuiltIn().fail(gp.sprint_error(error_message))
846
847 if test_mode:
848 # In test mode, we'll pretend the boot worked by assigning its
849 # required end state to the default state value.
Michael Walsh30dadae2017-02-27 14:25:52 -0600850 state = st.strip_anchor_state(boot_table[boot]['end'])
Michael Walsh6741f742017-02-20 16:16:38 -0600851 else:
852 # Assertion: We trust that the state data was made fresh by the
853 # caller.
854
Michael Walshb5839d02017-04-12 16:11:20 -0500855 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600856
857 if boot_table[boot]['method_type'] == "keyword":
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600858 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
Michael Walshb5839d02017-04-12 16:11:20 -0500859 boot_table[boot]['method'],
860 quiet=quiet)
Michael Walsh6741f742017-02-20 16:16:38 -0600861
862 if boot_table[boot]['bmc_reboot']:
863 st.wait_for_comm_cycle(int(state['epoch_seconds']))
Michael Walsh30dadae2017-02-27 14:25:52 -0600864 plug_in_setup()
865 rc, shell_rc, failed_plug_in_name = \
866 grpi.rprocess_plug_in_packages(call_point="post_reboot")
867 if rc != 0:
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600868 error_message = "Plug-in failed with non-zero return code.\n"
Michael Walsh986d8ae2019-07-17 10:02:23 -0500869 error_message += gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600870 set_default_siguser1()
Michael Walsh30dadae2017-02-27 14:25:52 -0600871 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600872 else:
873 match_state = st.anchor_state(state)
874 del match_state['epoch_seconds']
875 # Wait for the state to change in any way.
876 st.wait_state(match_state, wait_time=state_change_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500877 interval="10 seconds", invert=1)
Michael Walsh6741f742017-02-20 16:16:38 -0600878
Michael Walshb5839d02017-04-12 16:11:20 -0500879 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600880 if boot_table[boot]['end']['chassis'] == "Off":
881 boot_timeout = power_off_timeout
882 else:
883 boot_timeout = power_on_timeout
884 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500885 interval="10 seconds")
Michael Walsh6741f742017-02-20 16:16:38 -0600886
887 plug_in_setup()
888 rc, shell_rc, failed_plug_in_name = \
889 grpi.rprocess_plug_in_packages(call_point="post_boot")
890 if rc != 0:
891 error_message = "Plug-in failed with non-zero return code.\n" +\
Michael Walsh986d8ae2019-07-17 10:02:23 -0500892 gp.sprint_var(rc, fmt=gp.hexa())
Michael Walshf566fb12019-02-01 14:35:09 -0600893 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600894 BuiltIn().fail(gp.sprint_error(error_message))
895
Michael Walshf566fb12019-02-01 14:35:09 -0600896 # Restore original sigusr1 handler.
897 set_default_siguser1()
898
Michael Walsh6741f742017-02-20 16:16:38 -0600899
Michael Walsh6741f742017-02-20 16:16:38 -0600900def test_loop_body():
Michael Walsh6741f742017-02-20 16:16:38 -0600901 r"""
902 The main loop body for the loop in main_py.
903
904 Description of arguments:
905 boot_count The iteration number (starts at 1).
906 """
907
908 global boot_count
909 global state
910 global next_boot
911 global boot_success
Sunil M325eb542017-08-10 07:09:43 -0500912 global boot_end_time
Michael Walsh6741f742017-02-20 16:16:38 -0600913
Michael Walshb5839d02017-04-12 16:11:20 -0500914 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600915
916 next_boot = select_boot()
Michael Walshb5839d02017-04-12 16:11:20 -0500917 if next_boot == "":
918 return True
Michael Walsh6741f742017-02-20 16:16:38 -0600919
Michael Walshb5839d02017-04-12 16:11:20 -0500920 boot_count += 1
921 gp.qprint_timen("Starting boot " + str(boot_count) + ".")
Michael Walsh6741f742017-02-20 16:16:38 -0600922
Michael Walshe0cf8d72017-05-17 13:20:46 -0500923 pre_boot_plug_in_setup()
Michael Walsh6741f742017-02-20 16:16:38 -0600924
925 cmd_buf = ["run_boot", next_boot]
926 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
927 if boot_status == "FAIL":
Michael Walshb5839d02017-04-12 16:11:20 -0500928 gp.qprint(msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600929
Michael Walshb5839d02017-04-12 16:11:20 -0500930 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600931 if boot_status == "PASS":
932 boot_success = 1
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500933 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
934 + "\" succeeded.")
Michael Walsh6741f742017-02-20 16:16:38 -0600935 else:
936 boot_success = 0
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500937 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
938 + "\" failed.")
Sunil M325eb542017-08-10 07:09:43 -0500939
940 # Set boot_end_time for use by plug-ins.
941 boot_end_time = completion_msg[1:33]
942 gp.qprint_var(boot_end_time)
943
944 gp.qprint(completion_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600945
946 boot_results.update(next_boot, boot_status)
947
948 plug_in_setup()
949 # NOTE: A post_test_case call point failure is NOT counted as a boot
950 # failure.
951 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500952 call_point='post_test_case', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600953
954 plug_in_setup()
955 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500956 call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
Michael Walsh6741f742017-02-20 16:16:38 -0600957 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
Michael Walsh12059e22019-03-21 11:03:45 -0500958 if ffdc_check == "All" or\
Michael Walsh89de14a2018-10-01 16:51:37 -0500959 shell_rc == dump_ffdc_rc():
Michael Walsh83f4bc72017-04-20 16:49:43 -0500960 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
961 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500962 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500963 # Leave a record for caller that "soft" errors occurred.
964 soft_errors = 1
965 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600966
Michael Walshaabef1e2017-09-20 15:16:17 -0500967 if delete_errlogs:
968 # We need to purge error logs between boots or they build up.
Michael Walsh409ad352020-02-06 11:46:35 -0600969 grk.run_key(delete_errlogs_cmd, ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -0500970
Michael Walsh952f9b02017-03-09 13:11:14 -0600971 boot_results.print_report()
Michael Walshb5839d02017-04-12 16:11:20 -0500972 gp.qprint_timen("Finished boot " + str(boot_count) + ".")
Michael Walsh952f9b02017-03-09 13:11:14 -0600973
Michael Walsh6741f742017-02-20 16:16:38 -0600974 plug_in_setup()
975 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500976 call_point='stop_check', shell_rc=stop_test_rc(),
977 stop_on_non_zero_rc=1)
978 if shell_rc == stop_test_rc():
Michael Walsh3ba8ecd2018-04-24 11:33:25 -0500979 message = "Stopping as requested by user.\n"
Michael Walsh80dddde2019-10-22 13:54:38 -0500980 gp.qprint_time(message)
Michael Walsh3ba8ecd2018-04-24 11:33:25 -0500981 BuiltIn().fail(message)
Michael Walsh6741f742017-02-20 16:16:38 -0600982
Michael Walshd139f282017-04-04 18:00:23 -0500983 # This should help prevent ConnectionErrors.
Michael Walsh0960b382017-06-22 16:23:37 -0500984 grk.run_key_u("Close All Connections")
Michael Walshd139f282017-04-04 18:00:23 -0500985
Michael Walsh6741f742017-02-20 16:16:38 -0600986 return True
987
Michael Walsh6741f742017-02-20 16:16:38 -0600988
Michael Walsh83f4bc72017-04-20 16:49:43 -0500989def obmc_boot_test_teardown():
Michael Walsh6741f742017-02-20 16:16:38 -0600990 r"""
Michael Walshf75d4352019-12-05 17:01:20 -0600991 Clean up after the main keyword.
Michael Walsh6741f742017-02-20 16:16:38 -0600992 """
Michael Walshf75d4352019-12-05 17:01:20 -0600993 gp.qprint_executing()
994
995 if ga.psutil_imported:
996 ga.terminate_descendants()
Michael Walsh6741f742017-02-20 16:16:38 -0600997
998 if cp_setup_called:
999 plug_in_setup()
1000 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -05001001 call_point='cleanup', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -06001002
Michael Walsh600876d2017-05-30 17:58:58 -05001003 if 'boot_results_file_path' in globals():
Michael Walsh986d8ae2019-07-17 10:02:23 -05001004 # Save boot_results and boot_history objects to a file in case they are
Michael Walsh6c645742018-08-17 15:02:17 -05001005 # needed again.
Michael Walsh600876d2017-05-30 17:58:58 -05001006 gp.qprint_timen("Saving boot_results to the following path.")
1007 gp.qprint_var(boot_results_file_path)
Michael Walsh986d8ae2019-07-17 10:02:23 -05001008 pickle.dump((boot_results, boot_history),
Michael Walsh6c645742018-08-17 15:02:17 -05001009 open(boot_results_file_path, 'wb'),
Michael Walsh600876d2017-05-30 17:58:58 -05001010 pickle.HIGHEST_PROTOCOL)
Michael Walsh0b93fbf2017-03-02 14:42:41 -06001011
Michael Walshff340002017-08-29 11:18:27 -05001012 global save_stack
1013 # Restore any global values saved on the save_stack.
1014 for parm_name in main_func_parm_list:
1015 # Get the parm_value if it was saved on the stack.
1016 try:
1017 parm_value = save_stack.pop(parm_name)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -05001018 except BaseException:
Michael Walshff340002017-08-29 11:18:27 -05001019 # If it was not saved, no further action is required.
1020 continue
1021
1022 # Restore the saved value.
1023 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1024 "}\", parm_value)"
1025 gp.dpissuing(cmd_buf)
1026 exec(cmd_buf)
1027
1028 gp.dprintn(save_stack.sprint_obj())
1029
Michael Walsh6741f742017-02-20 16:16:38 -06001030
Michael Walshc9116812017-03-10 14:23:06 -06001031def test_teardown():
Michael Walshc9116812017-03-10 14:23:06 -06001032 r"""
1033 Clean up after this test case.
1034 """
1035
1036 gp.qprintn()
Michael Walshf75d4352019-12-05 17:01:20 -06001037 gp.qprint_executing()
1038
1039 if ga.psutil_imported:
1040 ga.terminate_descendants()
1041
Michael Walshc9116812017-03-10 14:23:06 -06001042 cmd_buf = ["Print Error",
1043 "A keyword timeout occurred ending this program.\n"]
1044 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1045
George Keishinga54e06f2020-06-12 10:42:41 -05001046 if redfish_supported:
1047 redfish.logout()
1048
Michael Walshc108e422019-03-28 12:27:18 -05001049 gp.qprint_pgm_footer()
Michael Walshb5839d02017-04-12 16:11:20 -05001050
Michael Walshc9116812017-03-10 14:23:06 -06001051
Michael Walsh89de14a2018-10-01 16:51:37 -05001052def post_stack():
1053 r"""
1054 Process post_stack plug-in programs.
1055 """
1056
1057 if not call_post_stack_plug:
1058 # The caller does not wish to have post_stack plug-in processing done.
1059 return
1060
1061 global boot_success
1062
1063 # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1064 pre_boot_plug_in_setup()
1065 # For the purposes of the following plug-ins, mark the "boot" as a success.
1066 boot_success = 1
1067 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001068 rc, shell_rc, failed_plug_in_name, history =\
1069 grpi.rprocess_plug_in_packages(call_point='post_stack',
1070 stop_on_plug_in_failure=0,
1071 return_history=True)
Michael Walsh986d8ae2019-07-17 10:02:23 -05001072 for doing_msg in history:
1073 update_boot_history(boot_history, doing_msg, max_boot_history)
Michael Walsh815b1d52018-10-30 13:32:26 -05001074 if rc != 0:
1075 boot_success = 0
Michael Walsh89de14a2018-10-01 16:51:37 -05001076
1077 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001078 rc, shell_rc, failed_plug_in_name =\
1079 grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1080 shell_rc=dump_ffdc_rc(),
1081 stop_on_plug_in_failure=1,
1082 stop_on_non_zero_rc=1)
1083 if shell_rc == dump_ffdc_rc():
Michael Walsh89de14a2018-10-01 16:51:37 -05001084 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1085 if status != 'PASS':
1086 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -05001087 # Leave a record for caller that "soft" errors occurred.
1088 soft_errors = 1
1089 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh89de14a2018-10-01 16:51:37 -05001090
1091 plug_in_setup()
1092 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1093 call_point='stop_check', shell_rc=stop_test_rc(),
1094 stop_on_non_zero_rc=1)
1095 if shell_rc == stop_test_rc():
1096 message = "Stopping as requested by user.\n"
Michael Walsh80dddde2019-10-22 13:54:38 -05001097 gp.qprint_time(message)
Michael Walsh89de14a2018-10-01 16:51:37 -05001098 BuiltIn().fail(message)
1099
1100
Michael Walshff340002017-08-29 11:18:27 -05001101def obmc_boot_test_py(loc_boot_stack=None,
1102 loc_stack_mode=None,
1103 loc_quiet=None):
Michael Walsh6741f742017-02-20 16:16:38 -06001104 r"""
1105 Do main program processing.
1106 """
1107
Michael Walshff340002017-08-29 11:18:27 -05001108 global save_stack
1109
Michael Walshf75d4352019-12-05 17:01:20 -06001110 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']})
1111
George Keishing36efbc02018-12-12 10:18:23 -06001112 gp.dprintn()
Michael Walshff340002017-08-29 11:18:27 -05001113 # Process function parms.
1114 for parm_name in main_func_parm_list:
1115 # Get parm's value.
George Keishing36efbc02018-12-12 10:18:23 -06001116 parm_value = eval("loc_" + parm_name)
1117 gp.dpvars(parm_name, parm_value)
Michael Walshff340002017-08-29 11:18:27 -05001118
George Keishing36efbc02018-12-12 10:18:23 -06001119 if parm_value is not None:
Michael Walshff340002017-08-29 11:18:27 -05001120 # Save the global value on a stack.
1121 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1122 parm_name + "}\"), \"" + parm_name + "\")"
1123 gp.dpissuing(cmd_buf)
1124 exec(cmd_buf)
1125
1126 # Set the global value to the passed value.
1127 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1128 "}\", loc_" + parm_name + ")"
1129 gp.dpissuing(cmd_buf)
1130 exec(cmd_buf)
1131
1132 gp.dprintn(save_stack.sprint_obj())
Michael Walshb5839d02017-04-12 16:11:20 -05001133
Michael Walsh6741f742017-02-20 16:16:38 -06001134 setup()
1135
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001136 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1137
Michael Walsha20da402017-03-31 16:27:45 -05001138 if ffdc_only:
1139 gp.qprint_timen("Caller requested ffdc_only.")
Michael Walsh986d8ae2019-07-17 10:02:23 -05001140 if do_pre_boot_plug_in_setup:
1141 pre_boot_plug_in_setup()
Michael Walsh83f4bc72017-04-20 16:49:43 -05001142 grk.run_key_u("my_ffdc")
Michael Walsh764d2f82017-04-27 16:01:08 -05001143 return
Michael Walsha20da402017-03-31 16:27:45 -05001144
Michael Walsh409ad352020-02-06 11:46:35 -06001145 if delete_errlogs:
1146 # Delete errlogs prior to doing any boot tests.
1147 grk.run_key(delete_errlogs_cmd, ignore=1)
1148
Michael Walsh6741f742017-02-20 16:16:38 -06001149 # Process caller's boot_stack.
1150 while (len(boot_stack) > 0):
1151 test_loop_body()
1152
Michael Walshb5839d02017-04-12 16:11:20 -05001153 gp.qprint_timen("Finished processing stack.")
Michael Walsh30dadae2017-02-27 14:25:52 -06001154
Michael Walsh89de14a2018-10-01 16:51:37 -05001155 post_stack()
1156
Michael Walsh6741f742017-02-20 16:16:38 -06001157 # Process caller's boot_list.
1158 if len(boot_list) > 0:
1159 for ix in range(1, max_num_tests + 1):
1160 test_loop_body()
1161
Michael Walshb5839d02017-04-12 16:11:20 -05001162 gp.qprint_timen("Completed all requested boot tests.")
1163
1164 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001165 new_fail = boot_fail - init_boot_fail
1166 if new_fail > boot_fail_threshold:
Michael Walshb5839d02017-04-12 16:11:20 -05001167 error_message = "Boot failures exceed the boot failure" +\
1168 " threshold:\n" +\
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001169 gp.sprint_var(new_fail) +\
Michael Walshb5839d02017-04-12 16:11:20 -05001170 gp.sprint_var(boot_fail_threshold)
1171 BuiltIn().fail(gp.sprint_error(error_message))