blob: e864571109a10dfce5a06c7a935563901bd1a54e [file] [log] [blame]
Michael Walsh0bbd8602016-11-22 11:31:49 -06001#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
Michael Walsh0b93fbf2017-03-02 14:42:41 -06007import os
8import imp
9import time
10import glob
11import random
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050012import re
Michael Walshf566fb12019-02-01 14:35:09 -060013import signal
George Keishingd54bbc22018-08-03 08:24:58 -050014try:
15 import cPickle as pickle
16except ImportError:
17 import pickle
Michael Walshdc80d672017-05-09 12:58:32 -050018import socket
Michael Walsh0b93fbf2017-03-02 14:42:41 -060019
20from robot.utils import DotDict
21from robot.libraries.BuiltIn import BuiltIn
22
Michael Walsh6741f742017-02-20 16:16:38 -060023from boot_data import *
Michael Walshc9116812017-03-10 14:23:06 -060024import gen_print as gp
Michael Walsh55302292017-01-10 11:43:02 -060025import gen_robot_plug_in as grpi
Michael Walsh6741f742017-02-20 16:16:38 -060026import gen_robot_valid as grv
27import gen_misc as gm
28import gen_cmd as gc
Michael Walshb5839d02017-04-12 16:11:20 -050029import gen_robot_keyword as grk
Michael Walsh55302292017-01-10 11:43:02 -060030import state as st
Michael Walshff340002017-08-29 11:18:27 -050031import var_stack as vs
Michael Walshc9bd2e82019-04-18 11:06:52 -050032import gen_plug_in_utils as gpu
Michael Walsh0bbd8602016-11-22 11:31:49 -060033
Michael Walsh0b93fbf2017-03-02 14:42:41 -060034base_path = os.path.dirname(os.path.dirname(
35 imp.find_module("gen_robot_print")[1])) +\
Michael Walshc9116812017-03-10 14:23:06 -060036 os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060037sys.path.append(base_path + "extended/")
38import run_keyword as rk
Michael Walsh0bbd8602016-11-22 11:31:49 -060039
Michael Walshe1e26442017-03-06 17:50:07 -060040# Setting master_pid correctly influences the behavior of plug-ins like
41# DB_Logging
42program_pid = os.getpid()
43master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -050044pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
Michael Walshe1e26442017-03-06 17:50:07 -060045
Michael Walshb5839d02017-04-12 16:11:20 -050046# Set up boot data structures.
47boot_table = create_boot_table()
48valid_boot_types = create_valid_boot_list(boot_table)
Michael Walsh0b93fbf2017-03-02 14:42:41 -060049
Michael Walsh6741f742017-02-20 16:16:38 -060050boot_lists = read_boot_lists()
Michael Walsh815b1d52018-10-30 13:32:26 -050051# The maximum number of entries that can be in the last_ten global variable.
52max_boot_history = 10
Michael Walsh6741f742017-02-20 16:16:38 -060053last_ten = []
Michael Walsh6741f742017-02-20 16:16:38 -060054
Michael Walsh7dc885b2018-03-14 17:51:59 -050055state = st.return_state_constant('default_state')
Michael Walsh6741f742017-02-20 16:16:38 -060056cp_setup_called = 0
57next_boot = ""
58base_tool_dir_path = os.path.normpath(os.environ.get(
59 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
Michael Walshb5839d02017-04-12 16:11:20 -050060
Michael Walsh6741f742017-02-20 16:16:38 -060061ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
Michael Walsh6741f742017-02-20 16:16:38 -060062boot_success = 0
Michael Walsh6741f742017-02-20 16:16:38 -060063status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
64if status_dir_path != "":
65 status_dir_path = os.path.normpath(status_dir_path) + os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060066default_power_on = "REST Power On"
67default_power_off = "REST Power Off"
Michael Walsh6741f742017-02-20 16:16:38 -060068boot_count = 0
Michael Walsh0bbd8602016-11-22 11:31:49 -060069
Michael Walsh85678942017-03-27 14:34:22 -050070LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
Michael Walshe1974b92017-08-03 13:39:51 -050071ffdc_prefix = ""
Sunil M325eb542017-08-10 07:09:43 -050072boot_start_time = ""
73boot_end_time = ""
Michael Walshff340002017-08-29 11:18:27 -050074save_stack = vs.var_stack('save_stack')
75main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
Michael Walsh85678942017-03-27 14:34:22 -050076
77
Michael Walsh89de14a2018-10-01 16:51:37 -050078def dump_ffdc_rc():
79 r"""
80 Return the constant dump ffdc test return code value.
81
82 When a plug-in call point program returns this value, it indicates that
83 this program should collect FFDC.
84 """
85
86 return 0x00000200
87
88
89def stop_test_rc():
90 r"""
91 Return the constant stop test return code value.
92
93 When a plug-in call point program returns this value, it indicates that
94 this program should stop running.
95 """
96
97 return 0x00000200
98
99
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500100def process_host(host,
101 host_var_name=""):
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500102 r"""
103 Process a host by getting the associated host name and IP address and
104 setting them in global variables.
105
106 If the caller does not pass the host_var_name, this function will try to
107 figure out the name of the variable used by the caller for the host parm.
108 Callers are advised to explicitly specify the host_var_name when calling
109 with an exec command. In such cases, the get_arg_name cannot figure out
110 the host variable name.
111
112 This function will then create similar global variable names by
113 removing "_host" and appending "_host_name" or "_ip" to the host variable
114 name.
115
116 Example:
117
118 If a call is made like this:
119 process_host(openbmc_host)
120
121 Global variables openbmc_host_name and openbmc_ip will be set.
122
123 Description of argument(s):
124 host A host name or IP. The name of the variable used should
125 have a suffix of "_host".
126 host_var_name The name of the variable being used as the host parm.
127 """
128
129 if host_var_name == "":
130 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
131
132 host_name_var_name = re.sub("host", "host_name", host_var_name)
133 ip_var_name = re.sub("host", "ip", host_var_name)
134 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
135 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
136 host + "')"
137 exec(cmd_buf)
138
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500139
Michael Walshb5839d02017-04-12 16:11:20 -0500140def process_pgm_parms():
Michael Walshb5839d02017-04-12 16:11:20 -0500141 r"""
142 Process the program parameters by assigning them all to corresponding
143 globals. Also, set some global values that depend on program parameters.
144 """
145
146 # Program parameter processing.
147 # Assign all program parms to python variables which are global to this
148 # module.
149
150 global parm_list
151 parm_list = BuiltIn().get_variable_value("${parm_list}")
152 # The following subset of parms should be processed as integers.
153 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
Michael Walsh89de14a2018-10-01 16:51:37 -0500154 'boot_fail_threshold', 'delete_errlogs',
155 'call_post_stack_plug', 'quiet', 'test_mode', 'debug']
Michael Walshb5839d02017-04-12 16:11:20 -0500156 for parm in parm_list:
157 if parm in int_list:
158 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
159 "}\", \"0\"))"
160 else:
161 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
162 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
Michael Walshff340002017-08-29 11:18:27 -0500163 gp.dpissuing(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500164 exec(cmd_buf)
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500165 if re.match(r".*_host$", parm):
166 cmd_buf = "process_host(" + parm + ", '" + parm + "')"
167 exec(cmd_buf)
168 if re.match(r".*_password$", parm):
169 # Register the value of any parm whose name ends in _password.
170 # This will cause the print functions to replace passwords with
171 # asterisks in the output.
172 cmd_buf = "gp.register_passwords(" + parm + ")"
173 exec(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500174
175 global ffdc_dir_path_style
176 global boot_list
177 global boot_stack
178 global boot_results_file_path
179 global boot_results
Michael Walsh6c645742018-08-17 15:02:17 -0500180 global last_ten
Michael Walshb5839d02017-04-12 16:11:20 -0500181 global ffdc_list_file_path
Michael Walshe0cf8d72017-05-17 13:20:46 -0500182 global ffdc_report_list_path
Michael Walsh600876d2017-05-30 17:58:58 -0500183 global ffdc_summary_list_path
Michael Walshb5839d02017-04-12 16:11:20 -0500184
185 if ffdc_dir_path_style == "":
186 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
187
188 # Convert these program parms to lists for easier processing..
George Keishing36efbc02018-12-12 10:18:23 -0600189 boot_list = list(filter(None, boot_list.split(":")))
190 boot_stack = list(filter(None, boot_stack.split(":")))
Michael Walshb5839d02017-04-12 16:11:20 -0500191
Michael Walsh903e0b22017-09-19 17:00:33 -0500192 cleanup_boot_results_file()
193 boot_results_file_path = create_boot_results_file_path(pgm_name,
194 openbmc_nickname,
195 master_pid)
Michael Walshb5839d02017-04-12 16:11:20 -0500196
197 if os.path.isfile(boot_results_file_path):
198 # We've been called before in this run so we'll load the saved
Michael Walsh6c645742018-08-17 15:02:17 -0500199 # boot_results and last_ten objects.
200 boot_results, last_ten =\
201 pickle.load(open(boot_results_file_path, 'rb'))
Michael Walshb5839d02017-04-12 16:11:20 -0500202 else:
203 boot_results = boot_results(boot_table, boot_pass, boot_fail)
204
205 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
206 "/FFDC_FILE_LIST"
Michael Walshe0cf8d72017-05-17 13:20:46 -0500207 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
208 "/FFDC_REPORT_FILE_LIST"
Michael Walshb5839d02017-04-12 16:11:20 -0500209
Michael Walsh600876d2017-05-30 17:58:58 -0500210 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
211 "/FFDC_SUMMARY_FILE_LIST"
212
Michael Walshb5839d02017-04-12 16:11:20 -0500213
Michael Walsh85678942017-03-27 14:34:22 -0500214def initial_plug_in_setup():
Michael Walsh85678942017-03-27 14:34:22 -0500215 r"""
216 Initialize all plug-in environment variables which do not change for the
217 duration of the program.
218
219 """
220
221 global LOG_LEVEL
222 BuiltIn().set_log_level("NONE")
223
224 BuiltIn().set_global_variable("${master_pid}", master_pid)
225 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
226 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
227 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
228 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
229 ffdc_list_file_path)
Michael Walshe0cf8d72017-05-17 13:20:46 -0500230 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
231 ffdc_report_list_path)
Michael Walsh600876d2017-05-30 17:58:58 -0500232 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
233 ffdc_summary_list_path)
Michael Walsh85678942017-03-27 14:34:22 -0500234
235 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
236 ffdc_dir_path_style)
237 BuiltIn().set_global_variable("${FFDC_CHECK}",
238 ffdc_check)
239
240 # For each program parameter, set the corresponding AUTOBOOT_ environment
241 # variable value. Also, set an AUTOBOOT_ environment variable for every
242 # element in additional_values.
243 additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
244 "status_dir_path", "base_tool_dir_path",
Michael Walsh600876d2017-05-30 17:58:58 -0500245 "ffdc_list_file_path", "ffdc_report_list_path",
Michael Walsh0a3bdb42019-01-31 16:21:44 +0000246 "ffdc_summary_list_path", "execdir"]
Michael Walsh85678942017-03-27 14:34:22 -0500247
248 plug_in_vars = parm_list + additional_values
249
250 for var_name in plug_in_vars:
251 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
252 var_name = var_name.upper()
253 if var_value is None:
254 var_value = ""
255 os.environ["AUTOBOOT_" + var_name] = str(var_value)
256
257 BuiltIn().set_log_level(LOG_LEVEL)
258
Michael Walsh68a61162017-04-25 11:54:06 -0500259 # Make sure the ffdc list directory exists.
260 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
261 if not os.path.exists(ffdc_list_dir_path):
262 os.makedirs(ffdc_list_dir_path)
Michael Walsh85678942017-03-27 14:34:22 -0500263
Michael Walsh85678942017-03-27 14:34:22 -0500264
Michael Walsh0bbd8602016-11-22 11:31:49 -0600265def plug_in_setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600266 r"""
Michael Walsh85678942017-03-27 14:34:22 -0500267 Initialize all changing plug-in environment variables for use by the
268 plug-in programs.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600269 """
270
Michael Walsh85678942017-03-27 14:34:22 -0500271 global LOG_LEVEL
272 global test_really_running
273
274 BuiltIn().set_log_level("NONE")
275
Michael Walsh6741f742017-02-20 16:16:38 -0600276 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600277 if boot_pass > 1:
278 test_really_running = 1
279 else:
280 test_really_running = 0
281
Michael Walsh6741f742017-02-20 16:16:38 -0600282 BuiltIn().set_global_variable("${test_really_running}",
283 test_really_running)
284 BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
Michael Walsh6741f742017-02-20 16:16:38 -0600285 BuiltIn().set_global_variable("${boot_pass}", boot_pass)
286 BuiltIn().set_global_variable("${boot_fail}", boot_fail)
287 BuiltIn().set_global_variable("${boot_success}", boot_success)
288 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
Sunil M325eb542017-08-10 07:09:43 -0500289 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
290 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
Michael Walsh4c9a6452016-12-13 16:03:11 -0600291
Michael Walsh0bbd8602016-11-22 11:31:49 -0600292 # For each program parameter, set the corresponding AUTOBOOT_ environment
293 # variable value. Also, set an AUTOBOOT_ environment variable for every
294 # element in additional_values.
295 additional_values = ["boot_type_desc", "boot_success", "boot_pass",
Sunil M325eb542017-08-10 07:09:43 -0500296 "boot_fail", "test_really_running", "ffdc_prefix",
297 "boot_start_time", "boot_end_time"]
Michael Walsh0bbd8602016-11-22 11:31:49 -0600298
Michael Walsh85678942017-03-27 14:34:22 -0500299 plug_in_vars = additional_values
Michael Walsh0bbd8602016-11-22 11:31:49 -0600300
301 for var_name in plug_in_vars:
302 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
303 var_name = var_name.upper()
304 if var_value is None:
305 var_value = ""
Michael Walsh6741f742017-02-20 16:16:38 -0600306 os.environ["AUTOBOOT_" + var_name] = str(var_value)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600307
Michael Walsh0bbd8602016-11-22 11:31:49 -0600308 if debug:
Michael Walsh6741f742017-02-20 16:16:38 -0600309 shell_rc, out_buf = \
310 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600311
Michael Walsh85678942017-03-27 14:34:22 -0500312 BuiltIn().set_log_level(LOG_LEVEL)
313
Michael Walsh0bbd8602016-11-22 11:31:49 -0600314
Michael Walshe0cf8d72017-05-17 13:20:46 -0500315def pre_boot_plug_in_setup():
316
317 # Clear the ffdc_list_file_path file. Plug-ins may now write to it.
318 try:
319 os.remove(ffdc_list_file_path)
320 except OSError:
321 pass
322
323 # Clear the ffdc_report_list_path file. Plug-ins may now write to it.
324 try:
325 os.remove(ffdc_report_list_path)
326 except OSError:
327 pass
328
Michael Walsh600876d2017-05-30 17:58:58 -0500329 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it.
330 try:
331 os.remove(ffdc_summary_list_path)
332 except OSError:
333 pass
334
Michael Walshe1974b92017-08-03 13:39:51 -0500335 global ffdc_prefix
336
337 seconds = time.time()
338 loc_time = time.localtime(seconds)
339 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
340
341 ffdc_prefix = openbmc_nickname + "." + time_string
342
Michael Walshe0cf8d72017-05-17 13:20:46 -0500343
Michael Walshf566fb12019-02-01 14:35:09 -0600344def default_sigusr1(signal_number=0,
345 frame=None):
346 r"""
347 Handle SIGUSR1 by doing nothing.
348
349 This function assists in debugging SIGUSR1 processing by printing messages
350 to stdout and to the log.html file.
351
352 Description of argument(s):
353 signal_number The signal number (should always be 10 for SIGUSR1).
354 frame The frame data.
355 """
356
357 gp.printn()
358 gp.print_executing()
359 gp.lprint_executing()
360
361
362def set_default_siguser1():
363 r"""
364 Set the default_sigusr1 function to be the SIGUSR1 handler.
365 """
366
367 gp.printn()
368 gp.print_executing()
369 gp.lprint_executing()
370 signal.signal(signal.SIGUSR1, default_sigusr1)
371
372
Michael Walsh6741f742017-02-20 16:16:38 -0600373def setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600374 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600375 Do general program setup tasks.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600376 """
377
Michael Walsh6741f742017-02-20 16:16:38 -0600378 global cp_setup_called
Michael Walsh81816742017-09-27 11:02:29 -0500379 global transitional_boot_selected
Michael Walsh0bbd8602016-11-22 11:31:49 -0600380
Michael Walshb5839d02017-04-12 16:11:20 -0500381 gp.qprintn()
382
Michael Walshf566fb12019-02-01 14:35:09 -0600383 set_default_siguser1()
Michael Walsh81816742017-09-27 11:02:29 -0500384 transitional_boot_selected = False
385
Michael Walsh83f4bc72017-04-20 16:49:43 -0500386 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
387 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
Michael Walshd061c042017-05-23 14:46:57 -0500388 # If we can't find process_plug_in_packages.py, ssh_pw or
389 # validate_plug_ins.py, then we don't have our repo bin in PATH.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500390 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
391 + " ssh_pw validate_plug_ins.py", quiet=1,
Michael Walshd061c042017-05-23 14:46:57 -0500392 print_output=0, show_err=0)
Michael Walshb5839d02017-04-12 16:11:20 -0500393 if shell_rc != 0:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500394 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
395 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
396 if robot_pgm_dir_path not in sys.path:
397 sys.path.append(robot_pgm_dir_path)
398 PYTHONPATH = os.environ.get("PYTHONPATH", "")
399 if PYTHONPATH == "":
400 os.environ['PYTHONPATH'] = robot_pgm_dir_path
401 else:
402 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
Michael Walsh6741f742017-02-20 16:16:38 -0600403
404 validate_parms()
405
Michael Walshc108e422019-03-28 12:27:18 -0500406 gp.qprint_pgm_header()
Michael Walsh6741f742017-02-20 16:16:38 -0600407
George Keishingefc3ff22017-12-12 11:49:25 -0600408 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF")
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500409
Michael Walsh85678942017-03-27 14:34:22 -0500410 initial_plug_in_setup()
411
Michael Walsh6741f742017-02-20 16:16:38 -0600412 plug_in_setup()
413 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
414 call_point='setup')
415 if rc != 0:
416 error_message = "Plug-in setup failed.\n"
Michael Walshc108e422019-03-28 12:27:18 -0500417 gp.print_error_report(error_message)
Michael Walsh6741f742017-02-20 16:16:38 -0600418 BuiltIn().fail(error_message)
419 # Setting cp_setup_called lets our Teardown know that it needs to call
420 # the cleanup plug-in call point.
421 cp_setup_called = 1
422
423 # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
424 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
Michael Walsh85678942017-03-27 14:34:22 -0500425 # FFDC_LOG_PATH is used by "FFDC" keyword.
426 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
Michael Walsh6741f742017-02-20 16:16:38 -0600427
Michael Walshdc80d672017-05-09 12:58:32 -0500428 # Also printed by FFDC.
429 global host_name
430 global host_ip
431 host = socket.gethostname()
432 host_name, host_ip = gm.get_host_name_ip(host)
433
Michael Walshb5839d02017-04-12 16:11:20 -0500434 gp.dprint_var(boot_table, 1)
435 gp.dprint_var(boot_lists)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600436
Michael Walsh0bbd8602016-11-22 11:31:49 -0600437
Michael Walsh6741f742017-02-20 16:16:38 -0600438def validate_parms():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600439 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600440 Validate all program parameters.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600441 """
442
Michael Walshb5839d02017-04-12 16:11:20 -0500443 process_pgm_parms()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600444
Michael Walshb5839d02017-04-12 16:11:20 -0500445 gp.qprintn()
446
447 global openbmc_model
Michael Walsh6741f742017-02-20 16:16:38 -0600448 grv.rvalid_value("openbmc_host")
449 grv.rvalid_value("openbmc_username")
450 grv.rvalid_value("openbmc_password")
Michael Walsh0a3bdb42019-01-31 16:21:44 +0000451 grv.rvalid_value("rest_username")
452 grv.rvalid_value("rest_password")
453 grv.rvalid_value("ipmi_username")
454 grv.rvalid_value("ipmi_password")
Michael Walsh6741f742017-02-20 16:16:38 -0600455 if os_host != "":
456 grv.rvalid_value("os_username")
457 grv.rvalid_value("os_password")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600458
Michael Walsh6741f742017-02-20 16:16:38 -0600459 if pdu_host != "":
460 grv.rvalid_value("pdu_username")
461 grv.rvalid_value("pdu_password")
Michael Walsh85678942017-03-27 14:34:22 -0500462 grv.rvalid_integer("pdu_slot_no")
Michael Walsh6741f742017-02-20 16:16:38 -0600463 if openbmc_serial_host != "":
464 grv.rvalid_integer("openbmc_serial_port")
Michael Walshb5839d02017-04-12 16:11:20 -0500465 if openbmc_model == "":
466 status, ret_values =\
467 grk.run_key_u("Get BMC System Model")
468 openbmc_model = ret_values
469 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
Michael Walsh6741f742017-02-20 16:16:38 -0600470 grv.rvalid_value("openbmc_model")
Michael Walshb5839d02017-04-12 16:11:20 -0500471 grv.rvalid_integer("max_num_tests")
Michael Walsh6741f742017-02-20 16:16:38 -0600472 grv.rvalid_integer("boot_pass")
473 grv.rvalid_integer("boot_fail")
474
475 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
476 BuiltIn().set_global_variable("${plug_in_packages_list}",
477 plug_in_packages_list)
478
Michael Walshb5839d02017-04-12 16:11:20 -0500479 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip'])
Michael Walsha20da402017-03-31 16:27:45 -0500480 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
Michael Walsh6741f742017-02-20 16:16:38 -0600481 error_message = "You must provide either a value for either the" +\
482 " boot_list or the boot_stack parm.\n"
483 BuiltIn().fail(gp.sprint_error(error_message))
484
485 valid_boot_list(boot_list, valid_boot_types)
486 valid_boot_list(boot_stack, valid_boot_types)
487
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500488 selected_PDU_boots = list(set(boot_list + boot_stack)
489 & set(boot_lists['PDU_reboot']))
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500490
491 if len(selected_PDU_boots) > 0 and pdu_host == "":
492 error_message = "You have selected the following boots which" +\
493 " require a PDU host but no value for pdu_host:\n"
494 error_message += gp.sprint_var(selected_PDU_boots)
495 error_message += gp.sprint_var(pdu_host, 2)
496 BuiltIn().fail(gp.sprint_error(error_message))
497
Michael Walsh6741f742017-02-20 16:16:38 -0600498 return
Michael Walsh0bbd8602016-11-22 11:31:49 -0600499
Michael Walsh0bbd8602016-11-22 11:31:49 -0600500
Michael Walsh6741f742017-02-20 16:16:38 -0600501def my_get_state():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600502 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600503 Get the system state plus a little bit of wrapping.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600504 """
505
Michael Walsh6741f742017-02-20 16:16:38 -0600506 global state
507
508 req_states = ['epoch_seconds'] + st.default_req_states
509
Michael Walshb5839d02017-04-12 16:11:20 -0500510 gp.qprint_timen("Getting system state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600511 if test_mode:
512 state['epoch_seconds'] = int(time.time())
513 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500514 state = st.get_state(req_states=req_states, quiet=quiet)
515 gp.qprint_var(state)
Michael Walsh341c21e2017-01-17 16:25:20 -0600516
Michael Walsh341c21e2017-01-17 16:25:20 -0600517
Michael Walsh45ca6e42017-09-14 17:29:12 -0500518def valid_state():
Michael Walsh45ca6e42017-09-14 17:29:12 -0500519 r"""
520 Verify that our state dictionary contains no blank values. If we don't get
521 valid state data, we cannot continue to work.
522 """
523
524 if st.compare_states(state, st.invalid_state_match, 'or'):
525 error_message = "The state dictionary contains blank fields which" +\
526 " is illegal.\n" + gp.sprint_var(state)
527 BuiltIn().fail(gp.sprint_error(error_message))
528
Michael Walsh45ca6e42017-09-14 17:29:12 -0500529
Michael Walsh6741f742017-02-20 16:16:38 -0600530def select_boot():
Michael Walsh341c21e2017-01-17 16:25:20 -0600531 r"""
532 Select a boot test to be run based on our current state and return the
533 chosen boot type.
534
535 Description of arguments:
Michael Walsh6741f742017-02-20 16:16:38 -0600536 state The state of the machine.
Michael Walsh341c21e2017-01-17 16:25:20 -0600537 """
538
Michael Walsh81816742017-09-27 11:02:29 -0500539 global transitional_boot_selected
Michael Walsh30dadae2017-02-27 14:25:52 -0600540 global boot_stack
541
Michael Walshb5839d02017-04-12 16:11:20 -0500542 gp.qprint_timen("Selecting a boot test.")
Michael Walsh6741f742017-02-20 16:16:38 -0600543
Michael Walsh81816742017-09-27 11:02:29 -0500544 if transitional_boot_selected and not boot_success:
545 prior_boot = next_boot
546 boot_candidate = boot_stack.pop()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500547 gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
548 + " transition to a valid state for '" + boot_candidate
549 + "' which was at the top of the boot_stack. Since"
550 + " the '" + next_boot + "' failed, the '"
551 + boot_candidate + "' has been removed from the stack"
552 + " to avoid and endless failure loop.")
Michael Walsh81816742017-09-27 11:02:29 -0500553 if len(boot_stack) == 0:
554 return ""
555
Michael Walsh6741f742017-02-20 16:16:38 -0600556 my_get_state()
Michael Walsh45ca6e42017-09-14 17:29:12 -0500557 valid_state()
Michael Walsh6741f742017-02-20 16:16:38 -0600558
Michael Walsh81816742017-09-27 11:02:29 -0500559 transitional_boot_selected = False
Michael Walsh6741f742017-02-20 16:16:38 -0600560 stack_popped = 0
561 if len(boot_stack) > 0:
562 stack_popped = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500563 gp.qprint_dashes()
564 gp.qprint_var(boot_stack)
565 gp.qprint_dashes()
566 skip_boot_printed = 0
567 while len(boot_stack) > 0:
568 boot_candidate = boot_stack.pop()
569 if stack_mode == 'normal':
570 break
571 else:
572 if st.compare_states(state, boot_table[boot_candidate]['end']):
573 if not skip_boot_printed:
Michael Walshff340002017-08-29 11:18:27 -0500574 gp.qprint_var(stack_mode)
575 gp.qprintn()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500576 gp.qprint_timen("Skipping the following boot tests"
577 + " which are unnecessary since their"
578 + " required end states match the"
579 + " current machine state:")
Michael Walshb5839d02017-04-12 16:11:20 -0500580 skip_boot_printed = 1
Michael Walshff340002017-08-29 11:18:27 -0500581 gp.qprint_var(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500582 boot_candidate = ""
583 if boot_candidate == "":
584 gp.qprint_dashes()
585 gp.qprint_var(boot_stack)
586 gp.qprint_dashes()
587 return boot_candidate
Michael Walsh6741f742017-02-20 16:16:38 -0600588 if st.compare_states(state, boot_table[boot_candidate]['start']):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500589 gp.qprint_timen("The machine state is valid for a '"
590 + boot_candidate + "' boot test.")
Michael Walshb5839d02017-04-12 16:11:20 -0500591 gp.qprint_dashes()
592 gp.qprint_var(boot_stack)
593 gp.qprint_dashes()
Michael Walsh6741f742017-02-20 16:16:38 -0600594 return boot_candidate
Michael Walsh341c21e2017-01-17 16:25:20 -0600595 else:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500596 gp.qprint_timen("The machine state does not match the required"
597 + " starting state for a '" + boot_candidate
598 + "' boot test:")
Michael Walshff340002017-08-29 11:18:27 -0500599 gp.qprint_varx("boot_table[" + boot_candidate + "][start]",
600 boot_table[boot_candidate]['start'], 1)
Michael Walsh6741f742017-02-20 16:16:38 -0600601 boot_stack.append(boot_candidate)
Michael Walsh81816742017-09-27 11:02:29 -0500602 transitional_boot_selected = True
Michael Walsh6741f742017-02-20 16:16:38 -0600603 popped_boot = boot_candidate
604
605 # Loop through your list selecting a boot_candidates
606 boot_candidates = []
607 for boot_candidate in boot_list:
608 if st.compare_states(state, boot_table[boot_candidate]['start']):
609 if stack_popped:
610 if st.compare_states(boot_table[boot_candidate]['end'],
Gunnar Mills096cd562018-03-26 10:19:12 -0500611 boot_table[popped_boot]['start']):
Michael Walsh6741f742017-02-20 16:16:38 -0600612 boot_candidates.append(boot_candidate)
613 else:
614 boot_candidates.append(boot_candidate)
615
616 if len(boot_candidates) == 0:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500617 gp.qprint_timen("The user's boot list contained no boot tests"
618 + " which are valid for the current machine state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600619 boot_candidate = default_power_on
620 if not st.compare_states(state, boot_table[default_power_on]['start']):
621 boot_candidate = default_power_off
622 boot_candidates.append(boot_candidate)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500623 gp.qprint_timen("Using default '" + boot_candidate
624 + "' boot type to transition to valid state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600625
Michael Walshb5839d02017-04-12 16:11:20 -0500626 gp.dprint_var(boot_candidates)
Michael Walsh6741f742017-02-20 16:16:38 -0600627
628 # Randomly select a boot from the candidate list.
629 boot = random.choice(boot_candidates)
Michael Walsh341c21e2017-01-17 16:25:20 -0600630
631 return boot
Michael Walsh0bbd8602016-11-22 11:31:49 -0600632
Michael Walsh55302292017-01-10 11:43:02 -0600633
Michael Walsh341c21e2017-01-17 16:25:20 -0600634def print_last_boots():
Michael Walsh341c21e2017-01-17 16:25:20 -0600635 r"""
636 Print the last ten boots done with their time stamps.
637 """
638
639 # indent 0, 90 chars wide, linefeed, char is "="
Michael Walshb5839d02017-04-12 16:11:20 -0500640 gp.qprint_dashes(0, 90)
641 gp.qprintn("Last 10 boots:\n")
Michael Walsh341c21e2017-01-17 16:25:20 -0600642
643 for boot_entry in last_ten:
Michael Walshc108e422019-03-28 12:27:18 -0500644 gp.qprint(boot_entry)
Michael Walshb5839d02017-04-12 16:11:20 -0500645 gp.qprint_dashes(0, 90)
Michael Walsh341c21e2017-01-17 16:25:20 -0600646
Michael Walsh341c21e2017-01-17 16:25:20 -0600647
Michael Walshb2e53ec2017-10-30 15:04:36 -0500648def print_defect_report(ffdc_file_list):
Michael Walsh341c21e2017-01-17 16:25:20 -0600649 r"""
650 Print a defect report.
Michael Walshb2e53ec2017-10-30 15:04:36 -0500651
652 Description of argument(s):
653 ffdc_file_list A list of files which were collected by our ffdc functions.
Michael Walsh341c21e2017-01-17 16:25:20 -0600654 """
655
Michael Walsh600876d2017-05-30 17:58:58 -0500656 # Making deliberate choice to NOT run plug_in_setup(). We don't want
657 # ffdc_prefix updated.
658 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
659 call_point='ffdc_report', stop_on_plug_in_failure=0)
660
Michael Walshe0cf8d72017-05-17 13:20:46 -0500661 # Get additional header data which may have been created by ffdc plug-ins.
662 # Also, delete the individual header files to cleanup.
663 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
664 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
665 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
666 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
667 show_err=0)
668
Michael Walshb2e53ec2017-10-30 15:04:36 -0500669 # Get additional summary data which may have been created by ffdc plug-ins.
Michael Walsh600876d2017-05-30 17:58:58 -0500670 # Also, delete the individual header files to cleanup.
671 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
672 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
673 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
674 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
675 show_err=0)
676
Michael Walshb2e53ec2017-10-30 15:04:36 -0500677 # ffdc_list_file_path contains a list of any ffdc files created by plug-
678 # ins, etc. Read that data into a list.
Michael Walsh341c21e2017-01-17 16:25:20 -0600679 try:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500680 plug_in_ffdc_list = \
681 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
George Keishing36efbc02018-12-12 10:18:23 -0600682 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
Michael Walsh341c21e2017-01-17 16:25:20 -0600683 except IOError:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500684 plug_in_ffdc_list = []
685
686 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
687 # in. Eliminate duplicates and sort the list.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500688 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
Michael Walshb2e53ec2017-10-30 15:04:36 -0500689
690 if status_file_path != "":
691 ffdc_file_list.insert(0, status_file_path)
692
693 # Convert the list to a printable list.
694 printable_ffdc_file_list = "\n".join(ffdc_file_list)
Michael Walsh341c21e2017-01-17 16:25:20 -0600695
Michael Walsh68a61162017-04-25 11:54:06 -0500696 # Open ffdc_file_list for writing. We will write a complete list of
697 # FFDC files to it for possible use by plug-ins like cp_stop_check.
698 ffdc_list_file = open(ffdc_list_file_path, 'w')
Michael Walshb2e53ec2017-10-30 15:04:36 -0500699 ffdc_list_file.write(printable_ffdc_file_list + "\n")
700 ffdc_list_file.close()
701
702 indent = 0
703 width = 90
704 linefeed = 1
705 char = "="
Michael Walsh68a61162017-04-25 11:54:06 -0500706
707 gp.qprintn()
Michael Walshb2e53ec2017-10-30 15:04:36 -0500708 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500709 gp.qprintn("Copy this data to the defect:\n")
710
Michael Walshe0cf8d72017-05-17 13:20:46 -0500711 if len(more_header_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500712 gp.qprintn(more_header_info)
Michael Walshdc80d672017-05-09 12:58:32 -0500713 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
714 openbmc_host_name, openbmc_ip, openbmc_username,
Michael Walsh0a3bdb42019-01-31 16:21:44 +0000715 openbmc_password, rest_username, rest_password, ipmi_username,
716 ipmi_password, os_host, os_host_name, os_ip, os_username,
Michael Walshdc80d672017-05-09 12:58:32 -0500717 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
718 pdu_password, pdu_slot_no, openbmc_serial_host,
719 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
Michael Walsh68a61162017-04-25 11:54:06 -0500720
721 gp.qprintn()
Michael Walsh68a61162017-04-25 11:54:06 -0500722 print_last_boots()
723 gp.qprintn()
724 gp.qprint_var(state)
Michael Walshb5839d02017-04-12 16:11:20 -0500725 gp.qprintn()
726 gp.qprintn("FFDC data files:")
Michael Walshb2e53ec2017-10-30 15:04:36 -0500727 gp.qprintn(printable_ffdc_file_list)
Michael Walshb5839d02017-04-12 16:11:20 -0500728 gp.qprintn()
Michael Walsh341c21e2017-01-17 16:25:20 -0600729
Michael Walsh600876d2017-05-30 17:58:58 -0500730 if len(ffdc_summary_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500731 gp.qprintn(ffdc_summary_info)
Michael Walsh600876d2017-05-30 17:58:58 -0500732
Michael Walshb2e53ec2017-10-30 15:04:36 -0500733 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500734
Michael Walsh6741f742017-02-20 16:16:38 -0600735
Michael Walsh6741f742017-02-20 16:16:38 -0600736def my_ffdc():
Michael Walsh6741f742017-02-20 16:16:38 -0600737 r"""
738 Collect FFDC data.
739 """
740
741 global state
742
743 plug_in_setup()
744 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500745 call_point='ffdc', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600746
747 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500748 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix="
749 + AUTOBOOT_FFDC_PREFIX
750 + " ffdc_function_list="
751 + ffdc_function_list, ignore=1)
Michael Walsh83f4bc72017-04-20 16:49:43 -0500752 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500753 gp.qprint_error("Call to ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500754 if type(ffdc_file_list) is not list:
755 ffdc_file_list = []
756 # Leave a record for caller that "soft" errors occurred.
757 soft_errors = 1
758 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600759
760 my_get_state()
761
Michael Walshb2e53ec2017-10-30 15:04:36 -0500762 print_defect_report(ffdc_file_list)
Michael Walsh6741f742017-02-20 16:16:38 -0600763
Michael Walsh6741f742017-02-20 16:16:38 -0600764
Michael Walsh6741f742017-02-20 16:16:38 -0600765def print_test_start_message(boot_keyword):
Michael Walsh6741f742017-02-20 16:16:38 -0600766 r"""
767 Print a message indicating what boot test is about to run.
768
769 Description of arguments:
770 boot_keyword The name of the boot which is to be run
771 (e.g. "BMC Power On").
772 """
773
774 global last_ten
Sunil M325eb542017-08-10 07:09:43 -0500775 global boot_start_time
Michael Walsh6741f742017-02-20 16:16:38 -0600776
777 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
Sunil M325eb542017-08-10 07:09:43 -0500778
779 # Set boot_start_time for use by plug-ins.
780 boot_start_time = doing_msg[1:33]
781 gp.qprint_var(boot_start_time)
782
Michael Walshb5839d02017-04-12 16:11:20 -0500783 gp.qprint(doing_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600784
785 last_ten.append(doing_msg)
786
Michael Walsh815b1d52018-10-30 13:32:26 -0500787 # Trim list to max number of entries.
788 del last_ten[:max(0, len(last_ten) - max_boot_history)]
Michael Walsh6741f742017-02-20 16:16:38 -0600789
Michael Walsh6741f742017-02-20 16:16:38 -0600790
Michael Walshf566fb12019-02-01 14:35:09 -0600791def stop_boot_test(signal_number=0,
792 frame=None):
793 r"""
794 Handle SIGUSR1 by aborting the boot test that is running.
795
796 Description of argument(s):
797 signal_number The signal number (should always be 10 for SIGUSR1).
798 frame The frame data.
799 """
800
801 gp.printn()
802 gp.print_executing()
803 gp.lprint_executing()
804
805 # Restore original sigusr1 handler.
806 set_default_siguser1()
807
808 message = "The caller has asked that the boot test be stopped and marked"
809 message += " as a failure."
810
811 function_stack = gm.get_function_stack()
812 if "wait_state" in function_stack:
813 st.set_wait_early_exit_message(message)
814 else:
815 BuiltIn().fail(gp.sprint_error(message))
816
817
Michael Walsh6741f742017-02-20 16:16:38 -0600818def run_boot(boot):
Michael Walsh6741f742017-02-20 16:16:38 -0600819 r"""
820 Run the specified boot.
821
822 Description of arguments:
823 boot The name of the boot test to be performed.
824 """
825
826 global state
827
Michael Walshf566fb12019-02-01 14:35:09 -0600828 signal.signal(signal.SIGUSR1, stop_boot_test)
829 gp.qprint_timen("stop_boot_test is armed.")
830
Michael Walsh6741f742017-02-20 16:16:38 -0600831 print_test_start_message(boot)
832
833 plug_in_setup()
834 rc, shell_rc, failed_plug_in_name = \
835 grpi.rprocess_plug_in_packages(call_point="pre_boot")
836 if rc != 0:
837 error_message = "Plug-in failed with non-zero return code.\n" +\
838 gp.sprint_var(rc, 1)
Michael Walshf566fb12019-02-01 14:35:09 -0600839 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600840 BuiltIn().fail(gp.sprint_error(error_message))
841
842 if test_mode:
843 # In test mode, we'll pretend the boot worked by assigning its
844 # required end state to the default state value.
Michael Walsh30dadae2017-02-27 14:25:52 -0600845 state = st.strip_anchor_state(boot_table[boot]['end'])
Michael Walsh6741f742017-02-20 16:16:38 -0600846 else:
847 # Assertion: We trust that the state data was made fresh by the
848 # caller.
849
Michael Walshb5839d02017-04-12 16:11:20 -0500850 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600851
852 if boot_table[boot]['method_type'] == "keyword":
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600853 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
Michael Walshb5839d02017-04-12 16:11:20 -0500854 boot_table[boot]['method'],
855 quiet=quiet)
Michael Walsh6741f742017-02-20 16:16:38 -0600856
857 if boot_table[boot]['bmc_reboot']:
858 st.wait_for_comm_cycle(int(state['epoch_seconds']))
Michael Walsh30dadae2017-02-27 14:25:52 -0600859 plug_in_setup()
860 rc, shell_rc, failed_plug_in_name = \
861 grpi.rprocess_plug_in_packages(call_point="post_reboot")
862 if rc != 0:
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600863 error_message = "Plug-in failed with non-zero return code.\n"
864 error_message += gp.sprint_var(rc, 1)
Michael Walshf566fb12019-02-01 14:35:09 -0600865 set_default_siguser1()
Michael Walsh30dadae2017-02-27 14:25:52 -0600866 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600867 else:
868 match_state = st.anchor_state(state)
869 del match_state['epoch_seconds']
870 # Wait for the state to change in any way.
871 st.wait_state(match_state, wait_time=state_change_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500872 interval="10 seconds", invert=1)
Michael Walsh6741f742017-02-20 16:16:38 -0600873
Michael Walshb5839d02017-04-12 16:11:20 -0500874 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600875 if boot_table[boot]['end']['chassis'] == "Off":
876 boot_timeout = power_off_timeout
877 else:
878 boot_timeout = power_on_timeout
879 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500880 interval="10 seconds")
Michael Walsh6741f742017-02-20 16:16:38 -0600881
882 plug_in_setup()
883 rc, shell_rc, failed_plug_in_name = \
884 grpi.rprocess_plug_in_packages(call_point="post_boot")
885 if rc != 0:
886 error_message = "Plug-in failed with non-zero return code.\n" +\
887 gp.sprint_var(rc, 1)
Michael Walshf566fb12019-02-01 14:35:09 -0600888 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600889 BuiltIn().fail(gp.sprint_error(error_message))
890
Michael Walshf566fb12019-02-01 14:35:09 -0600891 # Restore original sigusr1 handler.
892 set_default_siguser1()
893
Michael Walsh6741f742017-02-20 16:16:38 -0600894
Michael Walsh6741f742017-02-20 16:16:38 -0600895def test_loop_body():
Michael Walsh6741f742017-02-20 16:16:38 -0600896 r"""
897 The main loop body for the loop in main_py.
898
899 Description of arguments:
900 boot_count The iteration number (starts at 1).
901 """
902
903 global boot_count
904 global state
905 global next_boot
906 global boot_success
Sunil M325eb542017-08-10 07:09:43 -0500907 global boot_end_time
Michael Walsh6741f742017-02-20 16:16:38 -0600908
Michael Walshb5839d02017-04-12 16:11:20 -0500909 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600910
911 next_boot = select_boot()
Michael Walshb5839d02017-04-12 16:11:20 -0500912 if next_boot == "":
913 return True
Michael Walsh6741f742017-02-20 16:16:38 -0600914
Michael Walshb5839d02017-04-12 16:11:20 -0500915 boot_count += 1
916 gp.qprint_timen("Starting boot " + str(boot_count) + ".")
Michael Walsh6741f742017-02-20 16:16:38 -0600917
Michael Walshe0cf8d72017-05-17 13:20:46 -0500918 pre_boot_plug_in_setup()
Michael Walsh6741f742017-02-20 16:16:38 -0600919
920 cmd_buf = ["run_boot", next_boot]
921 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
922 if boot_status == "FAIL":
Michael Walshb5839d02017-04-12 16:11:20 -0500923 gp.qprint(msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600924
Michael Walshb5839d02017-04-12 16:11:20 -0500925 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600926 if boot_status == "PASS":
927 boot_success = 1
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500928 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
929 + "\" succeeded.")
Michael Walsh6741f742017-02-20 16:16:38 -0600930 else:
931 boot_success = 0
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500932 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
933 + "\" failed.")
Sunil M325eb542017-08-10 07:09:43 -0500934
935 # Set boot_end_time for use by plug-ins.
936 boot_end_time = completion_msg[1:33]
937 gp.qprint_var(boot_end_time)
938
939 gp.qprint(completion_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600940
941 boot_results.update(next_boot, boot_status)
942
943 plug_in_setup()
944 # NOTE: A post_test_case call point failure is NOT counted as a boot
945 # failure.
946 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500947 call_point='post_test_case', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600948
949 plug_in_setup()
950 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500951 call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
Michael Walsh6741f742017-02-20 16:16:38 -0600952 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
Michael Walsh12059e22019-03-21 11:03:45 -0500953 if ffdc_check == "All" or\
Michael Walsh89de14a2018-10-01 16:51:37 -0500954 shell_rc == dump_ffdc_rc():
Michael Walsh83f4bc72017-04-20 16:49:43 -0500955 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
956 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500957 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -0500958 # Leave a record for caller that "soft" errors occurred.
959 soft_errors = 1
960 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh6741f742017-02-20 16:16:38 -0600961
Michael Walshaabef1e2017-09-20 15:16:17 -0500962 if delete_errlogs:
963 # We need to purge error logs between boots or they build up.
964 grk.run_key("Delete Error logs", ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -0500965
Michael Walsh952f9b02017-03-09 13:11:14 -0600966 boot_results.print_report()
Michael Walshb5839d02017-04-12 16:11:20 -0500967 gp.qprint_timen("Finished boot " + str(boot_count) + ".")
Michael Walsh952f9b02017-03-09 13:11:14 -0600968
Michael Walsh6741f742017-02-20 16:16:38 -0600969 plug_in_setup()
970 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500971 call_point='stop_check', shell_rc=stop_test_rc(),
972 stop_on_non_zero_rc=1)
973 if shell_rc == stop_test_rc():
Michael Walsh3ba8ecd2018-04-24 11:33:25 -0500974 message = "Stopping as requested by user.\n"
975 gp.print_time(message)
976 BuiltIn().fail(message)
Michael Walsh6741f742017-02-20 16:16:38 -0600977
Michael Walshd139f282017-04-04 18:00:23 -0500978 # This should help prevent ConnectionErrors.
Michael Walsh0960b382017-06-22 16:23:37 -0500979 grk.run_key_u("Close All Connections")
Michael Walshd139f282017-04-04 18:00:23 -0500980
Michael Walsh6741f742017-02-20 16:16:38 -0600981 return True
982
Michael Walsh6741f742017-02-20 16:16:38 -0600983
Michael Walsh83f4bc72017-04-20 16:49:43 -0500984def obmc_boot_test_teardown():
Michael Walsh6741f742017-02-20 16:16:38 -0600985 r"""
Michael Walshc9116812017-03-10 14:23:06 -0600986 Clean up after the Main keyword.
Michael Walsh6741f742017-02-20 16:16:38 -0600987 """
988
989 if cp_setup_called:
990 plug_in_setup()
991 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500992 call_point='cleanup', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600993
Michael Walsh600876d2017-05-30 17:58:58 -0500994 if 'boot_results_file_path' in globals():
Michael Walsh6c645742018-08-17 15:02:17 -0500995 # Save boot_results and last_ten objects to a file in case they are
996 # needed again.
Michael Walsh600876d2017-05-30 17:58:58 -0500997 gp.qprint_timen("Saving boot_results to the following path.")
998 gp.qprint_var(boot_results_file_path)
Michael Walsh6c645742018-08-17 15:02:17 -0500999 pickle.dump((boot_results, last_ten),
1000 open(boot_results_file_path, 'wb'),
Michael Walsh600876d2017-05-30 17:58:58 -05001001 pickle.HIGHEST_PROTOCOL)
Michael Walsh0b93fbf2017-03-02 14:42:41 -06001002
Michael Walshff340002017-08-29 11:18:27 -05001003 global save_stack
1004 # Restore any global values saved on the save_stack.
1005 for parm_name in main_func_parm_list:
1006 # Get the parm_value if it was saved on the stack.
1007 try:
1008 parm_value = save_stack.pop(parm_name)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -05001009 except BaseException:
Michael Walshff340002017-08-29 11:18:27 -05001010 # If it was not saved, no further action is required.
1011 continue
1012
1013 # Restore the saved value.
1014 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1015 "}\", parm_value)"
1016 gp.dpissuing(cmd_buf)
1017 exec(cmd_buf)
1018
1019 gp.dprintn(save_stack.sprint_obj())
1020
Michael Walsh6741f742017-02-20 16:16:38 -06001021
Michael Walshc9116812017-03-10 14:23:06 -06001022def test_teardown():
Michael Walshc9116812017-03-10 14:23:06 -06001023 r"""
1024 Clean up after this test case.
1025 """
1026
1027 gp.qprintn()
1028 cmd_buf = ["Print Error",
1029 "A keyword timeout occurred ending this program.\n"]
1030 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1031
Michael Walshc108e422019-03-28 12:27:18 -05001032 gp.qprint_pgm_footer()
Michael Walshb5839d02017-04-12 16:11:20 -05001033
Michael Walshc9116812017-03-10 14:23:06 -06001034
Michael Walsh89de14a2018-10-01 16:51:37 -05001035def post_stack():
1036 r"""
1037 Process post_stack plug-in programs.
1038 """
1039
1040 if not call_post_stack_plug:
1041 # The caller does not wish to have post_stack plug-in processing done.
1042 return
1043
1044 global boot_success
1045
1046 # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1047 pre_boot_plug_in_setup()
1048 # For the purposes of the following plug-ins, mark the "boot" as a success.
1049 boot_success = 1
1050 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001051 rc, shell_rc, failed_plug_in_name, history =\
1052 grpi.rprocess_plug_in_packages(call_point='post_stack',
1053 stop_on_plug_in_failure=0,
1054 return_history=True)
1055 last_ten.extend(history)
1056 # Trim list to max number of entries.
1057 del last_ten[:max(0, len(last_ten) - max_boot_history)]
1058 if rc != 0:
1059 boot_success = 0
Michael Walsh89de14a2018-10-01 16:51:37 -05001060
1061 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001062 rc, shell_rc, failed_plug_in_name =\
1063 grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1064 shell_rc=dump_ffdc_rc(),
1065 stop_on_plug_in_failure=1,
1066 stop_on_non_zero_rc=1)
1067 if shell_rc == dump_ffdc_rc():
Michael Walsh89de14a2018-10-01 16:51:37 -05001068 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1069 if status != 'PASS':
1070 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walshc9bd2e82019-04-18 11:06:52 -05001071 # Leave a record for caller that "soft" errors occurred.
1072 soft_errors = 1
1073 gpu.save_plug_in_value(soft_errors, pgm_name)
Michael Walsh89de14a2018-10-01 16:51:37 -05001074
1075 plug_in_setup()
1076 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1077 call_point='stop_check', shell_rc=stop_test_rc(),
1078 stop_on_non_zero_rc=1)
1079 if shell_rc == stop_test_rc():
1080 message = "Stopping as requested by user.\n"
1081 gp.print_time(message)
1082 BuiltIn().fail(message)
1083
1084
Michael Walshff340002017-08-29 11:18:27 -05001085def obmc_boot_test_py(loc_boot_stack=None,
1086 loc_stack_mode=None,
1087 loc_quiet=None):
Michael Walsh6741f742017-02-20 16:16:38 -06001088 r"""
1089 Do main program processing.
1090 """
1091
Michael Walshff340002017-08-29 11:18:27 -05001092 global save_stack
1093
George Keishing36efbc02018-12-12 10:18:23 -06001094 gp.dprintn()
Michael Walshff340002017-08-29 11:18:27 -05001095 # Process function parms.
1096 for parm_name in main_func_parm_list:
1097 # Get parm's value.
George Keishing36efbc02018-12-12 10:18:23 -06001098 parm_value = eval("loc_" + parm_name)
1099 gp.dpvars(parm_name, parm_value)
Michael Walshff340002017-08-29 11:18:27 -05001100
George Keishing36efbc02018-12-12 10:18:23 -06001101 if parm_value is not None:
Michael Walshff340002017-08-29 11:18:27 -05001102 # Save the global value on a stack.
1103 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1104 parm_name + "}\"), \"" + parm_name + "\")"
1105 gp.dpissuing(cmd_buf)
1106 exec(cmd_buf)
1107
1108 # Set the global value to the passed value.
1109 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1110 "}\", loc_" + parm_name + ")"
1111 gp.dpissuing(cmd_buf)
1112 exec(cmd_buf)
1113
1114 gp.dprintn(save_stack.sprint_obj())
Michael Walshb5839d02017-04-12 16:11:20 -05001115
Michael Walsh6741f742017-02-20 16:16:38 -06001116 setup()
1117
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001118 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1119
Michael Walsha20da402017-03-31 16:27:45 -05001120 if ffdc_only:
1121 gp.qprint_timen("Caller requested ffdc_only.")
Michael Walshe0cf8d72017-05-17 13:20:46 -05001122 pre_boot_plug_in_setup()
Michael Walsh83f4bc72017-04-20 16:49:43 -05001123 grk.run_key_u("my_ffdc")
Michael Walsh764d2f82017-04-27 16:01:08 -05001124 return
Michael Walsha20da402017-03-31 16:27:45 -05001125
Michael Walsh6741f742017-02-20 16:16:38 -06001126 # Process caller's boot_stack.
1127 while (len(boot_stack) > 0):
1128 test_loop_body()
1129
Michael Walshb5839d02017-04-12 16:11:20 -05001130 gp.qprint_timen("Finished processing stack.")
Michael Walsh30dadae2017-02-27 14:25:52 -06001131
Michael Walsh89de14a2018-10-01 16:51:37 -05001132 post_stack()
1133
Michael Walsh6741f742017-02-20 16:16:38 -06001134 # Process caller's boot_list.
1135 if len(boot_list) > 0:
1136 for ix in range(1, max_num_tests + 1):
1137 test_loop_body()
1138
Michael Walshb5839d02017-04-12 16:11:20 -05001139 gp.qprint_timen("Completed all requested boot tests.")
1140
1141 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001142 new_fail = boot_fail - init_boot_fail
1143 if new_fail > boot_fail_threshold:
Michael Walshb5839d02017-04-12 16:11:20 -05001144 error_message = "Boot failures exceed the boot failure" +\
1145 " threshold:\n" +\
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001146 gp.sprint_var(new_fail) +\
Michael Walshb5839d02017-04-12 16:11:20 -05001147 gp.sprint_var(boot_fail_threshold)
1148 BuiltIn().fail(gp.sprint_error(error_message))