blob: d7c5e3672ce54b7fe5d5b37a00bcf31f5dd4e918 [file] [log] [blame]
Michael Walsh0bbd8602016-11-22 11:31:49 -06001#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
Michael Walsh0b93fbf2017-03-02 14:42:41 -06007import os
8import imp
9import time
10import glob
11import random
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050012import re
Michael Walshf566fb12019-02-01 14:35:09 -060013import signal
George Keishingd54bbc22018-08-03 08:24:58 -050014try:
15 import cPickle as pickle
16except ImportError:
17 import pickle
Michael Walshdc80d672017-05-09 12:58:32 -050018import socket
Michael Walsh0b93fbf2017-03-02 14:42:41 -060019
20from robot.utils import DotDict
21from robot.libraries.BuiltIn import BuiltIn
22
Michael Walsh6741f742017-02-20 16:16:38 -060023from boot_data import *
Michael Walshc9116812017-03-10 14:23:06 -060024import gen_print as gp
Michael Walsh0bbd8602016-11-22 11:31:49 -060025import gen_robot_print as grp
Michael Walsh55302292017-01-10 11:43:02 -060026import gen_robot_plug_in as grpi
Michael Walsh6741f742017-02-20 16:16:38 -060027import gen_robot_valid as grv
28import gen_misc as gm
29import gen_cmd as gc
Michael Walshb5839d02017-04-12 16:11:20 -050030import gen_robot_keyword as grk
Michael Walsh55302292017-01-10 11:43:02 -060031import state as st
Michael Walshff340002017-08-29 11:18:27 -050032import var_stack as vs
Michael Walsh0bbd8602016-11-22 11:31:49 -060033
Michael Walsh0b93fbf2017-03-02 14:42:41 -060034base_path = os.path.dirname(os.path.dirname(
35 imp.find_module("gen_robot_print")[1])) +\
Michael Walshc9116812017-03-10 14:23:06 -060036 os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060037sys.path.append(base_path + "extended/")
38import run_keyword as rk
Michael Walsh0bbd8602016-11-22 11:31:49 -060039
Michael Walshe1e26442017-03-06 17:50:07 -060040# Setting master_pid correctly influences the behavior of plug-ins like
41# DB_Logging
42program_pid = os.getpid()
43master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -050044pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
Michael Walshe1e26442017-03-06 17:50:07 -060045
Michael Walshb5839d02017-04-12 16:11:20 -050046# Set up boot data structures.
47boot_table = create_boot_table()
48valid_boot_types = create_valid_boot_list(boot_table)
Michael Walsh0b93fbf2017-03-02 14:42:41 -060049
Michael Walsh6741f742017-02-20 16:16:38 -060050boot_lists = read_boot_lists()
Michael Walsh815b1d52018-10-30 13:32:26 -050051# The maximum number of entries that can be in the last_ten global variable.
52max_boot_history = 10
Michael Walsh6741f742017-02-20 16:16:38 -060053last_ten = []
Michael Walsh6741f742017-02-20 16:16:38 -060054
Michael Walsh7dc885b2018-03-14 17:51:59 -050055state = st.return_state_constant('default_state')
Michael Walsh6741f742017-02-20 16:16:38 -060056cp_setup_called = 0
57next_boot = ""
58base_tool_dir_path = os.path.normpath(os.environ.get(
59 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
Michael Walshb5839d02017-04-12 16:11:20 -050060
Michael Walsh6741f742017-02-20 16:16:38 -060061ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
Michael Walsh6741f742017-02-20 16:16:38 -060062boot_success = 0
Michael Walsh6741f742017-02-20 16:16:38 -060063status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
64if status_dir_path != "":
65 status_dir_path = os.path.normpath(status_dir_path) + os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060066default_power_on = "REST Power On"
67default_power_off = "REST Power Off"
Michael Walsh6741f742017-02-20 16:16:38 -060068boot_count = 0
Michael Walsh0bbd8602016-11-22 11:31:49 -060069
Michael Walsh85678942017-03-27 14:34:22 -050070LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
Michael Walshe1974b92017-08-03 13:39:51 -050071ffdc_prefix = ""
Sunil M325eb542017-08-10 07:09:43 -050072boot_start_time = ""
73boot_end_time = ""
Michael Walshff340002017-08-29 11:18:27 -050074save_stack = vs.var_stack('save_stack')
75main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
Michael Walsh85678942017-03-27 14:34:22 -050076
77
Michael Walsh89de14a2018-10-01 16:51:37 -050078def dump_ffdc_rc():
79 r"""
80 Return the constant dump ffdc test return code value.
81
82 When a plug-in call point program returns this value, it indicates that
83 this program should collect FFDC.
84 """
85
86 return 0x00000200
87
88
89def stop_test_rc():
90 r"""
91 Return the constant stop test return code value.
92
93 When a plug-in call point program returns this value, it indicates that
94 this program should stop running.
95 """
96
97 return 0x00000200
98
99
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500100def process_host(host,
101 host_var_name=""):
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500102 r"""
103 Process a host by getting the associated host name and IP address and
104 setting them in global variables.
105
106 If the caller does not pass the host_var_name, this function will try to
107 figure out the name of the variable used by the caller for the host parm.
108 Callers are advised to explicitly specify the host_var_name when calling
109 with an exec command. In such cases, the get_arg_name cannot figure out
110 the host variable name.
111
112 This function will then create similar global variable names by
113 removing "_host" and appending "_host_name" or "_ip" to the host variable
114 name.
115
116 Example:
117
118 If a call is made like this:
119 process_host(openbmc_host)
120
121 Global variables openbmc_host_name and openbmc_ip will be set.
122
123 Description of argument(s):
124 host A host name or IP. The name of the variable used should
125 have a suffix of "_host".
126 host_var_name The name of the variable being used as the host parm.
127 """
128
129 if host_var_name == "":
130 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
131
132 host_name_var_name = re.sub("host", "host_name", host_var_name)
133 ip_var_name = re.sub("host", "ip", host_var_name)
134 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
135 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
136 host + "')"
137 exec(cmd_buf)
138
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500139
Michael Walshb5839d02017-04-12 16:11:20 -0500140def process_pgm_parms():
Michael Walshb5839d02017-04-12 16:11:20 -0500141 r"""
142 Process the program parameters by assigning them all to corresponding
143 globals. Also, set some global values that depend on program parameters.
144 """
145
146 # Program parameter processing.
147 # Assign all program parms to python variables which are global to this
148 # module.
149
150 global parm_list
151 parm_list = BuiltIn().get_variable_value("${parm_list}")
152 # The following subset of parms should be processed as integers.
153 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
Michael Walsh89de14a2018-10-01 16:51:37 -0500154 'boot_fail_threshold', 'delete_errlogs',
155 'call_post_stack_plug', 'quiet', 'test_mode', 'debug']
Michael Walshb5839d02017-04-12 16:11:20 -0500156 for parm in parm_list:
157 if parm in int_list:
158 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
159 "}\", \"0\"))"
160 else:
161 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
162 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
Michael Walshff340002017-08-29 11:18:27 -0500163 gp.dpissuing(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500164 exec(cmd_buf)
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500165 if re.match(r".*_host$", parm):
166 cmd_buf = "process_host(" + parm + ", '" + parm + "')"
167 exec(cmd_buf)
168 if re.match(r".*_password$", parm):
169 # Register the value of any parm whose name ends in _password.
170 # This will cause the print functions to replace passwords with
171 # asterisks in the output.
172 cmd_buf = "gp.register_passwords(" + parm + ")"
173 exec(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500174
175 global ffdc_dir_path_style
176 global boot_list
177 global boot_stack
178 global boot_results_file_path
179 global boot_results
Michael Walsh6c645742018-08-17 15:02:17 -0500180 global last_ten
Michael Walshb5839d02017-04-12 16:11:20 -0500181 global ffdc_list_file_path
Michael Walshe0cf8d72017-05-17 13:20:46 -0500182 global ffdc_report_list_path
Michael Walsh600876d2017-05-30 17:58:58 -0500183 global ffdc_summary_list_path
Michael Walshb5839d02017-04-12 16:11:20 -0500184
185 if ffdc_dir_path_style == "":
186 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
187
188 # Convert these program parms to lists for easier processing..
George Keishing36efbc02018-12-12 10:18:23 -0600189 boot_list = list(filter(None, boot_list.split(":")))
190 boot_stack = list(filter(None, boot_stack.split(":")))
Michael Walshb5839d02017-04-12 16:11:20 -0500191
Michael Walsh903e0b22017-09-19 17:00:33 -0500192 cleanup_boot_results_file()
193 boot_results_file_path = create_boot_results_file_path(pgm_name,
194 openbmc_nickname,
195 master_pid)
Michael Walshb5839d02017-04-12 16:11:20 -0500196
197 if os.path.isfile(boot_results_file_path):
198 # We've been called before in this run so we'll load the saved
Michael Walsh6c645742018-08-17 15:02:17 -0500199 # boot_results and last_ten objects.
200 boot_results, last_ten =\
201 pickle.load(open(boot_results_file_path, 'rb'))
Michael Walshb5839d02017-04-12 16:11:20 -0500202 else:
203 boot_results = boot_results(boot_table, boot_pass, boot_fail)
204
205 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
206 "/FFDC_FILE_LIST"
Michael Walshe0cf8d72017-05-17 13:20:46 -0500207 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
208 "/FFDC_REPORT_FILE_LIST"
Michael Walshb5839d02017-04-12 16:11:20 -0500209
Michael Walsh600876d2017-05-30 17:58:58 -0500210 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
211 "/FFDC_SUMMARY_FILE_LIST"
212
Michael Walshb5839d02017-04-12 16:11:20 -0500213
Michael Walsh85678942017-03-27 14:34:22 -0500214def initial_plug_in_setup():
Michael Walsh85678942017-03-27 14:34:22 -0500215 r"""
216 Initialize all plug-in environment variables which do not change for the
217 duration of the program.
218
219 """
220
221 global LOG_LEVEL
222 BuiltIn().set_log_level("NONE")
223
224 BuiltIn().set_global_variable("${master_pid}", master_pid)
225 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
226 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
227 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
228 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
229 ffdc_list_file_path)
Michael Walshe0cf8d72017-05-17 13:20:46 -0500230 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
231 ffdc_report_list_path)
Michael Walsh600876d2017-05-30 17:58:58 -0500232 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
233 ffdc_summary_list_path)
Michael Walsh85678942017-03-27 14:34:22 -0500234
235 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
236 ffdc_dir_path_style)
237 BuiltIn().set_global_variable("${FFDC_CHECK}",
238 ffdc_check)
239
240 # For each program parameter, set the corresponding AUTOBOOT_ environment
241 # variable value. Also, set an AUTOBOOT_ environment variable for every
242 # element in additional_values.
243 additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
244 "status_dir_path", "base_tool_dir_path",
Michael Walsh600876d2017-05-30 17:58:58 -0500245 "ffdc_list_file_path", "ffdc_report_list_path",
Michael Walsh0a3bdb42019-01-31 16:21:44 +0000246 "ffdc_summary_list_path", "execdir"]
Michael Walsh85678942017-03-27 14:34:22 -0500247
248 plug_in_vars = parm_list + additional_values
249
250 for var_name in plug_in_vars:
251 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
252 var_name = var_name.upper()
253 if var_value is None:
254 var_value = ""
255 os.environ["AUTOBOOT_" + var_name] = str(var_value)
256
257 BuiltIn().set_log_level(LOG_LEVEL)
258
Michael Walsh68a61162017-04-25 11:54:06 -0500259 # Make sure the ffdc list directory exists.
260 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
261 if not os.path.exists(ffdc_list_dir_path):
262 os.makedirs(ffdc_list_dir_path)
Michael Walsh85678942017-03-27 14:34:22 -0500263
Michael Walsh85678942017-03-27 14:34:22 -0500264
Michael Walsh0bbd8602016-11-22 11:31:49 -0600265def plug_in_setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600266 r"""
Michael Walsh85678942017-03-27 14:34:22 -0500267 Initialize all changing plug-in environment variables for use by the
268 plug-in programs.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600269 """
270
Michael Walsh85678942017-03-27 14:34:22 -0500271 global LOG_LEVEL
272 global test_really_running
273
274 BuiltIn().set_log_level("NONE")
275
Michael Walsh6741f742017-02-20 16:16:38 -0600276 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600277 if boot_pass > 1:
278 test_really_running = 1
279 else:
280 test_really_running = 0
281
Michael Walsh6741f742017-02-20 16:16:38 -0600282 BuiltIn().set_global_variable("${test_really_running}",
283 test_really_running)
284 BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
Michael Walsh6741f742017-02-20 16:16:38 -0600285 BuiltIn().set_global_variable("${boot_pass}", boot_pass)
286 BuiltIn().set_global_variable("${boot_fail}", boot_fail)
287 BuiltIn().set_global_variable("${boot_success}", boot_success)
288 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
Sunil M325eb542017-08-10 07:09:43 -0500289 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
290 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
Michael Walsh4c9a6452016-12-13 16:03:11 -0600291
Michael Walsh0bbd8602016-11-22 11:31:49 -0600292 # For each program parameter, set the corresponding AUTOBOOT_ environment
293 # variable value. Also, set an AUTOBOOT_ environment variable for every
294 # element in additional_values.
295 additional_values = ["boot_type_desc", "boot_success", "boot_pass",
Sunil M325eb542017-08-10 07:09:43 -0500296 "boot_fail", "test_really_running", "ffdc_prefix",
297 "boot_start_time", "boot_end_time"]
Michael Walsh0bbd8602016-11-22 11:31:49 -0600298
Michael Walsh85678942017-03-27 14:34:22 -0500299 plug_in_vars = additional_values
Michael Walsh0bbd8602016-11-22 11:31:49 -0600300
301 for var_name in plug_in_vars:
302 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
303 var_name = var_name.upper()
304 if var_value is None:
305 var_value = ""
Michael Walsh6741f742017-02-20 16:16:38 -0600306 os.environ["AUTOBOOT_" + var_name] = str(var_value)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600307
Michael Walsh0bbd8602016-11-22 11:31:49 -0600308 if debug:
Michael Walsh6741f742017-02-20 16:16:38 -0600309 shell_rc, out_buf = \
310 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600311
Michael Walsh85678942017-03-27 14:34:22 -0500312 BuiltIn().set_log_level(LOG_LEVEL)
313
Michael Walsh0bbd8602016-11-22 11:31:49 -0600314
Michael Walshe0cf8d72017-05-17 13:20:46 -0500315def pre_boot_plug_in_setup():
316
317 # Clear the ffdc_list_file_path file. Plug-ins may now write to it.
318 try:
319 os.remove(ffdc_list_file_path)
320 except OSError:
321 pass
322
323 # Clear the ffdc_report_list_path file. Plug-ins may now write to it.
324 try:
325 os.remove(ffdc_report_list_path)
326 except OSError:
327 pass
328
Michael Walsh600876d2017-05-30 17:58:58 -0500329 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it.
330 try:
331 os.remove(ffdc_summary_list_path)
332 except OSError:
333 pass
334
Michael Walshe1974b92017-08-03 13:39:51 -0500335 global ffdc_prefix
336
337 seconds = time.time()
338 loc_time = time.localtime(seconds)
339 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
340
341 ffdc_prefix = openbmc_nickname + "." + time_string
342
Michael Walshe0cf8d72017-05-17 13:20:46 -0500343
Michael Walshf566fb12019-02-01 14:35:09 -0600344def default_sigusr1(signal_number=0,
345 frame=None):
346 r"""
347 Handle SIGUSR1 by doing nothing.
348
349 This function assists in debugging SIGUSR1 processing by printing messages
350 to stdout and to the log.html file.
351
352 Description of argument(s):
353 signal_number The signal number (should always be 10 for SIGUSR1).
354 frame The frame data.
355 """
356
357 gp.printn()
358 gp.print_executing()
359 gp.lprint_executing()
360
361
362def set_default_siguser1():
363 r"""
364 Set the default_sigusr1 function to be the SIGUSR1 handler.
365 """
366
367 gp.printn()
368 gp.print_executing()
369 gp.lprint_executing()
370 signal.signal(signal.SIGUSR1, default_sigusr1)
371
372
Michael Walsh6741f742017-02-20 16:16:38 -0600373def setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600374 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600375 Do general program setup tasks.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600376 """
377
Michael Walsh6741f742017-02-20 16:16:38 -0600378 global cp_setup_called
Michael Walsh81816742017-09-27 11:02:29 -0500379 global transitional_boot_selected
Michael Walsh0bbd8602016-11-22 11:31:49 -0600380
Michael Walshb5839d02017-04-12 16:11:20 -0500381 gp.qprintn()
382
Michael Walshf566fb12019-02-01 14:35:09 -0600383 set_default_siguser1()
Michael Walsh81816742017-09-27 11:02:29 -0500384 transitional_boot_selected = False
385
Michael Walsh83f4bc72017-04-20 16:49:43 -0500386 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
387 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
Michael Walshd061c042017-05-23 14:46:57 -0500388 # If we can't find process_plug_in_packages.py, ssh_pw or
389 # validate_plug_ins.py, then we don't have our repo bin in PATH.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500390 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
391 + " ssh_pw validate_plug_ins.py", quiet=1,
Michael Walshd061c042017-05-23 14:46:57 -0500392 print_output=0, show_err=0)
Michael Walshb5839d02017-04-12 16:11:20 -0500393 if shell_rc != 0:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500394 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
395 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
396 if robot_pgm_dir_path not in sys.path:
397 sys.path.append(robot_pgm_dir_path)
398 PYTHONPATH = os.environ.get("PYTHONPATH", "")
399 if PYTHONPATH == "":
400 os.environ['PYTHONPATH'] = robot_pgm_dir_path
401 else:
402 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
Michael Walsh6741f742017-02-20 16:16:38 -0600403
404 validate_parms()
405
406 grp.rqprint_pgm_header()
407
George Keishingefc3ff22017-12-12 11:49:25 -0600408 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF")
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500409
Michael Walsh85678942017-03-27 14:34:22 -0500410 initial_plug_in_setup()
411
Michael Walsh6741f742017-02-20 16:16:38 -0600412 plug_in_setup()
413 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
414 call_point='setup')
415 if rc != 0:
416 error_message = "Plug-in setup failed.\n"
417 grp.rprint_error_report(error_message)
418 BuiltIn().fail(error_message)
419 # Setting cp_setup_called lets our Teardown know that it needs to call
420 # the cleanup plug-in call point.
421 cp_setup_called = 1
422
423 # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
424 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
Michael Walsh85678942017-03-27 14:34:22 -0500425 # FFDC_LOG_PATH is used by "FFDC" keyword.
426 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
Michael Walsh6741f742017-02-20 16:16:38 -0600427
Michael Walshdc80d672017-05-09 12:58:32 -0500428 # Also printed by FFDC.
429 global host_name
430 global host_ip
431 host = socket.gethostname()
432 host_name, host_ip = gm.get_host_name_ip(host)
433
Michael Walshb5839d02017-04-12 16:11:20 -0500434 gp.dprint_var(boot_table, 1)
435 gp.dprint_var(boot_lists)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600436
Michael Walsh0bbd8602016-11-22 11:31:49 -0600437
Michael Walsh6741f742017-02-20 16:16:38 -0600438def validate_parms():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600439 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600440 Validate all program parameters.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600441 """
442
Michael Walshb5839d02017-04-12 16:11:20 -0500443 process_pgm_parms()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600444
Michael Walshb5839d02017-04-12 16:11:20 -0500445 gp.qprintn()
446
447 global openbmc_model
Michael Walsh6741f742017-02-20 16:16:38 -0600448 grv.rvalid_value("openbmc_host")
449 grv.rvalid_value("openbmc_username")
450 grv.rvalid_value("openbmc_password")
Michael Walsh0a3bdb42019-01-31 16:21:44 +0000451 grv.rvalid_value("rest_username")
452 grv.rvalid_value("rest_password")
453 grv.rvalid_value("ipmi_username")
454 grv.rvalid_value("ipmi_password")
Michael Walsh6741f742017-02-20 16:16:38 -0600455 if os_host != "":
456 grv.rvalid_value("os_username")
457 grv.rvalid_value("os_password")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600458
Michael Walsh6741f742017-02-20 16:16:38 -0600459 if pdu_host != "":
460 grv.rvalid_value("pdu_username")
461 grv.rvalid_value("pdu_password")
Michael Walsh85678942017-03-27 14:34:22 -0500462 grv.rvalid_integer("pdu_slot_no")
Michael Walsh6741f742017-02-20 16:16:38 -0600463 if openbmc_serial_host != "":
464 grv.rvalid_integer("openbmc_serial_port")
Michael Walshb5839d02017-04-12 16:11:20 -0500465 if openbmc_model == "":
466 status, ret_values =\
467 grk.run_key_u("Get BMC System Model")
468 openbmc_model = ret_values
469 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
Michael Walsh6741f742017-02-20 16:16:38 -0600470 grv.rvalid_value("openbmc_model")
Michael Walshb5839d02017-04-12 16:11:20 -0500471 grv.rvalid_integer("max_num_tests")
Michael Walsh6741f742017-02-20 16:16:38 -0600472 grv.rvalid_integer("boot_pass")
473 grv.rvalid_integer("boot_fail")
474
475 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
476 BuiltIn().set_global_variable("${plug_in_packages_list}",
477 plug_in_packages_list)
478
Michael Walshb5839d02017-04-12 16:11:20 -0500479 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip'])
Michael Walsha20da402017-03-31 16:27:45 -0500480 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
Michael Walsh6741f742017-02-20 16:16:38 -0600481 error_message = "You must provide either a value for either the" +\
482 " boot_list or the boot_stack parm.\n"
483 BuiltIn().fail(gp.sprint_error(error_message))
484
485 valid_boot_list(boot_list, valid_boot_types)
486 valid_boot_list(boot_stack, valid_boot_types)
487
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500488 selected_PDU_boots = list(set(boot_list + boot_stack)
489 & set(boot_lists['PDU_reboot']))
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500490
491 if len(selected_PDU_boots) > 0 and pdu_host == "":
492 error_message = "You have selected the following boots which" +\
493 " require a PDU host but no value for pdu_host:\n"
494 error_message += gp.sprint_var(selected_PDU_boots)
495 error_message += gp.sprint_var(pdu_host, 2)
496 BuiltIn().fail(gp.sprint_error(error_message))
497
Michael Walsh6741f742017-02-20 16:16:38 -0600498 return
Michael Walsh0bbd8602016-11-22 11:31:49 -0600499
Michael Walsh0bbd8602016-11-22 11:31:49 -0600500
Michael Walsh6741f742017-02-20 16:16:38 -0600501def my_get_state():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600502 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600503 Get the system state plus a little bit of wrapping.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600504 """
505
Michael Walsh6741f742017-02-20 16:16:38 -0600506 global state
507
508 req_states = ['epoch_seconds'] + st.default_req_states
509
Michael Walshb5839d02017-04-12 16:11:20 -0500510 gp.qprint_timen("Getting system state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600511 if test_mode:
512 state['epoch_seconds'] = int(time.time())
513 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500514 state = st.get_state(req_states=req_states, quiet=quiet)
515 gp.qprint_var(state)
Michael Walsh341c21e2017-01-17 16:25:20 -0600516
Michael Walsh341c21e2017-01-17 16:25:20 -0600517
Michael Walsh45ca6e42017-09-14 17:29:12 -0500518def valid_state():
Michael Walsh45ca6e42017-09-14 17:29:12 -0500519 r"""
520 Verify that our state dictionary contains no blank values. If we don't get
521 valid state data, we cannot continue to work.
522 """
523
524 if st.compare_states(state, st.invalid_state_match, 'or'):
525 error_message = "The state dictionary contains blank fields which" +\
526 " is illegal.\n" + gp.sprint_var(state)
527 BuiltIn().fail(gp.sprint_error(error_message))
528
Michael Walsh45ca6e42017-09-14 17:29:12 -0500529
Michael Walsh6741f742017-02-20 16:16:38 -0600530def select_boot():
Michael Walsh341c21e2017-01-17 16:25:20 -0600531 r"""
532 Select a boot test to be run based on our current state and return the
533 chosen boot type.
534
535 Description of arguments:
Michael Walsh6741f742017-02-20 16:16:38 -0600536 state The state of the machine.
Michael Walsh341c21e2017-01-17 16:25:20 -0600537 """
538
Michael Walsh81816742017-09-27 11:02:29 -0500539 global transitional_boot_selected
Michael Walsh30dadae2017-02-27 14:25:52 -0600540 global boot_stack
541
Michael Walshb5839d02017-04-12 16:11:20 -0500542 gp.qprint_timen("Selecting a boot test.")
Michael Walsh6741f742017-02-20 16:16:38 -0600543
Michael Walsh81816742017-09-27 11:02:29 -0500544 if transitional_boot_selected and not boot_success:
545 prior_boot = next_boot
546 boot_candidate = boot_stack.pop()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500547 gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
548 + " transition to a valid state for '" + boot_candidate
549 + "' which was at the top of the boot_stack. Since"
550 + " the '" + next_boot + "' failed, the '"
551 + boot_candidate + "' has been removed from the stack"
552 + " to avoid and endless failure loop.")
Michael Walsh81816742017-09-27 11:02:29 -0500553 if len(boot_stack) == 0:
554 return ""
555
Michael Walsh6741f742017-02-20 16:16:38 -0600556 my_get_state()
Michael Walsh45ca6e42017-09-14 17:29:12 -0500557 valid_state()
Michael Walsh6741f742017-02-20 16:16:38 -0600558
Michael Walsh81816742017-09-27 11:02:29 -0500559 transitional_boot_selected = False
Michael Walsh6741f742017-02-20 16:16:38 -0600560 stack_popped = 0
561 if len(boot_stack) > 0:
562 stack_popped = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500563 gp.qprint_dashes()
564 gp.qprint_var(boot_stack)
565 gp.qprint_dashes()
566 skip_boot_printed = 0
567 while len(boot_stack) > 0:
568 boot_candidate = boot_stack.pop()
569 if stack_mode == 'normal':
570 break
571 else:
572 if st.compare_states(state, boot_table[boot_candidate]['end']):
573 if not skip_boot_printed:
Michael Walshff340002017-08-29 11:18:27 -0500574 gp.qprint_var(stack_mode)
575 gp.qprintn()
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500576 gp.qprint_timen("Skipping the following boot tests"
577 + " which are unnecessary since their"
578 + " required end states match the"
579 + " current machine state:")
Michael Walshb5839d02017-04-12 16:11:20 -0500580 skip_boot_printed = 1
Michael Walshff340002017-08-29 11:18:27 -0500581 gp.qprint_var(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500582 boot_candidate = ""
583 if boot_candidate == "":
584 gp.qprint_dashes()
585 gp.qprint_var(boot_stack)
586 gp.qprint_dashes()
587 return boot_candidate
Michael Walsh6741f742017-02-20 16:16:38 -0600588 if st.compare_states(state, boot_table[boot_candidate]['start']):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500589 gp.qprint_timen("The machine state is valid for a '"
590 + boot_candidate + "' boot test.")
Michael Walshb5839d02017-04-12 16:11:20 -0500591 gp.qprint_dashes()
592 gp.qprint_var(boot_stack)
593 gp.qprint_dashes()
Michael Walsh6741f742017-02-20 16:16:38 -0600594 return boot_candidate
Michael Walsh341c21e2017-01-17 16:25:20 -0600595 else:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500596 gp.qprint_timen("The machine state does not match the required"
597 + " starting state for a '" + boot_candidate
598 + "' boot test:")
Michael Walshff340002017-08-29 11:18:27 -0500599 gp.qprint_varx("boot_table[" + boot_candidate + "][start]",
600 boot_table[boot_candidate]['start'], 1)
Michael Walsh6741f742017-02-20 16:16:38 -0600601 boot_stack.append(boot_candidate)
Michael Walsh81816742017-09-27 11:02:29 -0500602 transitional_boot_selected = True
Michael Walsh6741f742017-02-20 16:16:38 -0600603 popped_boot = boot_candidate
604
605 # Loop through your list selecting a boot_candidates
606 boot_candidates = []
607 for boot_candidate in boot_list:
608 if st.compare_states(state, boot_table[boot_candidate]['start']):
609 if stack_popped:
610 if st.compare_states(boot_table[boot_candidate]['end'],
Gunnar Mills096cd562018-03-26 10:19:12 -0500611 boot_table[popped_boot]['start']):
Michael Walsh6741f742017-02-20 16:16:38 -0600612 boot_candidates.append(boot_candidate)
613 else:
614 boot_candidates.append(boot_candidate)
615
616 if len(boot_candidates) == 0:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500617 gp.qprint_timen("The user's boot list contained no boot tests"
618 + " which are valid for the current machine state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600619 boot_candidate = default_power_on
620 if not st.compare_states(state, boot_table[default_power_on]['start']):
621 boot_candidate = default_power_off
622 boot_candidates.append(boot_candidate)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500623 gp.qprint_timen("Using default '" + boot_candidate
624 + "' boot type to transition to valid state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600625
Michael Walshb5839d02017-04-12 16:11:20 -0500626 gp.dprint_var(boot_candidates)
Michael Walsh6741f742017-02-20 16:16:38 -0600627
628 # Randomly select a boot from the candidate list.
629 boot = random.choice(boot_candidates)
Michael Walsh341c21e2017-01-17 16:25:20 -0600630
631 return boot
Michael Walsh0bbd8602016-11-22 11:31:49 -0600632
Michael Walsh55302292017-01-10 11:43:02 -0600633
Michael Walsh341c21e2017-01-17 16:25:20 -0600634def print_last_boots():
Michael Walsh341c21e2017-01-17 16:25:20 -0600635 r"""
636 Print the last ten boots done with their time stamps.
637 """
638
639 # indent 0, 90 chars wide, linefeed, char is "="
Michael Walshb5839d02017-04-12 16:11:20 -0500640 gp.qprint_dashes(0, 90)
641 gp.qprintn("Last 10 boots:\n")
Michael Walsh341c21e2017-01-17 16:25:20 -0600642
643 for boot_entry in last_ten:
644 grp.rqprint(boot_entry)
Michael Walshb5839d02017-04-12 16:11:20 -0500645 gp.qprint_dashes(0, 90)
Michael Walsh341c21e2017-01-17 16:25:20 -0600646
Michael Walsh341c21e2017-01-17 16:25:20 -0600647
Michael Walshb2e53ec2017-10-30 15:04:36 -0500648def print_defect_report(ffdc_file_list):
Michael Walsh341c21e2017-01-17 16:25:20 -0600649 r"""
650 Print a defect report.
Michael Walshb2e53ec2017-10-30 15:04:36 -0500651
652 Description of argument(s):
653 ffdc_file_list A list of files which were collected by our ffdc functions.
Michael Walsh341c21e2017-01-17 16:25:20 -0600654 """
655
Michael Walsh600876d2017-05-30 17:58:58 -0500656 # Making deliberate choice to NOT run plug_in_setup(). We don't want
657 # ffdc_prefix updated.
658 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
659 call_point='ffdc_report', stop_on_plug_in_failure=0)
660
Michael Walshe0cf8d72017-05-17 13:20:46 -0500661 # Get additional header data which may have been created by ffdc plug-ins.
662 # Also, delete the individual header files to cleanup.
663 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
664 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
665 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
666 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
667 show_err=0)
668
Michael Walshb2e53ec2017-10-30 15:04:36 -0500669 # Get additional summary data which may have been created by ffdc plug-ins.
Michael Walsh600876d2017-05-30 17:58:58 -0500670 # Also, delete the individual header files to cleanup.
671 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
672 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
673 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
674 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
675 show_err=0)
676
Michael Walshb2e53ec2017-10-30 15:04:36 -0500677 # ffdc_list_file_path contains a list of any ffdc files created by plug-
678 # ins, etc. Read that data into a list.
Michael Walsh341c21e2017-01-17 16:25:20 -0600679 try:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500680 plug_in_ffdc_list = \
681 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
George Keishing36efbc02018-12-12 10:18:23 -0600682 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
Michael Walsh341c21e2017-01-17 16:25:20 -0600683 except IOError:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500684 plug_in_ffdc_list = []
685
686 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
687 # in. Eliminate duplicates and sort the list.
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500688 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
Michael Walshb2e53ec2017-10-30 15:04:36 -0500689
690 if status_file_path != "":
691 ffdc_file_list.insert(0, status_file_path)
692
693 # Convert the list to a printable list.
694 printable_ffdc_file_list = "\n".join(ffdc_file_list)
Michael Walsh341c21e2017-01-17 16:25:20 -0600695
Michael Walsh68a61162017-04-25 11:54:06 -0500696 # Open ffdc_file_list for writing. We will write a complete list of
697 # FFDC files to it for possible use by plug-ins like cp_stop_check.
698 ffdc_list_file = open(ffdc_list_file_path, 'w')
Michael Walshb2e53ec2017-10-30 15:04:36 -0500699 ffdc_list_file.write(printable_ffdc_file_list + "\n")
700 ffdc_list_file.close()
701
702 indent = 0
703 width = 90
704 linefeed = 1
705 char = "="
Michael Walsh68a61162017-04-25 11:54:06 -0500706
707 gp.qprintn()
Michael Walshb2e53ec2017-10-30 15:04:36 -0500708 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500709 gp.qprintn("Copy this data to the defect:\n")
710
Michael Walshe0cf8d72017-05-17 13:20:46 -0500711 if len(more_header_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500712 gp.qprintn(more_header_info)
Michael Walshdc80d672017-05-09 12:58:32 -0500713 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
714 openbmc_host_name, openbmc_ip, openbmc_username,
Michael Walsh0a3bdb42019-01-31 16:21:44 +0000715 openbmc_password, rest_username, rest_password, ipmi_username,
716 ipmi_password, os_host, os_host_name, os_ip, os_username,
Michael Walshdc80d672017-05-09 12:58:32 -0500717 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
718 pdu_password, pdu_slot_no, openbmc_serial_host,
719 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
Michael Walsh68a61162017-04-25 11:54:06 -0500720
721 gp.qprintn()
Michael Walsh68a61162017-04-25 11:54:06 -0500722 print_last_boots()
723 gp.qprintn()
724 gp.qprint_var(state)
Michael Walshb5839d02017-04-12 16:11:20 -0500725 gp.qprintn()
726 gp.qprintn("FFDC data files:")
Michael Walshb2e53ec2017-10-30 15:04:36 -0500727 gp.qprintn(printable_ffdc_file_list)
Michael Walshb5839d02017-04-12 16:11:20 -0500728 gp.qprintn()
Michael Walsh341c21e2017-01-17 16:25:20 -0600729
Michael Walsh600876d2017-05-30 17:58:58 -0500730 if len(ffdc_summary_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500731 gp.qprintn(ffdc_summary_info)
Michael Walsh600876d2017-05-30 17:58:58 -0500732
Michael Walshb2e53ec2017-10-30 15:04:36 -0500733 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500734
Michael Walsh6741f742017-02-20 16:16:38 -0600735
Michael Walsh6741f742017-02-20 16:16:38 -0600736def my_ffdc():
Michael Walsh6741f742017-02-20 16:16:38 -0600737 r"""
738 Collect FFDC data.
739 """
740
741 global state
742
743 plug_in_setup()
744 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500745 call_point='ffdc', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600746
747 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500748 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix="
749 + AUTOBOOT_FFDC_PREFIX
750 + " ffdc_function_list="
751 + ffdc_function_list, ignore=1)
Michael Walsh83f4bc72017-04-20 16:49:43 -0500752 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500753 gp.qprint_error("Call to ffdc failed.\n")
Michael Walsh6741f742017-02-20 16:16:38 -0600754
755 my_get_state()
756
Michael Walshb2e53ec2017-10-30 15:04:36 -0500757 print_defect_report(ffdc_file_list)
Michael Walsh6741f742017-02-20 16:16:38 -0600758
Michael Walsh6741f742017-02-20 16:16:38 -0600759
Michael Walsh6741f742017-02-20 16:16:38 -0600760def print_test_start_message(boot_keyword):
Michael Walsh6741f742017-02-20 16:16:38 -0600761 r"""
762 Print a message indicating what boot test is about to run.
763
764 Description of arguments:
765 boot_keyword The name of the boot which is to be run
766 (e.g. "BMC Power On").
767 """
768
769 global last_ten
Sunil M325eb542017-08-10 07:09:43 -0500770 global boot_start_time
Michael Walsh6741f742017-02-20 16:16:38 -0600771
772 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
Sunil M325eb542017-08-10 07:09:43 -0500773
774 # Set boot_start_time for use by plug-ins.
775 boot_start_time = doing_msg[1:33]
776 gp.qprint_var(boot_start_time)
777
Michael Walshb5839d02017-04-12 16:11:20 -0500778 gp.qprint(doing_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600779
780 last_ten.append(doing_msg)
781
Michael Walsh815b1d52018-10-30 13:32:26 -0500782 # Trim list to max number of entries.
783 del last_ten[:max(0, len(last_ten) - max_boot_history)]
Michael Walsh6741f742017-02-20 16:16:38 -0600784
Michael Walsh6741f742017-02-20 16:16:38 -0600785
Michael Walshf566fb12019-02-01 14:35:09 -0600786def stop_boot_test(signal_number=0,
787 frame=None):
788 r"""
789 Handle SIGUSR1 by aborting the boot test that is running.
790
791 Description of argument(s):
792 signal_number The signal number (should always be 10 for SIGUSR1).
793 frame The frame data.
794 """
795
796 gp.printn()
797 gp.print_executing()
798 gp.lprint_executing()
799
800 # Restore original sigusr1 handler.
801 set_default_siguser1()
802
803 message = "The caller has asked that the boot test be stopped and marked"
804 message += " as a failure."
805
806 function_stack = gm.get_function_stack()
807 if "wait_state" in function_stack:
808 st.set_wait_early_exit_message(message)
809 else:
810 BuiltIn().fail(gp.sprint_error(message))
811
812
Michael Walsh6741f742017-02-20 16:16:38 -0600813def run_boot(boot):
Michael Walsh6741f742017-02-20 16:16:38 -0600814 r"""
815 Run the specified boot.
816
817 Description of arguments:
818 boot The name of the boot test to be performed.
819 """
820
821 global state
822
Michael Walshf566fb12019-02-01 14:35:09 -0600823 signal.signal(signal.SIGUSR1, stop_boot_test)
824 gp.qprint_timen("stop_boot_test is armed.")
825
Michael Walsh6741f742017-02-20 16:16:38 -0600826 print_test_start_message(boot)
827
828 plug_in_setup()
829 rc, shell_rc, failed_plug_in_name = \
830 grpi.rprocess_plug_in_packages(call_point="pre_boot")
831 if rc != 0:
832 error_message = "Plug-in failed with non-zero return code.\n" +\
833 gp.sprint_var(rc, 1)
Michael Walshf566fb12019-02-01 14:35:09 -0600834 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600835 BuiltIn().fail(gp.sprint_error(error_message))
836
837 if test_mode:
838 # In test mode, we'll pretend the boot worked by assigning its
839 # required end state to the default state value.
Michael Walsh30dadae2017-02-27 14:25:52 -0600840 state = st.strip_anchor_state(boot_table[boot]['end'])
Michael Walsh6741f742017-02-20 16:16:38 -0600841 else:
842 # Assertion: We trust that the state data was made fresh by the
843 # caller.
844
Michael Walshb5839d02017-04-12 16:11:20 -0500845 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600846
847 if boot_table[boot]['method_type'] == "keyword":
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600848 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
Michael Walshb5839d02017-04-12 16:11:20 -0500849 boot_table[boot]['method'],
850 quiet=quiet)
Michael Walsh6741f742017-02-20 16:16:38 -0600851
852 if boot_table[boot]['bmc_reboot']:
853 st.wait_for_comm_cycle(int(state['epoch_seconds']))
Michael Walsh30dadae2017-02-27 14:25:52 -0600854 plug_in_setup()
855 rc, shell_rc, failed_plug_in_name = \
856 grpi.rprocess_plug_in_packages(call_point="post_reboot")
857 if rc != 0:
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600858 error_message = "Plug-in failed with non-zero return code.\n"
859 error_message += gp.sprint_var(rc, 1)
Michael Walshf566fb12019-02-01 14:35:09 -0600860 set_default_siguser1()
Michael Walsh30dadae2017-02-27 14:25:52 -0600861 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600862 else:
863 match_state = st.anchor_state(state)
864 del match_state['epoch_seconds']
865 # Wait for the state to change in any way.
866 st.wait_state(match_state, wait_time=state_change_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500867 interval="10 seconds", invert=1)
Michael Walsh6741f742017-02-20 16:16:38 -0600868
Michael Walshb5839d02017-04-12 16:11:20 -0500869 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600870 if boot_table[boot]['end']['chassis'] == "Off":
871 boot_timeout = power_off_timeout
872 else:
873 boot_timeout = power_on_timeout
874 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500875 interval="10 seconds")
Michael Walsh6741f742017-02-20 16:16:38 -0600876
877 plug_in_setup()
878 rc, shell_rc, failed_plug_in_name = \
879 grpi.rprocess_plug_in_packages(call_point="post_boot")
880 if rc != 0:
881 error_message = "Plug-in failed with non-zero return code.\n" +\
882 gp.sprint_var(rc, 1)
Michael Walshf566fb12019-02-01 14:35:09 -0600883 set_default_siguser1()
Michael Walsh6741f742017-02-20 16:16:38 -0600884 BuiltIn().fail(gp.sprint_error(error_message))
885
Michael Walshf566fb12019-02-01 14:35:09 -0600886 # Restore original sigusr1 handler.
887 set_default_siguser1()
888
Michael Walsh6741f742017-02-20 16:16:38 -0600889
Michael Walsh6741f742017-02-20 16:16:38 -0600890def test_loop_body():
Michael Walsh6741f742017-02-20 16:16:38 -0600891 r"""
892 The main loop body for the loop in main_py.
893
894 Description of arguments:
895 boot_count The iteration number (starts at 1).
896 """
897
898 global boot_count
899 global state
900 global next_boot
901 global boot_success
Sunil M325eb542017-08-10 07:09:43 -0500902 global boot_end_time
Michael Walsh6741f742017-02-20 16:16:38 -0600903
Michael Walshb5839d02017-04-12 16:11:20 -0500904 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600905
906 next_boot = select_boot()
Michael Walshb5839d02017-04-12 16:11:20 -0500907 if next_boot == "":
908 return True
Michael Walsh6741f742017-02-20 16:16:38 -0600909
Michael Walshb5839d02017-04-12 16:11:20 -0500910 boot_count += 1
911 gp.qprint_timen("Starting boot " + str(boot_count) + ".")
Michael Walsh6741f742017-02-20 16:16:38 -0600912
Michael Walshe0cf8d72017-05-17 13:20:46 -0500913 pre_boot_plug_in_setup()
Michael Walsh6741f742017-02-20 16:16:38 -0600914
915 cmd_buf = ["run_boot", next_boot]
916 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
917 if boot_status == "FAIL":
Michael Walshb5839d02017-04-12 16:11:20 -0500918 gp.qprint(msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600919
Michael Walshb5839d02017-04-12 16:11:20 -0500920 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600921 if boot_status == "PASS":
922 boot_success = 1
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500923 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
924 + "\" succeeded.")
Michael Walsh6741f742017-02-20 16:16:38 -0600925 else:
926 boot_success = 0
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500927 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
928 + "\" failed.")
Sunil M325eb542017-08-10 07:09:43 -0500929
930 # Set boot_end_time for use by plug-ins.
931 boot_end_time = completion_msg[1:33]
932 gp.qprint_var(boot_end_time)
933
934 gp.qprint(completion_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600935
936 boot_results.update(next_boot, boot_status)
937
938 plug_in_setup()
939 # NOTE: A post_test_case call point failure is NOT counted as a boot
940 # failure.
941 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500942 call_point='post_test_case', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600943
944 plug_in_setup()
945 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500946 call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
Michael Walsh6741f742017-02-20 16:16:38 -0600947 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
Michael Walsh89de14a2018-10-01 16:51:37 -0500948 if boot_status != "PASS" or ffdc_check == "All" or\
949 shell_rc == dump_ffdc_rc():
Michael Walsh83f4bc72017-04-20 16:49:43 -0500950 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
951 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500952 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walsh6741f742017-02-20 16:16:38 -0600953
Michael Walshaabef1e2017-09-20 15:16:17 -0500954 if delete_errlogs:
955 # We need to purge error logs between boots or they build up.
956 grk.run_key("Delete Error logs", ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -0500957
Michael Walsh952f9b02017-03-09 13:11:14 -0600958 boot_results.print_report()
Michael Walshb5839d02017-04-12 16:11:20 -0500959 gp.qprint_timen("Finished boot " + str(boot_count) + ".")
Michael Walsh952f9b02017-03-09 13:11:14 -0600960
Michael Walsh6741f742017-02-20 16:16:38 -0600961 plug_in_setup()
962 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh89de14a2018-10-01 16:51:37 -0500963 call_point='stop_check', shell_rc=stop_test_rc(),
964 stop_on_non_zero_rc=1)
965 if shell_rc == stop_test_rc():
Michael Walsh3ba8ecd2018-04-24 11:33:25 -0500966 message = "Stopping as requested by user.\n"
967 gp.print_time(message)
968 BuiltIn().fail(message)
Michael Walsh6741f742017-02-20 16:16:38 -0600969
Michael Walshd139f282017-04-04 18:00:23 -0500970 # This should help prevent ConnectionErrors.
Michael Walsh0960b382017-06-22 16:23:37 -0500971 grk.run_key_u("Close All Connections")
Michael Walshd139f282017-04-04 18:00:23 -0500972
Michael Walsh6741f742017-02-20 16:16:38 -0600973 return True
974
Michael Walsh6741f742017-02-20 16:16:38 -0600975
Michael Walsh83f4bc72017-04-20 16:49:43 -0500976def obmc_boot_test_teardown():
Michael Walsh6741f742017-02-20 16:16:38 -0600977 r"""
Michael Walshc9116812017-03-10 14:23:06 -0600978 Clean up after the Main keyword.
Michael Walsh6741f742017-02-20 16:16:38 -0600979 """
980
981 if cp_setup_called:
982 plug_in_setup()
983 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500984 call_point='cleanup', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600985
Michael Walsh600876d2017-05-30 17:58:58 -0500986 if 'boot_results_file_path' in globals():
Michael Walsh6c645742018-08-17 15:02:17 -0500987 # Save boot_results and last_ten objects to a file in case they are
988 # needed again.
Michael Walsh600876d2017-05-30 17:58:58 -0500989 gp.qprint_timen("Saving boot_results to the following path.")
990 gp.qprint_var(boot_results_file_path)
Michael Walsh6c645742018-08-17 15:02:17 -0500991 pickle.dump((boot_results, last_ten),
992 open(boot_results_file_path, 'wb'),
Michael Walsh600876d2017-05-30 17:58:58 -0500993 pickle.HIGHEST_PROTOCOL)
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600994
Michael Walshff340002017-08-29 11:18:27 -0500995 global save_stack
996 # Restore any global values saved on the save_stack.
997 for parm_name in main_func_parm_list:
998 # Get the parm_value if it was saved on the stack.
999 try:
1000 parm_value = save_stack.pop(parm_name)
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -05001001 except BaseException:
Michael Walshff340002017-08-29 11:18:27 -05001002 # If it was not saved, no further action is required.
1003 continue
1004
1005 # Restore the saved value.
1006 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1007 "}\", parm_value)"
1008 gp.dpissuing(cmd_buf)
1009 exec(cmd_buf)
1010
1011 gp.dprintn(save_stack.sprint_obj())
1012
Michael Walsh6741f742017-02-20 16:16:38 -06001013
Michael Walshc9116812017-03-10 14:23:06 -06001014def test_teardown():
Michael Walshc9116812017-03-10 14:23:06 -06001015 r"""
1016 Clean up after this test case.
1017 """
1018
1019 gp.qprintn()
1020 cmd_buf = ["Print Error",
1021 "A keyword timeout occurred ending this program.\n"]
1022 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1023
Michael Walshb5839d02017-04-12 16:11:20 -05001024 grp.rqprint_pgm_footer()
1025
Michael Walshc9116812017-03-10 14:23:06 -06001026
Michael Walsh89de14a2018-10-01 16:51:37 -05001027def post_stack():
1028 r"""
1029 Process post_stack plug-in programs.
1030 """
1031
1032 if not call_post_stack_plug:
1033 # The caller does not wish to have post_stack plug-in processing done.
1034 return
1035
1036 global boot_success
1037
1038 # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1039 pre_boot_plug_in_setup()
1040 # For the purposes of the following plug-ins, mark the "boot" as a success.
1041 boot_success = 1
1042 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001043 rc, shell_rc, failed_plug_in_name, history =\
1044 grpi.rprocess_plug_in_packages(call_point='post_stack',
1045 stop_on_plug_in_failure=0,
1046 return_history=True)
1047 last_ten.extend(history)
1048 # Trim list to max number of entries.
1049 del last_ten[:max(0, len(last_ten) - max_boot_history)]
1050 if rc != 0:
1051 boot_success = 0
Michael Walsh89de14a2018-10-01 16:51:37 -05001052
1053 plug_in_setup()
Michael Walsh815b1d52018-10-30 13:32:26 -05001054 rc, shell_rc, failed_plug_in_name =\
1055 grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1056 shell_rc=dump_ffdc_rc(),
1057 stop_on_plug_in_failure=1,
1058 stop_on_non_zero_rc=1)
1059 if shell_rc == dump_ffdc_rc():
Michael Walsh89de14a2018-10-01 16:51:37 -05001060 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1061 if status != 'PASS':
1062 gp.qprint_error("Call to my_ffdc failed.\n")
1063
1064 plug_in_setup()
1065 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1066 call_point='stop_check', shell_rc=stop_test_rc(),
1067 stop_on_non_zero_rc=1)
1068 if shell_rc == stop_test_rc():
1069 message = "Stopping as requested by user.\n"
1070 gp.print_time(message)
1071 BuiltIn().fail(message)
1072
1073
Michael Walshff340002017-08-29 11:18:27 -05001074def obmc_boot_test_py(loc_boot_stack=None,
1075 loc_stack_mode=None,
1076 loc_quiet=None):
Michael Walsh6741f742017-02-20 16:16:38 -06001077 r"""
1078 Do main program processing.
1079 """
1080
Michael Walshff340002017-08-29 11:18:27 -05001081 global save_stack
1082
George Keishing36efbc02018-12-12 10:18:23 -06001083 gp.dprintn()
Michael Walshff340002017-08-29 11:18:27 -05001084 # Process function parms.
1085 for parm_name in main_func_parm_list:
1086 # Get parm's value.
George Keishing36efbc02018-12-12 10:18:23 -06001087 parm_value = eval("loc_" + parm_name)
1088 gp.dpvars(parm_name, parm_value)
Michael Walshff340002017-08-29 11:18:27 -05001089
George Keishing36efbc02018-12-12 10:18:23 -06001090 if parm_value is not None:
Michael Walshff340002017-08-29 11:18:27 -05001091 # Save the global value on a stack.
1092 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1093 parm_name + "}\"), \"" + parm_name + "\")"
1094 gp.dpissuing(cmd_buf)
1095 exec(cmd_buf)
1096
1097 # Set the global value to the passed value.
1098 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1099 "}\", loc_" + parm_name + ")"
1100 gp.dpissuing(cmd_buf)
1101 exec(cmd_buf)
1102
1103 gp.dprintn(save_stack.sprint_obj())
Michael Walshb5839d02017-04-12 16:11:20 -05001104
Michael Walsh6741f742017-02-20 16:16:38 -06001105 setup()
1106
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001107 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1108
Michael Walsha20da402017-03-31 16:27:45 -05001109 if ffdc_only:
1110 gp.qprint_timen("Caller requested ffdc_only.")
Michael Walshe0cf8d72017-05-17 13:20:46 -05001111 pre_boot_plug_in_setup()
Michael Walsh83f4bc72017-04-20 16:49:43 -05001112 grk.run_key_u("my_ffdc")
Michael Walsh764d2f82017-04-27 16:01:08 -05001113 return
Michael Walsha20da402017-03-31 16:27:45 -05001114
Michael Walsh6741f742017-02-20 16:16:38 -06001115 # Process caller's boot_stack.
1116 while (len(boot_stack) > 0):
1117 test_loop_body()
1118
Michael Walshb5839d02017-04-12 16:11:20 -05001119 gp.qprint_timen("Finished processing stack.")
Michael Walsh30dadae2017-02-27 14:25:52 -06001120
Michael Walsh89de14a2018-10-01 16:51:37 -05001121 post_stack()
1122
Michael Walsh6741f742017-02-20 16:16:38 -06001123 # Process caller's boot_list.
1124 if len(boot_list) > 0:
1125 for ix in range(1, max_num_tests + 1):
1126 test_loop_body()
1127
Michael Walshb5839d02017-04-12 16:11:20 -05001128 gp.qprint_timen("Completed all requested boot tests.")
1129
1130 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001131 new_fail = boot_fail - init_boot_fail
1132 if new_fail > boot_fail_threshold:
Michael Walshb5839d02017-04-12 16:11:20 -05001133 error_message = "Boot failures exceed the boot failure" +\
1134 " threshold:\n" +\
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001135 gp.sprint_var(new_fail) +\
Michael Walshb5839d02017-04-12 16:11:20 -05001136 gp.sprint_var(boot_fail_threshold)
1137 BuiltIn().fail(gp.sprint_error(error_message))