blob: 35aa86b7f345013807e098785c9c92bfe5229cca [file] [log] [blame]
Michael Walsh0bbd8602016-11-22 11:31:49 -06001#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
Michael Walsh0b93fbf2017-03-02 14:42:41 -06007import os
8import imp
9import time
10import glob
11import random
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050012import re
Michael Walsh0b93fbf2017-03-02 14:42:41 -060013import cPickle as pickle
Michael Walshdc80d672017-05-09 12:58:32 -050014import socket
Michael Walsh0b93fbf2017-03-02 14:42:41 -060015
16from robot.utils import DotDict
17from robot.libraries.BuiltIn import BuiltIn
18
Michael Walsh6741f742017-02-20 16:16:38 -060019from boot_data import *
Michael Walshc9116812017-03-10 14:23:06 -060020import gen_print as gp
Michael Walsh0bbd8602016-11-22 11:31:49 -060021import gen_robot_print as grp
Michael Walsh55302292017-01-10 11:43:02 -060022import gen_robot_plug_in as grpi
Michael Walsh6741f742017-02-20 16:16:38 -060023import gen_robot_valid as grv
24import gen_misc as gm
25import gen_cmd as gc
Michael Walshb5839d02017-04-12 16:11:20 -050026import gen_robot_keyword as grk
Michael Walsh55302292017-01-10 11:43:02 -060027import state as st
Michael Walshff340002017-08-29 11:18:27 -050028import var_stack as vs
Michael Walsh0bbd8602016-11-22 11:31:49 -060029
Michael Walsh0b93fbf2017-03-02 14:42:41 -060030base_path = os.path.dirname(os.path.dirname(
31 imp.find_module("gen_robot_print")[1])) +\
Michael Walshc9116812017-03-10 14:23:06 -060032 os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060033sys.path.append(base_path + "extended/")
34import run_keyword as rk
Michael Walsh0bbd8602016-11-22 11:31:49 -060035
Michael Walshe1e26442017-03-06 17:50:07 -060036# Setting master_pid correctly influences the behavior of plug-ins like
37# DB_Logging
38program_pid = os.getpid()
39master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
Michael Walsh903e0b22017-09-19 17:00:33 -050040pgm_name = re.sub('\.py$', '', os.path.basename(__file__))
Michael Walshe1e26442017-03-06 17:50:07 -060041
Michael Walshb5839d02017-04-12 16:11:20 -050042# Set up boot data structures.
43boot_table = create_boot_table()
44valid_boot_types = create_valid_boot_list(boot_table)
Michael Walsh0b93fbf2017-03-02 14:42:41 -060045
Michael Walsh6741f742017-02-20 16:16:38 -060046boot_lists = read_boot_lists()
47last_ten = []
Michael Walsh6741f742017-02-20 16:16:38 -060048
49state = st.return_default_state()
50cp_setup_called = 0
51next_boot = ""
52base_tool_dir_path = os.path.normpath(os.environ.get(
53 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
Michael Walshb5839d02017-04-12 16:11:20 -050054
Michael Walsh6741f742017-02-20 16:16:38 -060055ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
Michael Walsh6741f742017-02-20 16:16:38 -060056boot_success = 0
Michael Walsh6741f742017-02-20 16:16:38 -060057status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
58if status_dir_path != "":
59 status_dir_path = os.path.normpath(status_dir_path) + os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060060default_power_on = "REST Power On"
61default_power_off = "REST Power Off"
Michael Walsh6741f742017-02-20 16:16:38 -060062boot_count = 0
Michael Walsh0bbd8602016-11-22 11:31:49 -060063
Michael Walsh85678942017-03-27 14:34:22 -050064LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
Michael Walshe1974b92017-08-03 13:39:51 -050065ffdc_prefix = ""
Sunil M325eb542017-08-10 07:09:43 -050066boot_start_time = ""
67boot_end_time = ""
Michael Walshff340002017-08-29 11:18:27 -050068save_stack = vs.var_stack('save_stack')
69main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
Michael Walsh85678942017-03-27 14:34:22 -050070
71
Michael Walsh0ad0f7f2017-05-04 14:39:58 -050072def process_host(host,
73 host_var_name=""):
74
75 r"""
76 Process a host by getting the associated host name and IP address and
77 setting them in global variables.
78
79 If the caller does not pass the host_var_name, this function will try to
80 figure out the name of the variable used by the caller for the host parm.
81 Callers are advised to explicitly specify the host_var_name when calling
82 with an exec command. In such cases, the get_arg_name cannot figure out
83 the host variable name.
84
85 This function will then create similar global variable names by
86 removing "_host" and appending "_host_name" or "_ip" to the host variable
87 name.
88
89 Example:
90
91 If a call is made like this:
92 process_host(openbmc_host)
93
94 Global variables openbmc_host_name and openbmc_ip will be set.
95
96 Description of argument(s):
97 host A host name or IP. The name of the variable used should
98 have a suffix of "_host".
99 host_var_name The name of the variable being used as the host parm.
100 """
101
102 if host_var_name == "":
103 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
104
105 host_name_var_name = re.sub("host", "host_name", host_var_name)
106 ip_var_name = re.sub("host", "ip", host_var_name)
107 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
108 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
109 host + "')"
110 exec(cmd_buf)
111
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500112
Michael Walshb5839d02017-04-12 16:11:20 -0500113def process_pgm_parms():
114
115 r"""
116 Process the program parameters by assigning them all to corresponding
117 globals. Also, set some global values that depend on program parameters.
118 """
119
120 # Program parameter processing.
121 # Assign all program parms to python variables which are global to this
122 # module.
123
124 global parm_list
125 parm_list = BuiltIn().get_variable_value("${parm_list}")
126 # The following subset of parms should be processed as integers.
127 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
Michael Walshaabef1e2017-09-20 15:16:17 -0500128 'boot_fail_threshold', 'delete_errlogs', 'quiet', 'test_mode',
129 'debug']
Michael Walshb5839d02017-04-12 16:11:20 -0500130 for parm in parm_list:
131 if parm in int_list:
132 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
133 "}\", \"0\"))"
134 else:
135 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
136 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
Michael Walshff340002017-08-29 11:18:27 -0500137 gp.dpissuing(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500138 exec(cmd_buf)
Michael Walsh0ad0f7f2017-05-04 14:39:58 -0500139 if re.match(r".*_host$", parm):
140 cmd_buf = "process_host(" + parm + ", '" + parm + "')"
141 exec(cmd_buf)
142 if re.match(r".*_password$", parm):
143 # Register the value of any parm whose name ends in _password.
144 # This will cause the print functions to replace passwords with
145 # asterisks in the output.
146 cmd_buf = "gp.register_passwords(" + parm + ")"
147 exec(cmd_buf)
Michael Walshb5839d02017-04-12 16:11:20 -0500148
149 global ffdc_dir_path_style
150 global boot_list
151 global boot_stack
152 global boot_results_file_path
153 global boot_results
154 global ffdc_list_file_path
Michael Walshe0cf8d72017-05-17 13:20:46 -0500155 global ffdc_report_list_path
Michael Walsh600876d2017-05-30 17:58:58 -0500156 global ffdc_summary_list_path
Michael Walshb5839d02017-04-12 16:11:20 -0500157
158 if ffdc_dir_path_style == "":
159 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
160
161 # Convert these program parms to lists for easier processing..
162 boot_list = filter(None, boot_list.split(":"))
163 boot_stack = filter(None, boot_stack.split(":"))
164
Michael Walsh903e0b22017-09-19 17:00:33 -0500165 cleanup_boot_results_file()
166 boot_results_file_path = create_boot_results_file_path(pgm_name,
167 openbmc_nickname,
168 master_pid)
Michael Walshb5839d02017-04-12 16:11:20 -0500169
170 if os.path.isfile(boot_results_file_path):
171 # We've been called before in this run so we'll load the saved
172 # boot_results object.
173 boot_results = pickle.load(open(boot_results_file_path, 'rb'))
174 else:
175 boot_results = boot_results(boot_table, boot_pass, boot_fail)
176
177 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
178 "/FFDC_FILE_LIST"
Michael Walshe0cf8d72017-05-17 13:20:46 -0500179 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
180 "/FFDC_REPORT_FILE_LIST"
Michael Walshb5839d02017-04-12 16:11:20 -0500181
Michael Walsh600876d2017-05-30 17:58:58 -0500182 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
183 "/FFDC_SUMMARY_FILE_LIST"
184
Michael Walshb5839d02017-04-12 16:11:20 -0500185
Michael Walsh85678942017-03-27 14:34:22 -0500186def initial_plug_in_setup():
187
188 r"""
189 Initialize all plug-in environment variables which do not change for the
190 duration of the program.
191
192 """
193
194 global LOG_LEVEL
195 BuiltIn().set_log_level("NONE")
196
197 BuiltIn().set_global_variable("${master_pid}", master_pid)
198 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
199 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
200 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
201 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
202 ffdc_list_file_path)
Michael Walshe0cf8d72017-05-17 13:20:46 -0500203 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
204 ffdc_report_list_path)
Michael Walsh600876d2017-05-30 17:58:58 -0500205 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
206 ffdc_summary_list_path)
Michael Walsh85678942017-03-27 14:34:22 -0500207
208 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
209 ffdc_dir_path_style)
210 BuiltIn().set_global_variable("${FFDC_CHECK}",
211 ffdc_check)
212
213 # For each program parameter, set the corresponding AUTOBOOT_ environment
214 # variable value. Also, set an AUTOBOOT_ environment variable for every
215 # element in additional_values.
216 additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
217 "status_dir_path", "base_tool_dir_path",
Michael Walsh600876d2017-05-30 17:58:58 -0500218 "ffdc_list_file_path", "ffdc_report_list_path",
219 "ffdc_summary_list_path"]
Michael Walsh85678942017-03-27 14:34:22 -0500220
221 plug_in_vars = parm_list + additional_values
222
223 for var_name in plug_in_vars:
224 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
225 var_name = var_name.upper()
226 if var_value is None:
227 var_value = ""
228 os.environ["AUTOBOOT_" + var_name] = str(var_value)
229
230 BuiltIn().set_log_level(LOG_LEVEL)
231
Michael Walsh68a61162017-04-25 11:54:06 -0500232 # Make sure the ffdc list directory exists.
233 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
234 if not os.path.exists(ffdc_list_dir_path):
235 os.makedirs(ffdc_list_dir_path)
Michael Walsh85678942017-03-27 14:34:22 -0500236
Michael Walsh85678942017-03-27 14:34:22 -0500237
Michael Walsh0bbd8602016-11-22 11:31:49 -0600238def plug_in_setup():
239
240 r"""
Michael Walsh85678942017-03-27 14:34:22 -0500241 Initialize all changing plug-in environment variables for use by the
242 plug-in programs.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600243 """
244
Michael Walsh85678942017-03-27 14:34:22 -0500245 global LOG_LEVEL
246 global test_really_running
247
248 BuiltIn().set_log_level("NONE")
249
Michael Walsh6741f742017-02-20 16:16:38 -0600250 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600251 if boot_pass > 1:
252 test_really_running = 1
253 else:
254 test_really_running = 0
255
Michael Walsh6741f742017-02-20 16:16:38 -0600256 BuiltIn().set_global_variable("${test_really_running}",
257 test_really_running)
258 BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
Michael Walsh6741f742017-02-20 16:16:38 -0600259 BuiltIn().set_global_variable("${boot_pass}", boot_pass)
260 BuiltIn().set_global_variable("${boot_fail}", boot_fail)
261 BuiltIn().set_global_variable("${boot_success}", boot_success)
262 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
Sunil M325eb542017-08-10 07:09:43 -0500263 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
264 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
Michael Walsh4c9a6452016-12-13 16:03:11 -0600265
Michael Walsh0bbd8602016-11-22 11:31:49 -0600266 # For each program parameter, set the corresponding AUTOBOOT_ environment
267 # variable value. Also, set an AUTOBOOT_ environment variable for every
268 # element in additional_values.
269 additional_values = ["boot_type_desc", "boot_success", "boot_pass",
Sunil M325eb542017-08-10 07:09:43 -0500270 "boot_fail", "test_really_running", "ffdc_prefix",
271 "boot_start_time", "boot_end_time"]
Michael Walsh0bbd8602016-11-22 11:31:49 -0600272
Michael Walsh85678942017-03-27 14:34:22 -0500273 plug_in_vars = additional_values
Michael Walsh0bbd8602016-11-22 11:31:49 -0600274
275 for var_name in plug_in_vars:
276 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
277 var_name = var_name.upper()
278 if var_value is None:
279 var_value = ""
Michael Walsh6741f742017-02-20 16:16:38 -0600280 os.environ["AUTOBOOT_" + var_name] = str(var_value)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600281
Michael Walsh0bbd8602016-11-22 11:31:49 -0600282 if debug:
Michael Walsh6741f742017-02-20 16:16:38 -0600283 shell_rc, out_buf = \
284 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600285
Michael Walsh85678942017-03-27 14:34:22 -0500286 BuiltIn().set_log_level(LOG_LEVEL)
287
Michael Walsh0bbd8602016-11-22 11:31:49 -0600288
Michael Walshe0cf8d72017-05-17 13:20:46 -0500289def pre_boot_plug_in_setup():
290
291 # Clear the ffdc_list_file_path file. Plug-ins may now write to it.
292 try:
293 os.remove(ffdc_list_file_path)
294 except OSError:
295 pass
296
297 # Clear the ffdc_report_list_path file. Plug-ins may now write to it.
298 try:
299 os.remove(ffdc_report_list_path)
300 except OSError:
301 pass
302
Michael Walsh600876d2017-05-30 17:58:58 -0500303 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it.
304 try:
305 os.remove(ffdc_summary_list_path)
306 except OSError:
307 pass
308
Michael Walshe1974b92017-08-03 13:39:51 -0500309 global ffdc_prefix
310
311 seconds = time.time()
312 loc_time = time.localtime(seconds)
313 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
314
315 ffdc_prefix = openbmc_nickname + "." + time_string
316
Michael Walshe0cf8d72017-05-17 13:20:46 -0500317
Michael Walsh6741f742017-02-20 16:16:38 -0600318def setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600319
320 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600321 Do general program setup tasks.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600322 """
323
Michael Walsh6741f742017-02-20 16:16:38 -0600324 global cp_setup_called
Michael Walsh81816742017-09-27 11:02:29 -0500325 global transitional_boot_selected
Michael Walsh0bbd8602016-11-22 11:31:49 -0600326
Michael Walshb5839d02017-04-12 16:11:20 -0500327 gp.qprintn()
328
Michael Walsh81816742017-09-27 11:02:29 -0500329 transitional_boot_selected = False
330
Michael Walsh83f4bc72017-04-20 16:49:43 -0500331 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
332 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
Michael Walshd061c042017-05-23 14:46:57 -0500333 # If we can't find process_plug_in_packages.py, ssh_pw or
334 # validate_plug_ins.py, then we don't have our repo bin in PATH.
335 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" +
336 " ssh_pw validate_plug_ins.py", quiet=1,
337 print_output=0, show_err=0)
Michael Walshb5839d02017-04-12 16:11:20 -0500338 if shell_rc != 0:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500339 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
340 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
341 if robot_pgm_dir_path not in sys.path:
342 sys.path.append(robot_pgm_dir_path)
343 PYTHONPATH = os.environ.get("PYTHONPATH", "")
344 if PYTHONPATH == "":
345 os.environ['PYTHONPATH'] = robot_pgm_dir_path
346 else:
347 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
Michael Walsh6741f742017-02-20 16:16:38 -0600348
349 validate_parms()
350
351 grp.rqprint_pgm_header()
352
Michael Walshfdc5ced2017-08-17 13:15:15 -0500353 grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE")
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500354
Michael Walsh85678942017-03-27 14:34:22 -0500355 initial_plug_in_setup()
356
Michael Walsh6741f742017-02-20 16:16:38 -0600357 plug_in_setup()
358 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
359 call_point='setup')
360 if rc != 0:
361 error_message = "Plug-in setup failed.\n"
362 grp.rprint_error_report(error_message)
363 BuiltIn().fail(error_message)
364 # Setting cp_setup_called lets our Teardown know that it needs to call
365 # the cleanup plug-in call point.
366 cp_setup_called = 1
367
368 # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
369 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
Michael Walsh85678942017-03-27 14:34:22 -0500370 # FFDC_LOG_PATH is used by "FFDC" keyword.
371 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
Michael Walsh6741f742017-02-20 16:16:38 -0600372
Michael Walshdc80d672017-05-09 12:58:32 -0500373 # Also printed by FFDC.
374 global host_name
375 global host_ip
376 host = socket.gethostname()
377 host_name, host_ip = gm.get_host_name_ip(host)
378
Michael Walshb5839d02017-04-12 16:11:20 -0500379 gp.dprint_var(boot_table, 1)
380 gp.dprint_var(boot_lists)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600381
Michael Walsh0bbd8602016-11-22 11:31:49 -0600382
Michael Walsh6741f742017-02-20 16:16:38 -0600383def validate_parms():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600384
385 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600386 Validate all program parameters.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600387 """
388
Michael Walshb5839d02017-04-12 16:11:20 -0500389 process_pgm_parms()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600390
Michael Walshb5839d02017-04-12 16:11:20 -0500391 gp.qprintn()
392
393 global openbmc_model
Michael Walsh6741f742017-02-20 16:16:38 -0600394 grv.rvalid_value("openbmc_host")
395 grv.rvalid_value("openbmc_username")
396 grv.rvalid_value("openbmc_password")
397 if os_host != "":
398 grv.rvalid_value("os_username")
399 grv.rvalid_value("os_password")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600400
Michael Walsh6741f742017-02-20 16:16:38 -0600401 if pdu_host != "":
402 grv.rvalid_value("pdu_username")
403 grv.rvalid_value("pdu_password")
Michael Walsh85678942017-03-27 14:34:22 -0500404 grv.rvalid_integer("pdu_slot_no")
Michael Walsh6741f742017-02-20 16:16:38 -0600405 if openbmc_serial_host != "":
406 grv.rvalid_integer("openbmc_serial_port")
Michael Walshb5839d02017-04-12 16:11:20 -0500407 if openbmc_model == "":
408 status, ret_values =\
409 grk.run_key_u("Get BMC System Model")
410 openbmc_model = ret_values
411 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
Michael Walsh6741f742017-02-20 16:16:38 -0600412 grv.rvalid_value("openbmc_model")
Michael Walshb5839d02017-04-12 16:11:20 -0500413 grv.rvalid_integer("max_num_tests")
Michael Walsh6741f742017-02-20 16:16:38 -0600414 grv.rvalid_integer("boot_pass")
415 grv.rvalid_integer("boot_fail")
416
417 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
418 BuiltIn().set_global_variable("${plug_in_packages_list}",
419 plug_in_packages_list)
420
Michael Walshb5839d02017-04-12 16:11:20 -0500421 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip'])
Michael Walsha20da402017-03-31 16:27:45 -0500422 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
Michael Walsh6741f742017-02-20 16:16:38 -0600423 error_message = "You must provide either a value for either the" +\
424 " boot_list or the boot_stack parm.\n"
425 BuiltIn().fail(gp.sprint_error(error_message))
426
427 valid_boot_list(boot_list, valid_boot_types)
428 valid_boot_list(boot_stack, valid_boot_types)
429
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500430 selected_PDU_boots = list(set(boot_list + boot_stack) &
431 set(boot_lists['PDU_reboot']))
432
433 if len(selected_PDU_boots) > 0 and pdu_host == "":
434 error_message = "You have selected the following boots which" +\
435 " require a PDU host but no value for pdu_host:\n"
436 error_message += gp.sprint_var(selected_PDU_boots)
437 error_message += gp.sprint_var(pdu_host, 2)
438 BuiltIn().fail(gp.sprint_error(error_message))
439
Michael Walsh6741f742017-02-20 16:16:38 -0600440 return
Michael Walsh0bbd8602016-11-22 11:31:49 -0600441
Michael Walsh0bbd8602016-11-22 11:31:49 -0600442
Michael Walsh6741f742017-02-20 16:16:38 -0600443def my_get_state():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600444
445 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600446 Get the system state plus a little bit of wrapping.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600447 """
448
Michael Walsh6741f742017-02-20 16:16:38 -0600449 global state
450
451 req_states = ['epoch_seconds'] + st.default_req_states
452
Michael Walshb5839d02017-04-12 16:11:20 -0500453 gp.qprint_timen("Getting system state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600454 if test_mode:
455 state['epoch_seconds'] = int(time.time())
456 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500457 state = st.get_state(req_states=req_states, quiet=quiet)
458 gp.qprint_var(state)
Michael Walsh341c21e2017-01-17 16:25:20 -0600459
Michael Walsh341c21e2017-01-17 16:25:20 -0600460
Michael Walsh45ca6e42017-09-14 17:29:12 -0500461def valid_state():
462
463 r"""
464 Verify that our state dictionary contains no blank values. If we don't get
465 valid state data, we cannot continue to work.
466 """
467
468 if st.compare_states(state, st.invalid_state_match, 'or'):
469 error_message = "The state dictionary contains blank fields which" +\
470 " is illegal.\n" + gp.sprint_var(state)
471 BuiltIn().fail(gp.sprint_error(error_message))
472
Michael Walsh45ca6e42017-09-14 17:29:12 -0500473
Michael Walsh6741f742017-02-20 16:16:38 -0600474def select_boot():
Michael Walsh341c21e2017-01-17 16:25:20 -0600475
476 r"""
477 Select a boot test to be run based on our current state and return the
478 chosen boot type.
479
480 Description of arguments:
Michael Walsh6741f742017-02-20 16:16:38 -0600481 state The state of the machine.
Michael Walsh341c21e2017-01-17 16:25:20 -0600482 """
483
Michael Walsh81816742017-09-27 11:02:29 -0500484 global transitional_boot_selected
Michael Walsh30dadae2017-02-27 14:25:52 -0600485 global boot_stack
486
Michael Walshb5839d02017-04-12 16:11:20 -0500487 gp.qprint_timen("Selecting a boot test.")
Michael Walsh6741f742017-02-20 16:16:38 -0600488
Michael Walsh81816742017-09-27 11:02:29 -0500489 if transitional_boot_selected and not boot_success:
490 prior_boot = next_boot
491 boot_candidate = boot_stack.pop()
492 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" +
493 " transition to a valid state for '" + boot_candidate +
494 "' which was at the top of the boot_stack. Since" +
495 " the '" + next_boot + "' failed, the '" +
496 boot_candidate + "' has been removed from the stack" +
497 " to avoid and endless failure loop.")
498 if len(boot_stack) == 0:
499 return ""
500
Michael Walsh6741f742017-02-20 16:16:38 -0600501 my_get_state()
Michael Walsh45ca6e42017-09-14 17:29:12 -0500502 valid_state()
Michael Walsh6741f742017-02-20 16:16:38 -0600503
Michael Walsh81816742017-09-27 11:02:29 -0500504 transitional_boot_selected = False
Michael Walsh6741f742017-02-20 16:16:38 -0600505 stack_popped = 0
506 if len(boot_stack) > 0:
507 stack_popped = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500508 gp.qprint_dashes()
509 gp.qprint_var(boot_stack)
510 gp.qprint_dashes()
511 skip_boot_printed = 0
512 while len(boot_stack) > 0:
513 boot_candidate = boot_stack.pop()
514 if stack_mode == 'normal':
515 break
516 else:
517 if st.compare_states(state, boot_table[boot_candidate]['end']):
518 if not skip_boot_printed:
Michael Walshff340002017-08-29 11:18:27 -0500519 gp.qprint_var(stack_mode)
520 gp.qprintn()
521 gp.qprint_timen("Skipping the following boot tests" +
522 " which are unnecessary since their" +
523 " required end states match the" +
524 " current machine state:")
Michael Walshb5839d02017-04-12 16:11:20 -0500525 skip_boot_printed = 1
Michael Walshff340002017-08-29 11:18:27 -0500526 gp.qprint_var(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500527 boot_candidate = ""
528 if boot_candidate == "":
529 gp.qprint_dashes()
530 gp.qprint_var(boot_stack)
531 gp.qprint_dashes()
532 return boot_candidate
Michael Walsh6741f742017-02-20 16:16:38 -0600533 if st.compare_states(state, boot_table[boot_candidate]['start']):
Michael Walshb5839d02017-04-12 16:11:20 -0500534 gp.qprint_timen("The machine state is valid for a '" +
535 boot_candidate + "' boot test.")
536 gp.qprint_dashes()
537 gp.qprint_var(boot_stack)
538 gp.qprint_dashes()
Michael Walsh6741f742017-02-20 16:16:38 -0600539 return boot_candidate
Michael Walsh341c21e2017-01-17 16:25:20 -0600540 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500541 gp.qprint_timen("The machine state does not match the required" +
542 " starting state for a '" + boot_candidate +
543 "' boot test:")
Michael Walshff340002017-08-29 11:18:27 -0500544 gp.qprint_varx("boot_table[" + boot_candidate + "][start]",
545 boot_table[boot_candidate]['start'], 1)
Michael Walsh6741f742017-02-20 16:16:38 -0600546 boot_stack.append(boot_candidate)
Michael Walsh81816742017-09-27 11:02:29 -0500547 transitional_boot_selected = True
Michael Walsh6741f742017-02-20 16:16:38 -0600548 popped_boot = boot_candidate
549
550 # Loop through your list selecting a boot_candidates
551 boot_candidates = []
552 for boot_candidate in boot_list:
553 if st.compare_states(state, boot_table[boot_candidate]['start']):
554 if stack_popped:
555 if st.compare_states(boot_table[boot_candidate]['end'],
556 boot_table[popped_boot]['start']):
557 boot_candidates.append(boot_candidate)
558 else:
559 boot_candidates.append(boot_candidate)
560
561 if len(boot_candidates) == 0:
Michael Walshb5839d02017-04-12 16:11:20 -0500562 gp.qprint_timen("The user's boot list contained no boot tests" +
563 " which are valid for the current machine state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600564 boot_candidate = default_power_on
565 if not st.compare_states(state, boot_table[default_power_on]['start']):
566 boot_candidate = default_power_off
567 boot_candidates.append(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500568 gp.qprint_timen("Using default '" + boot_candidate +
569 "' boot type to transition to valid state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600570
Michael Walshb5839d02017-04-12 16:11:20 -0500571 gp.dprint_var(boot_candidates)
Michael Walsh6741f742017-02-20 16:16:38 -0600572
573 # Randomly select a boot from the candidate list.
574 boot = random.choice(boot_candidates)
Michael Walsh341c21e2017-01-17 16:25:20 -0600575
576 return boot
Michael Walsh0bbd8602016-11-22 11:31:49 -0600577
Michael Walsh55302292017-01-10 11:43:02 -0600578
Michael Walsh341c21e2017-01-17 16:25:20 -0600579def print_last_boots():
580
581 r"""
582 Print the last ten boots done with their time stamps.
583 """
584
585 # indent 0, 90 chars wide, linefeed, char is "="
Michael Walshb5839d02017-04-12 16:11:20 -0500586 gp.qprint_dashes(0, 90)
587 gp.qprintn("Last 10 boots:\n")
Michael Walsh341c21e2017-01-17 16:25:20 -0600588
589 for boot_entry in last_ten:
590 grp.rqprint(boot_entry)
Michael Walshb5839d02017-04-12 16:11:20 -0500591 gp.qprint_dashes(0, 90)
Michael Walsh341c21e2017-01-17 16:25:20 -0600592
Michael Walsh341c21e2017-01-17 16:25:20 -0600593
Michael Walshb2e53ec2017-10-30 15:04:36 -0500594def print_defect_report(ffdc_file_list):
Michael Walsh341c21e2017-01-17 16:25:20 -0600595
596 r"""
597 Print a defect report.
Michael Walshb2e53ec2017-10-30 15:04:36 -0500598
599 Description of argument(s):
600 ffdc_file_list A list of files which were collected by our ffdc functions.
Michael Walsh341c21e2017-01-17 16:25:20 -0600601 """
602
Michael Walsh600876d2017-05-30 17:58:58 -0500603 # Making deliberate choice to NOT run plug_in_setup(). We don't want
604 # ffdc_prefix updated.
605 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
606 call_point='ffdc_report', stop_on_plug_in_failure=0)
607
Michael Walshe0cf8d72017-05-17 13:20:46 -0500608 # Get additional header data which may have been created by ffdc plug-ins.
609 # Also, delete the individual header files to cleanup.
610 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
611 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
612 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
613 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
614 show_err=0)
615
Michael Walshb2e53ec2017-10-30 15:04:36 -0500616 # Get additional summary data which may have been created by ffdc plug-ins.
Michael Walsh600876d2017-05-30 17:58:58 -0500617 # Also, delete the individual header files to cleanup.
618 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
619 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
620 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
621 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
622 show_err=0)
623
Michael Walshb2e53ec2017-10-30 15:04:36 -0500624 # ffdc_list_file_path contains a list of any ffdc files created by plug-
625 # ins, etc. Read that data into a list.
Michael Walsh341c21e2017-01-17 16:25:20 -0600626 try:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500627 plug_in_ffdc_list = \
628 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
629 plug_in_ffdc_list = filter(None, plug_in_ffdc_list)
Michael Walsh341c21e2017-01-17 16:25:20 -0600630 except IOError:
Michael Walshb2e53ec2017-10-30 15:04:36 -0500631 plug_in_ffdc_list = []
632
633 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
634 # in. Eliminate duplicates and sort the list.
635 ffdc_file_list = list(set(ffdc_file_list + plug_in_ffdc_list))
636 ffdc_file_list.sort()
637
638 if status_file_path != "":
639 ffdc_file_list.insert(0, status_file_path)
640
641 # Convert the list to a printable list.
642 printable_ffdc_file_list = "\n".join(ffdc_file_list)
Michael Walsh341c21e2017-01-17 16:25:20 -0600643
Michael Walsh68a61162017-04-25 11:54:06 -0500644 # Open ffdc_file_list for writing. We will write a complete list of
645 # FFDC files to it for possible use by plug-ins like cp_stop_check.
646 ffdc_list_file = open(ffdc_list_file_path, 'w')
Michael Walshb2e53ec2017-10-30 15:04:36 -0500647 ffdc_list_file.write(printable_ffdc_file_list + "\n")
648 ffdc_list_file.close()
649
650 indent = 0
651 width = 90
652 linefeed = 1
653 char = "="
Michael Walsh68a61162017-04-25 11:54:06 -0500654
655 gp.qprintn()
Michael Walshb2e53ec2017-10-30 15:04:36 -0500656 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500657 gp.qprintn("Copy this data to the defect:\n")
658
Michael Walshe0cf8d72017-05-17 13:20:46 -0500659 if len(more_header_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500660 gp.qprintn(more_header_info)
Michael Walshdc80d672017-05-09 12:58:32 -0500661 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
662 openbmc_host_name, openbmc_ip, openbmc_username,
663 openbmc_password, os_host, os_host_name, os_ip, os_username,
664 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
665 pdu_password, pdu_slot_no, openbmc_serial_host,
666 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
Michael Walsh68a61162017-04-25 11:54:06 -0500667
668 gp.qprintn()
Michael Walsh68a61162017-04-25 11:54:06 -0500669 print_last_boots()
670 gp.qprintn()
671 gp.qprint_var(state)
Michael Walshb5839d02017-04-12 16:11:20 -0500672 gp.qprintn()
673 gp.qprintn("FFDC data files:")
Michael Walshb2e53ec2017-10-30 15:04:36 -0500674 gp.qprintn(printable_ffdc_file_list)
Michael Walshb5839d02017-04-12 16:11:20 -0500675 gp.qprintn()
Michael Walsh341c21e2017-01-17 16:25:20 -0600676
Michael Walsh600876d2017-05-30 17:58:58 -0500677 if len(ffdc_summary_info) > 0:
Michael Walshff340002017-08-29 11:18:27 -0500678 gp.qprintn(ffdc_summary_info)
Michael Walsh600876d2017-05-30 17:58:58 -0500679
Michael Walshb2e53ec2017-10-30 15:04:36 -0500680 gp.qprint_dashes(indent, width, linefeed, char)
Michael Walsh68a61162017-04-25 11:54:06 -0500681
Michael Walsh6741f742017-02-20 16:16:38 -0600682
Michael Walsh6741f742017-02-20 16:16:38 -0600683def my_ffdc():
684
685 r"""
686 Collect FFDC data.
687 """
688
689 global state
690
691 plug_in_setup()
692 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500693 call_point='ffdc', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600694
695 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
Michael Walshb2e53ec2017-10-30 15:04:36 -0500696 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" +
697 AUTOBOOT_FFDC_PREFIX +
698 " ffdc_function_list=" +
699 ffdc_function_list, ignore=1)
Michael Walsh83f4bc72017-04-20 16:49:43 -0500700 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500701 gp.qprint_error("Call to ffdc failed.\n")
Michael Walsh6741f742017-02-20 16:16:38 -0600702
703 my_get_state()
704
Michael Walshb2e53ec2017-10-30 15:04:36 -0500705 print_defect_report(ffdc_file_list)
Michael Walsh6741f742017-02-20 16:16:38 -0600706
Michael Walsh6741f742017-02-20 16:16:38 -0600707
Michael Walsh6741f742017-02-20 16:16:38 -0600708def print_test_start_message(boot_keyword):
709
710 r"""
711 Print a message indicating what boot test is about to run.
712
713 Description of arguments:
714 boot_keyword The name of the boot which is to be run
715 (e.g. "BMC Power On").
716 """
717
718 global last_ten
Sunil M325eb542017-08-10 07:09:43 -0500719 global boot_start_time
Michael Walsh6741f742017-02-20 16:16:38 -0600720
721 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
Sunil M325eb542017-08-10 07:09:43 -0500722
723 # Set boot_start_time for use by plug-ins.
724 boot_start_time = doing_msg[1:33]
725 gp.qprint_var(boot_start_time)
726
Michael Walshb5839d02017-04-12 16:11:20 -0500727 gp.qprint(doing_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600728
729 last_ten.append(doing_msg)
730
731 if len(last_ten) > 10:
732 del last_ten[0]
733
Michael Walsh6741f742017-02-20 16:16:38 -0600734
Michael Walsh6741f742017-02-20 16:16:38 -0600735def run_boot(boot):
736
737 r"""
738 Run the specified boot.
739
740 Description of arguments:
741 boot The name of the boot test to be performed.
742 """
743
744 global state
745
746 print_test_start_message(boot)
747
748 plug_in_setup()
749 rc, shell_rc, failed_plug_in_name = \
750 grpi.rprocess_plug_in_packages(call_point="pre_boot")
751 if rc != 0:
752 error_message = "Plug-in failed with non-zero return code.\n" +\
753 gp.sprint_var(rc, 1)
754 BuiltIn().fail(gp.sprint_error(error_message))
755
756 if test_mode:
757 # In test mode, we'll pretend the boot worked by assigning its
758 # required end state to the default state value.
Michael Walsh30dadae2017-02-27 14:25:52 -0600759 state = st.strip_anchor_state(boot_table[boot]['end'])
Michael Walsh6741f742017-02-20 16:16:38 -0600760 else:
761 # Assertion: We trust that the state data was made fresh by the
762 # caller.
763
Michael Walshb5839d02017-04-12 16:11:20 -0500764 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600765
766 if boot_table[boot]['method_type'] == "keyword":
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600767 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
Michael Walshb5839d02017-04-12 16:11:20 -0500768 boot_table[boot]['method'],
769 quiet=quiet)
Michael Walsh6741f742017-02-20 16:16:38 -0600770
771 if boot_table[boot]['bmc_reboot']:
772 st.wait_for_comm_cycle(int(state['epoch_seconds']))
Michael Walsh30dadae2017-02-27 14:25:52 -0600773 plug_in_setup()
774 rc, shell_rc, failed_plug_in_name = \
775 grpi.rprocess_plug_in_packages(call_point="post_reboot")
776 if rc != 0:
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600777 error_message = "Plug-in failed with non-zero return code.\n"
778 error_message += gp.sprint_var(rc, 1)
Michael Walsh30dadae2017-02-27 14:25:52 -0600779 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600780 else:
781 match_state = st.anchor_state(state)
782 del match_state['epoch_seconds']
783 # Wait for the state to change in any way.
784 st.wait_state(match_state, wait_time=state_change_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500785 interval="10 seconds", invert=1)
Michael Walsh6741f742017-02-20 16:16:38 -0600786
Michael Walshb5839d02017-04-12 16:11:20 -0500787 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600788 if boot_table[boot]['end']['chassis'] == "Off":
789 boot_timeout = power_off_timeout
790 else:
791 boot_timeout = power_on_timeout
792 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
Michael Walsh600876d2017-05-30 17:58:58 -0500793 interval="10 seconds")
Michael Walsh6741f742017-02-20 16:16:38 -0600794
795 plug_in_setup()
796 rc, shell_rc, failed_plug_in_name = \
797 grpi.rprocess_plug_in_packages(call_point="post_boot")
798 if rc != 0:
799 error_message = "Plug-in failed with non-zero return code.\n" +\
800 gp.sprint_var(rc, 1)
801 BuiltIn().fail(gp.sprint_error(error_message))
802
Michael Walsh6741f742017-02-20 16:16:38 -0600803
Michael Walsh6741f742017-02-20 16:16:38 -0600804def test_loop_body():
805
806 r"""
807 The main loop body for the loop in main_py.
808
809 Description of arguments:
810 boot_count The iteration number (starts at 1).
811 """
812
813 global boot_count
814 global state
815 global next_boot
816 global boot_success
Sunil M325eb542017-08-10 07:09:43 -0500817 global boot_end_time
Michael Walsh6741f742017-02-20 16:16:38 -0600818
Michael Walshb5839d02017-04-12 16:11:20 -0500819 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600820
821 next_boot = select_boot()
Michael Walshb5839d02017-04-12 16:11:20 -0500822 if next_boot == "":
823 return True
Michael Walsh6741f742017-02-20 16:16:38 -0600824
Michael Walshb5839d02017-04-12 16:11:20 -0500825 boot_count += 1
826 gp.qprint_timen("Starting boot " + str(boot_count) + ".")
Michael Walsh6741f742017-02-20 16:16:38 -0600827
Michael Walshe0cf8d72017-05-17 13:20:46 -0500828 pre_boot_plug_in_setup()
Michael Walsh6741f742017-02-20 16:16:38 -0600829
830 cmd_buf = ["run_boot", next_boot]
831 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
832 if boot_status == "FAIL":
Michael Walshb5839d02017-04-12 16:11:20 -0500833 gp.qprint(msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600834
Michael Walshb5839d02017-04-12 16:11:20 -0500835 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600836 if boot_status == "PASS":
837 boot_success = 1
Michael Walshff340002017-08-29 11:18:27 -0500838 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot +
839 "\" succeeded.")
Michael Walsh6741f742017-02-20 16:16:38 -0600840 else:
841 boot_success = 0
Michael Walshff340002017-08-29 11:18:27 -0500842 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot +
843 "\" failed.")
Sunil M325eb542017-08-10 07:09:43 -0500844
845 # Set boot_end_time for use by plug-ins.
846 boot_end_time = completion_msg[1:33]
847 gp.qprint_var(boot_end_time)
848
849 gp.qprint(completion_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600850
851 boot_results.update(next_boot, boot_status)
852
853 plug_in_setup()
854 # NOTE: A post_test_case call point failure is NOT counted as a boot
855 # failure.
856 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500857 call_point='post_test_case', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600858
859 plug_in_setup()
860 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
861 call_point='ffdc_check', shell_rc=0x00000200,
862 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
863 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500864 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
865 if status != 'PASS':
Michael Walshff340002017-08-29 11:18:27 -0500866 gp.qprint_error("Call to my_ffdc failed.\n")
Michael Walsh6741f742017-02-20 16:16:38 -0600867
Michael Walshaabef1e2017-09-20 15:16:17 -0500868 if delete_errlogs:
869 # We need to purge error logs between boots or they build up.
870 grk.run_key("Delete Error logs", ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -0500871
Michael Walsh952f9b02017-03-09 13:11:14 -0600872 boot_results.print_report()
Michael Walshb5839d02017-04-12 16:11:20 -0500873 gp.qprint_timen("Finished boot " + str(boot_count) + ".")
Michael Walsh952f9b02017-03-09 13:11:14 -0600874
Michael Walsh6741f742017-02-20 16:16:38 -0600875 plug_in_setup()
876 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
877 call_point='stop_check')
878 if rc != 0:
879 error_message = "Stopping as requested by user.\n"
880 grp.rprint_error_report(error_message)
881 BuiltIn().fail(error_message)
882
Michael Walshd139f282017-04-04 18:00:23 -0500883 # This should help prevent ConnectionErrors.
Michael Walsh0960b382017-06-22 16:23:37 -0500884 grk.run_key_u("Close All Connections")
Michael Walshd139f282017-04-04 18:00:23 -0500885
Michael Walsh6741f742017-02-20 16:16:38 -0600886 return True
887
Michael Walsh6741f742017-02-20 16:16:38 -0600888
Michael Walsh83f4bc72017-04-20 16:49:43 -0500889def obmc_boot_test_teardown():
Michael Walsh6741f742017-02-20 16:16:38 -0600890
891 r"""
Michael Walshc9116812017-03-10 14:23:06 -0600892 Clean up after the Main keyword.
Michael Walsh6741f742017-02-20 16:16:38 -0600893 """
894
895 if cp_setup_called:
896 plug_in_setup()
897 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
Michael Walsh600876d2017-05-30 17:58:58 -0500898 call_point='cleanup', stop_on_plug_in_failure=0)
Michael Walsh6741f742017-02-20 16:16:38 -0600899
Michael Walsh600876d2017-05-30 17:58:58 -0500900 if 'boot_results_file_path' in globals():
901 # Save boot_results object to a file in case it is needed again.
902 gp.qprint_timen("Saving boot_results to the following path.")
903 gp.qprint_var(boot_results_file_path)
904 pickle.dump(boot_results, open(boot_results_file_path, 'wb'),
905 pickle.HIGHEST_PROTOCOL)
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600906
Michael Walshff340002017-08-29 11:18:27 -0500907 global save_stack
908 # Restore any global values saved on the save_stack.
909 for parm_name in main_func_parm_list:
910 # Get the parm_value if it was saved on the stack.
911 try:
912 parm_value = save_stack.pop(parm_name)
913 except:
914 # If it was not saved, no further action is required.
915 continue
916
917 # Restore the saved value.
918 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
919 "}\", parm_value)"
920 gp.dpissuing(cmd_buf)
921 exec(cmd_buf)
922
923 gp.dprintn(save_stack.sprint_obj())
924
Michael Walsh6741f742017-02-20 16:16:38 -0600925
Michael Walshc9116812017-03-10 14:23:06 -0600926def test_teardown():
927
928 r"""
929 Clean up after this test case.
930 """
931
932 gp.qprintn()
933 cmd_buf = ["Print Error",
934 "A keyword timeout occurred ending this program.\n"]
935 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
936
Michael Walshb5839d02017-04-12 16:11:20 -0500937 grp.rqprint_pgm_footer()
938
Michael Walshc9116812017-03-10 14:23:06 -0600939
Michael Walshff340002017-08-29 11:18:27 -0500940def obmc_boot_test_py(loc_boot_stack=None,
941 loc_stack_mode=None,
942 loc_quiet=None):
Michael Walsh6741f742017-02-20 16:16:38 -0600943
944 r"""
945 Do main program processing.
946 """
947
Michael Walshff340002017-08-29 11:18:27 -0500948 global save_stack
949
950 # Process function parms.
951 for parm_name in main_func_parm_list:
952 # Get parm's value.
953 cmd_buf = "parm_value = loc_" + parm_name
954 exec(cmd_buf)
955 gp.dpvar(parm_name)
956 gp.dpvar(parm_value)
957
958 if parm_value is None:
959 # Parm was not specified by the calling function so set it to its
960 # corresponding global value.
961 cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\
962 "(\"${" + parm_name + "}\")"
963 gp.dpissuing(cmd_buf)
964 exec(cmd_buf)
965 else:
966 # Save the global value on a stack.
967 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
968 parm_name + "}\"), \"" + parm_name + "\")"
969 gp.dpissuing(cmd_buf)
970 exec(cmd_buf)
971
972 # Set the global value to the passed value.
973 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
974 "}\", loc_" + parm_name + ")"
975 gp.dpissuing(cmd_buf)
976 exec(cmd_buf)
977
978 gp.dprintn(save_stack.sprint_obj())
Michael Walshb5839d02017-04-12 16:11:20 -0500979
Michael Walsh6741f742017-02-20 16:16:38 -0600980 setup()
981
Michael Walshcd9fbfd2017-09-19 12:00:08 -0500982 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
983
Michael Walsha20da402017-03-31 16:27:45 -0500984 if ffdc_only:
985 gp.qprint_timen("Caller requested ffdc_only.")
Michael Walshe0cf8d72017-05-17 13:20:46 -0500986 pre_boot_plug_in_setup()
Michael Walsh83f4bc72017-04-20 16:49:43 -0500987 grk.run_key_u("my_ffdc")
Michael Walsh764d2f82017-04-27 16:01:08 -0500988 return
Michael Walsha20da402017-03-31 16:27:45 -0500989
Michael Walsh6741f742017-02-20 16:16:38 -0600990 # Process caller's boot_stack.
991 while (len(boot_stack) > 0):
992 test_loop_body()
993
Michael Walshb5839d02017-04-12 16:11:20 -0500994 gp.qprint_timen("Finished processing stack.")
Michael Walsh30dadae2017-02-27 14:25:52 -0600995
Michael Walsh6741f742017-02-20 16:16:38 -0600996 # Process caller's boot_list.
997 if len(boot_list) > 0:
998 for ix in range(1, max_num_tests + 1):
999 test_loop_body()
1000
Michael Walshb5839d02017-04-12 16:11:20 -05001001 gp.qprint_timen("Completed all requested boot tests.")
1002
1003 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001004 new_fail = boot_fail - init_boot_fail
1005 if new_fail > boot_fail_threshold:
Michael Walshb5839d02017-04-12 16:11:20 -05001006 error_message = "Boot failures exceed the boot failure" +\
1007 " threshold:\n" +\
Michael Walshcd9fbfd2017-09-19 12:00:08 -05001008 gp.sprint_var(new_fail) +\
Michael Walshb5839d02017-04-12 16:11:20 -05001009 gp.sprint_var(boot_fail_threshold)
1010 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -06001011