blob: 731d23314b4276a96406ccc43dc4cf28a82bce7d [file] [log] [blame]
Michael Walsh0bbd8602016-11-22 11:31:49 -06001#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
Michael Walsh0b93fbf2017-03-02 14:42:41 -06007import os
8import imp
9import time
10import glob
11import random
12import cPickle as pickle
13
14from robot.utils import DotDict
15from robot.libraries.BuiltIn import BuiltIn
16
Michael Walsh6741f742017-02-20 16:16:38 -060017from boot_data import *
Michael Walshc9116812017-03-10 14:23:06 -060018import gen_print as gp
Michael Walsh0bbd8602016-11-22 11:31:49 -060019import gen_robot_print as grp
Michael Walsh55302292017-01-10 11:43:02 -060020import gen_robot_plug_in as grpi
Michael Walsh6741f742017-02-20 16:16:38 -060021import gen_robot_valid as grv
22import gen_misc as gm
23import gen_cmd as gc
Michael Walshb5839d02017-04-12 16:11:20 -050024import gen_robot_keyword as grk
Michael Walsh55302292017-01-10 11:43:02 -060025import state as st
Michael Walsh0bbd8602016-11-22 11:31:49 -060026
Michael Walsh0b93fbf2017-03-02 14:42:41 -060027base_path = os.path.dirname(os.path.dirname(
28 imp.find_module("gen_robot_print")[1])) +\
Michael Walshc9116812017-03-10 14:23:06 -060029 os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060030sys.path.append(base_path + "extended/")
31import run_keyword as rk
Michael Walsh0bbd8602016-11-22 11:31:49 -060032
Michael Walshe1e26442017-03-06 17:50:07 -060033# Setting master_pid correctly influences the behavior of plug-ins like
34# DB_Logging
35program_pid = os.getpid()
36master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
37
Michael Walshb5839d02017-04-12 16:11:20 -050038# Set up boot data structures.
39boot_table = create_boot_table()
40valid_boot_types = create_valid_boot_list(boot_table)
Michael Walsh0b93fbf2017-03-02 14:42:41 -060041
Michael Walsh6741f742017-02-20 16:16:38 -060042boot_lists = read_boot_lists()
43last_ten = []
Michael Walsh6741f742017-02-20 16:16:38 -060044
45state = st.return_default_state()
46cp_setup_called = 0
47next_boot = ""
48base_tool_dir_path = os.path.normpath(os.environ.get(
49 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
Michael Walshb5839d02017-04-12 16:11:20 -050050
Michael Walsh6741f742017-02-20 16:16:38 -060051ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
Michael Walsh6741f742017-02-20 16:16:38 -060052boot_success = 0
Michael Walsh6741f742017-02-20 16:16:38 -060053status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
54if status_dir_path != "":
55 status_dir_path = os.path.normpath(status_dir_path) + os.sep
Michael Walsh0b93fbf2017-03-02 14:42:41 -060056default_power_on = "REST Power On"
57default_power_off = "REST Power Off"
Michael Walsh6741f742017-02-20 16:16:38 -060058boot_count = 0
Michael Walsh0bbd8602016-11-22 11:31:49 -060059
Michael Walsh85678942017-03-27 14:34:22 -050060LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
61
62
63###############################################################################
Michael Walshb5839d02017-04-12 16:11:20 -050064def process_pgm_parms():
65
66 r"""
67 Process the program parameters by assigning them all to corresponding
68 globals. Also, set some global values that depend on program parameters.
69 """
70
71 # Program parameter processing.
72 # Assign all program parms to python variables which are global to this
73 # module.
74
75 global parm_list
76 parm_list = BuiltIn().get_variable_value("${parm_list}")
77 # The following subset of parms should be processed as integers.
78 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
79 'boot_fail_threshold', 'quiet', 'test_mode', 'debug']
80 for parm in parm_list:
81 if parm in int_list:
82 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
83 "}\", \"0\"))"
84 else:
85 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
86 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
87 exec(cmd_buf)
88
89 global ffdc_dir_path_style
90 global boot_list
91 global boot_stack
92 global boot_results_file_path
93 global boot_results
94 global ffdc_list_file_path
95
96 if ffdc_dir_path_style == "":
97 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
98
99 # Convert these program parms to lists for easier processing..
100 boot_list = filter(None, boot_list.split(":"))
101 boot_stack = filter(None, boot_stack.split(":"))
102
103 boot_results_file_path = "/tmp/" + openbmc_nickname + ":pid_" +\
104 str(master_pid) + ":boot_results"
105
106 if os.path.isfile(boot_results_file_path):
107 # We've been called before in this run so we'll load the saved
108 # boot_results object.
109 boot_results = pickle.load(open(boot_results_file_path, 'rb'))
110 else:
111 boot_results = boot_results(boot_table, boot_pass, boot_fail)
112
113 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
114 "/FFDC_FILE_LIST"
115
116###############################################################################
117
118
119###############################################################################
Michael Walsh85678942017-03-27 14:34:22 -0500120def initial_plug_in_setup():
121
122 r"""
123 Initialize all plug-in environment variables which do not change for the
124 duration of the program.
125
126 """
127
128 global LOG_LEVEL
129 BuiltIn().set_log_level("NONE")
130
131 BuiltIn().set_global_variable("${master_pid}", master_pid)
132 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
133 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
134 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
135 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
136 ffdc_list_file_path)
137
138 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
139 ffdc_dir_path_style)
140 BuiltIn().set_global_variable("${FFDC_CHECK}",
141 ffdc_check)
142
143 # For each program parameter, set the corresponding AUTOBOOT_ environment
144 # variable value. Also, set an AUTOBOOT_ environment variable for every
145 # element in additional_values.
146 additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
147 "status_dir_path", "base_tool_dir_path",
148 "ffdc_list_file_path"]
149
150 plug_in_vars = parm_list + additional_values
151
152 for var_name in plug_in_vars:
153 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
154 var_name = var_name.upper()
155 if var_value is None:
156 var_value = ""
157 os.environ["AUTOBOOT_" + var_name] = str(var_value)
158
159 BuiltIn().set_log_level(LOG_LEVEL)
160
Michael Walsh68a61162017-04-25 11:54:06 -0500161 # Make sure the ffdc list directory exists.
162 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
163 if not os.path.exists(ffdc_list_dir_path):
164 os.makedirs(ffdc_list_dir_path)
Michael Walsh85678942017-03-27 14:34:22 -0500165
166###############################################################################
167
Michael Walsh0bbd8602016-11-22 11:31:49 -0600168
169###############################################################################
Michael Walsh0bbd8602016-11-22 11:31:49 -0600170def plug_in_setup():
171
172 r"""
Michael Walsh85678942017-03-27 14:34:22 -0500173 Initialize all changing plug-in environment variables for use by the
174 plug-in programs.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600175 """
176
Michael Walsh85678942017-03-27 14:34:22 -0500177 global LOG_LEVEL
178 global test_really_running
179
180 BuiltIn().set_log_level("NONE")
181
Michael Walsh6741f742017-02-20 16:16:38 -0600182 boot_pass, boot_fail = boot_results.return_total_pass_fail()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600183 if boot_pass > 1:
184 test_really_running = 1
185 else:
186 test_really_running = 0
187
Michael Walsh0bbd8602016-11-22 11:31:49 -0600188 seconds = time.time()
189 loc_time = time.localtime(seconds)
190 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
191
Michael Walsh6741f742017-02-20 16:16:38 -0600192 ffdc_prefix = openbmc_nickname + "." + time_string
Michael Walsh0bbd8602016-11-22 11:31:49 -0600193
Michael Walsh6741f742017-02-20 16:16:38 -0600194 BuiltIn().set_global_variable("${test_really_running}",
195 test_really_running)
196 BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
Michael Walsh6741f742017-02-20 16:16:38 -0600197 BuiltIn().set_global_variable("${boot_pass}", boot_pass)
198 BuiltIn().set_global_variable("${boot_fail}", boot_fail)
199 BuiltIn().set_global_variable("${boot_success}", boot_success)
200 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
Michael Walsh4c9a6452016-12-13 16:03:11 -0600201
Michael Walsh0bbd8602016-11-22 11:31:49 -0600202 # For each program parameter, set the corresponding AUTOBOOT_ environment
203 # variable value. Also, set an AUTOBOOT_ environment variable for every
204 # element in additional_values.
205 additional_values = ["boot_type_desc", "boot_success", "boot_pass",
Michael Walsh85678942017-03-27 14:34:22 -0500206 "boot_fail", "test_really_running", "ffdc_prefix"]
Michael Walsh0bbd8602016-11-22 11:31:49 -0600207
Michael Walsh85678942017-03-27 14:34:22 -0500208 plug_in_vars = additional_values
Michael Walsh0bbd8602016-11-22 11:31:49 -0600209
210 for var_name in plug_in_vars:
211 var_value = BuiltIn().get_variable_value("${" + var_name + "}")
212 var_name = var_name.upper()
213 if var_value is None:
214 var_value = ""
Michael Walsh6741f742017-02-20 16:16:38 -0600215 os.environ["AUTOBOOT_" + var_name] = str(var_value)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600216
Michael Walsh0bbd8602016-11-22 11:31:49 -0600217 if debug:
Michael Walsh6741f742017-02-20 16:16:38 -0600218 shell_rc, out_buf = \
219 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600220
Michael Walsh85678942017-03-27 14:34:22 -0500221 BuiltIn().set_log_level(LOG_LEVEL)
222
Michael Walsh0bbd8602016-11-22 11:31:49 -0600223###############################################################################
224
225
226###############################################################################
Michael Walsh6741f742017-02-20 16:16:38 -0600227def setup():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600228
229 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600230 Do general program setup tasks.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600231 """
232
Michael Walsh6741f742017-02-20 16:16:38 -0600233 global cp_setup_called
Michael Walsh0bbd8602016-11-22 11:31:49 -0600234
Michael Walshb5839d02017-04-12 16:11:20 -0500235 gp.qprintn()
236
Michael Walsh83f4bc72017-04-20 16:49:43 -0500237 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
238 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
239 # If we can't find ssh_pw, then we don't have our repo bin in PATH.
Michael Walshb5839d02017-04-12 16:11:20 -0500240 shell_rc, out_buf = gc.cmd_fnc_u("which ssh_pw", quiet=1, print_output=0,
241 show_err=0)
242 if shell_rc != 0:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500243 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
244 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
245 if robot_pgm_dir_path not in sys.path:
246 sys.path.append(robot_pgm_dir_path)
247 PYTHONPATH = os.environ.get("PYTHONPATH", "")
248 if PYTHONPATH == "":
249 os.environ['PYTHONPATH'] = robot_pgm_dir_path
250 else:
251 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
Michael Walsh6741f742017-02-20 16:16:38 -0600252
253 validate_parms()
254
255 grp.rqprint_pgm_header()
256
Michael Walshb5839d02017-04-12 16:11:20 -0500257 grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE")
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500258
Michael Walsh85678942017-03-27 14:34:22 -0500259 initial_plug_in_setup()
260
Michael Walsh6741f742017-02-20 16:16:38 -0600261 plug_in_setup()
262 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
263 call_point='setup')
264 if rc != 0:
265 error_message = "Plug-in setup failed.\n"
266 grp.rprint_error_report(error_message)
267 BuiltIn().fail(error_message)
268 # Setting cp_setup_called lets our Teardown know that it needs to call
269 # the cleanup plug-in call point.
270 cp_setup_called = 1
271
272 # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
273 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
Michael Walsh85678942017-03-27 14:34:22 -0500274 # FFDC_LOG_PATH is used by "FFDC" keyword.
275 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
Michael Walsh6741f742017-02-20 16:16:38 -0600276
Michael Walshb5839d02017-04-12 16:11:20 -0500277 gp.dprint_var(boot_table, 1)
278 gp.dprint_var(boot_lists)
Michael Walsh0bbd8602016-11-22 11:31:49 -0600279
280###############################################################################
281
282
283###############################################################################
Michael Walsh6741f742017-02-20 16:16:38 -0600284def validate_parms():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600285
286 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600287 Validate all program parameters.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600288 """
289
Michael Walshb5839d02017-04-12 16:11:20 -0500290 process_pgm_parms()
Michael Walsh0bbd8602016-11-22 11:31:49 -0600291
Michael Walshb5839d02017-04-12 16:11:20 -0500292 gp.qprintn()
293
294 global openbmc_model
Michael Walsh6741f742017-02-20 16:16:38 -0600295 grv.rvalid_value("openbmc_host")
296 grv.rvalid_value("openbmc_username")
297 grv.rvalid_value("openbmc_password")
298 if os_host != "":
299 grv.rvalid_value("os_username")
300 grv.rvalid_value("os_password")
Michael Walsh0bbd8602016-11-22 11:31:49 -0600301
Michael Walsh6741f742017-02-20 16:16:38 -0600302 if pdu_host != "":
303 grv.rvalid_value("pdu_username")
304 grv.rvalid_value("pdu_password")
Michael Walsh85678942017-03-27 14:34:22 -0500305 grv.rvalid_integer("pdu_slot_no")
Michael Walsh6741f742017-02-20 16:16:38 -0600306 if openbmc_serial_host != "":
307 grv.rvalid_integer("openbmc_serial_port")
Michael Walshb5839d02017-04-12 16:11:20 -0500308 if openbmc_model == "":
309 status, ret_values =\
310 grk.run_key_u("Get BMC System Model")
311 openbmc_model = ret_values
312 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
Michael Walsh6741f742017-02-20 16:16:38 -0600313 grv.rvalid_value("openbmc_model")
Michael Walshb5839d02017-04-12 16:11:20 -0500314 grv.rvalid_integer("max_num_tests")
Michael Walsh6741f742017-02-20 16:16:38 -0600315 grv.rvalid_integer("boot_pass")
316 grv.rvalid_integer("boot_fail")
317
318 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
319 BuiltIn().set_global_variable("${plug_in_packages_list}",
320 plug_in_packages_list)
321
Michael Walshb5839d02017-04-12 16:11:20 -0500322 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip'])
Michael Walsha20da402017-03-31 16:27:45 -0500323 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
Michael Walsh6741f742017-02-20 16:16:38 -0600324 error_message = "You must provide either a value for either the" +\
325 " boot_list or the boot_stack parm.\n"
326 BuiltIn().fail(gp.sprint_error(error_message))
327
328 valid_boot_list(boot_list, valid_boot_types)
329 valid_boot_list(boot_stack, valid_boot_types)
330
Michael Walsh11cfc8c2017-03-31 09:40:55 -0500331 selected_PDU_boots = list(set(boot_list + boot_stack) &
332 set(boot_lists['PDU_reboot']))
333
334 if len(selected_PDU_boots) > 0 and pdu_host == "":
335 error_message = "You have selected the following boots which" +\
336 " require a PDU host but no value for pdu_host:\n"
337 error_message += gp.sprint_var(selected_PDU_boots)
338 error_message += gp.sprint_var(pdu_host, 2)
339 BuiltIn().fail(gp.sprint_error(error_message))
340
Michael Walsh6741f742017-02-20 16:16:38 -0600341 return
Michael Walsh0bbd8602016-11-22 11:31:49 -0600342
343###############################################################################
344
345
346###############################################################################
Michael Walsh6741f742017-02-20 16:16:38 -0600347def my_get_state():
Michael Walsh0bbd8602016-11-22 11:31:49 -0600348
349 r"""
Michael Walsh6741f742017-02-20 16:16:38 -0600350 Get the system state plus a little bit of wrapping.
Michael Walsh0bbd8602016-11-22 11:31:49 -0600351 """
352
Michael Walsh6741f742017-02-20 16:16:38 -0600353 global state
354
355 req_states = ['epoch_seconds'] + st.default_req_states
356
Michael Walshb5839d02017-04-12 16:11:20 -0500357 gp.qprint_timen("Getting system state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600358 if test_mode:
359 state['epoch_seconds'] = int(time.time())
360 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500361 state = st.get_state(req_states=req_states, quiet=quiet)
362 gp.qprint_var(state)
Michael Walsh341c21e2017-01-17 16:25:20 -0600363
364###############################################################################
365
366
367###############################################################################
Michael Walsh6741f742017-02-20 16:16:38 -0600368def select_boot():
Michael Walsh341c21e2017-01-17 16:25:20 -0600369
370 r"""
371 Select a boot test to be run based on our current state and return the
372 chosen boot type.
373
374 Description of arguments:
Michael Walsh6741f742017-02-20 16:16:38 -0600375 state The state of the machine.
Michael Walsh341c21e2017-01-17 16:25:20 -0600376 """
377
Michael Walsh30dadae2017-02-27 14:25:52 -0600378 global boot_stack
379
Michael Walshb5839d02017-04-12 16:11:20 -0500380 gp.qprint_timen("Selecting a boot test.")
Michael Walsh6741f742017-02-20 16:16:38 -0600381
382 my_get_state()
383
384 stack_popped = 0
385 if len(boot_stack) > 0:
386 stack_popped = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500387 gp.qprint_dashes()
388 gp.qprint_var(boot_stack)
389 gp.qprint_dashes()
390 skip_boot_printed = 0
391 while len(boot_stack) > 0:
392 boot_candidate = boot_stack.pop()
393 if stack_mode == 'normal':
394 break
395 else:
396 if st.compare_states(state, boot_table[boot_candidate]['end']):
397 if not skip_boot_printed:
398 gp.print_var(stack_mode)
399 gp.printn()
400 gp.print_timen("Skipping the following boot tests" +
401 " which are unnecessary since their" +
402 " required end states match the" +
403 " current machine state:")
404 skip_boot_printed = 1
405 gp.print_var(boot_candidate)
406 boot_candidate = ""
407 if boot_candidate == "":
408 gp.qprint_dashes()
409 gp.qprint_var(boot_stack)
410 gp.qprint_dashes()
411 return boot_candidate
Michael Walsh6741f742017-02-20 16:16:38 -0600412 if st.compare_states(state, boot_table[boot_candidate]['start']):
Michael Walshb5839d02017-04-12 16:11:20 -0500413 gp.qprint_timen("The machine state is valid for a '" +
414 boot_candidate + "' boot test.")
415 gp.qprint_dashes()
416 gp.qprint_var(boot_stack)
417 gp.qprint_dashes()
Michael Walsh6741f742017-02-20 16:16:38 -0600418 return boot_candidate
Michael Walsh341c21e2017-01-17 16:25:20 -0600419 else:
Michael Walshb5839d02017-04-12 16:11:20 -0500420 gp.qprint_timen("The machine state does not match the required" +
421 " starting state for a '" + boot_candidate +
422 "' boot test:")
423 gp.print_varx("boot_table[" + boot_candidate + "][start]",
424 boot_table[boot_candidate]['start'], 1)
Michael Walsh6741f742017-02-20 16:16:38 -0600425 boot_stack.append(boot_candidate)
426 popped_boot = boot_candidate
427
428 # Loop through your list selecting a boot_candidates
429 boot_candidates = []
430 for boot_candidate in boot_list:
431 if st.compare_states(state, boot_table[boot_candidate]['start']):
432 if stack_popped:
433 if st.compare_states(boot_table[boot_candidate]['end'],
434 boot_table[popped_boot]['start']):
435 boot_candidates.append(boot_candidate)
436 else:
437 boot_candidates.append(boot_candidate)
438
439 if len(boot_candidates) == 0:
Michael Walshb5839d02017-04-12 16:11:20 -0500440 gp.qprint_timen("The user's boot list contained no boot tests" +
441 " which are valid for the current machine state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600442 boot_candidate = default_power_on
443 if not st.compare_states(state, boot_table[default_power_on]['start']):
444 boot_candidate = default_power_off
445 boot_candidates.append(boot_candidate)
Michael Walshb5839d02017-04-12 16:11:20 -0500446 gp.qprint_timen("Using default '" + boot_candidate +
447 "' boot type to transition to valid state.")
Michael Walsh6741f742017-02-20 16:16:38 -0600448
Michael Walshb5839d02017-04-12 16:11:20 -0500449 gp.dprint_var(boot_candidates)
Michael Walsh6741f742017-02-20 16:16:38 -0600450
451 # Randomly select a boot from the candidate list.
452 boot = random.choice(boot_candidates)
Michael Walsh341c21e2017-01-17 16:25:20 -0600453
454 return boot
Michael Walsh0bbd8602016-11-22 11:31:49 -0600455
456###############################################################################
Michael Walsh55302292017-01-10 11:43:02 -0600457
458
459###############################################################################
Michael Walsh341c21e2017-01-17 16:25:20 -0600460def print_last_boots():
461
462 r"""
463 Print the last ten boots done with their time stamps.
464 """
465
466 # indent 0, 90 chars wide, linefeed, char is "="
Michael Walshb5839d02017-04-12 16:11:20 -0500467 gp.qprint_dashes(0, 90)
468 gp.qprintn("Last 10 boots:\n")
Michael Walsh341c21e2017-01-17 16:25:20 -0600469
470 for boot_entry in last_ten:
471 grp.rqprint(boot_entry)
Michael Walshb5839d02017-04-12 16:11:20 -0500472 gp.qprint_dashes(0, 90)
Michael Walsh341c21e2017-01-17 16:25:20 -0600473
474###############################################################################
475
476
477###############################################################################
Michael Walsh341c21e2017-01-17 16:25:20 -0600478def print_defect_report():
479
480 r"""
481 Print a defect report.
482 """
483
Michael Walsh341c21e2017-01-17 16:25:20 -0600484 # At some point I'd like to have the 'Call FFDC Methods' return a list
485 # of files it has collected. In that case, the following "ls" command
486 # would no longer be needed. For now, however, glob shows the files
487 # named in FFDC_LIST_FILE_PATH so I will refrain from printing those
488 # out (so we don't see duplicates in the list).
489
490 LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}")
491
492 output = '\n'.join(glob.glob(LOG_PREFIX + '*'))
Michael Walsh341c21e2017-01-17 16:25:20 -0600493 try:
Michael Walsh6741f742017-02-20 16:16:38 -0600494 ffdc_list = open(ffdc_list_file_path, 'r')
Michael Walsh341c21e2017-01-17 16:25:20 -0600495 except IOError:
496 ffdc_list = ""
497
Michael Walsh68a61162017-04-25 11:54:06 -0500498 # Open ffdc_file_list for writing. We will write a complete list of
499 # FFDC files to it for possible use by plug-ins like cp_stop_check.
500 ffdc_list_file = open(ffdc_list_file_path, 'w')
501
502 gp.qprintn()
503 # indent=0, width=90, linefeed=1, char="="
504 gp.qprint_dashes(0, 90, 1, "=")
505 gp.qprintn("Copy this data to the defect:\n")
506
507 grp.rqpvars(*parm_list)
508
509 gp.qprintn()
510
511 print_last_boots()
512 gp.qprintn()
513 gp.qprint_var(state)
514
Michael Walshb5839d02017-04-12 16:11:20 -0500515 gp.qprintn()
516 gp.qprintn("FFDC data files:")
Michael Walsh341c21e2017-01-17 16:25:20 -0600517 if status_file_path != "":
Michael Walshb5839d02017-04-12 16:11:20 -0500518 gp.qprintn(status_file_path)
Michael Walsh68a61162017-04-25 11:54:06 -0500519 ffdc_list_file.write(status_file_path + "\n")
Michael Walsh341c21e2017-01-17 16:25:20 -0600520
Michael Walshb5839d02017-04-12 16:11:20 -0500521 gp.qprintn(output)
522 # gp.qprintn(ffdc_list)
523 gp.qprintn()
Michael Walsh341c21e2017-01-17 16:25:20 -0600524
Michael Walshb5839d02017-04-12 16:11:20 -0500525 gp.qprint_dashes(0, 90, 1, "=")
Michael Walsh341c21e2017-01-17 16:25:20 -0600526
Michael Walsh68a61162017-04-25 11:54:06 -0500527 ffdc_list_file.write(output + "\n")
528 ffdc_list_file.close()
529
Michael Walsh341c21e2017-01-17 16:25:20 -0600530###############################################################################
Michael Walsh6741f742017-02-20 16:16:38 -0600531
532
533###############################################################################
534def my_ffdc():
535
536 r"""
537 Collect FFDC data.
538 """
539
540 global state
541
542 plug_in_setup()
543 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
544 call_point='ffdc', stop_on_plug_in_failure=1)
545
546 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
Michael Walsh83f4bc72017-04-20 16:49:43 -0500547 status, ret_values = grk.run_key_u("FFDC ffdc_prefix=" +
548 AUTOBOOT_FFDC_PREFIX +
549 " ffdc_function_list=" +
550 ffdc_function_list, ignore=1)
551 if status != 'PASS':
Michael Walsh3328caf2017-03-21 17:04:08 -0500552 gp.print_error("Call to ffdc failed.\n")
Michael Walsh6741f742017-02-20 16:16:38 -0600553
554 my_get_state()
555
556 print_defect_report()
557
558###############################################################################
559
560
561###############################################################################
562def print_test_start_message(boot_keyword):
563
564 r"""
565 Print a message indicating what boot test is about to run.
566
567 Description of arguments:
568 boot_keyword The name of the boot which is to be run
569 (e.g. "BMC Power On").
570 """
571
572 global last_ten
573
574 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
Michael Walshb5839d02017-04-12 16:11:20 -0500575 gp.qprint(doing_msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600576
577 last_ten.append(doing_msg)
578
579 if len(last_ten) > 10:
580 del last_ten[0]
581
582###############################################################################
583
584
585###############################################################################
586def run_boot(boot):
587
588 r"""
589 Run the specified boot.
590
591 Description of arguments:
592 boot The name of the boot test to be performed.
593 """
594
595 global state
596
597 print_test_start_message(boot)
598
599 plug_in_setup()
600 rc, shell_rc, failed_plug_in_name = \
601 grpi.rprocess_plug_in_packages(call_point="pre_boot")
602 if rc != 0:
603 error_message = "Plug-in failed with non-zero return code.\n" +\
604 gp.sprint_var(rc, 1)
605 BuiltIn().fail(gp.sprint_error(error_message))
606
607 if test_mode:
608 # In test mode, we'll pretend the boot worked by assigning its
609 # required end state to the default state value.
Michael Walsh30dadae2017-02-27 14:25:52 -0600610 state = st.strip_anchor_state(boot_table[boot]['end'])
Michael Walsh6741f742017-02-20 16:16:38 -0600611 else:
612 # Assertion: We trust that the state data was made fresh by the
613 # caller.
614
Michael Walshb5839d02017-04-12 16:11:20 -0500615 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600616
617 if boot_table[boot]['method_type'] == "keyword":
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600618 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
Michael Walshb5839d02017-04-12 16:11:20 -0500619 boot_table[boot]['method'],
620 quiet=quiet)
Michael Walsh6741f742017-02-20 16:16:38 -0600621
622 if boot_table[boot]['bmc_reboot']:
623 st.wait_for_comm_cycle(int(state['epoch_seconds']))
Michael Walsh30dadae2017-02-27 14:25:52 -0600624 plug_in_setup()
625 rc, shell_rc, failed_plug_in_name = \
626 grpi.rprocess_plug_in_packages(call_point="post_reboot")
627 if rc != 0:
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600628 error_message = "Plug-in failed with non-zero return code.\n"
629 error_message += gp.sprint_var(rc, 1)
Michael Walsh30dadae2017-02-27 14:25:52 -0600630 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600631 else:
632 match_state = st.anchor_state(state)
633 del match_state['epoch_seconds']
634 # Wait for the state to change in any way.
635 st.wait_state(match_state, wait_time=state_change_timeout,
636 interval="3 seconds", invert=1)
637
Michael Walshb5839d02017-04-12 16:11:20 -0500638 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600639 if boot_table[boot]['end']['chassis'] == "Off":
640 boot_timeout = power_off_timeout
641 else:
642 boot_timeout = power_on_timeout
643 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
644 interval="3 seconds")
645
646 plug_in_setup()
647 rc, shell_rc, failed_plug_in_name = \
648 grpi.rprocess_plug_in_packages(call_point="post_boot")
649 if rc != 0:
650 error_message = "Plug-in failed with non-zero return code.\n" +\
651 gp.sprint_var(rc, 1)
652 BuiltIn().fail(gp.sprint_error(error_message))
653
654###############################################################################
655
656
657###############################################################################
658def test_loop_body():
659
660 r"""
661 The main loop body for the loop in main_py.
662
663 Description of arguments:
664 boot_count The iteration number (starts at 1).
665 """
666
667 global boot_count
668 global state
669 global next_boot
670 global boot_success
671
Michael Walshb5839d02017-04-12 16:11:20 -0500672 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600673
674 next_boot = select_boot()
Michael Walshb5839d02017-04-12 16:11:20 -0500675 if next_boot == "":
676 return True
Michael Walsh6741f742017-02-20 16:16:38 -0600677
Michael Walshb5839d02017-04-12 16:11:20 -0500678 boot_count += 1
679 gp.qprint_timen("Starting boot " + str(boot_count) + ".")
Michael Walsh6741f742017-02-20 16:16:38 -0600680
681 # Clear the ffdc_list_file_path file. Plug-ins may now write to it.
682 try:
683 os.remove(ffdc_list_file_path)
684 except OSError:
685 pass
686
687 cmd_buf = ["run_boot", next_boot]
688 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
689 if boot_status == "FAIL":
Michael Walshb5839d02017-04-12 16:11:20 -0500690 gp.qprint(msg)
Michael Walsh6741f742017-02-20 16:16:38 -0600691
Michael Walshb5839d02017-04-12 16:11:20 -0500692 gp.qprintn()
Michael Walsh6741f742017-02-20 16:16:38 -0600693 if boot_status == "PASS":
694 boot_success = 1
Michael Walshb5839d02017-04-12 16:11:20 -0500695 gp.qprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.")
Michael Walsh6741f742017-02-20 16:16:38 -0600696 else:
697 boot_success = 0
Michael Walshb5839d02017-04-12 16:11:20 -0500698 gp.qprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.")
Michael Walsh6741f742017-02-20 16:16:38 -0600699
700 boot_results.update(next_boot, boot_status)
701
702 plug_in_setup()
703 # NOTE: A post_test_case call point failure is NOT counted as a boot
704 # failure.
705 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
706 call_point='post_test_case', stop_on_plug_in_failure=1)
707
708 plug_in_setup()
709 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
710 call_point='ffdc_check', shell_rc=0x00000200,
711 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
712 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200:
Michael Walsh83f4bc72017-04-20 16:49:43 -0500713 status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
714 if status != 'PASS':
Michael Walsh3328caf2017-03-21 17:04:08 -0500715 gp.print_error("Call to my_ffdc failed.\n")
Michael Walsh6741f742017-02-20 16:16:38 -0600716
Michael Walshd139f282017-04-04 18:00:23 -0500717 # We need to purge error logs between boots or they build up.
Michael Walshb5839d02017-04-12 16:11:20 -0500718 grk.run_key("Delete Error logs", ignore=1)
Michael Walshd139f282017-04-04 18:00:23 -0500719
Michael Walsh952f9b02017-03-09 13:11:14 -0600720 boot_results.print_report()
Michael Walshb5839d02017-04-12 16:11:20 -0500721 gp.qprint_timen("Finished boot " + str(boot_count) + ".")
Michael Walsh952f9b02017-03-09 13:11:14 -0600722
Michael Walsh6741f742017-02-20 16:16:38 -0600723 plug_in_setup()
724 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
725 call_point='stop_check')
726 if rc != 0:
727 error_message = "Stopping as requested by user.\n"
728 grp.rprint_error_report(error_message)
729 BuiltIn().fail(error_message)
730
Michael Walshd139f282017-04-04 18:00:23 -0500731 # This should help prevent ConnectionErrors.
Michael Walshb5839d02017-04-12 16:11:20 -0500732 grk.run_key_u("Delete All Sessions")
Michael Walshd139f282017-04-04 18:00:23 -0500733
Michael Walsh6741f742017-02-20 16:16:38 -0600734 return True
735
736###############################################################################
737
738
739###############################################################################
Michael Walsh83f4bc72017-04-20 16:49:43 -0500740def obmc_boot_test_teardown():
Michael Walsh6741f742017-02-20 16:16:38 -0600741
742 r"""
Michael Walshc9116812017-03-10 14:23:06 -0600743 Clean up after the Main keyword.
Michael Walsh6741f742017-02-20 16:16:38 -0600744 """
745
746 if cp_setup_called:
747 plug_in_setup()
748 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
749 call_point='cleanup', stop_on_plug_in_failure=1)
750
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600751 # Save boot_results object to a file in case it is needed again.
Michael Walshb5839d02017-04-12 16:11:20 -0500752 gp.qprint_timen("Saving boot_results to the following path.")
753 gp.qprint_var(boot_results_file_path)
Michael Walsh0b93fbf2017-03-02 14:42:41 -0600754 pickle.dump(boot_results, open(boot_results_file_path, 'wb'),
755 pickle.HIGHEST_PROTOCOL)
756
Michael Walsh6741f742017-02-20 16:16:38 -0600757###############################################################################
758
759
760###############################################################################
Michael Walshc9116812017-03-10 14:23:06 -0600761def test_teardown():
762
763 r"""
764 Clean up after this test case.
765 """
766
767 gp.qprintn()
768 cmd_buf = ["Print Error",
769 "A keyword timeout occurred ending this program.\n"]
770 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
771
Michael Walshb5839d02017-04-12 16:11:20 -0500772 grp.rqprint_pgm_footer()
773
Michael Walshc9116812017-03-10 14:23:06 -0600774###############################################################################
775
776
777###############################################################################
Michael Walsh83f4bc72017-04-20 16:49:43 -0500778def obmc_boot_test_py(alt_boot_stack=None):
Michael Walsh6741f742017-02-20 16:16:38 -0600779
780 r"""
781 Do main program processing.
782 """
783
Michael Walshb5839d02017-04-12 16:11:20 -0500784 if alt_boot_stack is not None:
785 BuiltIn().set_global_variable("${boot_stack}", alt_boot_stack)
786
Michael Walsh6741f742017-02-20 16:16:38 -0600787 setup()
788
Michael Walsha20da402017-03-31 16:27:45 -0500789 if ffdc_only:
790 gp.qprint_timen("Caller requested ffdc_only.")
Michael Walsh83f4bc72017-04-20 16:49:43 -0500791 grk.run_key_u("my_ffdc")
Michael Walsh764d2f82017-04-27 16:01:08 -0500792 return
Michael Walsha20da402017-03-31 16:27:45 -0500793
Michael Walsh6741f742017-02-20 16:16:38 -0600794 # Process caller's boot_stack.
795 while (len(boot_stack) > 0):
796 test_loop_body()
797
Michael Walshb5839d02017-04-12 16:11:20 -0500798 gp.qprint_timen("Finished processing stack.")
Michael Walsh30dadae2017-02-27 14:25:52 -0600799
Michael Walsh6741f742017-02-20 16:16:38 -0600800 # Process caller's boot_list.
801 if len(boot_list) > 0:
802 for ix in range(1, max_num_tests + 1):
803 test_loop_body()
804
Michael Walshb5839d02017-04-12 16:11:20 -0500805 gp.qprint_timen("Completed all requested boot tests.")
806
807 boot_pass, boot_fail = boot_results.return_total_pass_fail()
808 if boot_fail > boot_fail_threshold:
809 error_message = "Boot failures exceed the boot failure" +\
810 " threshold:\n" +\
811 gp.sprint_var(boot_fail) +\
812 gp.sprint_var(boot_fail_threshold)
813 BuiltIn().fail(gp.sprint_error(error_message))
Michael Walsh6741f742017-02-20 16:16:38 -0600814
815###############################################################################