George Keishing | e7e9171 | 2021-09-03 11:28:44 -0500 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 2 | |
| 3 | r""" |
| 4 | Check for stop conditions. Return code of 2 if stop conditions are found. |
| 5 | """ |
| 6 | |
Patrick Williams | a57fef4 | 2022-12-03 07:00:14 -0600 | [diff] [blame] | 7 | import argparse |
Michael Shepos | b3dffe8 | 2022-01-06 07:17:16 -0600 | [diff] [blame] | 8 | import os |
Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 9 | import re |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 10 | |
George Keishing | 7899a45 | 2023-02-15 02:46:54 -0600 | [diff] [blame] | 11 | from gen_arg import * # NOQA |
| 12 | from gen_call_robot import * # NOQA |
| 13 | from gen_cmd import * # NOQA |
| 14 | from gen_misc import * # NOQA |
| 15 | from gen_plug_in_utils import * # NOQA |
| 16 | from gen_print import * # NOQA |
| 17 | from gen_valid import * # NOQA |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 18 | |
Michael Walsh | 2ea965c | 2019-08-01 16:14:25 -0500 | [diff] [blame] | 19 | # Set exit_on_error for gen_valid functions. |
| 20 | set_exit_on_error(True) |
| 21 | |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 22 | # Initialize default plug-in parms.. |
| 23 | STOP_REST_FAIL = 0 |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 24 | STOP_COMMAND = "" |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 25 | stop_test_rc = 2 |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 26 | STOP_VERIFY_HARDWARE_FAIL = 0 |
| 27 | |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 28 | |
| 29 | # Create parser object to process command line parameters and args. |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 30 | parser = argparse.ArgumentParser( |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 31 | usage="%(prog)s [OPTIONS]", |
| 32 | description='If the "Stop" plug-in is selected by the user, %(prog)s' |
Patrick Williams | a57fef4 | 2022-12-03 07:00:14 -0600 | [diff] [blame] | 33 | + " is called by OBMC Boot Test after each boot test. If %(prog)s returns" |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 34 | + " " |
| 35 | + str(stop_test_rc) |
| 36 | + ", then OBMC Boot Test will stop. The user" |
Patrick Williams | a57fef4 | 2022-12-03 07:00:14 -0600 | [diff] [blame] | 37 | + " may set environment variable STOP_COMMAND to contain any valid bash" |
| 38 | + " command or program. %(prog)s will run this stop command. If the stop" |
| 39 | + " command returns non-zero, then %(prog)s will return " |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 40 | + str(stop_test_rc) |
| 41 | + ". %(prog)s recognizes some special values for" |
| 42 | + ' STOP_COMMAND: 1) "FAIL" means that OBMC Boot Test should stop' |
| 43 | + ' whenever a boot test fails. 2) "ALL" means that OBMC Boot Test' |
Patrick Williams | a57fef4 | 2022-12-03 07:00:14 -0600 | [diff] [blame] | 44 | + " should stop after any boot test. If environment variable" |
| 45 | + " STOP_REST_FAIL is set, OBMC Boot Test will stop if REST commands are" |
| 46 | + " no longer working.", |
Michael Walsh | 8d6bf60 | 2020-05-05 13:56:50 -0500 | [diff] [blame] | 47 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 48 | prefix_chars="-+", |
| 49 | ) |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 50 | |
Michael Walsh | 410b178 | 2019-10-22 15:56:18 -0500 | [diff] [blame] | 51 | # The stock_list will be passed to gen_get_options. We populate it with the names of stock parm options we |
| 52 | # want. These stock parms are pre-defined by gen_get_options. |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 53 | stock_list = [ |
| 54 | ("test_mode", get_plug_default("test_mode", 0)), |
| 55 | ("quiet", get_plug_default("quiet", 0)), |
| 56 | ("debug", get_plug_default("debug", 0)), |
| 57 | ] |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 58 | |
| 59 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 60 | def exit_function(signal_number=0, frame=None): |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 61 | r""" |
Michael Walsh | 410b178 | 2019-10-22 15:56:18 -0500 | [diff] [blame] | 62 | Execute whenever the program ends normally or with the signals that we catch (i.e. TERM, INT). |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 63 | |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 64 | This function will be called by gen_exit_function(). |
| 65 | """ |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 66 | |
Michael Walsh | a0ce75a | 2018-07-31 13:54:29 -0500 | [diff] [blame] | 67 | process_robot_output_files() |
| 68 | |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 69 | |
| 70 | def validate_parms(): |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 71 | r""" |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 72 | Validate program parameters, etc. |
| 73 | |
| 74 | This function will be called by gen_setup(). |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 75 | """ |
| 76 | |
| 77 | get_plug_vars() |
| 78 | |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 79 | |
Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 80 | def stop_check(): |
| 81 | r""" |
| 82 | Stop this program with the stop check return code. |
| 83 | """ |
| 84 | |
| 85 | if MASTER_PID != PROGRAM_PID: |
Michael Walsh | c213d49 | 2019-11-20 14:26:05 -0600 | [diff] [blame] | 86 | save_plug_in_value(stop_check_rc=stop_test_rc) |
Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 87 | exit(stop_test_rc) |
| 88 | |
| 89 | |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 90 | def rest_fail(): |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 91 | r""" |
Michael Walsh | 410b178 | 2019-10-22 15:56:18 -0500 | [diff] [blame] | 92 | If STOP_REST_FAIL, then this function will determine whether REST commands to the target are working. If |
| 93 | not, this function will stop the program by returning stop_test_rc. |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 94 | """ |
| 95 | |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 96 | if not STOP_REST_FAIL: |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 97 | return |
| 98 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 99 | REDFISH_SUPPORT_TRANS_STATE = int( |
| 100 | os.environ.get("REDFISH_SUPPORT_TRANS_STATE", 0) |
| 101 | ) or int(os.environ.get("AUTOBOOT_REDFISH_SUPPORT_TRANS_STATE", 0)) |
Michael Shepos | b3dffe8 | 2022-01-06 07:17:16 -0600 | [diff] [blame] | 102 | |
| 103 | if REDFISH_SUPPORT_TRANS_STATE: |
| 104 | interface = "redfish" |
| 105 | else: |
| 106 | interface = "rest" |
| 107 | |
| 108 | print_timen("Checking to see whether %s commands are working." % interface) |
Michael Walsh | 8ab9aea | 2018-11-01 16:30:45 -0500 | [diff] [blame] | 109 | init_robot_out_parms(get_plug_in_package_name() + "." + pgm_name + ".") |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 110 | lib_file_path = ( |
| 111 | init_robot_file_path("lib/utils.robot") |
| 112 | + ":" |
Michael Walsh | 8ab9aea | 2018-11-01 16:30:45 -0500 | [diff] [blame] | 113 | + init_robot_file_path("lib/gen_robot_print.py") |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 114 | ) |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 115 | set_mod_global(lib_file_path) |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 116 | timeout = "0 seconds" |
| 117 | interval = "1 second" |
| 118 | keyword_string = ( |
| 119 | "${match_state}= Create Dictionary %s=1 ;" % interface |
| 120 | + " ${state}= Wait State ${match_state} " |
| 121 | + timeout |
| 122 | + " " |
| 123 | + interval |
| 124 | + " quiet=${1} ; Rpvar state" |
| 125 | ) |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 126 | set_mod_global(keyword_string) |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 127 | cmd_buf = create_robot_cmd_string( |
| 128 | "extended/run_keyword.robot", |
| 129 | OPENBMC_HOST, |
| 130 | SSH_PORT, |
| 131 | HTTPS_PORT, |
| 132 | REST_USERNAME, |
| 133 | REST_PASSWORD, |
| 134 | OPENBMC_USERNAME, |
| 135 | OPENBMC_PASSWORD, |
| 136 | REDFISH_SUPPORT_TRANS_STATE, |
| 137 | keyword_string, |
| 138 | lib_file_path, |
| 139 | quiet, |
| 140 | test_mode, |
| 141 | debug, |
| 142 | outputdir, |
| 143 | output, |
| 144 | log, |
| 145 | report, |
| 146 | loglevel, |
| 147 | ) |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 148 | if not robot_cmd_fnc(cmd_buf): |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 149 | print_timen( |
| 150 | "The caller wishes to stop test execution if %s commands are" |
| 151 | " failing." % interface |
| 152 | ) |
Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 153 | stop_check() |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 154 | print_timen( |
| 155 | "%s commands are working so no reason as of yet to stop the test." |
| 156 | % interface |
| 157 | ) |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 158 | |
| 159 | |
| 160 | def esel_stop_check(): |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 161 | r""" |
Michael Walsh | 410b178 | 2019-10-22 15:56:18 -0500 | [diff] [blame] | 162 | Run the esel_stop_check program to determine whether any eSEL entries found warrant stopping the test |
| 163 | run. See esel_stop_check help text for details. |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 164 | """ |
| 165 | |
| 166 | if STOP_ESEL_STOP_FILE_PATH == "": |
| 167 | return |
| 168 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 169 | cmd_buf = ( |
| 170 | "esel_stop_check --esel_stop_file_path=" + STOP_ESEL_STOP_FILE_PATH |
| 171 | ) |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 172 | shell_rc, out_buf = shell_cmd(cmd_buf, show_err=0) |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 173 | if shell_rc == stop_test_rc: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 174 | print_timen( |
| 175 | "The caller wishes to stop test execution based on the presence of" |
| 176 | " certain esel entries." |
| 177 | ) |
Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 178 | stop_check() |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 179 | |
| 180 | |
Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 181 | def pel_stop_check(): |
| 182 | r""" |
| 183 | Determine whether any PEL entries found warrant stopping the test |
| 184 | run. |
| 185 | """ |
| 186 | |
| 187 | if STOP_PEL_STOP_FILE_PATH == "": |
| 188 | return |
| 189 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 190 | pel_txt_file_path = ( |
| 191 | os.environ.get("AUTOBOOT_FFDC_DIR_PATH", "") |
| 192 | + os.environ.get("AUTOBOOT_FFDC_PREFIX", "") |
| 193 | + "PEL_logs_list.json" |
| 194 | ) |
Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 195 | |
| 196 | if not os.path.isfile(pel_txt_file_path): |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 197 | qprint_timen( |
| 198 | "The following file was not present so no further" |
| 199 | + " action will be taken." |
| 200 | ) |
Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 201 | qprint_var(pel_txt_file_path) |
| 202 | return |
| 203 | |
| 204 | default_stop_dir_path = "/afs/rchland.ibm.com/projects/esw/dvt/" |
| 205 | |
| 206 | # If pel_stop_file_path is unqualified and cannot be found, pre-pend |
| 207 | # default_stop_dir_path for the user. |
| 208 | pel_stop_file_path = os.environ.get("STOP_PEL_STOP_FILE_PATH", "") |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 209 | if not os.path.isfile(pel_stop_file_path) and os.path.isfile( |
| 210 | default_stop_dir_path + pel_stop_file_path |
| 211 | ): |
Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 212 | pel_stop_file_path = default_stop_dir_path + pel_stop_file_path |
| 213 | qprint_timen("Using default stop file path.") |
| 214 | qprint_var(pel_stop_file_path) |
| 215 | |
| 216 | # First, read the file in and convert it to a list. |
| 217 | pel_stop_list = file_to_list(pel_stop_file_path, newlines=0, comments=0) |
| 218 | |
| 219 | if len(pel_stop_list) == 0: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 220 | print_timen( |
| 221 | "There are no records to process in " + pel_stop_file_path + "." |
| 222 | ) |
Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 223 | return |
| 224 | |
| 225 | pel_all_list = file_to_list(pel_txt_file_path, newlines=0, comments=0) |
| 226 | |
| 227 | if len(pel_all_list) == 0: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 228 | print_timen( |
| 229 | "There are no records to process in " + pel_txt_file_path + "." |
| 230 | ) |
Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 231 | return |
| 232 | |
| 233 | for stop_pel in pel_stop_list: |
| 234 | for pel_all in pel_all_list: |
| 235 | pel_match = re.search(".*SRC.*" + stop_pel + ".*", pel_all) |
| 236 | if pel_match: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 237 | print_timen( |
| 238 | "The caller wishes to stop test execution based on " |
| 239 | + "the presence of certain PEL entries." |
| 240 | ) |
Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 241 | stop_check() |
| 242 | |
| 243 | |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 244 | def main(): |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 245 | gen_setup() |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 246 | |
Michael Walsh | 80caea0 | 2019-06-21 11:31:41 -0500 | [diff] [blame] | 247 | print_plug_in_header() |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 248 | |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 249 | if STOP_COMMAND.upper() == "FAIL": |
| 250 | if AUTOBOOT_BOOT_SUCCESS == "0": |
| 251 | print_timen("The caller wishes to stop after each boot failure.") |
Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 252 | stop_check() |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 253 | elif STOP_COMMAND.upper() == "ALL": |
| 254 | print_timen("The caller wishes to stop after each boot test.") |
Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 255 | stop_check() |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 256 | elif len(STOP_COMMAND) > 0: |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 257 | shell_rc, out_buf = shell_cmd(STOP_COMMAND, quiet=quiet, show_err=0) |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 258 | if shell_rc != 0: |
| 259 | print_timen("The caller wishes to stop test execution.") |
Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 260 | stop_check() |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 261 | |
Michael Walsh | 3c4869a | 2019-07-17 10:29:31 -0500 | [diff] [blame] | 262 | rest_fail() |
| 263 | |
| 264 | esel_stop_check() |
| 265 | |
Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 266 | pel_stop_check() |
| 267 | |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 268 | if STOP_VERIFY_HARDWARE_FAIL: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 269 | hardware_error_found = restore_plug_in_value(0, "Verify_hardware") |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 270 | if hardware_error_found: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 271 | print_timen( |
| 272 | "The caller wishes to stop test execution when the" |
George Keishing | 7899a45 | 2023-02-15 02:46:54 -0600 | [diff] [blame] | 273 | " Verify_hardware plug-in detects a hardware error." |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 274 | ) |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 275 | stop_check() |
| 276 | |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 277 | qprint_timen("The caller does not wish to stop the test run.") |
Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 278 | |
Joy Onyerikwu | 004ad3c | 2018-06-11 16:29:56 -0500 | [diff] [blame] | 279 | |
Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 280 | main() |