| George Keishing | e7e9171 | 2021-09-03 11:28:44 -0500 | [diff] [blame] | 1 | #!/usr/bin/env python3 | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 2 |  | 
 | 3 | r""" | 
 | 4 | Check for stop conditions.  Return code of 2 if stop conditions are found. | 
 | 5 | """ | 
 | 6 |  | 
 | 7 | import sys | 
 | 8 | import subprocess | 
| Michael Shepos | b3dffe8 | 2022-01-06 07:17:16 -0600 | [diff] [blame] | 9 | import os | 
| Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 10 | import re | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 11 |  | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 12 | from gen_print import * | 
 | 13 | from gen_valid import * | 
 | 14 | from gen_arg import * | 
 | 15 | from gen_misc import * | 
 | 16 | from gen_cmd import * | 
 | 17 | from gen_plug_in_utils import * | 
 | 18 | from gen_call_robot import * | 
 | 19 |  | 
| Michael Walsh | 2ea965c | 2019-08-01 16:14:25 -0500 | [diff] [blame] | 20 | # Set exit_on_error for gen_valid functions. | 
 | 21 | set_exit_on_error(True) | 
 | 22 |  | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 23 | # Initialize default plug-in parms.. | 
 | 24 | STOP_REST_FAIL = 0 | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 25 | STOP_COMMAND = '' | 
 | 26 | stop_test_rc = 2 | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 27 | STOP_VERIFY_HARDWARE_FAIL = 0 | 
 | 28 |  | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 29 |  | 
 | 30 | # Create parser object to process command line parameters and args. | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 31 | parser = argparse.ArgumentParser( | 
 | 32 |     usage='%(prog)s [OPTIONS]', | 
 | 33 |     description="If the \"Stop\" plug-in is selected by the user, %(prog)s" + | 
 | 34 |     " is called by OBMC Boot Test after each boot test.  If %(prog)s returns" + | 
 | 35 |     " " + str(stop_test_rc) + ", then OBMC Boot Test will stop.  The user" + | 
 | 36 |     " may set environment variable STOP_COMMAND to contain any valid bash" + | 
 | 37 |     " command or program.  %(prog)s will run this stop command.  If the stop" + | 
 | 38 |     " command returns non-zero, then %(prog)s will return " + | 
 | 39 |     str(stop_test_rc) + ".  %(prog)s recognizes some special values for" + | 
 | 40 |     " STOP_COMMAND: 1) \"FAIL\" means that OBMC Boot Test should stop" + | 
 | 41 |     " whenever a boot test fails. 2) \"ALL\" means that OBMC Boot Test" + | 
 | 42 |     " should stop after any boot test.  If environment variable" + | 
 | 43 |     " STOP_REST_FAIL is set, OBMC Boot Test will stop if REST commands are" + | 
 | 44 |     " no longer working.", | 
| Michael Walsh | 8d6bf60 | 2020-05-05 13:56:50 -0500 | [diff] [blame] | 45 |     formatter_class=argparse.ArgumentDefaultsHelpFormatter, | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 46 |     prefix_chars='-+') | 
 | 47 |  | 
| Michael Walsh | 410b178 | 2019-10-22 15:56:18 -0500 | [diff] [blame] | 48 | # The stock_list will be passed to gen_get_options.  We populate it with the names of stock parm options we | 
 | 49 | # want.  These stock parms are pre-defined by gen_get_options. | 
| Michael Walsh | 1ea9b7a | 2020-04-02 13:39:25 -0500 | [diff] [blame] | 50 | stock_list = [("test_mode", get_plug_default("test_mode", 0)), | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 51 |               ("quiet", get_plug_default("quiet", 0)), | 
 | 52 |               ("debug", get_plug_default("debug", 0))] | 
 | 53 |  | 
 | 54 |  | 
 | 55 | def exit_function(signal_number=0, | 
 | 56 |                   frame=None): | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 57 |     r""" | 
| Michael Walsh | 410b178 | 2019-10-22 15:56:18 -0500 | [diff] [blame] | 58 |     Execute whenever the program ends normally or with the signals that we catch (i.e. TERM, INT). | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 59 |  | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 60 |     This function will be called by gen_exit_function(). | 
 | 61 |     """ | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 62 |  | 
| Michael Walsh | a0ce75a | 2018-07-31 13:54:29 -0500 | [diff] [blame] | 63 |     process_robot_output_files() | 
 | 64 |  | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 65 |  | 
 | 66 | def validate_parms(): | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 67 |     r""" | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 68 |     Validate program parameters, etc. | 
 | 69 |  | 
 | 70 |     This function will be called by gen_setup(). | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 71 |     """ | 
 | 72 |  | 
 | 73 |     get_plug_vars() | 
 | 74 |  | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 75 |  | 
| Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 76 | def stop_check(): | 
 | 77 |     r""" | 
 | 78 |     Stop this program with the stop check return code. | 
 | 79 |     """ | 
 | 80 |  | 
 | 81 |     if MASTER_PID != PROGRAM_PID: | 
| Michael Walsh | c213d49 | 2019-11-20 14:26:05 -0600 | [diff] [blame] | 82 |         save_plug_in_value(stop_check_rc=stop_test_rc) | 
| Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 83 |     exit(stop_test_rc) | 
 | 84 |  | 
 | 85 |  | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 86 | def rest_fail(): | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 87 |     r""" | 
| Michael Walsh | 410b178 | 2019-10-22 15:56:18 -0500 | [diff] [blame] | 88 |     If STOP_REST_FAIL, then this function will determine whether REST commands to the target are working.  If | 
 | 89 |     not, this function will stop the program by returning stop_test_rc. | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 90 |     """ | 
 | 91 |  | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 92 |     if not STOP_REST_FAIL: | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 93 |         return | 
 | 94 |  | 
| Michael Shepos | b3dffe8 | 2022-01-06 07:17:16 -0600 | [diff] [blame] | 95 |     REDFISH_SUPPORT_TRANS_STATE = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \ | 
 | 96 |         int(os.environ.get('AUTOBOOT_REDFISH_SUPPORT_TRANS_STATE', 0)) | 
 | 97 |  | 
 | 98 |     if REDFISH_SUPPORT_TRANS_STATE: | 
 | 99 |         interface = "redfish" | 
 | 100 |     else: | 
 | 101 |         interface = "rest" | 
 | 102 |  | 
 | 103 |     print_timen("Checking to see whether %s commands are working." % interface) | 
| Michael Walsh | 8ab9aea | 2018-11-01 16:30:45 -0500 | [diff] [blame] | 104 |     init_robot_out_parms(get_plug_in_package_name() + "." + pgm_name + ".") | 
| Michael Shepos | b3dffe8 | 2022-01-06 07:17:16 -0600 | [diff] [blame] | 105 |     lib_file_path = init_robot_file_path("lib/utils.robot") + ":" \ | 
| Michael Walsh | 8ab9aea | 2018-11-01 16:30:45 -0500 | [diff] [blame] | 106 |         + init_robot_file_path("lib/gen_robot_print.py") | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 107 |     set_mod_global(lib_file_path) | 
 | 108 |     timeout = '0 seconds' | 
 | 109 |     interval = '1 second' | 
| Michael Shepos | b3dffe8 | 2022-01-06 07:17:16 -0600 | [diff] [blame] | 110 |     keyword_string = "${match_state}=  Create Dictionary  %s=1 ;" % interface +\ | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 111 |         " ${state}=  Wait State  ${match_state}  " + timeout + "  " +\ | 
 | 112 |         interval + "  quiet=${1} ; Rpvar  state" | 
 | 113 |     set_mod_global(keyword_string) | 
| Michael Walsh | 046fe22 | 2019-11-08 14:33:53 -0600 | [diff] [blame] | 114 |     cmd_buf = create_robot_cmd_string("extended/run_keyword.robot", OPENBMC_HOST, SSH_PORT, HTTPS_PORT, | 
| Michael Shepos | a7d6614 | 2021-01-11 11:19:51 -0600 | [diff] [blame] | 115 |                                       REST_USERNAME, REST_PASSWORD, OPENBMC_USERNAME, OPENBMC_PASSWORD, | 
| Michael Shepos | b3dffe8 | 2022-01-06 07:17:16 -0600 | [diff] [blame] | 116 |                                       REDFISH_SUPPORT_TRANS_STATE, keyword_string, lib_file_path, quiet, | 
| Michael Walsh | 046fe22 | 2019-11-08 14:33:53 -0600 | [diff] [blame] | 117 |                                       test_mode, debug, outputdir, output, log, report, loglevel) | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 118 |     if not robot_cmd_fnc(cmd_buf): | 
| Michael Shepos | b3dffe8 | 2022-01-06 07:17:16 -0600 | [diff] [blame] | 119 |         print_timen("The caller wishes to stop test execution if %s commands are failing." % interface) | 
| Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 120 |         stop_check() | 
| Michael Shepos | b3dffe8 | 2022-01-06 07:17:16 -0600 | [diff] [blame] | 121 |     print_timen("%s commands are working so no reason as of yet to stop the test." % interface) | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 122 |  | 
 | 123 |  | 
 | 124 | def esel_stop_check(): | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 125 |     r""" | 
| Michael Walsh | 410b178 | 2019-10-22 15:56:18 -0500 | [diff] [blame] | 126 |     Run the esel_stop_check program to determine whether any eSEL entries found warrant stopping the test | 
 | 127 |     run.  See esel_stop_check help text for details. | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 128 |     """ | 
 | 129 |  | 
 | 130 |     if STOP_ESEL_STOP_FILE_PATH == "": | 
 | 131 |         return | 
 | 132 |  | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 133 |     cmd_buf = "esel_stop_check --esel_stop_file_path=" + STOP_ESEL_STOP_FILE_PATH | 
 | 134 |     shell_rc, out_buf = shell_cmd(cmd_buf, show_err=0) | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 135 |     if shell_rc == stop_test_rc: | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 136 |         print_timen("The caller wishes to stop test execution based on the presence of certain esel entries.") | 
| Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 137 |         stop_check() | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 138 |  | 
 | 139 |  | 
| Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 140 | def pel_stop_check(): | 
 | 141 |     r""" | 
 | 142 |     Determine whether any PEL entries found warrant stopping the test | 
 | 143 |     run. | 
 | 144 |     """ | 
 | 145 |  | 
 | 146 |     if STOP_PEL_STOP_FILE_PATH == "": | 
 | 147 |         return | 
 | 148 |  | 
 | 149 |     pel_txt_file_path = os.environ.get("AUTOBOOT_FFDC_DIR_PATH", "") + \ | 
 | 150 |      os.environ.get("AUTOBOOT_FFDC_PREFIX", "") + "PEL_logs_list.json" | 
 | 151 |  | 
 | 152 |     if not os.path.isfile(pel_txt_file_path): | 
 | 153 |         qprint_timen("The following file was not present so no further" + | 
 | 154 |                      " action will be taken.") | 
 | 155 |         qprint_var(pel_txt_file_path) | 
 | 156 |         return | 
 | 157 |  | 
 | 158 |     default_stop_dir_path = "/afs/rchland.ibm.com/projects/esw/dvt/" | 
 | 159 |  | 
 | 160 |     # If pel_stop_file_path is unqualified and cannot be found, pre-pend | 
 | 161 |     # default_stop_dir_path for the user. | 
 | 162 |     pel_stop_file_path = os.environ.get("STOP_PEL_STOP_FILE_PATH", "") | 
 | 163 |     if not os.path.isfile(pel_stop_file_path) and \ | 
 | 164 |        os.path.isfile(default_stop_dir_path + pel_stop_file_path): | 
 | 165 |         pel_stop_file_path = default_stop_dir_path + pel_stop_file_path | 
 | 166 |         qprint_timen("Using default stop file path.") | 
 | 167 |         qprint_var(pel_stop_file_path) | 
 | 168 |  | 
 | 169 |     # First, read the file in and convert it to a list. | 
 | 170 |     pel_stop_list = file_to_list(pel_stop_file_path, newlines=0, comments=0) | 
 | 171 |  | 
 | 172 |     if len(pel_stop_list) == 0: | 
 | 173 |         print_timen("There are no records to process in " + | 
 | 174 |                     pel_stop_file_path + ".") | 
 | 175 |         return | 
 | 176 |  | 
 | 177 |     pel_all_list = file_to_list(pel_txt_file_path, newlines=0, comments=0) | 
 | 178 |  | 
 | 179 |     if len(pel_all_list) == 0: | 
 | 180 |         print_timen("There are no records to process in " + | 
 | 181 |                     pel_txt_file_path + ".") | 
 | 182 |         return | 
 | 183 |  | 
 | 184 |     for stop_pel in pel_stop_list: | 
 | 185 |         for pel_all in pel_all_list: | 
 | 186 |             pel_match = re.search(".*SRC.*" + stop_pel + ".*", pel_all) | 
 | 187 |             if pel_match: | 
 | 188 |                 print_timen("The caller wishes to stop test execution based on the presence of certain PEL entries.") | 
 | 189 |                 stop_check() | 
 | 190 |  | 
 | 191 |  | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 192 | def main(): | 
 | 193 |  | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 194 |     gen_setup() | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 195 |  | 
| Michael Walsh | 80caea0 | 2019-06-21 11:31:41 -0500 | [diff] [blame] | 196 |     print_plug_in_header() | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 197 |  | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 198 |     if STOP_COMMAND.upper() == "FAIL": | 
 | 199 |         if AUTOBOOT_BOOT_SUCCESS == "0": | 
 | 200 |             print_timen("The caller wishes to stop after each boot failure.") | 
| Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 201 |             stop_check() | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 202 |     elif STOP_COMMAND.upper() == "ALL": | 
 | 203 |         print_timen("The caller wishes to stop after each boot test.") | 
| Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 204 |         stop_check() | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 205 |     elif len(STOP_COMMAND) > 0: | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 206 |         shell_rc, out_buf = shell_cmd(STOP_COMMAND, quiet=quiet, show_err=0) | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 207 |         if shell_rc != 0: | 
 | 208 |             print_timen("The caller wishes to stop test execution.") | 
| Michael Walsh | 2ce1dba | 2019-02-05 19:29:28 +0000 | [diff] [blame] | 209 |             stop_check() | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 210 |  | 
| Michael Walsh | 3c4869a | 2019-07-17 10:29:31 -0500 | [diff] [blame] | 211 |     rest_fail() | 
 | 212 |  | 
 | 213 |     esel_stop_check() | 
 | 214 |  | 
| Michael Shepos | f712967 | 2022-01-13 11:00:37 -0600 | [diff] [blame] | 215 |     pel_stop_check() | 
 | 216 |  | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 217 |     if STOP_VERIFY_HARDWARE_FAIL: | 
 | 218 |         hardware_error_found = restore_plug_in_value(0, 'Verify_hardware') | 
 | 219 |         if hardware_error_found: | 
 | 220 |             print_timen("The caller wishes to stop test execution when the Verify_hardware plug-in detects a" | 
 | 221 |                         + " hardware error.") | 
 | 222 |             stop_check() | 
 | 223 |  | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 224 |     qprint_timen("The caller does not wish to stop the test run.") | 
| Michael Walsh | 3ba8ecd | 2018-04-24 11:33:25 -0500 | [diff] [blame] | 225 |  | 
| Joy Onyerikwu | 004ad3c | 2018-06-11 16:29:56 -0500 | [diff] [blame] | 226 |  | 
| Michael Walsh | 71fec43 | 2020-04-09 17:25:31 -0500 | [diff] [blame] | 227 | main() |