George Keishing | e7e9171 | 2021-09-03 11:28:44 -0500 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 2 | |
| 3 | r""" |
| 4 | PEL functions. |
| 5 | """ |
| 6 | |
| 7 | import func_args as fa |
| 8 | import bmc_ssh_utils as bsu |
| 9 | import json |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 10 | import os |
| 11 | import sys |
| 12 | from robot.libraries.BuiltIn import BuiltIn |
| 13 | |
| 14 | base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 15 | sys.path.append(base_path + "/data/") |
| 16 | |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 17 | import pel_variables |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 18 | |
| 19 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 20 | class peltool_exception(Exception): |
| 21 | r""" |
| 22 | Base class for peltool related exceptions. |
| 23 | """ |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 24 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 25 | def __init__(self, message): |
| 26 | self.message = message |
| 27 | super().__init__(self.message) |
| 28 | |
| 29 | |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 30 | def peltool(option_string, parse_json=True, **bsu_options): |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 31 | r""" |
| 32 | Run peltool on the BMC with the caller's option string and return the result. |
| 33 | |
| 34 | Example: |
| 35 | |
| 36 | ${pel_results}= Peltool -l |
| 37 | Rprint Vars pel_results |
| 38 | |
| 39 | pel_results: |
| 40 | [0x50000031]: |
| 41 | [CompID]: 0x1000 |
| 42 | [PLID]: 0x50000031 |
| 43 | [Subsystem]: BMC Firmware |
| 44 | [Message]: An application had an internal failure |
| 45 | [SRC]: BD8D1002 |
| 46 | [Commit Time]: 02/25/2020 04:51:31 |
| 47 | [Sev]: Unrecoverable Error |
| 48 | [CreatorID]: BMC |
| 49 | |
| 50 | Description of argument(s): |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 51 | option_string A string of options which are to be processed by the peltool command. |
| 52 | parse_json Indicates that the raw JSON data should parsed into a list of |
| 53 | dictionaries. |
| 54 | bsu_options Options to be passed directly to bmc_execute_command. See its prolog for |
| 55 | details. |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 56 | """ |
| 57 | |
| 58 | bsu_options = fa.args_to_objects(bsu_options) |
| 59 | out_buf, stderr, rc = bsu.bmc_execute_command('peltool ' + option_string, **bsu_options) |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 60 | if parse_json: |
| 61 | try: |
| 62 | return json.loads(out_buf) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 63 | except ValueError: |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 64 | return {} |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 65 | return out_buf |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 66 | |
| 67 | |
Sridevi Ramesh | 2930050 | 2022-10-21 02:28:24 -0500 | [diff] [blame] | 68 | def get_pel_data_from_bmc(): |
| 69 | r""" |
| 70 | Returns PEL data from BMC else throws exception. |
| 71 | """ |
| 72 | try: |
| 73 | pel_data = peltool(" -l") |
| 74 | if not pel_data: |
| 75 | print("No PEL data present in BMC ...") |
| 76 | except Exception as e: |
| 77 | raise peltool_exception("Failed to get PEL data from BMC : " + str(e)) |
| 78 | return pel_data |
| 79 | |
| 80 | |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 81 | def fetch_all_pel_ids_for_src(src_id, severity): |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 82 | r""" |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 83 | Fetch all PEL IDs for the input SRC ID based on the severity type |
| 84 | in the list format. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 85 | |
| 86 | Description of arguments: |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 87 | src_id SRC ID (e.g. BC20E504). |
| 88 | severity PEL severity (e.g. "Predictive Error" |
| 89 | "Recovered Error"). |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 90 | """ |
| 91 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 92 | try: |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 93 | src_pel_ids = [] |
Sridevi Ramesh | c80a309 | 2022-11-09 05:04:27 -0600 | [diff] [blame] | 94 | pel_data = get_pel_data_from_bmc() |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 95 | pel_id_list = pel_data.keys() |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 96 | for pel_id in pel_id_list: |
| 97 | # Check if required SRC ID with severity is present |
| 98 | if ((pel_data[pel_id]["SRC"] == src_id) and (pel_data[pel_id]["Sev"] == severity)): |
| 99 | src_pel_ids.append(pel_id) |
| 100 | |
| 101 | if not src_pel_ids: |
| 102 | raise peltool_exception(src_id + " with severity " + severity + " not present") |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 103 | except Exception as e: |
| 104 | raise peltool_exception("Failed to fetch PEL ID for required SRC : " + str(e)) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 105 | return src_pel_ids |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 106 | |
| 107 | |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 108 | def verify_src_signature_and_threshold(pel_id, attn_type, signature_desc, th_limit): |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 109 | r""" |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 110 | Verifies SRC details for the given PEL ID based on the required |
| 111 | attention type, signature description, threshold limits. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 112 | |
| 113 | Description of arguments: |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 114 | pel_id PEL ID for the required SRC details to verify. |
| 115 | attn_type Attention type (e.g. RE, CS, UNIT_CS). |
| 116 | signature_desc Signature description of the error inject. |
| 117 | th_limit Threshold limit (e.g. 1, 5, 32). |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 118 | """ |
| 119 | |
| 120 | try: |
| 121 | pel_cmd = " -i " + pel_id |
| 122 | src_data = peltool(pel_cmd) |
| 123 | src_dict = src_data["Primary SRC"]["SRC Details"] |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 124 | usr_data = src_data["User Data 1"] |
| 125 | |
| 126 | # Example for signature in recoverable error |
| 127 | # |
| 128 | # "SRC Details": { |
| 129 | # "Attention Type": "RECOVERABLE", |
| 130 | # "Node": 0, |
| 131 | # "Target Type": "TYPE_OMIC", |
| 132 | # "Target Instance": 0, |
| 133 | # "Signature": "MC_OMI_DL_FIR[1]: OMI-DL0 UE on data flit" |
| 134 | # } |
| 135 | if (attn_type == "RE"): |
| 136 | if (src_dict["Attention Type"] != "RECOVERABLE"): |
| 137 | raise peltool_exception("Required Attention type " + attn_type + " not found") |
| 138 | |
| 139 | # Example for signature in system checkstop error |
| 140 | # |
| 141 | # "SRC Details": { |
| 142 | # "Primary Attention": "system checkstop", |
| 143 | # "Signature Description": { |
| 144 | # "Chip Desc": "node 0 proc 0 (P10 2.0)", |
| 145 | # "Signature": "EQ_L2_FIR(0)[7] L2 directory read UE", |
| 146 | # "Attn Type": "checkstop" |
| 147 | # } |
| 148 | |
| 149 | elif (attn_type == "CS"): |
| 150 | if (src_dict["Primary Attention"] != "system checkstop"): |
| 151 | raise peltool_exception("Required Attention type " + attn_type + " not found") |
| 152 | |
| 153 | elif (attn_type == "UNIT_CS"): |
| 154 | if (src_dict["Attention Type"] != "UNIT_CS"): |
| 155 | raise peltool_exception("Required Attention type " + attn_type + " not found") |
| 156 | else: |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 157 | raise peltool_exception("Required Attention type " + attn_type + " not found") |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 158 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 159 | if signature_desc not in src_dict["Signature"]: |
| 160 | raise peltool_exception("Required Signature " + signature_desc + " not found") |
| 161 | |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 162 | if (int(th_limit) != usr_data["Error Count"]): |
| 163 | raise peltool_exception("Required Threshold limit " + th_limit + " not found") |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 164 | |
| 165 | except Exception as e: |
| 166 | raise peltool_exception("Failed to verify SRC details : " + str(e)) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 167 | return True |
| 168 | |
| 169 | |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 170 | def fetch_all_src(): |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 171 | r""" |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 172 | Fetch all SRC IDs from peltool in the list format. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 173 | """ |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 174 | try: |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 175 | src_id = [] |
Sridevi Ramesh | c80a309 | 2022-11-09 05:04:27 -0600 | [diff] [blame] | 176 | pel_data = get_pel_data_from_bmc() |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 177 | if pel_data: |
| 178 | pel_id_list = pel_data.keys() |
| 179 | for pel_id in pel_id_list: |
| 180 | src_id.append(pel_data[pel_id]["SRC"]) |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 181 | print("SRC IDs: " + str(src_id)) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 182 | except Exception as e: |
| 183 | raise peltool_exception("Failed to fetch all SRCs : " + str(e)) |
| 184 | return src_id |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 185 | |
| 186 | |
| 187 | def check_for_unexpected_src(unexpected_src_list=[]): |
| 188 | r""" |
| 189 | From the given unexpected SRC list, check if any unexpected SRC created |
| 190 | on the BMC. Returns 0 if no SRC found else throws exception. |
| 191 | |
| 192 | Description of arguments: |
| 193 | unexpected_src_list Give unexpected SRCs in the list format. |
| 194 | e.g.: ["BBXXYYYY", "AAXXYYYY"]. |
| 195 | """ |
| 196 | try: |
| 197 | unexpected_src_count = 0 |
| 198 | if not unexpected_src_list: |
| 199 | print("Unexpected SRC list is empty.") |
| 200 | src_data = fetch_all_src() |
| 201 | for src in unexpected_src_list: |
| 202 | if src in src_data: |
| 203 | print("Found an unexpected SRC : " + src) |
| 204 | unexpected_src_count = unexpected_src_count + 1 |
| 205 | if (unexpected_src_count >= 1): |
| 206 | raise peltool_exception("Unexpected SRC found.") |
| 207 | |
| 208 | except Exception as e: |
| 209 | raise peltool_exception("Failed to verify unexpected SRC list : " + str(e)) |
| 210 | return unexpected_src_count |
Anusha Dathatri | e8801a3 | 2022-11-28 04:56:31 -0600 | [diff] [blame] | 211 | |
| 212 | |
| 213 | def filter_unexpected_srcs(expected_srcs=None): |
| 214 | r""" |
| 215 | Return list of SRCs found in BMC after filtering expected SRCs. |
| 216 | If expected_srcs is None then all SRCs found in system are returned. |
| 217 | |
| 218 | Description of arguments: |
| 219 | expected_srcs List of expected SRCs. E.g. ["BBXXYYYY", "AAXXYYYY"]. |
| 220 | """ |
| 221 | |
| 222 | srcs_found = fetch_all_src() |
| 223 | if not expected_srcs: |
| 224 | expected_srcs = [] |
| 225 | print(expected_srcs) |
| 226 | return list(set(srcs_found) - set(expected_srcs)) |
Anusha Dathatri | 5636ad1 | 2022-11-29 04:26:00 -0600 | [diff] [blame] | 227 | |
| 228 | |
| 229 | def get_bmc_event_log_id_for_pel(pel_id): |
| 230 | r""" |
| 231 | Return BMC event log ID for the given PEL ID. |
| 232 | |
| 233 | Description of arguments: |
| 234 | pel_id PEL ID. E.g. 0x50000021. |
| 235 | """ |
| 236 | |
| 237 | pel_data = peltool("-i " + pel_id) |
| 238 | print(pel_data) |
| 239 | bmc_id_for_pel = pel_data["Private Header"]["BMC Event Log Id"] |
| 240 | return bmc_id_for_pel |