George Keishing | e7e9171 | 2021-09-03 11:28:44 -0500 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 2 | |
| 3 | r""" |
| 4 | PEL functions. |
| 5 | """ |
| 6 | |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 7 | import json |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 8 | import os |
| 9 | import sys |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 10 | from datetime import datetime |
| 11 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 12 | import bmc_ssh_utils as bsu |
| 13 | import func_args as fa |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 14 | |
| 15 | base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 16 | sys.path.append(base_path + "/data/") |
| 17 | |
George Keishing | 0967989 | 2022-12-08 08:21:52 -0600 | [diff] [blame] | 18 | import pel_variables # NOQA |
George Keishing | 37c58c8 | 2022-12-08 07:42:54 -0600 | [diff] [blame] | 19 | |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 20 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 21 | class peltool_exception(Exception): |
| 22 | r""" |
| 23 | Base class for peltool related exceptions. |
| 24 | """ |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 25 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 26 | def __init__(self, message): |
| 27 | self.message = message |
| 28 | super().__init__(self.message) |
| 29 | |
| 30 | |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 31 | def peltool(option_string, parse_json=True, **bsu_options): |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 32 | r""" |
| 33 | Run peltool on the BMC with the caller's option string and return the result. |
| 34 | |
| 35 | Example: |
| 36 | |
| 37 | ${pel_results}= Peltool -l |
| 38 | Rprint Vars pel_results |
| 39 | |
| 40 | pel_results: |
| 41 | [0x50000031]: |
| 42 | [CompID]: 0x1000 |
| 43 | [PLID]: 0x50000031 |
| 44 | [Subsystem]: BMC Firmware |
| 45 | [Message]: An application had an internal failure |
| 46 | [SRC]: BD8D1002 |
| 47 | [Commit Time]: 02/25/2020 04:51:31 |
| 48 | [Sev]: Unrecoverable Error |
| 49 | [CreatorID]: BMC |
| 50 | |
| 51 | Description of argument(s): |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 52 | option_string A string of options which are to be processed by the peltool command. |
| 53 | parse_json Indicates that the raw JSON data should parsed into a list of |
| 54 | dictionaries. |
| 55 | bsu_options Options to be passed directly to bmc_execute_command. See its prolog for |
| 56 | details. |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 57 | """ |
| 58 | |
| 59 | bsu_options = fa.args_to_objects(bsu_options) |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 60 | out_buf, stderr, rc = bsu.bmc_execute_command( |
| 61 | "peltool " + option_string, **bsu_options |
| 62 | ) |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 63 | if parse_json: |
| 64 | try: |
| 65 | return json.loads(out_buf) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 66 | except ValueError: |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 67 | return {} |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 68 | return out_buf |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 69 | |
| 70 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 71 | def get_pel_data_from_bmc( |
| 72 | include_hidden_pels=False, include_informational_pels=False |
| 73 | ): |
Sridevi Ramesh | 2930050 | 2022-10-21 02:28:24 -0500 | [diff] [blame] | 74 | r""" |
| 75 | Returns PEL data from BMC else throws exception. |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 76 | |
| 77 | Description of arguments: |
| 78 | include_hidden_pels True/False (default: False). |
| 79 | Set True to get hidden PELs else False. |
| 80 | include_informational_pels True/False (default: False). |
| 81 | Set True to get informational PELs else False. |
Sridevi Ramesh | 2930050 | 2022-10-21 02:28:24 -0500 | [diff] [blame] | 82 | """ |
| 83 | try: |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 84 | pel_cmd = " -l" |
| 85 | if include_hidden_pels: |
| 86 | pel_cmd = pel_cmd + " -h" |
| 87 | if include_informational_pels: |
| 88 | pel_cmd = pel_cmd + " -f" |
| 89 | pel_data = peltool(pel_cmd) |
Sridevi Ramesh | 2930050 | 2022-10-21 02:28:24 -0500 | [diff] [blame] | 90 | if not pel_data: |
| 91 | print("No PEL data present in BMC ...") |
| 92 | except Exception as e: |
| 93 | raise peltool_exception("Failed to get PEL data from BMC : " + str(e)) |
| 94 | return pel_data |
| 95 | |
| 96 | |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 97 | def verify_no_pel_exists_on_bmc(): |
| 98 | r""" |
| 99 | Verify that no PEL exists in BMC. Raise an exception if it does. |
| 100 | """ |
| 101 | |
| 102 | try: |
| 103 | pel_data = get_pel_data_from_bmc() |
| 104 | |
| 105 | if len(pel_data) == 0: |
| 106 | return True |
| 107 | else: |
| 108 | print("PEL data present. \n", pel_data) |
| 109 | raise peltool_exception("PEL data present in BMC") |
| 110 | except Exception as e: |
| 111 | raise peltool_exception("Failed to get PEL data from BMC : " + str(e)) |
| 112 | |
| 113 | |
| 114 | def compare_pel_and_redfish_event_log(pel_record, event_record): |
| 115 | r""" |
| 116 | Compare PEL log attributes like "SRC", "Created at" with Redfish |
| 117 | event log attributes like "EventId", "Created". |
| 118 | Return False if they do not match. |
| 119 | |
| 120 | Description of arguments: |
| 121 | pel_record PEL record. |
| 122 | event_record Redfish event which is equivalent of PEL record. |
| 123 | """ |
| 124 | |
| 125 | try: |
| 126 | # Below is format of PEL record / event record and following |
| 127 | # i.e. "SRC", "Created at" from |
| 128 | # PEL record is compared with "EventId", "Created" from event record. |
| 129 | |
| 130 | # PEL Log attributes |
| 131 | # SRC : XXXXXXXX |
| 132 | # Created at : 11/14/2022 12:38:04 |
| 133 | |
| 134 | # Event log attributes |
| 135 | # EventId : XXXXXXXX XXXXXXXX XXXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX |
| 136 | |
| 137 | # Created : 2022-11-14T12:38:04+00:00 |
| 138 | |
| 139 | print("\nPEL records : {0}".format(pel_record)) |
| 140 | print("\nEvent records : {0}".format(event_record)) |
| 141 | |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 142 | pel_src = pel_record["pel_data"]["SRC"] |
| 143 | pel_created_time = pel_record["pel_detail_data"]["Private Header"][ |
| 144 | "Created at" |
| 145 | ] |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 146 | |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 147 | event_ids = (event_record["EventId"]).split(" ") |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 148 | |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 149 | event_time_format = (event_record["Created"]).split("T") |
| 150 | event_date = (event_time_format[0]).split("-") |
| 151 | event_date = datetime( |
| 152 | int(event_date[0]), int(event_date[1]), int(event_date[2]) |
| 153 | ) |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 154 | event_date = event_date.strftime("%m/%d/%Y") |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 155 | event_sub_time_format = (event_time_format[1]).split("+") |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 156 | event_date_time = event_date + " " + event_sub_time_format[0] |
| 157 | |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 158 | event_created_time = event_date_time.replace("-", "/") |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 159 | |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 160 | print( |
| 161 | "\nPEL SRC : {0} | PEL Created Time : {1}".format( |
| 162 | pel_src, pel_created_time |
| 163 | ) |
| 164 | ) |
| 165 | print( |
| 166 | "\nError event ID : {0} | Error Log Created Time : {1}".format( |
| 167 | event_ids[0], event_created_time |
| 168 | ) |
| 169 | ) |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 170 | |
| 171 | if pel_src == event_ids[0] and pel_created_time == event_created_time: |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 172 | print( |
| 173 | "\nPEL SRC and created date time match with " |
| 174 | "event ID, created time" |
| 175 | ) |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 176 | else: |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 177 | raise peltool_exception( |
| 178 | "\nPEL SRC and created date time did not " |
| 179 | "match with event ID, created time" |
| 180 | ) |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 181 | except Exception as e: |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 182 | raise peltool_exception( |
George Keishing | e16f158 | 2022-12-15 07:32:21 -0600 | [diff] [blame] | 183 | "Exception occurred during PEL and Event log " |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame] | 184 | "comparison for SRC or event ID and created " |
| 185 | "time : " |
| 186 | + str(e) |
| 187 | ) |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 188 | |
| 189 | |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 190 | def fetch_all_pel_ids_for_src(src_id, severity, include_hidden_pels=False): |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 191 | r""" |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 192 | Fetch all PEL IDs for the input SRC ID based on the severity type |
| 193 | in the list format. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 194 | |
| 195 | Description of arguments: |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 196 | src_id SRC ID (e.g. BCXXYYYY). |
| 197 | severity PEL severity (e.g. "Predictive Error" |
| 198 | "Recovered Error"). |
| 199 | include_hidden_pels True/False (default: False). |
| 200 | Set True to get hidden PELs else False. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 201 | """ |
| 202 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 203 | try: |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 204 | src_pel_ids = [] |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 205 | pel_data = get_pel_data_from_bmc(include_hidden_pels) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 206 | pel_id_list = pel_data.keys() |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 207 | for pel_id in pel_id_list: |
| 208 | # Check if required SRC ID with severity is present |
Sridevi Ramesh | f329997 | 2022-12-18 03:27:22 -0600 | [diff] [blame^] | 209 | if src_id in pel_data[pel_id]["SRC"]: |
| 210 | if pel_data[pel_id]["Sev"] == severity: |
| 211 | src_pel_ids.append(pel_id) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 212 | |
| 213 | if not src_pel_ids: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 214 | raise peltool_exception( |
| 215 | src_id + " with severity " + severity + " not present" |
| 216 | ) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 217 | except Exception as e: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 218 | raise peltool_exception( |
| 219 | "Failed to fetch PEL ID for required SRC : " + str(e) |
| 220 | ) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 221 | return src_pel_ids |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 222 | |
| 223 | |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 224 | def fetch_all_src(include_hidden_pels=False): |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 225 | r""" |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 226 | Fetch all SRC IDs from peltool in the list format. |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 227 | |
| 228 | include_hidden_pels True/False (default: False). |
| 229 | Set True to get hidden PELs else False. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 230 | """ |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 231 | try: |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 232 | src_id = [] |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 233 | pel_data = get_pel_data_from_bmc(include_hidden_pels) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 234 | if pel_data: |
| 235 | pel_id_list = pel_data.keys() |
| 236 | for pel_id in pel_id_list: |
| 237 | src_id.append(pel_data[pel_id]["SRC"]) |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 238 | print("SRC IDs: " + str(src_id)) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 239 | except Exception as e: |
| 240 | raise peltool_exception("Failed to fetch all SRCs : " + str(e)) |
| 241 | return src_id |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 242 | |
| 243 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 244 | def check_for_unexpected_src( |
| 245 | unexpected_src_list=[], include_hidden_pels=False |
| 246 | ): |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 247 | r""" |
| 248 | From the given unexpected SRC list, check if any unexpected SRC created |
| 249 | on the BMC. Returns 0 if no SRC found else throws exception. |
| 250 | |
| 251 | Description of arguments: |
| 252 | unexpected_src_list Give unexpected SRCs in the list format. |
| 253 | e.g.: ["BBXXYYYY", "AAXXYYYY"]. |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 254 | |
| 255 | include_hidden_pels True/False (default: False). |
| 256 | Set True to get hidden PELs else False. |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 257 | """ |
| 258 | try: |
| 259 | unexpected_src_count = 0 |
| 260 | if not unexpected_src_list: |
| 261 | print("Unexpected SRC list is empty.") |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 262 | src_data = fetch_all_src(include_hidden_pels) |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 263 | for src in unexpected_src_list: |
| 264 | if src in src_data: |
| 265 | print("Found an unexpected SRC : " + src) |
| 266 | unexpected_src_count = unexpected_src_count + 1 |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 267 | if unexpected_src_count >= 1: |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 268 | raise peltool_exception("Unexpected SRC found.") |
| 269 | |
| 270 | except Exception as e: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 271 | raise peltool_exception( |
| 272 | "Failed to verify unexpected SRC list : " + str(e) |
| 273 | ) |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 274 | return unexpected_src_count |
Anusha Dathatri | e8801a3 | 2022-11-28 04:56:31 -0600 | [diff] [blame] | 275 | |
| 276 | |
| 277 | def filter_unexpected_srcs(expected_srcs=None): |
| 278 | r""" |
| 279 | Return list of SRCs found in BMC after filtering expected SRCs. |
| 280 | If expected_srcs is None then all SRCs found in system are returned. |
| 281 | |
| 282 | Description of arguments: |
| 283 | expected_srcs List of expected SRCs. E.g. ["BBXXYYYY", "AAXXYYYY"]. |
| 284 | """ |
| 285 | |
| 286 | srcs_found = fetch_all_src() |
| 287 | if not expected_srcs: |
| 288 | expected_srcs = [] |
| 289 | print(expected_srcs) |
| 290 | return list(set(srcs_found) - set(expected_srcs)) |
Anusha Dathatri | 5636ad1 | 2022-11-29 04:26:00 -0600 | [diff] [blame] | 291 | |
| 292 | |
| 293 | def get_bmc_event_log_id_for_pel(pel_id): |
| 294 | r""" |
| 295 | Return BMC event log ID for the given PEL ID. |
| 296 | |
| 297 | Description of arguments: |
| 298 | pel_id PEL ID. E.g. 0x50000021. |
| 299 | """ |
| 300 | |
| 301 | pel_data = peltool("-i " + pel_id) |
| 302 | print(pel_data) |
| 303 | bmc_id_for_pel = pel_data["Private Header"]["BMC Event Log Id"] |
| 304 | return bmc_id_for_pel |