George Keishing | e7e9171 | 2021-09-03 11:28:44 -0500 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 2 | |
| 3 | r""" |
| 4 | PEL functions. |
| 5 | """ |
| 6 | |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 7 | from robot.libraries.BuiltIn import BuiltIn |
| 8 | from datetime import datetime |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 9 | import json |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 10 | import os |
| 11 | import sys |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 12 | import bmc_ssh_utils as bsu |
| 13 | import func_args as fa |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 14 | |
| 15 | base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 16 | sys.path.append(base_path + "/data/") |
| 17 | |
George Keishing | 0967989 | 2022-12-08 08:21:52 -0600 | [diff] [blame] | 18 | import pel_variables # NOQA |
George Keishing | 37c58c8 | 2022-12-08 07:42:54 -0600 | [diff] [blame] | 19 | |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 20 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 21 | class peltool_exception(Exception): |
| 22 | r""" |
| 23 | Base class for peltool related exceptions. |
| 24 | """ |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 25 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 26 | def __init__(self, message): |
| 27 | self.message = message |
| 28 | super().__init__(self.message) |
| 29 | |
| 30 | |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 31 | def peltool(option_string, parse_json=True, **bsu_options): |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 32 | r""" |
| 33 | Run peltool on the BMC with the caller's option string and return the result. |
| 34 | |
| 35 | Example: |
| 36 | |
| 37 | ${pel_results}= Peltool -l |
| 38 | Rprint Vars pel_results |
| 39 | |
| 40 | pel_results: |
| 41 | [0x50000031]: |
| 42 | [CompID]: 0x1000 |
| 43 | [PLID]: 0x50000031 |
| 44 | [Subsystem]: BMC Firmware |
| 45 | [Message]: An application had an internal failure |
| 46 | [SRC]: BD8D1002 |
| 47 | [Commit Time]: 02/25/2020 04:51:31 |
| 48 | [Sev]: Unrecoverable Error |
| 49 | [CreatorID]: BMC |
| 50 | |
| 51 | Description of argument(s): |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 52 | option_string A string of options which are to be processed by the peltool command. |
| 53 | parse_json Indicates that the raw JSON data should parsed into a list of |
| 54 | dictionaries. |
| 55 | bsu_options Options to be passed directly to bmc_execute_command. See its prolog for |
| 56 | details. |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 57 | """ |
| 58 | |
| 59 | bsu_options = fa.args_to_objects(bsu_options) |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 60 | out_buf, stderr, rc = bsu.bmc_execute_command( |
| 61 | "peltool " + option_string, **bsu_options |
| 62 | ) |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 63 | if parse_json: |
| 64 | try: |
| 65 | return json.loads(out_buf) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 66 | except ValueError: |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 67 | return {} |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 68 | return out_buf |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 69 | |
| 70 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 71 | def get_pel_data_from_bmc( |
| 72 | include_hidden_pels=False, include_informational_pels=False |
| 73 | ): |
Sridevi Ramesh | 2930050 | 2022-10-21 02:28:24 -0500 | [diff] [blame] | 74 | r""" |
| 75 | Returns PEL data from BMC else throws exception. |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 76 | |
| 77 | Description of arguments: |
| 78 | include_hidden_pels True/False (default: False). |
| 79 | Set True to get hidden PELs else False. |
| 80 | include_informational_pels True/False (default: False). |
| 81 | Set True to get informational PELs else False. |
Sridevi Ramesh | 2930050 | 2022-10-21 02:28:24 -0500 | [diff] [blame] | 82 | """ |
| 83 | try: |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 84 | pel_cmd = " -l" |
| 85 | if include_hidden_pels: |
| 86 | pel_cmd = pel_cmd + " -h" |
| 87 | if include_informational_pels: |
| 88 | pel_cmd = pel_cmd + " -f" |
| 89 | pel_data = peltool(pel_cmd) |
Sridevi Ramesh | 2930050 | 2022-10-21 02:28:24 -0500 | [diff] [blame] | 90 | if not pel_data: |
| 91 | print("No PEL data present in BMC ...") |
| 92 | except Exception as e: |
| 93 | raise peltool_exception("Failed to get PEL data from BMC : " + str(e)) |
| 94 | return pel_data |
| 95 | |
| 96 | |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 97 | def verify_no_pel_exists_on_bmc(): |
| 98 | r""" |
| 99 | Verify that no PEL exists in BMC. Raise an exception if it does. |
| 100 | """ |
| 101 | |
| 102 | try: |
| 103 | pel_data = get_pel_data_from_bmc() |
| 104 | |
| 105 | if len(pel_data) == 0: |
| 106 | return True |
| 107 | else: |
| 108 | print("PEL data present. \n", pel_data) |
| 109 | raise peltool_exception("PEL data present in BMC") |
| 110 | except Exception as e: |
| 111 | raise peltool_exception("Failed to get PEL data from BMC : " + str(e)) |
| 112 | |
| 113 | |
| 114 | def compare_pel_and_redfish_event_log(pel_record, event_record): |
| 115 | r""" |
| 116 | Compare PEL log attributes like "SRC", "Created at" with Redfish |
| 117 | event log attributes like "EventId", "Created". |
| 118 | Return False if they do not match. |
| 119 | |
| 120 | Description of arguments: |
| 121 | pel_record PEL record. |
| 122 | event_record Redfish event which is equivalent of PEL record. |
| 123 | """ |
| 124 | |
| 125 | try: |
| 126 | # Below is format of PEL record / event record and following |
| 127 | # i.e. "SRC", "Created at" from |
| 128 | # PEL record is compared with "EventId", "Created" from event record. |
| 129 | |
| 130 | # PEL Log attributes |
| 131 | # SRC : XXXXXXXX |
| 132 | # Created at : 11/14/2022 12:38:04 |
| 133 | |
| 134 | # Event log attributes |
| 135 | # EventId : XXXXXXXX XXXXXXXX XXXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX |
| 136 | |
| 137 | # Created : 2022-11-14T12:38:04+00:00 |
| 138 | |
| 139 | print("\nPEL records : {0}".format(pel_record)) |
| 140 | print("\nEvent records : {0}".format(event_record)) |
| 141 | |
| 142 | pel_src = pel_record['pel_data']['SRC'] |
| 143 | pel_created_time = \ |
| 144 | pel_record['pel_detail_data']['Private Header']['Created at'] |
| 145 | |
| 146 | event_ids = (event_record['EventId']).split(' ') |
| 147 | |
| 148 | event_time_format = (event_record['Created']).split('T') |
| 149 | event_date = (event_time_format[0]).split('-') |
| 150 | event_date = \ |
| 151 | datetime(int(event_date[0]), int(event_date[1]), int(event_date[2])) |
| 152 | event_date = event_date.strftime("%m/%d/%Y") |
| 153 | event_sub_time_format = (event_time_format[1]).split('+') |
| 154 | event_date_time = event_date + " " + event_sub_time_format[0] |
| 155 | |
| 156 | event_created_time = event_date_time.replace('-', '/') |
| 157 | |
| 158 | print("\nPEL SRC : {0} | PEL Created Time : {1}". |
| 159 | format(pel_src, pel_created_time)) |
| 160 | print("\nError event ID : {0} | Error Log Created Time : {1}". |
| 161 | format(event_ids[0], event_created_time)) |
| 162 | |
| 163 | if pel_src == event_ids[0] and pel_created_time == event_created_time: |
| 164 | print("\nPEL SRC and created date time match with " |
| 165 | "event ID, created time") |
| 166 | else: |
| 167 | raise peltool_exception("\nPEL SRC and created date time did not " |
| 168 | "match with event ID, created time") |
| 169 | except Exception as e: |
| 170 | raise peltool_exception("Exception occured during PEL and Event log " |
| 171 | "comparison for SRC or event ID and created " |
| 172 | "time : " + str(e)) |
| 173 | |
| 174 | |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 175 | def fetch_all_pel_ids_for_src(src_id, severity, include_hidden_pels=False): |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 176 | r""" |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 177 | Fetch all PEL IDs for the input SRC ID based on the severity type |
| 178 | in the list format. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 179 | |
| 180 | Description of arguments: |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 181 | src_id SRC ID (e.g. BCXXYYYY). |
| 182 | severity PEL severity (e.g. "Predictive Error" |
| 183 | "Recovered Error"). |
| 184 | include_hidden_pels True/False (default: False). |
| 185 | Set True to get hidden PELs else False. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 186 | """ |
| 187 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 188 | try: |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 189 | src_pel_ids = [] |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 190 | pel_data = get_pel_data_from_bmc(include_hidden_pels) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 191 | pel_id_list = pel_data.keys() |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 192 | for pel_id in pel_id_list: |
| 193 | # Check if required SRC ID with severity is present |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 194 | if (pel_data[pel_id]["SRC"] == src_id) and ( |
| 195 | pel_data[pel_id]["Sev"] == severity |
| 196 | ): |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 197 | src_pel_ids.append(pel_id) |
| 198 | |
| 199 | if not src_pel_ids: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 200 | raise peltool_exception( |
| 201 | src_id + " with severity " + severity + " not present" |
| 202 | ) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 203 | except Exception as e: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 204 | raise peltool_exception( |
| 205 | "Failed to fetch PEL ID for required SRC : " + str(e) |
| 206 | ) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 207 | return src_pel_ids |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 208 | |
| 209 | |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 210 | def fetch_all_src(include_hidden_pels=False): |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 211 | r""" |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 212 | Fetch all SRC IDs from peltool in the list format. |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 213 | |
| 214 | include_hidden_pels True/False (default: False). |
| 215 | Set True to get hidden PELs else False. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 216 | """ |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 217 | try: |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 218 | src_id = [] |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 219 | pel_data = get_pel_data_from_bmc(include_hidden_pels) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 220 | if pel_data: |
| 221 | pel_id_list = pel_data.keys() |
| 222 | for pel_id in pel_id_list: |
| 223 | src_id.append(pel_data[pel_id]["SRC"]) |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 224 | print("SRC IDs: " + str(src_id)) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 225 | except Exception as e: |
| 226 | raise peltool_exception("Failed to fetch all SRCs : " + str(e)) |
| 227 | return src_id |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 228 | |
| 229 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 230 | def check_for_unexpected_src( |
| 231 | unexpected_src_list=[], include_hidden_pels=False |
| 232 | ): |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 233 | r""" |
| 234 | From the given unexpected SRC list, check if any unexpected SRC created |
| 235 | on the BMC. Returns 0 if no SRC found else throws exception. |
| 236 | |
| 237 | Description of arguments: |
| 238 | unexpected_src_list Give unexpected SRCs in the list format. |
| 239 | e.g.: ["BBXXYYYY", "AAXXYYYY"]. |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 240 | |
| 241 | include_hidden_pels True/False (default: False). |
| 242 | Set True to get hidden PELs else False. |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 243 | """ |
| 244 | try: |
| 245 | unexpected_src_count = 0 |
| 246 | if not unexpected_src_list: |
| 247 | print("Unexpected SRC list is empty.") |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 248 | src_data = fetch_all_src(include_hidden_pels) |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 249 | for src in unexpected_src_list: |
| 250 | if src in src_data: |
| 251 | print("Found an unexpected SRC : " + src) |
| 252 | unexpected_src_count = unexpected_src_count + 1 |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 253 | if unexpected_src_count >= 1: |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 254 | raise peltool_exception("Unexpected SRC found.") |
| 255 | |
| 256 | except Exception as e: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 257 | raise peltool_exception( |
| 258 | "Failed to verify unexpected SRC list : " + str(e) |
| 259 | ) |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 260 | return unexpected_src_count |
Anusha Dathatri | e8801a3 | 2022-11-28 04:56:31 -0600 | [diff] [blame] | 261 | |
| 262 | |
| 263 | def filter_unexpected_srcs(expected_srcs=None): |
| 264 | r""" |
| 265 | Return list of SRCs found in BMC after filtering expected SRCs. |
| 266 | If expected_srcs is None then all SRCs found in system are returned. |
| 267 | |
| 268 | Description of arguments: |
| 269 | expected_srcs List of expected SRCs. E.g. ["BBXXYYYY", "AAXXYYYY"]. |
| 270 | """ |
| 271 | |
| 272 | srcs_found = fetch_all_src() |
| 273 | if not expected_srcs: |
| 274 | expected_srcs = [] |
| 275 | print(expected_srcs) |
| 276 | return list(set(srcs_found) - set(expected_srcs)) |
Anusha Dathatri | 5636ad1 | 2022-11-29 04:26:00 -0600 | [diff] [blame] | 277 | |
| 278 | |
| 279 | def get_bmc_event_log_id_for_pel(pel_id): |
| 280 | r""" |
| 281 | Return BMC event log ID for the given PEL ID. |
| 282 | |
| 283 | Description of arguments: |
| 284 | pel_id PEL ID. E.g. 0x50000021. |
| 285 | """ |
| 286 | |
| 287 | pel_data = peltool("-i " + pel_id) |
| 288 | print(pel_data) |
| 289 | bmc_id_for_pel = pel_data["Private Header"]["BMC Event Log Id"] |
| 290 | return bmc_id_for_pel |