George Keishing | e7e9171 | 2021-09-03 11:28:44 -0500 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 2 | |
| 3 | r""" |
| 4 | PEL functions. |
| 5 | """ |
| 6 | |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 7 | import json |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 8 | import os |
| 9 | import sys |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 10 | from datetime import datetime |
| 11 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 12 | import bmc_ssh_utils as bsu |
| 13 | import func_args as fa |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 14 | from robot.libraries.BuiltIn import BuiltIn |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 15 | |
| 16 | base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 17 | sys.path.append(base_path + "/data/") |
| 18 | |
George Keishing | 0967989 | 2022-12-08 08:21:52 -0600 | [diff] [blame] | 19 | import pel_variables # NOQA |
George Keishing | 37c58c8 | 2022-12-08 07:42:54 -0600 | [diff] [blame] | 20 | |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 21 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 22 | class peltool_exception(Exception): |
| 23 | r""" |
| 24 | Base class for peltool related exceptions. |
| 25 | """ |
George Keishing | 48ffa2c | 2022-02-17 01:42:41 -0600 | [diff] [blame] | 26 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 27 | def __init__(self, message): |
| 28 | self.message = message |
| 29 | super().__init__(self.message) |
| 30 | |
| 31 | |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 32 | def peltool(option_string, parse_json=True, **bsu_options): |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 33 | r""" |
| 34 | Run peltool on the BMC with the caller's option string and return the result. |
| 35 | |
| 36 | Example: |
| 37 | |
| 38 | ${pel_results}= Peltool -l |
| 39 | Rprint Vars pel_results |
| 40 | |
| 41 | pel_results: |
| 42 | [0x50000031]: |
| 43 | [CompID]: 0x1000 |
| 44 | [PLID]: 0x50000031 |
| 45 | [Subsystem]: BMC Firmware |
| 46 | [Message]: An application had an internal failure |
| 47 | [SRC]: BD8D1002 |
| 48 | [Commit Time]: 02/25/2020 04:51:31 |
| 49 | [Sev]: Unrecoverable Error |
| 50 | [CreatorID]: BMC |
| 51 | |
| 52 | Description of argument(s): |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 53 | option_string A string of options which are to be processed by the peltool command. |
| 54 | parse_json Indicates that the raw JSON data should parsed into a list of |
| 55 | dictionaries. |
| 56 | bsu_options Options to be passed directly to bmc_execute_command. See its prolog for |
| 57 | details. |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 58 | """ |
| 59 | |
| 60 | bsu_options = fa.args_to_objects(bsu_options) |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 61 | out_buf, stderr, rc = bsu.bmc_execute_command( |
| 62 | "peltool " + option_string, **bsu_options |
| 63 | ) |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 64 | if parse_json: |
| 65 | try: |
| 66 | return json.loads(out_buf) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 67 | except ValueError: |
Michael Walsh | a20876d | 2020-03-18 16:32:37 -0500 | [diff] [blame] | 68 | return {} |
Rahul Maheshwari | faa5d20 | 2020-02-24 23:32:57 -0600 | [diff] [blame] | 69 | return out_buf |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 70 | |
| 71 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 72 | def get_pel_data_from_bmc( |
| 73 | include_hidden_pels=False, include_informational_pels=False |
| 74 | ): |
Sridevi Ramesh | 2930050 | 2022-10-21 02:28:24 -0500 | [diff] [blame] | 75 | r""" |
| 76 | Returns PEL data from BMC else throws exception. |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 77 | |
| 78 | Description of arguments: |
| 79 | include_hidden_pels True/False (default: False). |
| 80 | Set True to get hidden PELs else False. |
| 81 | include_informational_pels True/False (default: False). |
| 82 | Set True to get informational PELs else False. |
Sridevi Ramesh | 2930050 | 2022-10-21 02:28:24 -0500 | [diff] [blame] | 83 | """ |
| 84 | try: |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 85 | pel_cmd = " -l" |
| 86 | if include_hidden_pels: |
| 87 | pel_cmd = pel_cmd + " -h" |
| 88 | if include_informational_pels: |
| 89 | pel_cmd = pel_cmd + " -f" |
| 90 | pel_data = peltool(pel_cmd) |
Sridevi Ramesh | 2930050 | 2022-10-21 02:28:24 -0500 | [diff] [blame] | 91 | if not pel_data: |
| 92 | print("No PEL data present in BMC ...") |
| 93 | except Exception as e: |
| 94 | raise peltool_exception("Failed to get PEL data from BMC : " + str(e)) |
| 95 | return pel_data |
| 96 | |
| 97 | |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 98 | def verify_no_pel_exists_on_bmc(): |
| 99 | r""" |
| 100 | Verify that no PEL exists in BMC. Raise an exception if it does. |
| 101 | """ |
| 102 | |
| 103 | try: |
| 104 | pel_data = get_pel_data_from_bmc() |
| 105 | |
| 106 | if len(pel_data) == 0: |
| 107 | return True |
| 108 | else: |
| 109 | print("PEL data present. \n", pel_data) |
| 110 | raise peltool_exception("PEL data present in BMC") |
| 111 | except Exception as e: |
| 112 | raise peltool_exception("Failed to get PEL data from BMC : " + str(e)) |
| 113 | |
| 114 | |
| 115 | def compare_pel_and_redfish_event_log(pel_record, event_record): |
| 116 | r""" |
| 117 | Compare PEL log attributes like "SRC", "Created at" with Redfish |
| 118 | event log attributes like "EventId", "Created". |
| 119 | Return False if they do not match. |
| 120 | |
| 121 | Description of arguments: |
| 122 | pel_record PEL record. |
| 123 | event_record Redfish event which is equivalent of PEL record. |
| 124 | """ |
| 125 | |
| 126 | try: |
| 127 | # Below is format of PEL record / event record and following |
| 128 | # i.e. "SRC", "Created at" from |
| 129 | # PEL record is compared with "EventId", "Created" from event record. |
| 130 | |
| 131 | # PEL Log attributes |
| 132 | # SRC : XXXXXXXX |
| 133 | # Created at : 11/14/2022 12:38:04 |
| 134 | |
| 135 | # Event log attributes |
| 136 | # EventId : XXXXXXXX XXXXXXXX XXXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX |
| 137 | |
| 138 | # Created : 2022-11-14T12:38:04+00:00 |
| 139 | |
| 140 | print("\nPEL records : {0}".format(pel_record)) |
| 141 | print("\nEvent records : {0}".format(event_record)) |
| 142 | |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 143 | pel_src = pel_record["pel_data"]["SRC"] |
| 144 | pel_created_time = pel_record["pel_detail_data"]["Private Header"][ |
| 145 | "Created at" |
| 146 | ] |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 147 | |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 148 | event_ids = (event_record["EventId"]).split(" ") |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 149 | |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 150 | event_time_format = (event_record["Created"]).split("T") |
| 151 | event_date = (event_time_format[0]).split("-") |
| 152 | event_date = datetime( |
| 153 | int(event_date[0]), int(event_date[1]), int(event_date[2]) |
| 154 | ) |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 155 | event_date = event_date.strftime("%m/%d/%Y") |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 156 | event_sub_time_format = (event_time_format[1]).split("+") |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 157 | event_date_time = event_date + " " + event_sub_time_format[0] |
| 158 | |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 159 | event_created_time = event_date_time.replace("-", "/") |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 160 | |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 161 | print( |
| 162 | "\nPEL SRC : {0} | PEL Created Time : {1}".format( |
| 163 | pel_src, pel_created_time |
| 164 | ) |
| 165 | ) |
| 166 | print( |
| 167 | "\nError event ID : {0} | Error Log Created Time : {1}".format( |
| 168 | event_ids[0], event_created_time |
| 169 | ) |
| 170 | ) |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 171 | |
| 172 | if pel_src == event_ids[0] and pel_created_time == event_created_time: |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 173 | print( |
| 174 | "\nPEL SRC and created date time match with " |
| 175 | "event ID, created time" |
| 176 | ) |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 177 | else: |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 178 | raise peltool_exception( |
| 179 | "\nPEL SRC and created date time did not " |
| 180 | "match with event ID, created time" |
| 181 | ) |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 182 | except Exception as e: |
Patrick Williams | b813b55 | 2022-12-12 10:38:46 -0600 | [diff] [blame^] | 183 | raise peltool_exception( |
| 184 | "Exception occured during PEL and Event log " |
| 185 | "comparison for SRC or event ID and created " |
| 186 | "time : " |
| 187 | + str(e) |
| 188 | ) |
Sushil Singh | 28cedb1 | 2022-12-12 01:33:32 -0600 | [diff] [blame] | 189 | |
| 190 | |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 191 | def fetch_all_pel_ids_for_src(src_id, severity, include_hidden_pels=False): |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 192 | r""" |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 193 | Fetch all PEL IDs for the input SRC ID based on the severity type |
| 194 | in the list format. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 195 | |
| 196 | Description of arguments: |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 197 | src_id SRC ID (e.g. BCXXYYYY). |
| 198 | severity PEL severity (e.g. "Predictive Error" |
| 199 | "Recovered Error"). |
| 200 | include_hidden_pels True/False (default: False). |
| 201 | Set True to get hidden PELs else False. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 202 | """ |
| 203 | |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 204 | try: |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 205 | src_pel_ids = [] |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 206 | pel_data = get_pel_data_from_bmc(include_hidden_pels) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 207 | pel_id_list = pel_data.keys() |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 208 | for pel_id in pel_id_list: |
| 209 | # Check if required SRC ID with severity is present |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 210 | if (pel_data[pel_id]["SRC"] == src_id) and ( |
| 211 | pel_data[pel_id]["Sev"] == severity |
| 212 | ): |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 213 | src_pel_ids.append(pel_id) |
| 214 | |
| 215 | if not src_pel_ids: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 216 | raise peltool_exception( |
| 217 | src_id + " with severity " + severity + " not present" |
| 218 | ) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 219 | except Exception as e: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 220 | raise peltool_exception( |
| 221 | "Failed to fetch PEL ID for required SRC : " + str(e) |
| 222 | ) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 223 | return src_pel_ids |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 224 | |
| 225 | |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 226 | def fetch_all_src(include_hidden_pels=False): |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 227 | r""" |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 228 | Fetch all SRC IDs from peltool in the list format. |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 229 | |
| 230 | include_hidden_pels True/False (default: False). |
| 231 | Set True to get hidden PELs else False. |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 232 | """ |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 233 | try: |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 234 | src_id = [] |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 235 | pel_data = get_pel_data_from_bmc(include_hidden_pels) |
Sridevi Ramesh | c6dd799 | 2022-02-10 01:06:47 -0600 | [diff] [blame] | 236 | if pel_data: |
| 237 | pel_id_list = pel_data.keys() |
| 238 | for pel_id in pel_id_list: |
| 239 | src_id.append(pel_data[pel_id]["SRC"]) |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 240 | print("SRC IDs: " + str(src_id)) |
Sridevi Ramesh | 6e0e091 | 2021-11-16 11:17:37 -0600 | [diff] [blame] | 241 | except Exception as e: |
| 242 | raise peltool_exception("Failed to fetch all SRCs : " + str(e)) |
| 243 | return src_id |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 244 | |
| 245 | |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 246 | def check_for_unexpected_src( |
| 247 | unexpected_src_list=[], include_hidden_pels=False |
| 248 | ): |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 249 | r""" |
| 250 | From the given unexpected SRC list, check if any unexpected SRC created |
| 251 | on the BMC. Returns 0 if no SRC found else throws exception. |
| 252 | |
| 253 | Description of arguments: |
| 254 | unexpected_src_list Give unexpected SRCs in the list format. |
| 255 | e.g.: ["BBXXYYYY", "AAXXYYYY"]. |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 256 | |
| 257 | include_hidden_pels True/False (default: False). |
| 258 | Set True to get hidden PELs else False. |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 259 | """ |
| 260 | try: |
| 261 | unexpected_src_count = 0 |
| 262 | if not unexpected_src_list: |
| 263 | print("Unexpected SRC list is empty.") |
Sridevi Ramesh | d1cb325 | 2022-11-28 10:04:05 -0600 | [diff] [blame] | 264 | src_data = fetch_all_src(include_hidden_pels) |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 265 | for src in unexpected_src_list: |
| 266 | if src in src_data: |
| 267 | print("Found an unexpected SRC : " + src) |
| 268 | unexpected_src_count = unexpected_src_count + 1 |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 269 | if unexpected_src_count >= 1: |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 270 | raise peltool_exception("Unexpected SRC found.") |
| 271 | |
| 272 | except Exception as e: |
Patrick Williams | 20f3871 | 2022-12-08 06:18:26 -0600 | [diff] [blame] | 273 | raise peltool_exception( |
| 274 | "Failed to verify unexpected SRC list : " + str(e) |
| 275 | ) |
Sridevi Ramesh | 6bf2363 | 2022-11-25 05:55:38 -0600 | [diff] [blame] | 276 | return unexpected_src_count |
Anusha Dathatri | e8801a3 | 2022-11-28 04:56:31 -0600 | [diff] [blame] | 277 | |
| 278 | |
| 279 | def filter_unexpected_srcs(expected_srcs=None): |
| 280 | r""" |
| 281 | Return list of SRCs found in BMC after filtering expected SRCs. |
| 282 | If expected_srcs is None then all SRCs found in system are returned. |
| 283 | |
| 284 | Description of arguments: |
| 285 | expected_srcs List of expected SRCs. E.g. ["BBXXYYYY", "AAXXYYYY"]. |
| 286 | """ |
| 287 | |
| 288 | srcs_found = fetch_all_src() |
| 289 | if not expected_srcs: |
| 290 | expected_srcs = [] |
| 291 | print(expected_srcs) |
| 292 | return list(set(srcs_found) - set(expected_srcs)) |
Anusha Dathatri | 5636ad1 | 2022-11-29 04:26:00 -0600 | [diff] [blame] | 293 | |
| 294 | |
| 295 | def get_bmc_event_log_id_for_pel(pel_id): |
| 296 | r""" |
| 297 | Return BMC event log ID for the given PEL ID. |
| 298 | |
| 299 | Description of arguments: |
| 300 | pel_id PEL ID. E.g. 0x50000021. |
| 301 | """ |
| 302 | |
| 303 | pel_data = peltool("-i " + pel_id) |
| 304 | print(pel_data) |
| 305 | bmc_id_for_pel = pel_data["Private Header"]["BMC Event Log Id"] |
| 306 | return bmc_id_for_pel |