blob: a06a518ad03e47343660646d488aa89adae68a53 [file] [log] [blame]
George Keishinge7e91712021-09-03 11:28:44 -05001#!/usr/bin/env python3
Rahul Maheshwarifaa5d202020-02-24 23:32:57 -06002
3r"""
4PEL functions.
5"""
6
Rahul Maheshwarifaa5d202020-02-24 23:32:57 -06007import json
George Keishing48ffa2c2022-02-17 01:42:41 -06008import os
9import sys
Patrick Williamsb813b552022-12-12 10:38:46 -060010from datetime import datetime
11
Patrick Williams20f38712022-12-08 06:18:26 -060012import bmc_ssh_utils as bsu
13import func_args as fa
George Keishing48ffa2c2022-02-17 01:42:41 -060014
15base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
16sys.path.append(base_path + "/data/")
17
George Keishing09679892022-12-08 08:21:52 -060018import pel_variables # NOQA
George Keishing37c58c82022-12-08 07:42:54 -060019
Rahul Maheshwarifaa5d202020-02-24 23:32:57 -060020
Anusha Dathatri3914da72023-02-09 06:01:05 -060021class PeltoolException(Exception):
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -060022 r"""
23 Base class for peltool related exceptions.
24 """
George Keishing48ffa2c2022-02-17 01:42:41 -060025
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -060026 def __init__(self, message):
27 self.message = message
28 super().__init__(self.message)
29
30
Sridevi Ramesh45b45c12024-12-17 01:38:19 -060031def peltool(
32 option_string, peltool_extension=None, parse_json=True, **bsu_options
33):
Rahul Maheshwarifaa5d202020-02-24 23:32:57 -060034 r"""
35 Run peltool on the BMC with the caller's option string and return the result.
36
37 Example:
38
39 ${pel_results}= Peltool -l
40 Rprint Vars pel_results
41
42 pel_results:
43 [0x50000031]:
44 [CompID]: 0x1000
45 [PLID]: 0x50000031
46 [Subsystem]: BMC Firmware
47 [Message]: An application had an internal failure
48 [SRC]: BD8D1002
49 [Commit Time]: 02/25/2020 04:51:31
50 [Sev]: Unrecoverable Error
51 [CreatorID]: BMC
52
53 Description of argument(s):
Anusha Dathatri3914da72023-02-09 06:01:05 -060054 option_string A string of options which are to be
55 processed by the peltool command.
Sridevi Ramesh45b45c12024-12-17 01:38:19 -060056 peltool_extension Provide peltool extension format.
57 Default: None.
Anusha Dathatri3914da72023-02-09 06:01:05 -060058 parse_json Indicates that the raw JSON data should
59 parsed into a list of dictionaries.
60 bsu_options Options to be passed directly to
61 bmc_execute_command. See its prolog for
Michael Walsha20876d2020-03-18 16:32:37 -050062 details.
Rahul Maheshwarifaa5d202020-02-24 23:32:57 -060063 """
64
65 bsu_options = fa.args_to_objects(bsu_options)
Sridevi Ramesh45b45c12024-12-17 01:38:19 -060066 peltool_cmd = "peltool"
67 if peltool_extension:
68 peltool_cmd = peltool_cmd + peltool_extension
69
Anusha Dathatri3914da72023-02-09 06:01:05 -060070 out_buf, _, _ = bsu.bmc_execute_command(
Sridevi Ramesh45b45c12024-12-17 01:38:19 -060071 peltool_cmd + " " + option_string, **bsu_options
Patrick Williams20f38712022-12-08 06:18:26 -060072 )
Michael Walsha20876d2020-03-18 16:32:37 -050073 if parse_json:
74 try:
75 return json.loads(out_buf)
Anusha Dathatri1e8757a2023-09-21 13:02:31 -050076 except ValueError as e:
77 if type(out_buf) is str:
78 return out_buf
79 else:
80 print(str(e))
81 return {}
Rahul Maheshwarifaa5d202020-02-24 23:32:57 -060082 return out_buf
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -060083
84
Patrick Williams20f38712022-12-08 06:18:26 -060085def get_pel_data_from_bmc(
86 include_hidden_pels=False, include_informational_pels=False
87):
Sridevi Ramesh29300502022-10-21 02:28:24 -050088 r"""
89 Returns PEL data from BMC else throws exception.
Sridevi Rameshd1cb3252022-11-28 10:04:05 -060090
91 Description of arguments:
92 include_hidden_pels True/False (default: False).
93 Set True to get hidden PELs else False.
94 include_informational_pels True/False (default: False).
95 Set True to get informational PELs else False.
Sridevi Ramesh29300502022-10-21 02:28:24 -050096 """
97 try:
Sridevi Rameshd1cb3252022-11-28 10:04:05 -060098 pel_cmd = " -l"
99 if include_hidden_pels:
100 pel_cmd = pel_cmd + " -h"
101 if include_informational_pels:
102 pel_cmd = pel_cmd + " -f"
103 pel_data = peltool(pel_cmd)
Sridevi Ramesh29300502022-10-21 02:28:24 -0500104 if not pel_data:
105 print("No PEL data present in BMC ...")
Anusha Dathatri3914da72023-02-09 06:01:05 -0600106 except Exception as exception:
107 raise PeltoolException(
108 "Failed to get PEL data from BMC : " + str(exception)
109 ) from exception
Sridevi Ramesh29300502022-10-21 02:28:24 -0500110 return pel_data
111
112
Sushil Singh28cedb12022-12-12 01:33:32 -0600113def verify_no_pel_exists_on_bmc():
114 r"""
115 Verify that no PEL exists in BMC. Raise an exception if it does.
116 """
117
118 try:
119 pel_data = get_pel_data_from_bmc()
120
121 if len(pel_data) == 0:
122 return True
Anusha Dathatri3914da72023-02-09 06:01:05 -0600123
124 print("PEL data present. \n", pel_data)
125 raise PeltoolException("PEL data present in BMC")
126 except Exception as exception:
127 raise PeltoolException(
128 "Failed to get PEL data from BMC : " + str(exception)
129 ) from exception
Sushil Singh28cedb12022-12-12 01:33:32 -0600130
131
132def compare_pel_and_redfish_event_log(pel_record, event_record):
133 r"""
134 Compare PEL log attributes like "SRC", "Created at" with Redfish
135 event log attributes like "EventId", "Created".
136 Return False if they do not match.
137
138 Description of arguments:
139 pel_record PEL record.
140 event_record Redfish event which is equivalent of PEL record.
141 """
142
143 try:
144 # Below is format of PEL record / event record and following
145 # i.e. "SRC", "Created at" from
146 # PEL record is compared with "EventId", "Created" from event record.
147
148 # PEL Log attributes
149 # SRC : XXXXXXXX
150 # Created at : 11/14/2022 12:38:04
151
152 # Event log attributes
153 # EventId : XXXXXXXX XXXXXXXX XXXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX
154
155 # Created : 2022-11-14T12:38:04+00:00
156
Anusha Dathatri3914da72023-02-09 06:01:05 -0600157 print(f"\nPEL records : {pel_record}")
158 print(f"\nEvent records : {event_record}")
Sushil Singh28cedb12022-12-12 01:33:32 -0600159
Patrick Williamsb813b552022-12-12 10:38:46 -0600160 pel_src = pel_record["pel_data"]["SRC"]
161 pel_created_time = pel_record["pel_detail_data"]["Private Header"][
162 "Created at"
163 ]
Sushil Singh28cedb12022-12-12 01:33:32 -0600164
Patrick Williamsb813b552022-12-12 10:38:46 -0600165 event_ids = (event_record["EventId"]).split(" ")
Sushil Singh28cedb12022-12-12 01:33:32 -0600166
Patrick Williamsb813b552022-12-12 10:38:46 -0600167 event_time_format = (event_record["Created"]).split("T")
168 event_date = (event_time_format[0]).split("-")
169 event_date = datetime(
170 int(event_date[0]), int(event_date[1]), int(event_date[2])
171 )
Sushil Singh28cedb12022-12-12 01:33:32 -0600172 event_date = event_date.strftime("%m/%d/%Y")
Patrick Williamsb813b552022-12-12 10:38:46 -0600173 event_sub_time_format = (event_time_format[1]).split("+")
Sushil Singh28cedb12022-12-12 01:33:32 -0600174 event_date_time = event_date + " " + event_sub_time_format[0]
175
Patrick Williamsb813b552022-12-12 10:38:46 -0600176 event_created_time = event_date_time.replace("-", "/")
Sushil Singh28cedb12022-12-12 01:33:32 -0600177
Anusha Dathatri3914da72023-02-09 06:01:05 -0600178 print(f"\nPEL SRC : {pel_src} | PEL Created Time : {pel_created_time}")
179
Patrick Williamsb813b552022-12-12 10:38:46 -0600180 print(
Anusha Dathatri3914da72023-02-09 06:01:05 -0600181 f"\nError event ID : {event_ids[0]} | Error Log Created Time "
182 + ": {event_created_time}"
Patrick Williamsb813b552022-12-12 10:38:46 -0600183 )
Sushil Singh28cedb12022-12-12 01:33:32 -0600184
185 if pel_src == event_ids[0] and pel_created_time == event_created_time:
Patrick Williamsb813b552022-12-12 10:38:46 -0600186 print(
187 "\nPEL SRC and created date time match with "
188 "event ID, created time"
189 )
Sushil Singh28cedb12022-12-12 01:33:32 -0600190 else:
Anusha Dathatri3914da72023-02-09 06:01:05 -0600191 raise PeltoolException(
Patrick Williamsb813b552022-12-12 10:38:46 -0600192 "\nPEL SRC and created date time did not "
193 "match with event ID, created time"
194 )
Anusha Dathatri3914da72023-02-09 06:01:05 -0600195 except Exception as exception:
196 raise PeltoolException(
George Keishinge16f1582022-12-15 07:32:21 -0600197 "Exception occurred during PEL and Event log "
Patrick Williamsb813b552022-12-12 10:38:46 -0600198 "comparison for SRC or event ID and created "
George Keishing7899a452023-02-15 02:46:54 -0600199 "time : " + str(exception)
Anusha Dathatri3914da72023-02-09 06:01:05 -0600200 ) from exception
Sushil Singh28cedb12022-12-12 01:33:32 -0600201
202
Sridevi Rameshd1cb3252022-11-28 10:04:05 -0600203def fetch_all_pel_ids_for_src(src_id, severity, include_hidden_pels=False):
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -0600204 r"""
Sridevi Rameshc6dd7992022-02-10 01:06:47 -0600205 Fetch all PEL IDs for the input SRC ID based on the severity type
206 in the list format.
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -0600207
208 Description of arguments:
Sridevi Rameshd1cb3252022-11-28 10:04:05 -0600209 src_id SRC ID (e.g. BCXXYYYY).
210 severity PEL severity (e.g. "Predictive Error"
211 "Recovered Error").
212 include_hidden_pels True/False (default: False).
213 Set True to get hidden PELs else False.
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -0600214 """
215
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -0600216 try:
Sridevi Rameshc6dd7992022-02-10 01:06:47 -0600217 src_pel_ids = []
Sridevi Rameshd1cb3252022-11-28 10:04:05 -0600218 pel_data = get_pel_data_from_bmc(include_hidden_pels)
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -0600219 pel_id_list = pel_data.keys()
Sridevi Rameshc6dd7992022-02-10 01:06:47 -0600220 for pel_id in pel_id_list:
221 # Check if required SRC ID with severity is present
Sridevi Rameshf3299972022-12-18 03:27:22 -0600222 if src_id in pel_data[pel_id]["SRC"]:
223 if pel_data[pel_id]["Sev"] == severity:
224 src_pel_ids.append(pel_id)
Sridevi Rameshc6dd7992022-02-10 01:06:47 -0600225
226 if not src_pel_ids:
Anusha Dathatri3914da72023-02-09 06:01:05 -0600227 raise PeltoolException(
Patrick Williams20f38712022-12-08 06:18:26 -0600228 src_id + " with severity " + severity + " not present"
229 )
Anusha Dathatri3914da72023-02-09 06:01:05 -0600230 except Exception as exception:
231 raise PeltoolException(
232 "Failed to fetch PEL ID for required SRC : " + str(exception)
233 ) from exception
Sridevi Rameshc6dd7992022-02-10 01:06:47 -0600234 return src_pel_ids
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -0600235
236
Sridevi Rameshd1cb3252022-11-28 10:04:05 -0600237def fetch_all_src(include_hidden_pels=False):
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -0600238 r"""
Sridevi Rameshc6dd7992022-02-10 01:06:47 -0600239 Fetch all SRC IDs from peltool in the list format.
Sridevi Rameshd1cb3252022-11-28 10:04:05 -0600240
241 include_hidden_pels True/False (default: False).
242 Set True to get hidden PELs else False.
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -0600243 """
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -0600244 try:
Sridevi Rameshc6dd7992022-02-10 01:06:47 -0600245 src_id = []
Sridevi Rameshd1cb3252022-11-28 10:04:05 -0600246 pel_data = get_pel_data_from_bmc(include_hidden_pels)
Sridevi Rameshc6dd7992022-02-10 01:06:47 -0600247 if pel_data:
248 pel_id_list = pel_data.keys()
249 for pel_id in pel_id_list:
250 src_id.append(pel_data[pel_id]["SRC"])
Sridevi Ramesh6bf23632022-11-25 05:55:38 -0600251 print("SRC IDs: " + str(src_id))
Anusha Dathatri3914da72023-02-09 06:01:05 -0600252 except Exception as exception:
253 raise PeltoolException(
254 "Failed to fetch all SRCs : " + str(exception)
255 ) from exception
Sridevi Ramesh6e0e0912021-11-16 11:17:37 -0600256 return src_id
Sridevi Ramesh6bf23632022-11-25 05:55:38 -0600257
258
Patrick Williams20f38712022-12-08 06:18:26 -0600259def check_for_unexpected_src(
Anusha Dathatri3914da72023-02-09 06:01:05 -0600260 unexpected_src_list=None, include_hidden_pels=False
Patrick Williams20f38712022-12-08 06:18:26 -0600261):
Sridevi Ramesh6bf23632022-11-25 05:55:38 -0600262 r"""
263 From the given unexpected SRC list, check if any unexpected SRC created
264 on the BMC. Returns 0 if no SRC found else throws exception.
265
266 Description of arguments:
267 unexpected_src_list Give unexpected SRCs in the list format.
268 e.g.: ["BBXXYYYY", "AAXXYYYY"].
Sridevi Rameshd1cb3252022-11-28 10:04:05 -0600269
270 include_hidden_pels True/False (default: False).
271 Set True to get hidden PELs else False.
Sridevi Ramesh6bf23632022-11-25 05:55:38 -0600272 """
273 try:
274 unexpected_src_count = 0
275 if not unexpected_src_list:
276 print("Unexpected SRC list is empty.")
Sridevi Rameshd1cb3252022-11-28 10:04:05 -0600277 src_data = fetch_all_src(include_hidden_pels)
Sridevi Ramesh6bf23632022-11-25 05:55:38 -0600278 for src in unexpected_src_list:
279 if src in src_data:
280 print("Found an unexpected SRC : " + src)
281 unexpected_src_count = unexpected_src_count + 1
Patrick Williams20f38712022-12-08 06:18:26 -0600282 if unexpected_src_count >= 1:
Anusha Dathatri3914da72023-02-09 06:01:05 -0600283 raise PeltoolException("Unexpected SRC found.")
Sridevi Ramesh6bf23632022-11-25 05:55:38 -0600284
Anusha Dathatri3914da72023-02-09 06:01:05 -0600285 except Exception as exception:
286 raise PeltoolException(
287 "Failed to verify unexpected SRC list : " + str(exception)
288 ) from exception
Sridevi Ramesh6bf23632022-11-25 05:55:38 -0600289 return unexpected_src_count
Anusha Dathatrie8801a32022-11-28 04:56:31 -0600290
291
292def filter_unexpected_srcs(expected_srcs=None):
293 r"""
294 Return list of SRCs found in BMC after filtering expected SRCs.
295 If expected_srcs is None then all SRCs found in system are returned.
296
297 Description of arguments:
298 expected_srcs List of expected SRCs. E.g. ["BBXXYYYY", "AAXXYYYY"].
299 """
300
Shiwani Kumari8b0a9632025-06-09 01:19:55 -0500301 temp_srcs_found = fetch_all_src()
302
Anusha Dathatrie8801a32022-11-28 04:56:31 -0600303 if not expected_srcs:
304 expected_srcs = []
Shiwani Kumari8b0a9632025-06-09 01:19:55 -0500305 srcs_found = list()
306 # Remove any extra space after SRC ID
307 for item in temp_srcs_found:
308 var = item.split()
309 srcs_found.append(var[0])
310 print(srcs_found)
Anusha Dathatrie8801a32022-11-28 04:56:31 -0600311 print(expected_srcs)
312 return list(set(srcs_found) - set(expected_srcs))
Anusha Dathatri5636ad12022-11-29 04:26:00 -0600313
314
315def get_bmc_event_log_id_for_pel(pel_id):
316 r"""
317 Return BMC event log ID for the given PEL ID.
318
319 Description of arguments:
320 pel_id PEL ID. E.g. 0x50000021.
321 """
322
323 pel_data = peltool("-i " + pel_id)
324 print(pel_data)
325 bmc_id_for_pel = pel_data["Private Header"]["BMC Event Log Id"]
326 return bmc_id_for_pel
Anusha Dathatri3914da72023-02-09 06:01:05 -0600327
328
329def get_latest_pels(number_of_pels=1):
330 r"""
331 Return latest PEL IDs.
332
333 Description of arguments:
334 number_of_pels Number of PELS to be returned.
335 """
336
337 pel_data = peltool("-lr")
338 pel_ids = list(pel_data.keys())
339 return pel_ids[:number_of_pels]
Sridevi Rameshdd76dfb2025-04-17 05:35:19 -0500340
341
342def fetch_all_pel_ids_based_on_error_message(
343 error_msg, include_hidden_pels=False, include_informational_pels=False
344):
345 r"""
346 Fetch all PEL IDs based on the input error message and return
347 in the list format.
348
349 Description of arguments:
350 error_msg Error message
351 include_hidden_pels True/False (default: False).
352 Set True to get hidden PELs else False.
353 include_informational_pels True/False (default: False).
354 Set True to get informational PELs else False
355 """
356
357 try:
358 err_pel_ids = []
359 pel_data = get_pel_data_from_bmc(
360 include_hidden_pels, include_informational_pels
361 )
362 pel_id_list = pel_data.keys()
363 for pel_id in pel_id_list:
364 # Check if required PEL with error message is created.
365 if error_msg in pel_data[pel_id]["Message"]:
366 err_pel_ids.append(pel_id)
367
368 if not err_pel_ids:
369 raise PeltoolException(
370 "Failed to get PEL ID with error message : " + error_msg
371 )
372 except Exception as exception:
373 raise PeltoolException(
374 "Failed to fetch PEL ID for required SRC : " + str(exception)
375 ) from exception
376 return err_pel_ids
Sridevi Ramesh912b91b2025-06-23 03:37:41 -0500377
378
379def check_if_pel_transmitted_to_host(pel_id):
380 r"""
381 Return True if PEL is transmitted to Host else False.
382
383 Description of arguments:
384 pel_id PEL ID. E.g. 0x50000021.
385 """
386
387 try:
388 pel_data = peltool("-i " + pel_id)
389 print(pel_data)
390 host_state = pel_data["User Header"]["Host Transmission"]
391 if host_state != "Acked":
392 return False
393 except Exception as exception:
394 raise PeltoolException(
395 "Failed to parse PEL data : " + str(exception)
396 ) from exception
397 return True