psutil: Add PSU Event Log Reporting Methods
This commit introduces new methods in the Updater class to log errors
and create Platform Event Logs for PSU failures and I2C-related
issues. The following methods are added:
- createServiceableEventLog:
Creates a serviceable Platform Event Log using
xyz.openbmc_project.Logging.Create. Takes an error name, severity, and
additional data as parameters. Retrieves the logging service and calls
the D-Bus method to create the log.
- getI2CAdditionalData:
Retrieves I2C-related callout data, including I2C bus ID, address, and
error number. Formats the ID and address as hexadecimal strings and
returns them as a map.
- callOutI2CEventLog:
Reports a Event Log for I2C failures. Collects PSU inventory path,
priority, and I2C-specific callout data. Merges any additional
provided data and creates a Event Log.
- callOutPsuEventLog:
Reports a Event Log for general PSU failures. Includes PSU inventory
callout information and priority.
- callOutSWEventLog:
Reports Event Log for software-related PSU file issues.Logs errors
using predefined PSU firmware file issue messages. These changes
improve fault logging and troubleshooting capabilities in PSU
management by ensuring proper logging and event recording.
- callOutGoodEventLog:
Reports a successful PSU firmware update Event Log along with the
firmware level.
- Added accessor functions to provide control over Event logging,
allowing:
- Enabling/disabling Event logging at runtime.
- Tracking if Event Log has been logged in the current session.
Accessor functions:
- enableEventLogging()
- disableEventLogging()
- isEventLogEnabled()
- enableEventLoggedThisSession()
- isEventLoggedThisSession()
Test:
Verified each function correctly generates Event Log in openBmc. The
test was conducted by writing a standalone test program that:
1 - Calls createServiceableEventLog() with various error names,
severity levels, and additionalData then check the generated Event Log.
2 - Calls getI2CAdditionalData() and verifies the returned data
contains the correct I2C bus ID, Address and error number.
3 - Calls callOutI2CEventLog() with simulated I2C error and checks the
generated Event Log.
4 - Calls callOutPsuEventLog() and callOutSWEventLog() with different
input data and verifies the correct error messages are logged.
Change-Id: Id52b3e1c0b2a3b09ae3ac5d93bc53e02d335d6c7
Signed-off-by: Faisal Awada <faisal@us.ibm.com>
diff --git a/tools/power-utils/aei_updater.cpp b/tools/power-utils/aei_updater.cpp
index 27da440..a98a1d8 100644
--- a/tools/power-utils/aei_updater.cpp
+++ b/tools/power-utils/aei_updater.cpp
@@ -22,10 +22,12 @@
#include "types.hpp"
#include "updater.hpp"
#include "utility.hpp"
+#include "utils.hpp"
#include <phosphor-logging/lg2.hpp>
#include <fstream>
+#include <system_error>
namespace aeiUpdater
{
@@ -72,8 +74,15 @@
int AeiUpdater::doUpdate()
{
i2cInterface = Updater::getI2C();
+ enableEventLogging();
if (i2cInterface == nullptr)
{
+ // Report serviceable error
+ std::map<std::string, std::string> additionalData = {
+ {"I2C_INTERFACE", "I2C interface is null pointer."}};
+ // Callout PSU & I2C
+ callOutI2CEventLog(additionalData);
+
throw std::runtime_error("I2C interface error");
}
if (!getFirmwarePath() || !isFirmwareFileValid())
@@ -83,24 +92,22 @@
bool downloadFwFailed = false; // Download Firmware status
int retryProcessTwo(0);
int retryProcessOne(0);
- int serviceableEvent(0);
+ disableEventLogging();
while ((retryProcessTwo < MAX_RETRIES) && (retryProcessOne < MAX_RETRIES))
{
- retryProcessTwo++;
// Write AEI PSU ISP key
if (!writeIspKey())
{
- serviceableEvent++;
- if (serviceableEvent == MAX_RETRIES)
- {
- createServiceableError(
- "createServiceableError: ISP key failed");
- }
lg2::error("Failed to set ISP Key");
downloadFwFailed = true; // Download Firmware status
- continue;
+ break;
}
+ if (retryProcessTwo == (MAX_RETRIES - 1))
+ {
+ enableEventLogging();
+ }
+ retryProcessTwo++;
while (retryProcessOne < MAX_RETRIES)
{
downloadFwFailed = false; // Download Firmware status
@@ -117,7 +124,7 @@
// Reset ISP status
if (writeIspStatusReset())
{
- // Start PSU frimware download.
+ // Start PSU firmware download.
if (downloadPsuFirmware())
{
if (!verifyDownloadFWStatus())
@@ -128,11 +135,16 @@
}
else
{
+ // One of the block write commands failed, retry download
+ // procedure one time starting with re-writing initial ISP
+ // mode. If it fails again, log serviceable error.
if (retryProcessOne == MAX_RETRIES)
{
- // no more retries report serviceable error
- createServiceableError(
- "serviceableError: Download firmware failed");
+ // Callout PSU failed to update FW
+ std::map<std::string, std::string> additionalData = {
+ {"UPDATE_FAILED", "Download firmware failed"}};
+
+ callOutPsuEventLog(additionalData);
ispReboot(); // Try to set PSU to normal mode
}
downloadFwFailed = true;
@@ -148,6 +160,7 @@
}
ispReboot();
+
if (ispReadRebootStatus() && !downloadFwFailed)
{
// Download completed successful
@@ -156,6 +169,8 @@
}
else
{
+ // Retry the whole download process starting with the key and
+ // if fails again then report event log
if ((retryProcessOne < (MAX_RETRIES - 1)) &&
(retryProcessTwo < (MAX_RETRIES - 1)))
{
@@ -169,6 +184,10 @@
{
return 1;
}
+ enableEventLogging();
+ bindUnbind(true);
+ updater::internal::delay(100);
+ callOutGoodEventLog();
return 0; // Update successful
}
@@ -177,24 +196,46 @@
// ISP Key to unlock programming mode ( ASCII for "artY").
constexpr std::array<uint8_t, 4> unlockData = {0x61, 0x72, 0x74,
0x59}; // ISP Key "artY"
- try
+ for (int retry = 0; retry < MAX_RETRIES; ++retry)
{
- // Send ISP Key to unlock device for firmware update
- i2cInterface->write(KEY_REGISTER, unlockData.size(), unlockData.data());
- return true;
+ try
+ {
+ // Send ISP Key to unlock device for firmware update
+ i2cInterface->write(KEY_REGISTER, unlockData.size(),
+ unlockData.data());
+ disableEventLogging();
+ return true;
+ }
+ catch (const i2c::I2CException& e)
+ {
+ // Log failure if I2C write fails.
+ lg2::error("I2C write failed: {ERROR}", "ERROR", e);
+ std::map<std::string, std::string> additionalData = {
+ {"I2C_ISP_KEY", "ISP key failed due to I2C exception"}};
+ callOutI2CEventLog(additionalData, e.what(), e.errorCode);
+ enableEventLogging(); // enable event logging if fail again call out
+ // PSU & I2C
+ }
+
+ catch (const std::exception& e)
+ {
+ lg2::error("Exception write failed: {ERROR}", "ERROR", e);
+ std::map<std::string, std::string> additionalData = {
+ {"ISP_KEY", "ISP key failed due to exception"},
+ {"EXCEPTION", e.what()}};
+ callOutPsuEventLog(additionalData);
+ enableEventLogging(); // enable Event Logging if fail again call out
+ // PSU
+ }
}
- catch (const std::exception& e)
- {
- // Log failure if I2C write fails.
- lg2::error("I2C write failed: {ERROR}", "ERROR", e);
- return false;
- }
+ return false;
}
bool AeiUpdater::writeIspMode()
{
// Attempt to set device in ISP mode with retries.
uint8_t ispStatus = 0x0;
+ uint8_t exceptionCount = 0;
for (int retry = 0; retry < MAX_RETRIES; ++retry)
{
try
@@ -209,17 +250,57 @@
if (ispStatus & B_ISP_MODE)
{
lg2::info("Set ISP Mode");
+ disableEventLogging();
return true;
}
+ enableEventLogging();
+ }
+ catch (const i2c::I2CException& e)
+ {
+ exceptionCount++;
+ // Log I2C error with each retry attempt.
+ lg2::error("I2C exception during ISP mode write/read: {ERROR}",
+ "ERROR", e);
+ if (exceptionCount == MAX_RETRIES)
+ {
+ enableEventLogging();
+ std::map<std::string, std::string> additionalData = {
+ {"I2C_FIRMWARE_STATUS",
+ "Download firmware failed during writeIspMode due to I2C exception"}};
+ // Callout PSU & I2C
+ callOutI2CEventLog(additionalData, e.what(), e.errorCode);
+ return false; // Failed to set ISP Mode
+ }
}
catch (const std::exception& e)
{
- // Log I2C error with each retry attempt.
- lg2::error("I2C error during ISP mode write/read: {ERROR}", "ERROR",
+ exceptionCount++;
+ // Log error with each retry attempt.
+ lg2::error("Exception during ISP mode write/read: {ERROR}", "ERROR",
e);
+ if (exceptionCount == MAX_RETRIES)
+ {
+ enableEventLogging();
+ std::map<std::string, std::string> additionalData = {
+ {"FIRMWARE_STATUS",
+ "Download firmware failed during writeIspMode due to exception"},
+ {"EXCEPTION", e.what()}};
+ // Callout PSU
+ callOutPsuEventLog(additionalData);
+ return false; // Failed to set ISP Mode
+ }
}
}
- createServiceableError("createServiceableError: ISP Status failed");
+
+ if (exceptionCount != MAX_RETRIES)
+ {
+ // Callout PSU
+ std::map<std::string, std::string> additionalData = {
+ {"FIRMWARE_STATUS",
+ "Download firmware failed during writeIspMode"}};
+ callOutPsuEventLog(additionalData);
+ }
+
lg2::error("Failed to set ISP Mode");
return false; // Failed to set ISP Mode after retries
}
@@ -228,6 +309,7 @@
{
// Reset ISP status register before firmware download.
uint8_t ispStatus = 0;
+ uint8_t exceptionCount = 0;
for (int retry = 0; retry < MAX_RETRIES; retry++)
{
try
@@ -236,13 +318,42 @@
CMD_RESET_SEQ); // Start reset sequence.
retry = MAX_RETRIES;
}
- catch (const std::exception& e)
+ catch (const i2c::I2CException& e)
{
+ exceptionCount++;
// Log any errors encountered during reset sequence.
lg2::error("I2C Write ISP reset failed: {ERROR}", "ERROR", e);
+ if (exceptionCount == MAX_RETRIES)
+ {
+ enableEventLogging();
+ std::map<std::string, std::string> additionalData = {
+ {"I2C_ISP_RESET", "I2C exception during ISP status reset"}};
+ // Callout PSU & I2C
+ callOutI2CEventLog(additionalData, e.what(), e.errorCode);
+ ispReboot();
+ return false;
+ }
+ }
+ catch (const std::exception& e)
+ {
+ exceptionCount++;
+ // Log any errors encountered during reset sequence.
+ lg2::error("Write ISP reset failed: {ERROR}", "ERROR", e);
+ if (exceptionCount == MAX_RETRIES)
+ {
+ enableEventLogging();
+ std::map<std::string, std::string> additionalData = {
+ {"ISP_RESET", "Exception during ISP status reset"},
+ {"EXCEPTION", e.what()}};
+ // Callout PSU
+ callOutPsuEventLog(additionalData);
+ ispReboot();
+ return false;
+ }
}
}
+ exceptionCount = 0;
for (int retry = 0; retry < MAX_RETRIES; ++retry)
{
try
@@ -250,22 +361,57 @@
i2cInterface->read(STATUS_REGISTER, ispStatus);
if (ispStatus == B_ISP_MODE)
{
- lg2::info("Read/Write ISP reset");
+ lg2::info("write/read ISP reset");
+ disableEventLogging();
return true; // ISP status reset successfully.
}
i2cInterface->write(STATUS_REGISTER,
CMD_CLEAR_STATUS); // Clear status if
// not reset.
- lg2::error("Read/Write ISP reset failed");
+ lg2::error("Write ISP reset failed");
+ enableEventLogging();
+ }
+ catch (const i2c::I2CException& e)
+ {
+ exceptionCount++;
+ // Log any errors encountered during reset sequence.
+ lg2::error(
+ "I2C Write/Read or Write error during ISP reset: {ERROR}",
+ "ERROR", e);
+ if (exceptionCount == MAX_RETRIES)
+ {
+ enableEventLogging();
+ std::map<std::string, std::string> additionalData = {
+ {"I2C_ISP_READ_STATUS",
+ "I2C exception during read ISP status"}};
+ // Callout PSU & I2C
+ callOutI2CEventLog(additionalData, e.what(), e.errorCode);
+ }
}
catch (const std::exception& e)
{
+ exceptionCount++;
// Log any errors encountered during reset sequence.
- lg2::error("I2C Read/Write error during ISP reset: {ERROR}",
+ lg2::error("Write/Read or Write error during ISP reset: {ERROR}",
"ERROR", e);
+ if (exceptionCount == MAX_RETRIES)
+ {
+ enableEventLogging();
+ std::map<std::string, std::string> additionalData = {
+ {"ISP_READ_STATUS", "Exception during read ISP status"},
+ {"EXCEPTION", e.what()}};
+ // Callout PSU
+ callOutPsuEventLog(additionalData);
+ }
}
}
- createServiceableError("createServiceableError: Failed to reset ISP");
+ if (exceptionCount != MAX_RETRIES)
+ {
+ std::map<std::string, std::string> additionalData = {
+ {"ISP_REST_FAILED", "Failed to read ISP expected status"}};
+ // Callout PSU
+ callOutPsuEventLog(additionalData);
+ }
lg2::error("Failed to reset ISP Status");
ispReboot();
return false;
@@ -276,6 +422,10 @@
fspath = updater::internal::getFWFilenamePath(getImageDir());
if (fspath.empty())
{
+ std::map<std::string, std::string> additionalData = {
+ {"FILE_PATH", "Firmware file path is null"}};
+ // Callout BMC0001 procedure
+ callOutSWEventLog(additionalData);
lg2::error("Firmware file path not found");
return false;
}
@@ -286,6 +436,11 @@
{
if (!updater::internal::validateFWFile(fspath))
{
+ std::map<std::string, std::string> additionalData = {
+ {"FIRMWARE_VALID",
+ "Firmware validation failed, FW file path = " + fspath}};
+ // Callout BMC0001 procedure
+ callOutSWEventLog(additionalData);
lg2::error("Firmware validation failed, fspath={PATH}", "PATH", fspath);
return false;
}
@@ -297,6 +452,11 @@
auto inputFile = updater::internal::openFirmwareFile(fspath);
if (!inputFile)
{
+ std::map<std::string, std::string> additionalData = {
+ {"FIRMWARE_OPEN",
+ "Firmware file failed to open, FW file path = " + fspath}};
+ // Callout BMC0001 procedure
+ callOutSWEventLog(additionalData);
lg2::error("Failed to open firmware file");
}
return inputFile;
@@ -336,6 +496,16 @@
auto inputFile = openFirmwareFile();
if (!inputFile)
{
+ if (isEventLogEnabled())
+ {
+ // Callout BMC0001 procedure
+ std::map<std::string, std::string> additionalData = {
+ {"FW_FAILED_TO_OPEN", "Firmware file failed to open"},
+ {"FW_FILE_PATH", fspath}};
+
+ callOutSWEventLog(additionalData);
+ ispReboot(); // Try to set PSU to normal mode
+ }
lg2::error("Unable to open firmware file {FILE}", "FILE", fspath);
return false;
}
@@ -377,7 +547,6 @@
"BYTESREAD", bytesRead);
return false; // Failed
}
-
return true;
}
@@ -385,6 +554,8 @@
uint8_t regAddr, const uint8_t expectedReadSize, uint8_t* readData,
const int retries, const int delayTime)
{
+ uint8_t exceptionCount = 0;
+ uint32_t bigEndianValue = 0;
for (int i = 0; i < retries; ++i)
{
uint8_t readReplySize = 0;
@@ -406,22 +577,46 @@
}
else
{
- uint32_t littleEndianValue =
- *reinterpret_cast<uint32_t*>(readData);
- uint32_t bigEndianValue =
- ((littleEndianValue & 0x000000FF) << 24) |
- ((littleEndianValue & 0x0000FF00) << 8) |
- ((littleEndianValue & 0x00FF0000) >> 8) |
- ((littleEndianValue & 0xFF000000) >> 24);
+ bigEndianValue = (readData[0] << 24) | (readData[1] << 16) |
+ (readData[2] << 8) | (readData[3]);
lg2::error("Write/read block {NUM} failed", "NUM",
bigEndianValue);
}
}
+ catch (const i2c::I2CException& e)
+ {
+ exceptionCount++;
+ if (exceptionCount == MAX_RETRIES)
+ {
+ std::map<std::string, std::string> additionalData = {
+ {"I2C_WRITE_READ",
+ "I2C exception while flashing the firmware."}};
+ // Callout PSU & I2C
+ callOutI2CEventLog(additionalData, e.what(), e.errorCode);
+ }
+ lg2::error("I2C exception write/read block failed: {ERROR}",
+ "ERROR", e.what());
+ }
catch (const std::exception& e)
{
- lg2::error("I2C write/read block failed: {ERROR}", "ERROR", e);
+ exceptionCount++;
+ if (exceptionCount == MAX_RETRIES)
+ {
+ std::map<std::string, std::string> additionalData = {
+ {"WRITE_READ", "Exception while flashing the firmware."},
+ {"EXCEPTION", e.what()}};
+ // Callout PSU
+ callOutPsuEventLog(additionalData);
+ }
+ lg2::error("Exception write/read block failed: {ERROR}", "ERROR",
+ e.what());
}
}
+ std::map<std::string, std::string> additionalData = {
+ {"WRITE_READ",
+ "Download firmware failed block: " + std::to_string(bigEndianValue)}};
+ // Callout PSU
+ callOutPsuEventLog(additionalData);
return false;
}
@@ -481,27 +676,56 @@
bool AeiUpdater::ispReadRebootStatus()
{
- try
+ for (int retry = 0; retry < MAX_RETRIES; ++retry)
{
- // Read from the status register to verify reboot
- uint8_t data = 1; // Initialize data to a non-zero value
- i2cInterface->read(STATUS_REGISTER, data);
-
- // If the reboot was successful, the read data should be 0
- if (data == SUCCESSFUL_ISP_REBOOT_STATUS)
+ try
{
- lg2::info("ISP Status Reboot successful.");
- return true;
+ // Read from the status register to verify reboot
+ uint8_t data = 1; // Initialize data to a non-zero value
+ i2cInterface->read(STATUS_REGISTER, data);
+
+ // If the reboot was successful, the read data should be 0
+ if (data == SUCCESSFUL_ISP_REBOOT_STATUS)
+ {
+ lg2::info("ISP Status Reboot successful.");
+ return true;
+ }
}
- }
- catch (const std::exception& e)
- {
- lg2::error("I2C read error during reboot attempt: {ERROR}", "ERROR", e);
+ catch (const i2c::I2CException& e)
+ {
+ if (isEventLogEnabled())
+ {
+ std::map<std::string, std::string> additionalData = {
+ {"I2C_READ_REBOOT",
+ "I2C exception while reading ISP reboot status"}};
+
+ // Callout PSU & I2C
+ callOutI2CEventLog(additionalData, e.what(), e.errorCode);
+ }
+ lg2::error("I2C read error during reboot attempt: {ERROR}", "ERROR",
+ e);
+ }
+ catch (const std::exception& e)
+ {
+ if (isEventLogEnabled())
+ {
+ std::map<std::string, std::string> additionalData = {
+ {"READ_REBOOT",
+ "Exception while reading ISP reboot status"},
+ {"EXCEPTION", e.what()}};
+
+ // Callout PSU
+ callOutPsuEventLog(additionalData);
+ }
+ lg2::error("Read exception during reboot attempt: {ERROR}", "ERROR",
+ e);
+ }
+ // Reboot the PSU
+ ispReboot(); // Try to set PSU to normal mode
}
// If we reach here, all retries have failed
lg2::error("Failed to reboot ISP status after max retries.");
return false;
}
-
} // namespace aeiUpdater