pseq: Check STATUS_VOUT first to find pgood fault
Enhance the support for finding the correct voltage rail when a pgood
fault occurs.
First check the PMBus STATUS_VOUT register for all rails. Check the
rails in power-on-sequence order.
Checking STATUS_VOUT is usually the most accurate method. For example,
if a pgood fault occurs, the power sequencer device may automatically
shut off related rails. Ideally the device will only set fault bits in
STATUS_VOUT for the rail with the pgood fault. However, all the related
rails will likely appear to be faulted by the other methods.
If no fault is found by checking STATUS_VOUT, then check the GPIOs and
output voltage for all rails. Check the rails in power-on-sequence
order.
Tested:
* Verified all automated test cases run successfully.
Change-Id: Ida8732db573013f1b72edac8ed54e3cfc38da146
Signed-off-by: Shawn McCarney <shawnmm@us.ibm.com>
diff --git a/phosphor-power-sequencer/test/standard_device_tests.cpp b/phosphor-power-sequencer/test/standard_device_tests.cpp
index 2eb3a42..1afb3bd 100644
--- a/phosphor-power-sequencer/test/standard_device_tests.cpp
+++ b/phosphor-power-sequencer/test/standard_device_tests.cpp
@@ -72,6 +72,10 @@
MOCK_METHOD(uint8_t, getStatusVout, (uint8_t page), (override));
MOCK_METHOD(double, getReadVout, (uint8_t page), (override));
MOCK_METHOD(double, getVoutUVFaultLimit, (uint8_t page), (override));
+
+ // Override empty implementation with mock so we can verify it is called
+ MOCK_METHOD(void, prepareForPgoodFaultDetection, (Services & services),
+ (override));
};
/**
@@ -119,6 +123,28 @@
checkStatusVout, compareVoltageToLimit, gpio);
}
+/**
+ * Creates a Rail object that checks for a pgood fault using output voltage.
+ *
+ * @param name Unique name for the rail
+ * @param isPowerSupplyRail Specifies whether the rail is produced by a
+ power supply
+ * @param pageNum PMBus PAGE number of the rail
+ * @return Rail object
+ */
+std::unique_ptr<Rail> createRailOutputVoltage(const std::string& name,
+ bool isPowerSupplyRail,
+ uint8_t pageNum)
+{
+ std::optional<std::string> presence{};
+ std::optional<uint8_t> page{pageNum};
+ bool checkStatusVout{false};
+ bool compareVoltageToLimit{true};
+ std::optional<GPIO> gpio{};
+ return std::make_unique<Rail>(name, presence, page, isPowerSupplyRail,
+ checkStatusVout, compareVoltageToLimit, gpio);
+}
+
TEST(StandardDeviceTests, Constructor)
{
// Empty vector of rails
@@ -134,7 +160,7 @@
{
std::vector<std::unique_ptr<Rail>> rails{};
rails.emplace_back(createRailGPIO("PSU", true, 3));
- rails.emplace_back(createRailStatusVout("VDD", false, 5));
+ rails.emplace_back(createRailOutputVoltage("VDD", false, 5));
rails.emplace_back(createRailStatusVout("VIO", false, 7));
StandardDeviceImpl device{"abc_pseq", std::move(rails)};
@@ -168,7 +194,7 @@
{
std::vector<std::unique_ptr<Rail>> rails{};
rails.emplace_back(createRailGPIO("PSU", true, 3));
- rails.emplace_back(createRailStatusVout("VDD", false, 5));
+ rails.emplace_back(createRailOutputVoltage("VDD", false, 5));
rails.emplace_back(createRailStatusVout("VIO", false, 7));
StandardDeviceImpl device{"abc_pseq", std::move(rails)};
@@ -185,15 +211,19 @@
{
std::vector<std::unique_ptr<Rail>> rails{};
rails.emplace_back(createRailGPIO("PSU", true, 2));
- rails.emplace_back(createRailStatusVout("VDD", false, 5));
+ rails.emplace_back(createRailOutputVoltage("VDD", false, 5));
rails.emplace_back(createRailStatusVout("VIO", false, 7));
StandardDeviceImpl device{"abc_pseq", std::move(rails)};
+ EXPECT_CALL(device, prepareForPgoodFaultDetection).Times(1);
std::vector<int> gpioValues{1, 1, 1};
EXPECT_CALL(device, getGPIOValues)
.Times(1)
.WillOnce(Return(gpioValues));
- EXPECT_CALL(device, getStatusVout(5)).Times(1).WillOnce(Return(0x00));
+ EXPECT_CALL(device, getReadVout(5)).Times(1).WillOnce(Return(1.2));
+ EXPECT_CALL(device, getVoutUVFaultLimit(5))
+ .Times(1)
+ .WillOnce(Return(1.1));
EXPECT_CALL(device, getStatusVout(7)).Times(1).WillOnce(Return(0x00));
MockServices services{};
@@ -206,20 +236,23 @@
EXPECT_EQ(additionalData.size(), 0);
}
- // First rail has a pgood fault
+ // First rail has a pgood fault detected via GPIO
// Is a PSU rail: No PSU error specified
{
std::vector<std::unique_ptr<Rail>> rails{};
rails.emplace_back(createRailGPIO("PSU", true, 2));
- rails.emplace_back(createRailStatusVout("VDD", false, 5));
+ rails.emplace_back(createRailOutputVoltage("VDD", false, 5));
rails.emplace_back(createRailStatusVout("VIO", false, 7));
StandardDeviceImpl device{"abc_pseq", std::move(rails)};
+ EXPECT_CALL(device, prepareForPgoodFaultDetection).Times(1);
std::vector<int> gpioValues{1, 1, 0};
EXPECT_CALL(device, getGPIOValues)
.Times(1)
.WillOnce(Return(gpioValues));
- EXPECT_CALL(device, getStatusVout).Times(0);
+ EXPECT_CALL(device, getReadVout(5)).Times(0);
+ EXPECT_CALL(device, getVoutUVFaultLimit(5)).Times(0);
+ EXPECT_CALL(device, getStatusVout(7)).Times(1).WillOnce(Return(0x00));
MockServices services{};
EXPECT_CALL(services,
@@ -252,20 +285,23 @@
EXPECT_EQ(additionalData["GPIO_VALUE"], "0");
}
- // First rail has a pgood fault
+ // First rail has a pgood fault detected via GPIO
// Is a PSU rail: PSU error specified
{
std::vector<std::unique_ptr<Rail>> rails{};
rails.emplace_back(createRailGPIO("PSU", true, 2));
- rails.emplace_back(createRailStatusVout("VDD", false, 5));
+ rails.emplace_back(createRailOutputVoltage("VDD", false, 5));
rails.emplace_back(createRailStatusVout("VIO", false, 7));
StandardDeviceImpl device{"abc_pseq", std::move(rails)};
+ EXPECT_CALL(device, prepareForPgoodFaultDetection).Times(1);
std::vector<int> gpioValues{1, 1, 0};
EXPECT_CALL(device, getGPIOValues)
.Times(1)
.WillOnce(Return(gpioValues));
- EXPECT_CALL(device, getStatusVout).Times(0);
+ EXPECT_CALL(device, getReadVout(5)).Times(0);
+ EXPECT_CALL(device, getVoutUVFaultLimit(5)).Times(0);
+ EXPECT_CALL(device, getStatusVout(7)).Times(1).WillOnce(Return(0x00));
MockServices services{};
EXPECT_CALL(services,
@@ -297,21 +333,25 @@
EXPECT_EQ(additionalData["GPIO_VALUE"], "0");
}
- // Middle rail has a pgood fault
+ // Second rail has a pgood fault detected via output voltage
// Not a PSU rail: PSU error specified
{
std::vector<std::unique_ptr<Rail>> rails{};
rails.emplace_back(createRailGPIO("PSU", true, 2));
- rails.emplace_back(createRailStatusVout("VDD", false, 5));
+ rails.emplace_back(createRailOutputVoltage("VDD", false, 5));
rails.emplace_back(createRailStatusVout("VIO", false, 7));
StandardDeviceImpl device{"abc_pseq", std::move(rails)};
+ EXPECT_CALL(device, prepareForPgoodFaultDetection).Times(1);
std::vector<int> gpioValues{1, 1, 1};
EXPECT_CALL(device, getGPIOValues)
.Times(1)
.WillOnce(Return(gpioValues));
- EXPECT_CALL(device, getStatusVout(5)).Times(1).WillOnce(Return(0x10));
- EXPECT_CALL(device, getStatusVout(7)).Times(0);
+ EXPECT_CALL(device, getReadVout(5)).Times(1).WillOnce(Return(1.1));
+ EXPECT_CALL(device, getVoutUVFaultLimit(5))
+ .Times(1)
+ .WillOnce(Return(1.2));
+ EXPECT_CALL(device, getStatusVout(7)).Times(1).WillOnce(Return(0x00));
EXPECT_CALL(device, getStatusWord(5)).Times(1).WillOnce(Return(0xbeef));
MockServices services{};
@@ -329,7 +369,8 @@
.Times(1);
EXPECT_CALL(
services,
- logErrorMsg("Rail VDD has fault bits set in STATUS_VOUT: 0x10"))
+ logErrorMsg(
+ "Rail VDD output voltage 1.1V is <= UV fault limit 1.2V"))
.Times(1);
std::string powerSupplyError{"Undervoltage fault: PSU1"};
@@ -338,30 +379,33 @@
additionalData);
EXPECT_EQ(error,
"xyz.openbmc_project.Power.Error.PowerSequencerVoltageFault");
- EXPECT_EQ(additionalData.size(), 5);
+ EXPECT_EQ(additionalData.size(), 6);
EXPECT_EQ(additionalData["DEVICE_NAME"], "abc_pseq");
EXPECT_EQ(additionalData["GPIO_VALUES"], "[1, 1, 1]");
EXPECT_EQ(additionalData["RAIL_NAME"], "VDD");
- EXPECT_EQ(additionalData["STATUS_VOUT"], "0x10");
+ EXPECT_EQ(additionalData["READ_VOUT"], "1.1");
+ EXPECT_EQ(additionalData["VOUT_UV_FAULT_LIMIT"], "1.2");
EXPECT_EQ(additionalData["STATUS_WORD"], "0xbeef");
}
- // Last rail has a pgood fault
+ // Third rail has a pgood fault detected via STATUS_VOUT
// Device returns 0 GPIO values
// Does not halt pgood fault detection because GPIO values not used by rails
{
std::vector<std::unique_ptr<Rail>> rails{};
rails.emplace_back(createRailStatusVout("PSU", true, 3));
- rails.emplace_back(createRailStatusVout("VDD", false, 5));
+ rails.emplace_back(createRailOutputVoltage("VDD", false, 5));
rails.emplace_back(createRailStatusVout("VIO", false, 7));
StandardDeviceImpl device{"abc_pseq", std::move(rails)};
+ EXPECT_CALL(device, prepareForPgoodFaultDetection).Times(1);
std::vector<int> gpioValues{};
EXPECT_CALL(device, getGPIOValues)
.Times(1)
.WillOnce(Return(gpioValues));
EXPECT_CALL(device, getStatusVout(3)).Times(1).WillOnce(Return(0x00));
- EXPECT_CALL(device, getStatusVout(5)).Times(1).WillOnce(Return(0x00));
+ EXPECT_CALL(device, getReadVout(5)).Times(0);
+ EXPECT_CALL(device, getVoutUVFaultLimit(5)).Times(0);
EXPECT_CALL(device, getStatusVout(7)).Times(1).WillOnce(Return(0x11));
EXPECT_CALL(device, getStatusWord(7)).Times(1).WillOnce(Return(0xbeef));
@@ -393,21 +437,23 @@
EXPECT_EQ(additionalData["STATUS_WORD"], "0xbeef");
}
- // Last rail has a pgood fault
+ // Third rail has a pgood fault detected via STATUS_VOUT
// Exception occurs trying to obtain GPIO values from device
// Does not halt pgood fault detection because GPIO values not used by rails
{
std::vector<std::unique_ptr<Rail>> rails{};
rails.emplace_back(createRailStatusVout("PSU", true, 3));
- rails.emplace_back(createRailStatusVout("VDD", false, 5));
+ rails.emplace_back(createRailOutputVoltage("VDD", false, 5));
rails.emplace_back(createRailStatusVout("VIO", false, 7));
StandardDeviceImpl device{"abc_pseq", std::move(rails)};
+ EXPECT_CALL(device, prepareForPgoodFaultDetection).Times(1);
EXPECT_CALL(device, getGPIOValues)
.Times(1)
.WillOnce(Throw(std::runtime_error{"Unable to acquire GPIO line"}));
EXPECT_CALL(device, getStatusVout(3)).Times(1).WillOnce(Return(0x00));
- EXPECT_CALL(device, getStatusVout(5)).Times(1).WillOnce(Return(0x00));
+ EXPECT_CALL(device, getReadVout(5)).Times(0);
+ EXPECT_CALL(device, getVoutUVFaultLimit(5)).Times(0);
EXPECT_CALL(device, getStatusVout(7)).Times(1).WillOnce(Return(0x11));
EXPECT_CALL(device, getStatusWord(7)).Times(1).WillOnce(Return(0xbeef));
@@ -439,22 +485,130 @@
EXPECT_EQ(additionalData["STATUS_WORD"], "0xbeef");
}
+ // All three rails appear to have a pgood fault. Verify third rail is
+ // selected, even though it is last in the power on sequence, because it is
+ // checked using STATUS_VOUT. That check happens before the other checks.
+ {
+ std::vector<std::unique_ptr<Rail>> rails{};
+ rails.emplace_back(createRailGPIO("PSU", true, 2));
+ rails.emplace_back(createRailGPIO("VDD", false, 1));
+ rails.emplace_back(createRailStatusVout("VIO", false, 7));
+ StandardDeviceImpl device{"abc_pseq", std::move(rails)};
+
+ EXPECT_CALL(device, prepareForPgoodFaultDetection).Times(1);
+ std::vector<int> gpioValues{0, 0, 0};
+ EXPECT_CALL(device, getGPIOValues)
+ .Times(1)
+ .WillOnce(Return(gpioValues));
+ EXPECT_CALL(device, getStatusVout(7)).Times(1).WillOnce(Return(0x11));
+ EXPECT_CALL(device, getStatusWord(7)).Times(1).WillOnce(Return(0xbeef));
+
+ MockServices services{};
+ EXPECT_CALL(services,
+ logInfoMsg("Device abc_pseq GPIO values: [0, 0, 0]"))
+ .Times(1);
+ EXPECT_CALL(
+ services,
+ logErrorMsg(
+ "Pgood fault found in rail monitored by device abc_pseq"))
+ .Times(1);
+ EXPECT_CALL(services, logInfoMsg("Rail VIO STATUS_WORD: 0xbeef"))
+ .Times(1);
+ EXPECT_CALL(services, logErrorMsg("Pgood fault detected in rail VIO"))
+ .Times(1);
+ EXPECT_CALL(
+ services,
+ logErrorMsg("Rail VIO has fault bits set in STATUS_VOUT: 0x11"))
+ .Times(1);
+
+ std::string powerSupplyError{};
+ std::map<std::string, std::string> additionalData{};
+ std::string error = device.findPgoodFault(services, powerSupplyError,
+ additionalData);
+ EXPECT_EQ(error,
+ "xyz.openbmc_project.Power.Error.PowerSequencerVoltageFault");
+ EXPECT_EQ(additionalData.size(), 5);
+ EXPECT_EQ(additionalData["DEVICE_NAME"], "abc_pseq");
+ EXPECT_EQ(additionalData["GPIO_VALUES"], "[0, 0, 0]");
+ EXPECT_EQ(additionalData["RAIL_NAME"], "VIO");
+ EXPECT_EQ(additionalData["STATUS_VOUT"], "0x11");
+ EXPECT_EQ(additionalData["STATUS_WORD"], "0xbeef");
+ }
+
+ // Two rails appear to have a pgood fault. One is found via output voltage
+ // and one is found via a GPIO. Verify the first rail in the sequence with
+ // a fault is selected.
+ {
+ std::vector<std::unique_ptr<Rail>> rails{};
+ rails.emplace_back(createRailStatusVout("VIO", false, 7));
+ rails.emplace_back(createRailOutputVoltage("VDD", false, 5));
+ rails.emplace_back(createRailGPIO("PSU", true, 2));
+ StandardDeviceImpl device{"abc_pseq", std::move(rails)};
+
+ EXPECT_CALL(device, prepareForPgoodFaultDetection).Times(1);
+ std::vector<int> gpioValues{1, 1, 0};
+ EXPECT_CALL(device, getGPIOValues)
+ .Times(1)
+ .WillOnce(Return(gpioValues));
+ EXPECT_CALL(device, getStatusVout(7)).Times(1).WillOnce(Return(0x00));
+ EXPECT_CALL(device, getReadVout(5)).Times(1).WillOnce(Return(1.1));
+ EXPECT_CALL(device, getVoutUVFaultLimit(5))
+ .Times(1)
+ .WillOnce(Return(1.2));
+ EXPECT_CALL(device, getStatusWord(5)).Times(1).WillOnce(Return(0xbeef));
+
+ MockServices services{};
+ EXPECT_CALL(services,
+ logInfoMsg("Device abc_pseq GPIO values: [1, 1, 0]"))
+ .Times(1);
+ EXPECT_CALL(
+ services,
+ logErrorMsg(
+ "Pgood fault found in rail monitored by device abc_pseq"))
+ .Times(1);
+ EXPECT_CALL(services, logInfoMsg("Rail VDD STATUS_WORD: 0xbeef"))
+ .Times(1);
+ EXPECT_CALL(services, logErrorMsg("Pgood fault detected in rail VDD"))
+ .Times(1);
+ EXPECT_CALL(
+ services,
+ logErrorMsg(
+ "Rail VDD output voltage 1.1V is <= UV fault limit 1.2V"))
+ .Times(1);
+
+ std::string powerSupplyError{};
+ std::map<std::string, std::string> additionalData{};
+ std::string error = device.findPgoodFault(services, powerSupplyError,
+ additionalData);
+ EXPECT_EQ(error,
+ "xyz.openbmc_project.Power.Error.PowerSequencerVoltageFault");
+ EXPECT_EQ(additionalData.size(), 6);
+ EXPECT_EQ(additionalData["DEVICE_NAME"], "abc_pseq");
+ EXPECT_EQ(additionalData["GPIO_VALUES"], "[1, 1, 0]");
+ EXPECT_EQ(additionalData["RAIL_NAME"], "VDD");
+ EXPECT_EQ(additionalData["READ_VOUT"], "1.1");
+ EXPECT_EQ(additionalData["VOUT_UV_FAULT_LIMIT"], "1.2");
+ EXPECT_EQ(additionalData["STATUS_WORD"], "0xbeef");
+ }
+
// Exception is thrown during pgood fault detection
{
std::vector<std::unique_ptr<Rail>> rails{};
rails.emplace_back(createRailGPIO("PSU", true, 2));
- rails.emplace_back(createRailStatusVout("VDD", false, 5));
+ rails.emplace_back(createRailOutputVoltage("VDD", false, 5));
rails.emplace_back(createRailStatusVout("VIO", false, 7));
StandardDeviceImpl device{"abc_pseq", std::move(rails)};
+ EXPECT_CALL(device, prepareForPgoodFaultDetection).Times(1);
std::vector<int> gpioValues{1, 1, 1};
EXPECT_CALL(device, getGPIOValues)
.Times(1)
.WillOnce(Return(gpioValues));
- EXPECT_CALL(device, getStatusVout(5))
+ EXPECT_CALL(device, getReadVout(5)).Times(0);
+ EXPECT_CALL(device, getVoutUVFaultLimit(5)).Times(0);
+ EXPECT_CALL(device, getStatusVout(7))
.Times(1)
.WillOnce(Throw(std::runtime_error{"File does not exist"}));
- EXPECT_CALL(device, getStatusVout(7)).Times(0);
MockServices services{};
@@ -470,7 +624,7 @@
EXPECT_STREQ(
e.what(),
"Unable to determine if a pgood fault occurred in device abc_pseq: "
- "Unable to read STATUS_VOUT value for rail VDD: "
+ "Unable to read STATUS_VOUT value for rail VIO: "
"File does not exist");
}
}