regulators: Implement phase_fault_detection object
Implement the new phase_fault_detection object in the JSON configuration
file.
Create gtests to test the new class.
Signed-off-by: Shawn McCarney <shawnmm@us.ibm.com>
Change-Id: I364281c83fbcc7fd26d2d7decb6d8338484af4e1
diff --git a/phosphor-regulators/src/meson.build b/phosphor-regulators/src/meson.build
index d9ba765..581343f 100644
--- a/phosphor-regulators/src/meson.build
+++ b/phosphor-regulators/src/meson.build
@@ -17,6 +17,7 @@
'ffdc_file.cpp',
'id_map.cpp',
'journal.cpp',
+ 'phase_fault_detection.cpp',
'pmbus_utils.cpp',
'presence_detection.cpp',
'presence_service.cpp',
diff --git a/phosphor-regulators/src/phase_fault_detection.cpp b/phosphor-regulators/src/phase_fault_detection.cpp
new file mode 100644
index 0000000..4ca276b
--- /dev/null
+++ b/phosphor-regulators/src/phase_fault_detection.cpp
@@ -0,0 +1,144 @@
+/**
+ * Copyright © 2021 IBM Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "phase_fault_detection.hpp"
+
+#include "action_utils.hpp"
+#include "chassis.hpp"
+#include "device.hpp"
+#include "error_logging.hpp"
+#include "error_logging_utils.hpp"
+#include "exception_utils.hpp"
+#include "journal.hpp"
+#include "system.hpp"
+
+#include <exception>
+#include <map>
+
+namespace phosphor::power::regulators
+{
+
+/**
+ * Maximum number of action errors to write to the journal.
+ */
+constexpr unsigned short maxActionErrorCount{3};
+
+/**
+ * Number of consecutive phase faults required to log an error. This provides
+ * "de-glitching" to ignore transient hardware problems.
+ */
+constexpr unsigned short requiredConsecutiveFaults{2};
+
+void PhaseFaultDetection::execute(Services& services, System& system,
+ Chassis& /*chassis*/, Device& regulator)
+{
+ try
+ {
+ // Find the device ID to use. If the deviceID data member is empty, use
+ // the ID of the specified regulator.
+ const std::string& effectiveDeviceID =
+ deviceID.empty() ? regulator.getID() : deviceID;
+
+ // Create ActionEnvironment
+ ActionEnvironment environment{system.getIDMap(), effectiveDeviceID,
+ services};
+
+ // Execute the actions to detect phase faults
+ action_utils::execute(actions, environment);
+
+ // Check for any N or N+1 phase faults that were detected
+ checkForPhaseFault(PhaseFaultType::n, services, regulator, environment);
+ checkForPhaseFault(PhaseFaultType::n_plus_1, services, regulator,
+ environment);
+ }
+ catch (const std::exception& e)
+ {
+ // Log error messages in journal if we haven't hit the max
+ if (actionErrorCount < maxActionErrorCount)
+ {
+ ++actionErrorCount;
+ services.getJournal().logError(exception_utils::getMessages(e));
+ services.getJournal().logError(
+ "Unable to detect phase faults in regulator " +
+ regulator.getID());
+ }
+
+ // Create error log entry if this type hasn't already been logged
+ error_logging_utils::logError(std::current_exception(),
+ Entry::Level::Warning, services,
+ errorHistory);
+ }
+}
+
+void PhaseFaultDetection::checkForPhaseFault(PhaseFaultType faultType,
+ Services& services,
+ Device& regulator,
+ ActionEnvironment& environment)
+{
+ // Find ErrorType that corresponds to PhaseFaultType; used by ErrorHistory
+ ErrorType errorType = toErrorType(faultType);
+
+ // If this error has not been logged yet
+ if (!errorHistory.wasLogged(errorType))
+ {
+ // Create reference to consecutive fault count data member
+ unsigned short& faultCount =
+ (faultType == PhaseFaultType::n) ? nFaultCount : nPlus1FaultCount;
+
+ // Check if the phase fault was detected
+ if (environment.getPhaseFaults().count(faultType) == 0)
+ {
+ // Phase fault not detected; reset consecutive fault count
+ faultCount = 0;
+ }
+ else
+ {
+ // Phase fault detected; increment consecutive fault count
+ ++faultCount;
+
+ // Log error message in journal
+ services.getJournal().logError(
+ toString(faultType) + " phase fault detected in regulator " +
+ regulator.getID() + ": count=" + std::to_string(faultCount));
+
+ // If the required number of consecutive faults have been detected
+ if (faultCount >= requiredConsecutiveFaults)
+ {
+ // Log phase fault error and update ErrorHistory
+ logPhaseFault(faultType, services, regulator, environment);
+ errorHistory.setWasLogged(errorType, true);
+ }
+ }
+ }
+}
+
+void PhaseFaultDetection::logPhaseFault(PhaseFaultType faultType,
+ Services& services, Device& regulator,
+ ActionEnvironment& environment)
+{
+ ErrorLogging& errorLogging = services.getErrorLogging();
+ Entry::Level severity = (faultType == PhaseFaultType::n)
+ ? Entry::Level::Warning
+ : Entry::Level::Informational;
+ Journal& journal = services.getJournal();
+ const std::string& inventoryPath = regulator.getFRU();
+ const std::map<std::string, std::string>& additionalData =
+ environment.getAdditionalErrorData();
+ errorLogging.logPhaseFault(severity, journal, faultType, inventoryPath,
+ additionalData);
+}
+
+} // namespace phosphor::power::regulators
diff --git a/phosphor-regulators/src/phase_fault_detection.hpp b/phosphor-regulators/src/phase_fault_detection.hpp
new file mode 100644
index 0000000..467d8c7
--- /dev/null
+++ b/phosphor-regulators/src/phase_fault_detection.hpp
@@ -0,0 +1,208 @@
+/**
+ * Copyright © 2021 IBM Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "action.hpp"
+#include "action_environment.hpp"
+#include "error_history.hpp"
+#include "phase_fault.hpp"
+#include "services.hpp"
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace phosphor::power::regulators
+{
+
+// Forward declarations to avoid circular dependencies
+class Chassis;
+class Device;
+class System;
+
+/**
+ * @class PhaseFaultDetection
+ *
+ * Detects and logs redundant phase faults in a voltage regulator.
+ *
+ * A voltage regulator is sometimes called a "phase controller" because it
+ * controls one or more phases that perform the actual voltage regulation.
+ *
+ * A regulator may have redundant phases. If a redundant phase fails, the
+ * regulator will continue to provide the desired output voltage. However, a
+ * phase fault error should be logged warning the user that the regulator has
+ * lost redundancy.
+ *
+ * The technique used to detect a phase fault varies depending on the regulator
+ * hardware. Often a bit is checked in a status register. The status register
+ * could exist in the regulator or in a related I/O expander.
+ *
+ * Phase fault detection is performed every 15 seconds. A phase fault must be
+ * detected two consecutive times (15 seconds apart) before an error is logged.
+ * This provides "de-glitching" to ignore transient hardware problems.
+ *
+ * Phase faults are detected by executing actions.
+ */
+class PhaseFaultDetection
+{
+ public:
+ // Specify which compiler-generated methods we want
+ PhaseFaultDetection() = delete;
+ PhaseFaultDetection(const PhaseFaultDetection&) = delete;
+ PhaseFaultDetection(PhaseFaultDetection&&) = delete;
+ PhaseFaultDetection& operator=(const PhaseFaultDetection&) = delete;
+ PhaseFaultDetection& operator=(PhaseFaultDetection&&) = delete;
+ ~PhaseFaultDetection() = default;
+
+ /**
+ * Constructor.
+ *
+ * @param actions Actions that detect phase faults in the regulator.
+ * @param deviceID Unique ID of the device to use when detecting phase
+ * faults. If not specified, the regulator will be used.
+ */
+ explicit PhaseFaultDetection(std::vector<std::unique_ptr<Action>> actions,
+ const std::string& deviceID = "") :
+ actions{std::move(actions)},
+ deviceID{deviceID}
+ {
+ }
+
+ /**
+ * Clears all error history.
+ *
+ * All data on previously logged errors will be deleted. If errors occur
+ * again in the future they will be logged again.
+ *
+ * This method is normally called when the system is being powered on.
+ */
+ void clearErrorHistory()
+ {
+ errorHistory.clear();
+ actionErrorCount = 0;
+ nFaultCount = 0;
+ nPlus1FaultCount = 0;
+ }
+
+ /**
+ * Executes the actions that detect phase faults in the regulator.
+ *
+ * If the required number of consecutive phase faults are detected, an error
+ * is logged.
+ *
+ * @param services system services like error logging and the journal
+ * @param system system that contains the chassis
+ * @param chassis chassis that contains the regulator device
+ * @param regulator voltage regulator device
+ */
+ void execute(Services& services, System& system, Chassis& chassis,
+ Device& regulator);
+
+ /**
+ * Returns the actions that detect phase faults in the regulator.
+ *
+ * @return actions
+ */
+ const std::vector<std::unique_ptr<Action>>& getActions() const
+ {
+ return actions;
+ }
+
+ /**
+ * Returns the unique ID of the device to use when detecting phase
+ * faults.
+ *
+ * If the value is "", the regulator will be used.
+ *
+ * @return device ID
+ */
+ const std::string& getDeviceID() const
+ {
+ return deviceID;
+ }
+
+ private:
+ /**
+ * Checks if the specified phase fault type was detected.
+ *
+ * If the fault type was detected, increments the counter tracking
+ * consecutive faults. If the required number of consecutive faults have
+ * been detected, logs a phase fault error.
+ *
+ * The ActionEnvironment contains the set of phase fault types that were
+ * detected (if any).
+ *
+ * @param faultType phase fault type to check
+ * @param services system services like error logging and the journal
+ * @param regulator voltage regulator device
+ * @param environment action execution environment
+ */
+ void checkForPhaseFault(PhaseFaultType faultType, Services& services,
+ Device& regulator, ActionEnvironment& environment);
+
+ /**
+ * Logs an error for the specified phase fault type.
+ *
+ * @param faultType phase fault type that occurred
+ * @param services system services like error logging and the journal
+ * @param regulator voltage regulator device
+ * @param environment action execution environment
+ */
+ void logPhaseFault(PhaseFaultType faultType, Services& services,
+ Device& regulator, ActionEnvironment& environment);
+
+ /**
+ * Actions that detect phase faults in the regulator.
+ */
+ std::vector<std::unique_ptr<Action>> actions{};
+
+ /**
+ * Unique ID of the device to use when detecting phase faults.
+ *
+ * Sometimes a separate device, such as an I/O expander, is accessed to
+ * obtain the phase fault status for a regulator.
+ *
+ * If the value is "", the regulator will be used.
+ */
+ const std::string deviceID{};
+
+ /**
+ * History of which error types have been logged.
+ *
+ * Since phase fault detection runs repeatedly based on a timer, each error
+ * type is only logged once.
+ */
+ ErrorHistory errorHistory{};
+
+ /**
+ * Number of errors that have occurred while executing actions, resulting in
+ * an exception.
+ */
+ unsigned short actionErrorCount{0};
+
+ /**
+ * Number of consecutive N phase faults that have been detected.
+ */
+ unsigned short nFaultCount{0};
+
+ /**
+ * Number of consecutive N+1 phase faults that have been detected.
+ */
+ unsigned short nPlus1FaultCount{0};
+};
+
+} // namespace phosphor::power::regulators