Add journal traces to occ-control PELs
When creating a PEL, the last lines of the journal for the app will get
saved into the PEL for additional debug.
Change-Id: Ifa05a00ffdc57833859d719d0e7d8b81ccadb5c8
Signed-off-by: Chris Cain <cjcain@us.ibm.com>
diff --git a/occ_ffdc.cpp b/occ_ffdc.cpp
index 633914b..f83f74a 100644
--- a/occ_ffdc.cpp
+++ b/occ_ffdc.cpp
@@ -10,6 +10,7 @@
#include <sys/ioctl.h>
#include <unistd.h>
+#include <nlohmann/json.hpp>
#include <org/open_power/OCC/Device/error.hpp>
#include <phosphor-logging/elog.hpp>
#include <phosphor-logging/log.hpp>
@@ -25,7 +26,6 @@
static constexpr size_t sbe_status_header_size = 8;
static constexpr auto loggingObjectPath = "/xyz/openbmc_project/logging";
-static constexpr auto loggingInterface = "xyz.openbmc_project.Logging.Create";
static constexpr auto opLoggingInterface = "org.open_power.Logging.PEL";
using namespace phosphor::logging;
@@ -55,6 +55,10 @@
static_cast<uint8_t>(0xCB), static_cast<uint8_t>(0x01), fd));
}
+ // Add journal traces to PEL FFDC
+ auto occJournalFile =
+ addJournalEntries(pelFFDCInfo, "openpower-occ-control", 25);
+
std::map<std::string, std::string> additionalData;
additionalData.emplace("SRC6", std::to_string(src6));
additionalData.emplace("_PID", std::to_string(getpid()));
@@ -69,6 +73,7 @@
auto method =
bus.new_method_call(service.c_str(), loggingObjectPath,
opLoggingInterface, "CreatePELWithFFDCFiles");
+
// Set level to Notice (Informational). Error should trigger an OCC
// reset and if it does not recover, HTMGT/HBRT will create an
// unrecoverable error.
@@ -76,6 +81,7 @@
sdbusplus::xyz::openbmc_project::Logging::server::convertForMessage(
sdbusplus::xyz::openbmc_project::Logging::server::Entry::Level::
Notice);
+
method.append(path, level, additionalData, pelFFDCInfo);
auto response = bus.call(method);
std::tuple<uint32_t, uint32_t> reply = {0, 0};
@@ -119,10 +125,17 @@
try
{
+ FFDCFiles ffdc;
+ // Add journal traces to PEL FFDC
+ auto occJournalFile =
+ addJournalEntries(ffdc, "openpower-occ-control", 25);
+
std::string service =
- utils::getService(loggingObjectPath, loggingInterface);
- auto method = bus.new_method_call(service.c_str(), loggingObjectPath,
- loggingInterface, "Create");
+ utils::getService(loggingObjectPath, opLoggingInterface);
+ auto method =
+ bus.new_method_call(service.c_str(), loggingObjectPath,
+ opLoggingInterface, "CreatePELWithFFDCFiles");
+
// Set level to Notice (Informational). Error should trigger an OCC
// reset and if it does not recover, HTMGT/HBRT will create an
// unrecoverable error.
@@ -130,7 +143,8 @@
sdbusplus::xyz::openbmc_project::Logging::server::convertForMessage(
sdbusplus::xyz::openbmc_project::Logging::server::Entry::Level::
Notice);
- method.append(path, level, additionalData);
+
+ method.append(path, level, additionalData, ffdc);
bus.call(method);
}
catch (const sdbusplus::exception_t& e)
@@ -141,7 +155,7 @@
}
}
-// Reads the FFDC file and create an error log
+// Reads the SBE FFDC file and create an error log
void FFDC::analyzeEvent()
{
int tfd = -1;
@@ -216,5 +230,210 @@
"SBE command reported error", tfd);
}
+// Create file with the latest journal entries for specified executable
+std::unique_ptr<FFDCFile> FFDC::addJournalEntries(FFDCFiles& fileList,
+ const std::string& executable,
+ unsigned int lines)
+{
+ auto journalFile = makeJsonFFDCFile(getJournalEntries(lines, executable));
+ if (journalFile && journalFile->fd() != -1)
+ {
+ log<level::DEBUG>(
+ fmt::format(
+ "addJournalEntries: Added up to {} journal entries for {}",
+ lines, executable)
+ .c_str());
+ fileList.emplace_back(FFDCFormat::JSON, 0x01, 0x01, journalFile->fd());
+ }
+ else
+ {
+ log<level::ERR>(
+ fmt::format(
+ "addJournalEntries: Failed to add journal entries for {}",
+ executable)
+ .c_str());
+ }
+ return journalFile;
+}
+
+// Write JSON data into FFDC file and return the file
+std::unique_ptr<FFDCFile> FFDC::makeJsonFFDCFile(const nlohmann::json& ffdcData)
+{
+ std::string tmpFile = fs::temp_directory_path() / "OCC_JOURNAL_XXXXXX";
+ auto fd = mkostemp(tmpFile.data(), O_RDWR);
+ if (fd != -1)
+ {
+ auto jsonString = ffdcData.dump();
+ auto rc = write(fd, jsonString.data(), jsonString.size());
+ close(fd);
+ if (rc != -1)
+ {
+ fs::path jsonFile{tmpFile};
+ return std::make_unique<FFDCFile>(jsonFile);
+ }
+ else
+ {
+ auto e = errno;
+ log<level::ERR>(
+ fmt::format(
+ "makeJsonFFDCFile: Failed call to write JSON FFDC file, errno={}",
+ e)
+ .c_str());
+ }
+ }
+ else
+ {
+ auto e = errno;
+ log<level::ERR>(
+ fmt::format("makeJsonFFDCFile: Failed called to mkostemp, errno={}",
+ e)
+ .c_str());
+ }
+ return nullptr;
+}
+
+// Collect the latest journal entries for a specified executable
+nlohmann::json FFDC::getJournalEntries(int numLines, std::string executable)
+{
+ // Sleep 100ms; otherwise recent journal entries sometimes not available
+ using namespace std::chrono_literals;
+ std::this_thread::sleep_for(100ms);
+
+ std::vector<std::string> entries;
+
+ // Open the journal
+ sd_journal* journal;
+ int rc = sd_journal_open(&journal, SD_JOURNAL_LOCAL_ONLY);
+ if (rc < 0)
+ {
+ // Build one line string containing field values
+ entries.push_back("[Internal error: sd_journal_open(), rc=" +
+ std::string(strerror(rc)) + "]");
+ return nlohmann::json(entries);
+ }
+
+ // Create object to automatically close journal
+ JournalCloser closer{journal};
+
+ // Add match so we only loop over entries with specified field value
+ std::string field{"SYSLOG_IDENTIFIER"};
+ std::string match{field + '=' + executable};
+ rc = sd_journal_add_match(journal, match.c_str(), 0);
+ if (rc < 0)
+ {
+ // Build one line string containing field values
+ entries.push_back("[Internal error: sd_journal_add_match(), rc=" +
+ std::string(strerror(rc)) + "]");
+ }
+ else
+ {
+ int count{1};
+ entries.reserve(numLines);
+ std::string syslogID, pid, message, timeStamp;
+
+ // Loop through journal entries from newest to oldest
+ SD_JOURNAL_FOREACH_BACKWARDS(journal)
+ {
+ // Get relevant journal entry fields
+ timeStamp = getTimeStamp(journal);
+ syslogID = getFieldValue(journal, "SYSLOG_IDENTIFIER");
+ pid = getFieldValue(journal, "_PID");
+ message = getFieldValue(journal, "MESSAGE");
+
+ // Build one line string containing field values
+ entries.push_back(timeStamp + " " + syslogID + "[" + pid +
+ "]: " + message);
+
+ // Stop after number of lines was read
+ if (count++ >= numLines)
+ {
+ break;
+ }
+ }
+ }
+
+ // put the journal entries in chronological order
+ std::reverse(entries.begin(), entries.end());
+
+ return nlohmann::json(entries);
+}
+
+std::string FFDC::getTimeStamp(sd_journal* journal)
+{
+ // Get realtime (wallclock) timestamp of current journal entry. The
+ // timestamp is in microseconds since the epoch.
+ uint64_t usec{0};
+ int rc = sd_journal_get_realtime_usec(journal, &usec);
+ if (rc < 0)
+ {
+ return "[Internal error: sd_journal_get_realtime_usec(), rc=" +
+ std::string(strerror(rc)) + "]";
+ }
+
+ // Convert to number of seconds since the epoch
+ time_t secs = usec / 1000000;
+
+ // Convert seconds to tm struct required by strftime()
+ struct tm* timeStruct = localtime(&secs);
+ if (timeStruct == nullptr)
+ {
+ return "[Internal error: localtime() returned nullptr]";
+ }
+
+ // Convert tm struct into a date/time string
+ char timeStamp[80];
+ strftime(timeStamp, sizeof(timeStamp), "%b %d %H:%M:%S", timeStruct);
+
+ return timeStamp;
+}
+
+std::string FFDC::getFieldValue(sd_journal* journal, const std::string& field)
+{
+ std::string value{};
+
+ // Get field data from current journal entry
+ const void* data{nullptr};
+ size_t length{0};
+ int rc = sd_journal_get_data(journal, field.c_str(), &data, &length);
+ if (rc < 0)
+ {
+ if (-rc == ENOENT)
+ {
+ // Current entry does not include this field; return empty value
+ return value;
+ }
+ else
+ {
+ return "[Internal error: sd_journal_get_data() rc=" +
+ std::string(strerror(rc)) + "]";
+ }
+ }
+
+ // Get value from field data. Field data in format "FIELD=value".
+ std::string dataString{static_cast<const char*>(data), length};
+ std::string::size_type pos = dataString.find('=');
+ if ((pos != std::string::npos) && ((pos + 1) < dataString.size()))
+ {
+ // Value is substring after the '='
+ value = dataString.substr(pos + 1);
+ }
+
+ return value;
+}
+
+// Create temporary file that will automatically get removed when destructed
+FFDCFile::FFDCFile(const fs::path& name) :
+ _fd(open(name.c_str(), O_RDONLY)), _name(name)
+{
+ if (_fd() == -1)
+ {
+ auto e = errno;
+ log<level::ERR>(
+ fmt::format("FFDCFile: Could not open FFDC file {}. errno {}",
+ _name.string(), e)
+ .c_str());
+ }
+}
+
} // namespace occ
} // namespace open_power
diff --git a/occ_ffdc.hpp b/occ_ffdc.hpp
index a2b1200..61bd714 100644
--- a/occ_ffdc.hpp
+++ b/occ_ffdc.hpp
@@ -2,13 +2,76 @@
#include "config.h"
+#include "file.hpp"
#include "occ_errors.hpp"
+#include <systemd/sd-journal.h>
+
+#include <nlohmann/json.hpp>
+#include <xyz/openbmc_project/Logging/Create/server.hpp>
+
+using FFDCFormat =
+ sdbusplus::xyz::openbmc_project::Logging::server::Create::FFDCFormat;
+using FFDCFiles = std::vector<
+ std::tuple<FFDCFormat, uint8_t, uint8_t, sdbusplus::message::unix_fd>>;
+
namespace open_power
{
namespace occ
{
+/** @class FFDCFile
+ * @brief Represents a single file that will get opened when created and
+ * deleted when the object is destructed
+ */
+class FFDCFile
+{
+ public:
+ FFDCFile() = delete;
+ FFDCFile(const FFDCFile&) = delete;
+ FFDCFile& operator=(const FFDCFile&) = delete;
+ FFDCFile(FFDCFile&&) = delete;
+ FFDCFile& operator=(FFDCFile&&) = delete;
+
+ /**
+ * @brief Constructor
+ *
+ * Opens the file and saves the descriptor
+ *
+ * @param[in] name - The filename
+ */
+ explicit FFDCFile(const std::filesystem::path& name);
+
+ /**
+ * @brief Destructor - Deletes the file
+ */
+ ~FFDCFile()
+ {
+ std::filesystem::remove(_name);
+ }
+
+ /**
+ * @brief Returns the file descriptor
+ *
+ * @return int - The descriptor
+ */
+ int fd()
+ {
+ return _fd();
+ }
+
+ private:
+ /**
+ * @brief The file descriptor holder
+ */
+ FileDescriptor _fd;
+
+ /**
+ * @brief The filename
+ */
+ const std::filesystem::path _name;
+};
+
/** @class FFDC
* @brief Monitors for SBE FFDC availability
*/
@@ -64,6 +127,20 @@
static void createOCCResetPEL(unsigned int instance, const char* path,
int err, const char* callout);
+ /**
+ * @brief Create a file containing the latest journal traces for the
+ * specified executable and add it to the file list.
+ *
+ * @param[in] fileList - where to add the new file
+ * @param[in] executable - name of app to collect
+ * @param[in] lines - number of journal lines to save
+ *
+ * @return std::unique_ptr<FFDCFile> - The file object
+ */
+ static std::unique_ptr<FFDCFile>
+ addJournalEntries(FFDCFiles& fileList, const std::string& executable,
+ unsigned int lines);
+
private:
/** @brief OCC instance number. Ex, 0,1, etc */
unsigned int instance;
@@ -79,6 +156,77 @@
* content denotes an error condition
*/
void analyzeEvent() override;
+
+ /**
+ * @brief Returns an FFDCFile containing the JSON data
+ *
+ * @param[in] ffdcData - The JSON data to write to a file
+ *
+ * @return std::unique_ptr<FFDCFile> - The file object
+ */
+ static std::unique_ptr<FFDCFile>
+ makeJsonFFDCFile(const nlohmann::json& ffdcData);
+
+ /**
+ * @brief Returns a JSON structure containing the previous N journal
+ * entries.
+ *
+ * @param[in] numLines - Number of lines of journal to retrieve
+ * @param[in] executable - name of app to collect for
+ *
+ * @return JSON object that was created
+ */
+ static nlohmann::json getJournalEntries(int numLines,
+ std::string executable);
+
+ /**
+ * @brief Gets the realtime (wallclock) timestamp for the current journal
+ * entry.
+ *
+ * @param journal current journal entry
+ * @return timestamp as a date/time string
+ */
+ static std::string getTimeStamp(sd_journal* journal);
+
+ /**
+ * @brief Gets the value of the specified field for the current journal
+ * entry.
+ *
+ * Returns an empty string if the current journal entry does not have the
+ * specified field.
+ *
+ * @param journal current journal entry
+ * @param field journal field name
+ * @return field value
+ */
+ static std::string getFieldValue(sd_journal* journal,
+ const std::string& field);
+};
+
+/**
+ * @class JournalCloser
+ * @brief Automatically closes the journal when the object goes out of scope.
+ */
+class JournalCloser
+{
+ public:
+ // Specify which compiler-generated methods we want
+ JournalCloser() = delete;
+ JournalCloser(const JournalCloser&) = delete;
+ JournalCloser(JournalCloser&&) = delete;
+ JournalCloser& operator=(const JournalCloser&) = delete;
+ JournalCloser& operator=(JournalCloser&&) = delete;
+
+ JournalCloser(sd_journal* journal) : journal{journal}
+ {}
+
+ ~JournalCloser()
+ {
+ sd_journal_close(journal);
+ }
+
+ private:
+ sd_journal* journal{nullptr};
};
} // namespace occ