PEL: Python3 module support for UserData parsing
This change enables the calling of python3 parsers for UserData section.
Modules will be searched under the namespace "udparsers" from python3
sys.path directories (including the current directory).
Example:
/usr/lib/python3.8/site-packages/udparsers/bxxxx/bxxxx.py
or
./udparsers/bxxxx/bxxxx.py
where
b = Creator Subsystem ID for Hostboot
xxxx = Component ID of UserData section
The parsers will need to provide a function called "parseUDToJson" with
input parameters:
1. (int) Sub-section type
2. (int) Section version
3. (memoryview): Data
The return value needs to be a valid JSON string.
Signed-off-by: Harisuddin Mohamed Isa <harisuddin@gmail.com>
Change-Id: I4d3523083bc48ad0c329c525d83c4a61d36ff611
diff --git a/configure.ac b/configure.ac
index 6dad3da..6cebfae 100644
--- a/configure.ac
+++ b/configure.ac
@@ -214,8 +214,23 @@
[],
[AC_MSG_ERROR([Could not find fifo_map.hpp])])
- AX_PKG_CHECK_MODULES([LIBPLDM], [libpldm])]
+ AX_PKG_CHECK_MODULES([LIBPLDM], [libpldm])
PKG_CHECK_MODULES([FMT], [fmt])
+ AC_ARG_VAR([PYTHON_INCLUDE], [Include flags for python, bypassing python-config])
+ AC_ARG_VAR([PKG_CONFIG], [Path to pkg-config])
+ AS_IF([test -z "$PYTHON_INCLUDE"], [
+ AS_IF([test -z "$PKG_CONFIG"], [
+ AC_PATH_PROGS([PKG_CONFIG],
+ [pkg-config],
+ [no],
+ [`dirname $PYTHON`])
+ AS_IF([test "$PKG_CONFIG" = no], [AC_MSG_ERROR([cannot find pkg-config.])])
+ ])
+ AC_MSG_CHECKING([python include flags])
+ PYTHON_INCLUDE=`$PKG_CONFIG --cflags python3`
+ AC_MSG_RESULT([$PYTHON_INCLUDE])
+ ])
+ AX_APPEND_COMPILE_FLAGS([$PYTHON_INCLUDE], [CXXFLAGS])]
)
AC_ARG_ENABLE([dont-send-pels-to-host],
diff --git a/extensions/openpower-pels/openpower-pels.mk b/extensions/openpower-pels/openpower-pels.mk
index 0bb9fd0..1468935 100644
--- a/extensions/openpower-pels/openpower-pels.mk
+++ b/extensions/openpower-pels/openpower-pels.mk
@@ -73,5 +73,6 @@
extensions/openpower-pels/tools/peltool.cpp \
extensions/openpower-pels/user_data.cpp \
extensions/openpower-pels/user_data_json.cpp
+peltool_LDFLAGS = "-lpython$(PYTHON_VERSION)"
peltool_LDADD = libpel.la
peltool_CXXFLAGS = "-DPELTOOL"
diff --git a/extensions/openpower-pels/pel.cpp b/extensions/openpower-pels/pel.cpp
index 0845aba..3295123 100644
--- a/extensions/openpower-pels/pel.cpp
+++ b/extensions/openpower-pels/pel.cpp
@@ -265,7 +265,9 @@
void PEL::printSectionInJSON(const Section& section, std::string& buf,
std::map<uint16_t, size_t>& pluralSections,
- message::Registry& registry) const
+ message::Registry& registry,
+ const std::vector<std::string>& plugins,
+ uint8_t creatorID) const
{
char tmpB[5];
uint8_t id[] = {static_cast<uint8_t>(section.header().id >> 8),
@@ -286,9 +288,26 @@
if (section.valid())
{
- auto json = (sectionID == "PS" || sectionID == "SS")
- ? section.getJSON(registry)
- : section.getJSON();
+ std::optional<std::string> json;
+ if (sectionID == "PS" || sectionID == "SS")
+ {
+ json = section.getJSON(registry);
+ }
+ else if (sectionID == "UD")
+ {
+ std::string subsystem = getNumberString("%c", tolower(creatorID));
+ std::string component =
+ getNumberString("%04x", section.header().componentID);
+ if (std::find(plugins.begin(), plugins.end(),
+ subsystem + component) != plugins.end())
+ {
+ json = section.getJSON(creatorID);
+ }
+ }
+ else
+ {
+ json = section.getJSON();
+ }
buf += "\"" + sectionName + "\": {\n";
@@ -355,16 +374,18 @@
return sections;
}
-void PEL::toJSON(message::Registry& registry) const
+void PEL::toJSON(message::Registry& registry,
+ const std::vector<std::string>& plugins) const
{
auto sections = getPluralSections();
std::string buf = "{\n";
- printSectionInJSON(*(_ph.get()), buf, sections, registry);
- printSectionInJSON(*(_uh.get()), buf, sections, registry);
+ printSectionInJSON(*(_ph.get()), buf, sections, registry, plugins);
+ printSectionInJSON(*(_uh.get()), buf, sections, registry, plugins);
for (auto& section : this->optionalSections())
{
- printSectionInJSON(*(section.get()), buf, sections, registry);
+ printSectionInJSON(*(section.get()), buf, sections, registry, plugins,
+ _ph->creatorID());
}
buf += "}";
std::size_t found = buf.rfind(",");
diff --git a/extensions/openpower-pels/pel.hpp b/extensions/openpower-pels/pel.hpp
index ac1bddc..0e8ee1a 100644
--- a/extensions/openpower-pels/pel.hpp
+++ b/extensions/openpower-pels/pel.hpp
@@ -239,8 +239,10 @@
/**
* @brief Output a PEL in JSON.
* @param[in] registry - Registry object reference
+ * @param[in] plugins - Vector of strings of plugins found in filesystem
*/
- void toJSON(message::Registry& registry) const;
+ void toJSON(message::Registry& registry,
+ const std::vector<std::string>& plugins) const;
/**
* @brief Sets the host transmission state in the User Header
@@ -333,10 +335,14 @@
* @param[in|out] pluralSections - Map used to track sections counts for
* when there is more than 1.
* @param[in] registry - Registry object reference
+ * @param[in] plugins - Vector of strings of plugins found in filesystem
+ * @param[in] creatorID - Creator Subsystem ID (only for UserData section)
*/
void printSectionInJSON(const Section& section, std::string& buf,
std::map<uint16_t, size_t>& pluralSections,
- message::Registry& registry) const;
+ message::Registry& registry,
+ const std::vector<std::string>& plugins,
+ uint8_t creatorID = 0) const;
/**
* @brief The PEL Private Header section
diff --git a/extensions/openpower-pels/section.hpp b/extensions/openpower-pels/section.hpp
index bf08aa5..0d079f2 100644
--- a/extensions/openpower-pels/section.hpp
+++ b/extensions/openpower-pels/section.hpp
@@ -72,6 +72,17 @@
}
/**
+ * @brief Get section in JSON. Derived classes to override when required to.
+ * @param[in] creatorID - Creator Subsystem ID from Private Header
+ * @return std::optional<std::string> - If a section comes with a JSON
+ * representation, this would return the string for it.
+ */
+ virtual std::optional<std::string> getJSON(uint8_t creatorID) const
+ {
+ return std::nullopt;
+ }
+
+ /**
* @brief Shrinks a PEL section
*
* If this is even possible for a section depends on which section
diff --git a/extensions/openpower-pels/tools/peltool.cpp b/extensions/openpower-pels/tools/peltool.cpp
index 3abdc74..983d589 100644
--- a/extensions/openpower-pels/tools/peltool.cpp
+++ b/extensions/openpower-pels/tools/peltool.cpp
@@ -22,6 +22,8 @@
#include "../pel_types.hpp"
#include "../pel_values.hpp"
+#include <Python.h>
+
#include <CLI/CLI.hpp>
#include <bitset>
#include <fstream>
@@ -205,6 +207,56 @@
}
/**
+ * @brief Initialize Python interpreter and gather all UD parser modules under
+ * the paths found in Python sys.path and the current user directory.
+ * This is to prevent calling a non-existant module which causes Python
+ * to print an import error message and breaking JSON output.
+ *
+ * @return std::vector<std::string> Vector of plugins found in filesystem
+ */
+std::vector<std::string> getPlugins()
+{
+ Py_Initialize();
+ std::vector<std::string> plugins;
+ std::vector<std::string> siteDirs;
+ PyObject* pName = PyUnicode_FromString("sys");
+ PyObject* pModule = PyImport_Import(pName);
+ Py_XDECREF(pName);
+ PyObject* pDict = PyModule_GetDict(pModule);
+ Py_XDECREF(pModule);
+ PyObject* pResult = PyDict_GetItemString(pDict, "path");
+ PyObject* pValue = PyUnicode_FromString(".");
+ PyList_Append(pResult, pValue);
+ Py_XDECREF(pValue);
+ auto list_size = PyList_Size(pResult);
+ for (auto i = 0; i < list_size; i++)
+ {
+ PyObject* item = PyList_GetItem(pResult, i);
+ PyObject* pBytes = PyUnicode_AsEncodedString(item, "utf-8", "~E~");
+ const char* output = PyBytes_AS_STRING(pBytes);
+ Py_XDECREF(pBytes);
+ std::string tmpStr(output);
+ siteDirs.push_back(tmpStr);
+ }
+ for (const auto& dir : siteDirs)
+ {
+ if (fs::exists(dir + "/udparsers"))
+ {
+ for (const auto& entry : fs::directory_iterator(dir + "/udparsers"))
+ {
+ if (entry.is_directory() and
+ fs::exists(entry.path().string() + "/" +
+ entry.path().stem().string() + ".py"))
+ {
+ plugins.push_back(entry.path().stem());
+ }
+ }
+ }
+ }
+ return plugins;
+}
+
+/**
* @brief Creates JSON string of a PEL entry if fullPEL is false or prints to
* stdout the full PEL in JSON if fullPEL is true
* @param[in] itr - std::map iterator of <uint32_t, BCDTime>
@@ -213,12 +265,14 @@
* @param[in] fullPEL - Boolean to print full JSON representation of PEL
* @param[in] foundPEL - Boolean to check if any PEL is present
* @param[in] scrubRegex - SRC regex object
+ * @param[in] plugins - Vector of strings of plugins found in filesystem
* @return std::string - JSON string of PEL entry (empty if fullPEL is true)
*/
template <typename T>
std::string genPELJSON(T itr, bool hidden, bool includeInfo, bool fullPEL,
bool& foundPEL,
- const std::optional<std::regex>& scrubRegex)
+ const std::optional<std::regex>& scrubRegex,
+ const std::vector<std::string>& plugins)
{
std::size_t found;
std::string val;
@@ -274,7 +328,7 @@
{
std::cout << ",\n\n";
}
- pel.toJSON(registry);
+ pel.toJSON(registry, plugins);
}
else
{
@@ -365,6 +419,7 @@
{
std::string listStr;
std::map<uint32_t, BCDTime> PELs;
+ std::vector<std::string> plugins;
listStr = "{\n";
for (auto it = fs::directory_iterator(EXTENSION_PERSIST_DIR "/pels/logs");
it != fs::directory_iterator(); ++it)
@@ -380,10 +435,14 @@
}
}
bool foundPEL = false;
+ if (fullPEL)
+ {
+ plugins = getPlugins();
+ }
auto buildJSON = [&listStr, &hidden, &includeInfo, &fullPEL, &foundPEL,
- &scrubRegex](const auto& i) {
- listStr +=
- genPELJSON(i, hidden, includeInfo, fullPEL, foundPEL, scrubRegex);
+ &scrubRegex, &plugins](const auto& i) {
+ listStr += genPELJSON(i, hidden, includeInfo, fullPEL, foundPEL,
+ scrubRegex, plugins);
};
if (order)
{
@@ -544,7 +603,8 @@
{
if (pel.valid())
{
- pel.toJSON(registry);
+ auto plugins = getPlugins();
+ pel.toJSON(registry, plugins);
}
else
{
@@ -724,8 +784,9 @@
std::vector<uint8_t> data = getFileData(fileName);
if (!data.empty())
{
+ auto plugins = getPlugins();
PEL pel{data};
- pel.toJSON(registry);
+ pel.toJSON(registry, plugins);
}
else
{
@@ -765,5 +826,6 @@
{
std::cout << app.help("", CLI::AppFormatMode::All) << std::endl;
}
+ Py_Finalize();
return 0;
}
diff --git a/extensions/openpower-pels/user_data.cpp b/extensions/openpower-pels/user_data.cpp
index 0ae227b..092bfb6 100644
--- a/extensions/openpower-pels/user_data.cpp
+++ b/extensions/openpower-pels/user_data.cpp
@@ -95,11 +95,11 @@
}
}
-std::optional<std::string> UserData::getJSON() const
+std::optional<std::string> UserData::getJSON(uint8_t creatorID) const
{
#ifdef PELTOOL
return user_data::getJSON(_header.componentID, _header.subType,
- _header.version, _data);
+ _header.version, _data, creatorID);
#endif
return std::nullopt;
}
diff --git a/extensions/openpower-pels/user_data.hpp b/extensions/openpower-pels/user_data.hpp
index b1c9084..e27342e 100644
--- a/extensions/openpower-pels/user_data.hpp
+++ b/extensions/openpower-pels/user_data.hpp
@@ -82,11 +82,11 @@
/**
* @brief Get the section contents in JSON
- *
+ * @param[in] creatorID - Creator Subsystem ID from Private Header
* @return The JSON as a string if a parser was found,
* otherwise std::nullopt.
*/
- std::optional<std::string> getJSON() const override;
+ std::optional<std::string> getJSON(uint8_t creatorID) const override;
/**
* @brief Shrink the section
diff --git a/extensions/openpower-pels/user_data_json.cpp b/extensions/openpower-pels/user_data_json.cpp
index e133cdd..408f5c3 100644
--- a/extensions/openpower-pels/user_data_json.cpp
+++ b/extensions/openpower-pels/user_data_json.cpp
@@ -22,6 +22,8 @@
#include "stream.hpp"
#include "user_data_formats.hpp"
+#include <Python.h>
+
#include <fifo_map.hpp>
#include <iomanip>
#include <nlohmann/json.hpp>
@@ -39,6 +41,11 @@
using fifoMap = nlohmann::fifo_map<K, V, nlohmann::fifo_map_compare<K>, A>;
using fifoJSON = nlohmann::basic_json<fifoMap>;
+void pyDecRef(PyObject* pyObj)
+{
+ Py_XDECREF(pyObj);
+}
+
/**
* @brief Returns a JSON string for use by PEL::printSectionInJSON().
*
@@ -228,19 +235,151 @@
return std::nullopt;
}
+/**
+ * @brief Call Python modules to parse the data into a JSON string
+ *
+ * The module to call is based on the Creator Subsystem ID and the Component
+ * ID under the namespace "udparsers". For example: "udparsers.xyyyy.xyyyy"
+ * where "x" is the Creator Subsystem ID and "yyyy" is the Component ID.
+ *
+ * All modules must provide the following:
+ * Function: parseUDToJson
+ * Argument list:
+ * 1. (int) Sub-section type
+ * 2. (int) Section version
+ * 3. (memoryview): Data
+ *-Return data:
+ * 1. (str) JSON string
+ *
+ * @param[in] componentID - The comp ID from the UserData section header
+ * @param[in] subType - The subtype from the UserData section header
+ * @param[in] version - The version from the UserData section header
+ * @param[in] data - The data itself
+ * @param[in] creatorID - The creatorID from the PrivateHeader section
+ * @return std::optional<std::string> - The JSON string if it could be created,
+ * else std::nullopt
+ */
+std::optional<std::string> getPythonJSON(uint16_t componentID, uint8_t subType,
+ uint8_t version,
+ const std::vector<uint8_t>& data,
+ uint8_t creatorID)
+{
+ PyObject *pName, *pModule, *pDict, *pFunc, *pArgs, *pData, *pResult,
+ *pBytes, *eType, *eValue, *eTraceback;
+ std::string pErrStr;
+ std::string module = getNumberString("%c", tolower(creatorID)) +
+ getNumberString("%04x", componentID);
+ pName = PyUnicode_FromString(
+ std::string("udparsers." + module + "." + module).c_str());
+ std::unique_ptr<PyObject, decltype(&pyDecRef)> modNamePtr(pName, &pyDecRef);
+ pModule = PyImport_Import(pName);
+ std::unique_ptr<PyObject, decltype(&pyDecRef)> modPtr(pModule, &pyDecRef);
+ if (pModule == NULL)
+ {
+ pErrStr = "No error string found";
+ PyErr_Fetch(&eType, &eValue, &eTraceback);
+ if (eValue)
+ {
+ PyObject* pStr = PyObject_Str(eValue);
+ if (pStr)
+ {
+ pErrStr = PyUnicode_AsUTF8(pStr);
+ }
+ Py_XDECREF(pStr);
+ }
+ }
+ else
+ {
+ pDict = PyModule_GetDict(pModule);
+ pFunc = PyDict_GetItemString(pDict, "parseUDToJson");
+ if (PyCallable_Check(pFunc))
+ {
+ auto ud = data.data();
+ pArgs = PyTuple_New(3);
+ std::unique_ptr<PyObject, decltype(&pyDecRef)> argPtr(pArgs,
+ &pyDecRef);
+ PyTuple_SetItem(pArgs, 0,
+ PyLong_FromUnsignedLong((unsigned long)subType));
+ PyTuple_SetItem(pArgs, 1,
+ PyLong_FromUnsignedLong((unsigned long)version));
+ pData = PyMemoryView_FromMemory(
+ reinterpret_cast<char*>(const_cast<unsigned char*>(ud)),
+ data.size(), PyBUF_READ);
+ std::unique_ptr<PyObject, decltype(&pyDecRef)> dataPtr(pData,
+ &pyDecRef);
+ PyTuple_SetItem(pArgs, 2, pData);
+ pResult = PyObject_CallObject(pFunc, pArgs);
+ std::unique_ptr<PyObject, decltype(&pyDecRef)> resPtr(pResult,
+ &pyDecRef);
+ if (pResult)
+ {
+ pBytes = PyUnicode_AsEncodedString(pResult, "utf-8", "~E~");
+ std::unique_ptr<PyObject, decltype(&pyDecRef)> pyBytePtr(
+ pBytes, &pyDecRef);
+ const char* output = PyBytes_AS_STRING(pBytes);
+ try
+ {
+ fifoJSON json = nlohmann::json::parse(output);
+ return prettyJSON(componentID, subType, version, json);
+ }
+ catch (std::exception& e)
+ {
+ log<level::ERR>("Bad JSON from parser",
+ entry("ERROR=%s", e.what()),
+ entry("PARSER_MODULE=%s", module.c_str()),
+ entry("SUBTYPE=0x%X", subType),
+ entry("VERSION=%d", version),
+ entry("DATA_LENGTH=%lu\n", data.size()));
+ return std::nullopt;
+ }
+ }
+ else
+ {
+ pErrStr = "No error string found";
+ PyErr_Fetch(&eType, &eValue, &eTraceback);
+ if (eValue)
+ {
+ PyObject* pStr = PyObject_Str(eValue);
+ if (pStr)
+ {
+ pErrStr = PyUnicode_AsUTF8(pStr);
+ }
+ Py_XDECREF(pStr);
+ }
+ }
+ }
+ }
+ if (!pErrStr.empty())
+ {
+ log<level::ERR>("Python exception thrown by parser",
+ entry("ERROR=%s", pErrStr.c_str()),
+ entry("PARSER_MODULE=%s", module.c_str()),
+ entry("SUBTYPE=0x%X", subType),
+ entry("VERSION=%d", version),
+ entry("DATA_LENGTH=%lu\n", data.size()));
+ }
+ Py_XDECREF(eType);
+ Py_XDECREF(eValue);
+ Py_XDECREF(eTraceback);
+ return std::nullopt;
+}
+
std::optional<std::string> getJSON(uint16_t componentID, uint8_t subType,
uint8_t version,
- const std::vector<uint8_t>& data)
+ const std::vector<uint8_t>& data,
+ uint8_t creatorID)
{
try
{
- switch (componentID)
+ if (pv::creatorIDs.at(getNumberString("%c", creatorID)) == "BMC" &&
+ componentID == static_cast<uint16_t>(ComponentID::phosphorLogging))
{
- case static_cast<uint16_t>(ComponentID::phosphorLogging):
- return getBuiltinFormatJSON(componentID, subType, version,
- data);
- default:
- break;
+ return getBuiltinFormatJSON(componentID, subType, version, data);
+ }
+ else
+ {
+ return getPythonJSON(componentID, subType, version, data,
+ creatorID);
}
}
catch (std::exception& e)
diff --git a/extensions/openpower-pels/user_data_json.hpp b/extensions/openpower-pels/user_data_json.hpp
index 64fac79..5a0cc19 100644
--- a/extensions/openpower-pels/user_data_json.hpp
+++ b/extensions/openpower-pels/user_data_json.hpp
@@ -14,12 +14,14 @@
* @param[in] subType - The subtype from the UserData section header
* @param[in] version - The version from the UserData section header
* @param[in] data - The section data
+ * @param[in] creatorID - Creator Subsystem ID from Private Header
*
* @return std::optional<std::string> - The JSON string if it could be created,
* else std::nullopt.
*/
std::optional<std::string> getJSON(uint16_t componentID, uint8_t subType,
uint8_t version,
- const std::vector<uint8_t>& data);
+ const std::vector<uint8_t>& data,
+ uint8_t creatorID);
} // namespace openpower::pels::user_data