NVMeContext: Split out NVMeMCTPContext
The MCTP-oriented implementation of NVMeContext requires linking against
a libmctp build that provides an SMBus binding implementation. The SMBus
binding implementation isn't upstream, and neither are the kernel
interfaces it relies on.
Later, an NVMe MI Basic implementation will be introduced. NVMe MI Basic
only requires support for SMBus block reads rather than MCTP, though is
not as fully featured as the MCTP interface and is not required to be
supported by implementers of the NVMe specification.
For now, reduce NVMeContext to a base class that NVMeMCTPContext
extends and only use the NVMeMCTPContext type where explicitly
required. This opens up the opportunity to make MCTP support a
build-time configurable option.
Signed-off-by: Andrew Jeffery <andrew@aj.id.au>
Change-Id: I6920f79ca389481f3be43ee9a0d336bf8f72d55b
diff --git a/src/NVMeMCTPContext.cpp b/src/NVMeMCTPContext.cpp
new file mode 100644
index 0000000..40bb8ac
--- /dev/null
+++ b/src/NVMeMCTPContext.cpp
@@ -0,0 +1,432 @@
+/*
+// Copyright (c) 2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include "NVMeMCTPContext.hpp"
+
+#include "NVMeDevice.hpp"
+
+#include <crc32c.h>
+#include <libmctp-smbus.h>
+#include <libmctp.h>
+
+#include <boost/container/flat_map.hpp>
+
+static constexpr bool debug = false;
+
+static void rxMessage(uint8_t eid, void* data, void* msg, size_t len);
+
+namespace nvmeMCTP
+{
+struct mctp_binding_smbus* smbus = mctp_smbus_init();
+struct mctp* mctp = mctp_init();
+
+static boost::container::flat_map<int, int> inFds;
+static boost::container::flat_map<int, int> outFds;
+
+int getInFd(int rootBus)
+{
+ auto findBus = inFds.find(rootBus);
+ if (findBus != inFds.end())
+ {
+ return findBus->second;
+ }
+ int fd = mctp_smbus_open_in_bus(smbus, rootBus);
+ if (fd < 0)
+ {
+ std::cerr << "Error opening IN Bus " << rootBus << "\n";
+ }
+ inFds[rootBus] = fd;
+ return fd;
+}
+
+int getOutFd(int bus)
+{
+ auto findBus = outFds.find(bus);
+ if (findBus != outFds.end())
+ {
+ return findBus->second;
+ }
+ int fd = mctp_smbus_open_out_bus(smbus, bus);
+ if (fd < 0)
+ {
+ std::cerr << "Error opening Out Bus " << bus << "\n";
+ }
+ outFds[bus] = fd;
+ return fd;
+}
+
+// we don't close the outFd as multiple sensors could be sharing the fd, we need
+// to close the inFd as it can only be used on 1 socket at a time
+void closeInFd(int rootBus)
+{
+ auto findFd = inFds.find(rootBus);
+ if (findFd == inFds.end())
+ {
+ return;
+ }
+ close(findFd->second);
+ inFds.erase(rootBus);
+}
+
+int getRootBus(int inFd)
+{
+ // we assume that we won't have too many FDs, so looping is OK
+ for (const auto [root, fd] : inFds)
+ {
+ if (fd == inFd)
+ {
+ return root;
+ }
+ }
+
+ return -1;
+}
+
+void init()
+{
+ if (mctp == nullptr || smbus == nullptr)
+ {
+ throw std::runtime_error("Unable to init mctp");
+ }
+ mctp_smbus_register_bus(smbus, nvmeMCTP::mctp, 0);
+ mctp_set_rx_all(mctp, rxMessage, nullptr);
+}
+
+} // namespace nvmeMCTP
+
+static void rxMessage(uint8_t eid, void*, void* msg, size_t len)
+{
+ int inFd = mctp_smbus_get_in_fd(nvmeMCTP::smbus);
+ int rootBus = nvmeMCTP::getRootBus(inFd);
+
+ NVMEMap& nvmeMap = getNVMEMap();
+ auto findMap = nvmeMap.find(rootBus);
+ if (findMap == nvmeMap.end())
+ {
+ std::cerr << "Unable to lookup root bus " << rootBus << "\n";
+ return;
+ }
+
+ if (debug)
+ {
+ std::cout << "Eid from the received messaged: " << eid << "\n";
+ }
+
+ findMap->second->processResponse(msg, len);
+}
+
+static int verifyIntegrity(uint8_t* msg, size_t len)
+{
+ uint32_t msgIntegrity = {0};
+ if (len < NVME_MI_MSG_RESPONSE_HEADER_SIZE + sizeof(msgIntegrity))
+ {
+ std::cerr << "Not enough bytes for nvme header and trailer\n";
+ return -1;
+ }
+
+ msgIntegrity = (msg[len - 4]) + (msg[len - 3] << 8) + (msg[len - 2] << 16) +
+ (msg[len - 1] << 24);
+
+ uint32_t calculateIntegrity = crc32c(msg, len - sizeof(msgIntegrity));
+ if (msgIntegrity != calculateIntegrity)
+ {
+ std::cerr << "CRC mismatch. Got=" << msgIntegrity
+ << " Expected=" << calculateIntegrity << "\n";
+ return -1;
+ }
+ return 0;
+}
+
+static double getTemperatureReading(int8_t reading)
+{
+
+ if (reading == static_cast<int8_t>(0x80) ||
+ reading == static_cast<int8_t>(0x81))
+ {
+ // 0x80 = No temperature data or temperature data is more the 5 s
+ // old 0x81 = Temperature sensor failure
+ return std::numeric_limits<double>::quiet_NaN();
+ }
+
+ return reading;
+}
+
+void NVMeMCTPContext::processResponse(void* msg, size_t len)
+{
+ struct nvme_mi_msg_response_header header
+ {};
+
+ if (msg == nullptr)
+ {
+ std::cerr << "Bad message received\n";
+ return;
+ }
+
+ if (len <= 0)
+ {
+ std::cerr << "Received message not long enough\n";
+ return;
+ }
+
+ uint8_t* messageData = static_cast<uint8_t*>(msg);
+
+ if ((*messageData & NVME_MI_MESSAGE_TYPE_MASK) != NVME_MI_MESSAGE_TYPE)
+ {
+ std::cerr << "Got unknown type message_type="
+ << (*messageData & NVME_MI_MESSAGE_TYPE_MASK) << "\n";
+ return;
+ }
+
+ if (len < NVME_MI_MSG_RESPONSE_HEADER_SIZE + sizeof(uint32_t))
+ {
+ std::cerr << "Not enough bytes for NVMe header and trailer\n";
+ return;
+ }
+
+ if (verifyIntegrity(messageData, len) != 0)
+ {
+ std::cerr << "Verification of message integrity failed\n";
+ return;
+ }
+
+ header.message_type = messageData[0];
+ header.flags = messageData[1];
+ header.status = messageData[4];
+
+ if (header.status == NVME_MI_HDR_STATUS_MORE_PROCESSING_REQUIRED)
+ {
+ return;
+ }
+
+ if (header.status != NVME_MI_HDR_STATUS_SUCCESS)
+ {
+ std::cerr << "Command failed with status= " << header.status << "\n";
+ return;
+ }
+
+ messageData += NVME_MI_MSG_RESPONSE_HEADER_SIZE;
+ size_t messageLength =
+ len - NVME_MI_MSG_RESPONSE_HEADER_SIZE - sizeof(uint32_t);
+ if (((header.flags >> NVME_MI_HDR_FLAG_MSG_TYPE_SHIFT) &
+ NVME_MI_HDR_FLAG_MSG_TYPE_MASK) != NVME_MI_HDR_MESSAGE_TYPE_MI_COMMAND)
+ {
+ std::cerr << "Not MI type comamnd\n";
+ return;
+ }
+
+ if (messageLength < NVME_MI_HEALTH_STATUS_POLL_MSG_MIN)
+ {
+ std::cerr << "Got improperly sized health status poll\n";
+ return;
+ }
+
+ std::shared_ptr<NVMeSensor> sensorInfo = sensors.front();
+ if (debug)
+ {
+ std::cout << "Temperature Reading: "
+ << getTemperatureReading(messageData[5])
+ << " Celsius for device " << sensorInfo->name << "\n";
+ }
+
+ double value = getTemperatureReading(messageData[5]);
+ if (!std::isfinite(value))
+ {
+ sensorInfo->markAvailable(false);
+ sensorInfo->incrementError();
+ }
+ else
+ {
+ sensorInfo->updateValue(value);
+ }
+
+ if (debug)
+ {
+ std::cout << "Cancelling the timer now\n";
+ }
+
+ // move to back of scan queue
+ sensors.pop_front();
+ sensors.emplace_back(sensorInfo);
+
+ mctpResponseTimer.cancel();
+}
+
+static int nvmeMessageTransmit(mctp& mctp, nvme_mi_msg_request& req)
+{
+ std::array<uint8_t, NVME_MI_MSG_BUFFER_SIZE> messageBuf = {};
+
+ req.header.flags |= NVME_MI_HDR_MESSAGE_TYPE_MI_COMMAND
+ << NVME_MI_HDR_FLAG_MSG_TYPE_SHIFT;
+ req.header.message_type =
+ NVME_MI_MESSAGE_TYPE | NVME_MI_MCTP_INTEGRITY_CHECK;
+
+ uint32_t integrity = 0;
+ size_t msgSize = NVME_MI_MSG_REQUEST_HEADER_SIZE + req.request_data_len +
+ sizeof(integrity);
+
+ if (sizeof(messageBuf) < msgSize)
+ {
+ return EXIT_FAILURE;
+ }
+
+ messageBuf[0] = req.header.message_type;
+ messageBuf[1] = req.header.flags;
+ // Reserved bytes 2-3
+
+ messageBuf[4] = req.header.opcode;
+ // reserved bytes 5-7
+ messageBuf[8] = req.header.dword0 & 0xff;
+ messageBuf[9] = (req.header.dword0 >> 8) & 0xff;
+ messageBuf[10] = (req.header.dword0 >> 16) & 0xff;
+ messageBuf[11] = (req.header.dword0 >> 24) & 0xff;
+
+ messageBuf[12] = req.header.dword1 & 0xff;
+ messageBuf[13] = (req.header.dword1 >> 8) & 0xff;
+ messageBuf[14] = (req.header.dword1 >> 16) & 0xff;
+ messageBuf[15] = (req.header.dword1 >> 24) & 0xff;
+
+ std::copy_n(req.request_data, req.request_data_len,
+ messageBuf.data() +
+ static_cast<uint8_t>(NVME_MI_MSG_REQUEST_HEADER_SIZE));
+
+ msgSize = NVME_MI_MSG_REQUEST_HEADER_SIZE + req.request_data_len;
+ integrity = crc32c(messageBuf.data(),
+ NVME_MI_MSG_REQUEST_HEADER_SIZE + req.request_data_len);
+ messageBuf[msgSize] = integrity & 0xff;
+ messageBuf[msgSize + 1] = (integrity >> 8) & 0xff;
+ messageBuf[msgSize + 2] = (integrity >> 16) & 0xff;
+ messageBuf[msgSize + 3] = (integrity >> 24) & 0xff;
+ msgSize += sizeof(integrity);
+
+ return mctp_message_tx(&mctp, 0, messageBuf.data(), msgSize);
+}
+
+void NVMeMCTPContext::readResponse()
+{
+ nvmeSlaveSocket.async_wait(
+ boost::asio::ip::tcp::socket::wait_error,
+ [this](const boost::system::error_code errorCode) {
+ if (errorCode)
+ {
+ return;
+ }
+
+ mctp_smbus_set_in_fd(nvmeMCTP::smbus, nvmeMCTP::getInFd(rootBus));
+
+ // through libmctp this will invoke rxMessage
+ mctp_smbus_read(nvmeMCTP::smbus);
+ });
+}
+
+void NVMeMCTPContext::readAndProcessNVMeSensor()
+{
+ struct nvme_mi_msg_request requestMsg = {};
+ requestMsg.header.opcode = NVME_MI_OPCODE_HEALTH_STATUS_POLL;
+ requestMsg.header.dword0 = 0;
+ requestMsg.header.dword1 = 0;
+
+ int mctpResponseTimeout = 1;
+
+ if (sensors.empty())
+ {
+ return;
+ }
+
+ std::shared_ptr<NVMeSensor>& sensor = sensors.front();
+
+ // setup the timeout timer
+ mctpResponseTimer.expires_from_now(
+ boost::posix_time::seconds(mctpResponseTimeout));
+
+ mctpResponseTimer.async_wait(
+ [sensor, this](const boost::system::error_code errorCode) {
+ if (errorCode)
+ {
+ // timer cancelled successfully
+ return;
+ }
+
+ sensor->incrementError();
+
+ // cycle it back
+ sensors.pop_front();
+ sensors.emplace_back(sensor);
+
+ nvmeSlaveSocket.cancel();
+ });
+
+ readResponse();
+
+ if (debug)
+ {
+ std::cout << "Sending message to read data from Drive on bus: "
+ << sensor->bus << " , rootBus: " << rootBus
+ << " device: " << sensor->name << "\n";
+ }
+
+ mctp_smbus_set_out_fd(nvmeMCTP::smbus, nvmeMCTP::getOutFd(sensor->bus));
+ int rc = nvmeMessageTransmit(*nvmeMCTP::mctp, requestMsg);
+
+ if (rc != 0)
+ {
+ std::cerr << "Error sending request message to NVMe device\n";
+ }
+}
+
+NVMeMCTPContext::NVMeMCTPContext(boost::asio::io_service& io, int rootBus) :
+ NVMeContext::NVMeContext(io, rootBus), nvmeSlaveSocket(io),
+ mctpResponseTimer(io)
+{
+ nvmeSlaveSocket.assign(boost::asio::ip::tcp::v4(),
+ nvmeMCTP::getInFd(rootBus));
+}
+
+void NVMeMCTPContext::pollNVMeDevices()
+{
+ scanTimer.expires_from_now(boost::posix_time::seconds(1));
+ scanTimer.async_wait(
+ [self{shared_from_this()}](const boost::system::error_code errorCode) {
+ if (errorCode == boost::asio::error::operation_aborted)
+ {
+ return; // we're being canceled
+ }
+ else if (errorCode)
+ {
+ std::cerr << "Error:" << errorCode.message() << "\n";
+ return;
+ }
+ else
+ {
+ self->readAndProcessNVMeSensor();
+ }
+
+ self->pollNVMeDevices();
+ });
+}
+
+void NVMeMCTPContext::close()
+{
+ this->NVMeContext::close();
+
+ mctpResponseTimer.cancel();
+ nvmeSlaveSocket.cancel();
+ nvmeMCTP::closeInFd(rootBus);
+}
+
+NVMeMCTPContext::~NVMeMCTPContext()
+{
+ close();
+}