Make use of filebody for dump offload
Logservice has been rewritten to use file_body to offload dump files
from BMC.
There are two kind of dump files, BMC dump and System dump.While BMC
dump just requires default support from beast::file_body, System dump
requires base64 encoding support from beast. But beast::file_body do not
have ready-made support for base64 encoding. So a custom file_body has
been written for the base64 encoding.
The openFile apis in crow::Response do not have support for unix file
descriptor. Since dump files are accesses via descriptors, added new
openFile api that accepts descriptors.
Tested:
Functionality test have been executed to verify the bmc dump offload.
Did sanity test by invoking bmcweb pages via browser.
Change-Id: I24192657c03d8b2f0394d31e7424c6796ba3227a
Signed-off-by: Abhilash Raju <abhilash.kollam@gmail.com>
diff --git a/http/http_file_body.hpp b/http/http_file_body.hpp
new file mode 100644
index 0000000..eaafd5d
--- /dev/null
+++ b/http/http_file_body.hpp
@@ -0,0 +1,136 @@
+#pragma once
+
+#include "utility.hpp"
+
+#include <boost/beast/core/file_posix.hpp>
+#include <boost/beast/http/message.hpp>
+#include <boost/system/error_code.hpp>
+
+namespace bmcweb
+{
+struct FileBody
+{
+ class writer;
+ class value_type;
+
+ static std::uint64_t size(const value_type& body);
+};
+
+enum class EncodingType
+{
+ Raw,
+ Base64,
+};
+
+class FileBody::value_type
+{
+ boost::beast::file_posix fileHandle;
+
+ std::uint64_t fileSize = 0;
+
+ public:
+ EncodingType encodingType = EncodingType::Raw;
+
+ ~value_type() = default;
+ value_type() = default;
+ explicit value_type(EncodingType enc) : encodingType(enc) {}
+ value_type(value_type&& other) = default;
+ value_type& operator=(value_type&& other) = default;
+ value_type(const value_type& other) = delete;
+ value_type& operator=(const value_type& other) = delete;
+
+ boost::beast::file_posix& file()
+ {
+ return fileHandle;
+ }
+
+ std::uint64_t size() const
+ {
+ return fileSize;
+ }
+
+ void open(const char* path, boost::beast::file_mode mode,
+ boost::system::error_code& ec)
+ {
+ fileHandle.open(path, mode, ec);
+ fileSize = fileHandle.size(ec);
+ }
+
+ void setFd(int fd, boost::system::error_code& ec)
+ {
+ fileHandle.native_handle(fd);
+ fileSize = fileHandle.size(ec);
+ }
+};
+
+inline std::uint64_t FileBody::size(const value_type& body)
+{
+ return body.size();
+}
+
+class FileBody::writer
+{
+ public:
+ using const_buffers_type = boost::asio::const_buffer;
+
+ private:
+ std::string buf;
+ crow::utility::Base64Encoder encoder;
+
+ value_type& body;
+ std::uint64_t remain;
+ constexpr static size_t readBufSize = 4096;
+ std::array<char, readBufSize> fileReadBuf{};
+
+ public:
+ template <bool IsRequest, class Fields>
+ writer(boost::beast::http::header<IsRequest, Fields>& /*header*/,
+ value_type& bodyIn) :
+ body(bodyIn),
+ remain(body.size())
+ {}
+
+ static void init(boost::beast::error_code& ec)
+ {
+ ec = {};
+ }
+
+ boost::optional<std::pair<const_buffers_type, bool>>
+ get(boost::beast::error_code& ec)
+ {
+ size_t toRead = fileReadBuf.size();
+ if (remain < toRead)
+ {
+ toRead = static_cast<size_t>(remain);
+ }
+ size_t read = body.file().read(fileReadBuf.data(), toRead, ec);
+ if (read != toRead || ec)
+ {
+ return boost::none;
+ }
+ remain -= read;
+
+ std::string_view chunkView(fileReadBuf.data(), read);
+
+ std::pair<const_buffers_type, bool> ret;
+ ret.second = remain > 0;
+ if (body.encodingType == EncodingType::Base64)
+ {
+ buf.clear();
+ buf.reserve(
+ crow::utility::Base64Encoder::encodedSize(chunkView.size()));
+ encoder.encode(chunkView, buf);
+ if (!ret.second)
+ {
+ encoder.finalize(buf);
+ }
+ ret.first = const_buffers_type(buf.data(), buf.size());
+ }
+ else
+ {
+ ret.first = const_buffers_type(chunkView.data(), chunkView.size());
+ }
+ return ret;
+ }
+};
+} // namespace bmcweb
diff --git a/http/http_response.hpp b/http/http_response.hpp
index ec54a90..4015899 100644
--- a/http/http_response.hpp
+++ b/http/http_response.hpp
@@ -1,8 +1,8 @@
#pragma once
+#include "http_file_body.hpp"
#include "logging.hpp"
#include "utils/hex_utils.hpp"
-#include <boost/beast/http/file_body.hpp>
#include <boost/beast/http/message.hpp>
#include <boost/beast/http/message_generator.hpp>
#include <boost/beast/http/string_body.hpp>
@@ -27,7 +27,7 @@
friend class crow::Connection;
using string_response = http::response<http::string_body>;
- using file_response = http::response<http::file_body>;
+ using file_response = http::response<bmcweb::FileBody>;
// Use boost variant2 because it doesn't have valueless by exception
boost::variant2::variant<string_response, file_response> response;
@@ -356,25 +356,45 @@
response);
}
- bool openFile(const std::filesystem::path& path)
+ bool openFile(const std::filesystem::path& path,
+ bmcweb::EncodingType enc = bmcweb::EncodingType::Raw)
{
- http::file_body::value_type file;
+ file_response::body_type::value_type body(enc);
boost::beast::error_code ec;
- file.open(path.c_str(), boost::beast::file_mode::read, ec);
+ body.open(path.c_str(), boost::beast::file_mode::read, ec);
if (ec)
{
return false;
}
+ updateFileBody(std::move(body));
+ return true;
+ }
+
+ bool openFd(int fd, bmcweb::EncodingType enc = bmcweb::EncodingType::Raw)
+ {
+ file_response::body_type::value_type body(enc);
+ boost::beast::error_code ec;
+ body.setFd(fd, ec);
+ if (ec)
+ {
+ BMCWEB_LOG_ERROR("Failed to set fd");
+ return false;
+ }
+ updateFileBody(std::move(body));
+ return true;
+ }
+
+ private:
+ void updateFileBody(file_response::body_type::value_type file)
+ {
// store the headers on stack temporarily so we can reconstruct the new
// base with the old headers copied in.
http::header<false> headTemp = std::move(fields());
file_response& fileResponse =
response.emplace<file_response>(std::move(headTemp));
fileResponse.body() = std::move(file);
- return true;
}
- private:
std::optional<std::string> expectedHash;
bool completed = false;
std::function<void(Response&)> completeRequestHandler;
diff --git a/redfish-core/lib/log_services.hpp b/redfish-core/lib/log_services.hpp
index 371ae44..24f0251 100644
--- a/redfish-core/lib/log_services.hpp
+++ b/redfish-core/lib/log_services.hpp
@@ -45,6 +45,7 @@
#include <array>
#include <charconv>
+#include <cstddef>
#include <filesystem>
#include <iterator>
#include <optional>
@@ -745,7 +746,35 @@
std::format("{}/entry/{}", getDumpPath(dumpType), entryID),
"xyz.openbmc_project.Object.Delete", "Delete");
}
+inline bool checkSizeLimit(int fd, crow::Response& res)
+{
+ long long int size = lseek(fd, 0, SEEK_END);
+ if (size <= 0)
+ {
+ BMCWEB_LOG_ERROR("Failed to get size of file, lseek() returned {}",
+ size);
+ messages::internalError(res);
+ return false;
+ }
+ // Arbitrary max size of 20MB to accommodate BMC dumps
+ constexpr long long int maxFileSize = 20LL * 1024LL * 1024LL;
+ if (size > maxFileSize)
+ {
+ BMCWEB_LOG_ERROR("File size {} exceeds maximum allowed size of {}",
+ size, maxFileSize);
+ messages::internalError(res);
+ return false;
+ }
+ off_t rc = lseek(fd, 0, SEEK_SET);
+ if (rc < 0)
+ {
+ BMCWEB_LOG_ERROR("Failed to reset file offset to 0");
+ messages::internalError(res);
+ return false;
+ }
+ return true;
+}
inline void
downloadEntryCallback(const std::shared_ptr<bmcweb::AsyncResp>& asyncResp,
const std::string& entryID,
@@ -781,59 +810,29 @@
messages::internalError(asyncResp->res);
return;
}
-
- long long int size = lseek(fd, 0, SEEK_END);
- if (size <= 0)
+ if (!checkSizeLimit(fd, asyncResp->res))
{
- BMCWEB_LOG_ERROR("Failed to get size of file, lseek() returned {}",
- size);
- messages::internalError(asyncResp->res);
close(fd);
return;
}
-
- // Arbitrary max size of 20MB to accommodate BMC dumps
- constexpr int maxFileSize = 20 * 1024 * 1024;
- if (size > maxFileSize)
- {
- BMCWEB_LOG_ERROR("File size {} exceeds maximum allowed size of {}",
- size, maxFileSize);
- messages::internalError(asyncResp->res);
- close(fd);
- return;
- }
- long long int rc = lseek(fd, 0, SEEK_SET);
- if (rc < 0)
- {
- BMCWEB_LOG_ERROR("Failed to reset file offset to 0");
- messages::internalError(asyncResp->res);
- close(fd);
- return;
- }
-
- std::string body;
- body.resize(static_cast<size_t>(size), '\0');
- rc = read(fd, body.data(), body.size());
- if ((rc == -1) || (rc != size))
- {
- BMCWEB_LOG_ERROR("Failed to read in file");
- messages::internalError(asyncResp->res);
- close(fd);
- return;
- }
- close(fd);
if (downloadEntryType == "System")
{
- // Base64 encode response.
- asyncResp->res.write(crow::utility::base64encode(body));
+ if (!asyncResp->res.openFd(fd, bmcweb::EncodingType::Base64))
+ {
+ messages::internalError(asyncResp->res);
+ close(fd);
+ return;
+ }
asyncResp->res.addHeader(
boost::beast::http::field::content_transfer_encoding, "Base64");
+ return;
}
- else
+ if (!asyncResp->res.openFd(fd))
{
- asyncResp->res.write(std::move(body));
+ messages::internalError(asyncResp->res);
+ close(fd);
+ return;
}
-
asyncResp->res.addHeader(boost::beast::http::field::content_type,
"application/octet-stream");
}
diff --git a/test/http/http_response_test.cpp b/test/http/http_response_test.cpp
index c644ea9..1ee3853 100644
--- a/test/http/http_response_test.cpp
+++ b/test/http/http_response_test.cpp
@@ -22,17 +22,76 @@
EXPECT_EQ(res.result(), boost::beast::http::status::ok);
}
-std::string makeFile()
+std::string makeFile(std::string_view sampleData)
{
std::filesystem::path path = std::filesystem::temp_directory_path();
path /= "bmcweb_http_response_test_XXXXXXXXXXX";
std::string stringPath = path.string();
int fd = mkstemp(stringPath.data());
EXPECT_GT(fd, 0);
- std::string_view sample = "sample text";
- EXPECT_EQ(write(fd, sample.data(), sample.size()), sample.size());
+ EXPECT_EQ(write(fd, sampleData.data(), sampleData.size()),
+ sampleData.size());
+ close(fd);
return stringPath;
}
+
+void readHeader(boost::beast::http::serializer<false, bmcweb::FileBody>& sr)
+{
+ while (!sr.is_header_done())
+ {
+ boost::system::error_code ec;
+ sr.next(ec, [&sr](const boost::system::error_code& ec2,
+ const auto& buffer) {
+ ASSERT_FALSE(ec2);
+ sr.consume(boost::beast::buffer_bytes(buffer));
+ });
+ ASSERT_FALSE(ec);
+ }
+}
+
+std::string collectFromBuffers(
+ const auto& buffer,
+ boost::beast::http::serializer<false, bmcweb::FileBody>& sr)
+{
+ std::string ret;
+
+ for (auto iter = boost::asio::buffer_sequence_begin(buffer);
+ iter != boost::asio::buffer_sequence_end(buffer); ++iter)
+ {
+ const auto& innerBuf = *iter;
+ auto view = std::string_view(static_cast<const char*>(innerBuf.data()),
+ innerBuf.size());
+ ret += view;
+ sr.consume(innerBuf.size());
+ }
+ return ret;
+}
+
+std::string
+ readBody(boost::beast::http::serializer<false, bmcweb::FileBody>& sr)
+{
+ std::string ret;
+ while (!sr.is_done())
+ {
+ boost::system::error_code ec;
+ sr.next(ec, [&sr, &ret](const boost::system::error_code& ec2,
+ const auto& buffer) {
+ ASSERT_FALSE(ec2);
+ ret += collectFromBuffers(buffer, sr);
+ });
+ EXPECT_FALSE(ec);
+ }
+
+ return ret;
+}
+std::string getData(crow::Response::file_response& m)
+{
+ boost::beast::http::serializer<false, bmcweb::FileBody> sr{m};
+ std::stringstream ret;
+ sr.split(true);
+ readHeader(sr);
+ return readBody(sr);
+}
TEST(HttpResponse, Defaults)
{
crow::Response res;
@@ -60,17 +119,41 @@
{
crow::Response res;
addHeaders(res);
- std::string path = makeFile();
+ std::string path = makeFile("sample text");
res.openFile(path);
verifyHeaders(res);
std::filesystem::remove(path);
}
+TEST(HttpResponse, FileBodyWithFd)
+{
+ crow::Response res;
+ addHeaders(res);
+ std::string path = makeFile("sample text");
+ FILE* fd = fopen(path.c_str(), "r+");
+ res.openFd(fileno(fd));
+ verifyHeaders(res);
+ fclose(fd);
+ std::filesystem::remove(path);
+}
+
+TEST(HttpResponse, Base64FileBodyWithFd)
+{
+ crow::Response res;
+ addHeaders(res);
+ std::string path = makeFile("sample text");
+ FILE* fd = fopen(path.c_str(), "r+");
+ res.openFd(fileno(fd), bmcweb::EncodingType::Base64);
+ verifyHeaders(res);
+ fclose(fd);
+ std::filesystem::remove(path);
+}
+
TEST(HttpResponse, BodyTransitions)
{
crow::Response res;
addHeaders(res);
- std::string path = makeFile();
+ std::string path = makeFile("sample text");
res.openFile(path);
EXPECT_EQ(boost::variant2::holds_alternative<crow::Response::file_response>(
@@ -88,4 +171,67 @@
verifyHeaders(res);
std::filesystem::remove(path);
}
+
+void testFileData(crow::Response& res, const std::string& data)
+{
+ auto& fb =
+ boost::variant2::get<crow::Response::file_response>(res.response);
+ EXPECT_EQ(getData(fb), data);
+}
+
+TEST(HttpResponse, Base64FileBodyWriter)
+{
+ crow::Response res;
+ std::string data = "sample text";
+ std::string path = makeFile(data);
+ FILE* f = fopen(path.c_str(), "r+");
+ res.openFd(fileno(f), bmcweb::EncodingType::Base64);
+ testFileData(res, crow::utility::base64encode(data));
+ fclose(f);
+ std::filesystem::remove(path);
+}
+
+std::string generateBigdata()
+{
+ std::string result;
+ while (result.size() < 10000)
+ {
+ result += "sample text";
+ }
+ return result;
+}
+
+TEST(HttpResponse, Base64FileBodyWriterLarge)
+{
+ crow::Response res;
+ std::string data = generateBigdata();
+ std::string path = makeFile(data);
+ {
+ boost::beast::file_posix file;
+ boost::system::error_code ec;
+ file.open(path.c_str(), boost::beast::file_mode::read, ec);
+ EXPECT_EQ(ec.value(), 0);
+ res.openFd(file.native_handle(), bmcweb::EncodingType::Base64);
+ testFileData(res, crow::utility::base64encode(data));
+ }
+
+ std::filesystem::remove(path);
+}
+
+TEST(HttpResponse, FileBodyWriterLarge)
+{
+ crow::Response res;
+ std::string data = generateBigdata();
+ std::string path = makeFile(data);
+ {
+ boost::beast::file_posix file;
+ boost::system::error_code ec;
+ file.open(path.c_str(), boost::beast::file_mode::read, ec);
+ EXPECT_EQ(ec.value(), 0);
+ res.openFd(file.native_handle());
+ testFileData(res, data);
+ }
+ std::filesystem::remove(path);
+}
+
} // namespace