Implement zstd decompression
Given the size of Redfish schemas these days, it would be nice to be
able to store them on disk in a zstd format. Unfortunately, not all
clients support zstd at this time.
This commit implements reading of zstd files from disk, as well as
decompressing zstd in the case where the client does not support zstd as
a return type.
Tested:
Implanted an artificial zstd file into the system, and observed correct
decompression both with an allow-encoding header of empty string and
zstd.
Change-Id: I8b631bb943de99002fdd6745340aec010ee591ff
Signed-off-by: Ed Tanous <etanous@nvidia.com>
diff --git a/http/complete_response_fields.hpp b/http/complete_response_fields.hpp
index 486b699..41314b8 100644
--- a/http/complete_response_fields.hpp
+++ b/http/complete_response_fields.hpp
@@ -3,6 +3,7 @@
#pragma once
#include "boost_formatters.hpp"
+#include "http_body.hpp"
#include "http_response.hpp"
#include "http_utility.hpp"
#include "json_html_serializer.hpp"
@@ -20,7 +21,48 @@
namespace crow
{
-inline void completeResponseFields(std::string_view accepts, Response& res)
+inline void handleEncoding(std::string_view acceptEncoding, Response& res)
+{
+ using bmcweb::CompressionType;
+ using enum bmcweb::CompressionType;
+ using http_helpers::Encoding;
+ using enum http_helpers::Encoding;
+ // If the payload is currently compressed, see if we can avoid
+ // decompressing it by sending it to the client directly
+ switch (res.response.body().compressionType)
+ {
+ case Zstd:
+ {
+ std::array<Encoding, 1> allowedEnc{ZSTD};
+ Encoding encoding =
+ http_helpers::getPreferredEncoding(acceptEncoding, allowedEnc);
+
+ if (encoding == ZSTD)
+ {
+ // If the client supports returning zstd directly, allow that.
+ res.response.body().clientCompressionType = Zstd;
+ }
+ }
+ break;
+ case Gzip:
+ {
+ std::array<Encoding, 1> allowedEnc{GZIP};
+ Encoding encoding =
+ http_helpers::getPreferredEncoding(acceptEncoding, allowedEnc);
+ if (encoding != GZIP)
+ {
+ BMCWEB_LOG_WARNING(
+ "Unimplemented: Returning gzip payload to client that did not explicitly allow it.");
+ }
+ }
+ break;
+ default:
+ break;
+ }
+}
+
+inline void completeResponseFields(
+ std::string_view accepts, std::string_view acceptEncoding, Response& res)
{
BMCWEB_LOG_INFO("Response: {}", res.resultInt());
addSecurityHeaders(res);
@@ -56,5 +98,7 @@
2, ' ', true, nlohmann::json::error_handler_t::replace));
}
}
+
+ handleEncoding(acceptEncoding, res);
}
} // namespace crow
diff --git a/http/http2_connection.hpp b/http/http2_connection.hpp
index 99604f5..e9e9cd8 100644
--- a/http/http2_connection.hpp
+++ b/http/http2_connection.hpp
@@ -52,6 +52,7 @@
std::shared_ptr<Request> req = std::make_shared<Request>();
std::optional<bmcweb::HttpBody::reader> reqReader;
std::string accept;
+ std::string acceptEnc;
Response res;
std::optional<bmcweb::HttpBody::writer> writer;
};
@@ -205,7 +206,7 @@
Response& res = stream.res;
res = std::move(completedRes);
- completeResponseFields(stream.accept, res);
+ completeResponseFields(stream.accept, stream.acceptEnc, res);
res.addHeader(boost::beast::http::field::date, getCachedDateStr());
res.preparePayload();
@@ -277,7 +278,9 @@
}
}
crow::Request& thisReq = *it->second.req;
- it->second.accept = thisReq.getHeaderValue("Accept");
+ using boost::beast::http::field;
+ it->second.accept = thisReq.getHeaderValue(field::accept);
+ it->second.acceptEnc = thisReq.getHeaderValue(field::accept_encoding);
BMCWEB_LOG_DEBUG("Handling {} \"{}\"", logPtr(&thisReq),
thisReq.url().encoded_path());
@@ -520,7 +523,7 @@
{
BMCWEB_LOG_DEBUG("create stream for id {}", frame.hd.stream_id);
- streams.emplace(frame.hd.stream_id, Http2StreamData());
+ streams[frame.hd.stream_id];
}
return 0;
}
diff --git a/http/http_body.hpp b/http/http_body.hpp
index d76ed49..288c1fe 100644
--- a/http/http_body.hpp
+++ b/http/http_body.hpp
@@ -5,6 +5,7 @@
#include "duplicatable_file_handle.hpp"
#include "logging.hpp"
#include "utility.hpp"
+#include "zstd_decompressor.hpp"
#include <fcntl.h>
@@ -49,6 +50,13 @@
Base64,
};
+enum class CompressionType
+{
+ Raw,
+ Gzip,
+ Zstd,
+};
+
class HttpBody::value_type
{
DuplicatableFileHandle fileHandle;
@@ -60,6 +68,19 @@
explicit value_type(std::string_view s) : strBody(s) {}
explicit value_type(EncodingType e) : encodingType(e) {}
EncodingType encodingType = EncodingType::Raw;
+ CompressionType compressionType = CompressionType::Raw;
+ CompressionType clientCompressionType = CompressionType::Raw;
+
+ ~value_type() = default;
+
+ explicit value_type(EncodingType enc, CompressionType comp) :
+ encodingType(enc), compressionType(comp)
+ {}
+
+ value_type(const value_type& other) noexcept = default;
+ value_type& operator=(const value_type& other) noexcept = default;
+ value_type(value_type&& other) noexcept = default;
+ value_type& operator=(value_type&& other) noexcept = default;
const boost::beast::file_posix& file() const
{
@@ -156,6 +177,8 @@
std::string buf;
crow::utility::Base64Encoder encoder;
+ std::optional<ZstdDecompressor> zstd;
+
value_type& body;
size_t sent = 0;
// 64KB This number is arbitrary, and selected to try to optimize for larger
@@ -169,7 +192,13 @@
template <bool IsRequest, class Fields>
writer(boost::beast::http::header<IsRequest, Fields>& /*header*/,
value_type& bodyIn) : body(bodyIn)
- {}
+ {
+ if (body.compressionType == CompressionType::Zstd &&
+ body.clientCompressionType != CompressionType::Zstd)
+ {
+ zstd.emplace();
+ }
+ }
static void init(boost::beast::error_code& ec)
{
@@ -235,6 +264,18 @@
{
ret.first = const_buffers_type(chunkView.data(), chunkView.size());
}
+
+ if (zstd)
+ {
+ std::optional<const_buffers_type> decompressed =
+ zstd->decompress(ret.first);
+ if (!decompressed)
+ {
+ return boost::none;
+ }
+ ret.first = *decompressed;
+ }
+
return ret;
}
};
diff --git a/http/http_connection.hpp b/http/http_connection.hpp
index 807ca35..0c8ad0e 100644
--- a/http/http_connection.hpp
+++ b/http/http_connection.hpp
@@ -383,14 +383,16 @@
return;
}
req->session = userSession;
- accept = req->getHeaderValue("Accept");
+ using boost::beast::http::field;
+ accept = req->getHeaderValue(field::accept);
+ acceptEncoding = req->getHeaderValue(field::accept_encoding);
// Fetch the client IP address
req->ipAddress = ip;
// Check for HTTP version 1.1.
if (req->version() == 11)
{
- if (req->getHeaderValue(boost::beast::http::field::host).empty())
+ if (req->getHeaderValue(field::host).empty())
{
res.result(boost::beast::http::status::bad_request);
completeRequest(res);
@@ -490,7 +492,7 @@
res = std::move(thisRes);
res.keepAlive(keepAlive);
- completeResponseFields(accept, res);
+ completeResponseFields(accept, acceptEncoding, res);
res.addHeader(boost::beast::http::field::date, getCachedDateStr());
doWrite();
@@ -931,6 +933,8 @@
std::shared_ptr<crow::Request> req;
std::string accept;
std::string http2settings;
+ std::string acceptEncoding;
+
crow::Response res;
std::shared_ptr<persistent_data::UserSession> userSession;
diff --git a/http/http_response.hpp b/http/http_response.hpp
index 4f2b0f0..46155a3 100644
--- a/http/http_response.hpp
+++ b/http/http_response.hpp
@@ -306,12 +306,15 @@
expectedHash = hash;
}
- OpenCode openFile(const std::filesystem::path& path,
- bmcweb::EncodingType enc = bmcweb::EncodingType::Raw)
+ OpenCode openFile(
+ const std::filesystem::path& path,
+ bmcweb::EncodingType enc = bmcweb::EncodingType::Raw,
+ bmcweb::CompressionType comp = bmcweb::CompressionType::Raw)
{
boost::beast::error_code ec;
response.body().open(path.c_str(), boost::beast::file_mode::read, ec);
response.body().encodingType = enc;
+ response.body().compressionType = comp;
if (ec)
{
BMCWEB_LOG_ERROR("Failed to open file {}, ec={}", path.c_str(),
diff --git a/http/zstd_decompressor.cpp b/http/zstd_decompressor.cpp
new file mode 100644
index 0000000..aa42392
--- /dev/null
+++ b/http/zstd_decompressor.cpp
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright OpenBMC Authors
+
+#include "zstd_decompressor.hpp"
+
+#include "logging.hpp"
+
+#ifdef HAVE_ZSTD
+#include <zstd.h>
+#endif
+#include <boost/asio/buffer.hpp>
+
+#include <cstddef>
+#include <optional>
+
+#ifdef HAVE_ZSTD
+ZstdDecompressor::ZstdDecompressor() : dctx(ZSTD_createDStream())
+{
+ ZSTD_initDStream(dctx);
+}
+#else
+ZstdDecompressor::ZstdDecompressor() {};
+#endif
+
+std::optional<boost::asio::const_buffer> ZstdDecompressor::decompress(
+ [[maybe_unused]] boost::asio::const_buffer buffIn)
+{
+#ifdef HAVE_ZSTD
+ compressionBuf.clear();
+ ZSTD_inBuffer input = {buffIn.data(), buffIn.size(), 0};
+
+ // Note, this loop is prone to compression bombs, decompressing chunks that
+ // appear very small, but decompress to be very large, given that they're
+ // highly decompressible. This algorithm assumes that at this time, the
+ // whole file will fit in ram.
+ while (input.pos != input.size)
+ {
+ constexpr size_t frameSize = 4096;
+ auto buffer = compressionBuf.prepare(frameSize);
+ ZSTD_outBuffer output = {buffer.data(), buffer.size(), 0};
+ const size_t ret = ZSTD_decompressStream(dctx, &output, &input);
+ if (ZSTD_isError(ret) != 0)
+ {
+ BMCWEB_LOG_ERROR("Decompression Failed with code {}:{}", ret,
+ ZSTD_getErrorName(ret));
+ return std::nullopt;
+ }
+ compressionBuf.commit(output.pos);
+ }
+ return compressionBuf.cdata();
+#else
+ BMCWEB_LOG_CRITICAL("Attempt to decompress, but libzstd not enabled");
+
+ return std::nullopt;
+#endif
+}
+
+ZstdDecompressor::~ZstdDecompressor()
+{
+#ifdef HAVE_ZSTD
+ ZSTD_freeDStream(dctx);
+#endif
+}
diff --git a/http/zstd_decompressor.hpp b/http/zstd_decompressor.hpp
new file mode 100644
index 0000000..777a4b8
--- /dev/null
+++ b/http/zstd_decompressor.hpp
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright OpenBMC Authors
+
+#pragma once
+
+#ifdef HAVE_ZSTD
+#include <zstd.h>
+#endif
+
+#include <boost/asio/buffer.hpp>
+#include <boost/beast/core/flat_buffer.hpp>
+
+#include <optional>
+
+class ZstdDecompressor
+{
+ boost::beast::flat_buffer compressionBuf;
+
+#ifdef HAVE_ZSTD
+ ZSTD_DCtx* dctx;
+#endif
+
+ public:
+ ZstdDecompressor(const ZstdDecompressor&) = delete;
+ ZstdDecompressor(ZstdDecompressor&&) = delete;
+ ZstdDecompressor& operator=(const ZstdDecompressor&) = delete;
+ ZstdDecompressor& operator=(ZstdDecompressor&&) = delete;
+
+ ZstdDecompressor();
+ std::optional<boost::asio::const_buffer> decompress(
+ boost::asio::const_buffer buffIn);
+ ~ZstdDecompressor();
+};