Implement zstd decompression
Given the size of Redfish schemas these days, it would be nice to be
able to store them on disk in a zstd format. Unfortunately, not all
clients support zstd at this time.
This commit implements reading of zstd files from disk, as well as
decompressing zstd in the case where the client does not support zstd as
a return type.
Tested:
Implanted an artificial zstd file into the system, and observed correct
decompression both with an allow-encoding header of empty string and
zstd.
Change-Id: I8b631bb943de99002fdd6745340aec010ee591ff
Signed-off-by: Ed Tanous <etanous@nvidia.com>
diff --git a/test/http/http_body_test.cpp b/test/http/http_body_test.cpp
index 75c81fa..889161c 100644
--- a/test/http/http_body_test.cpp
+++ b/test/http/http_body_test.cpp
@@ -62,13 +62,14 @@
// Move constructor
HttpBody::value_type value2 = value;
EXPECT_EQ(value2.encodingType, EncodingType::Raw);
+ EXPECT_EQ(value2.compressionType, CompressionType::Raw);
EXPECT_EQ(value2.str(), "teststring");
EXPECT_EQ(value2.payloadSize(), 10);
}
TEST(HttpHttpBodyValueType, MoveFile)
{
- HttpBody::value_type value(EncodingType::Base64);
+ HttpBody::value_type value(EncodingType::Base64, CompressionType::Raw);
TemporaryFileHandle temporaryFile("teststring");
boost::system::error_code ec;
value.open(temporaryFile.stringPath.c_str(), boost::beast::file_mode::read,
@@ -80,6 +81,7 @@
size_t out = value2.file().read(buffer.data(), buffer.size(), ec);
ASSERT_FALSE(ec);
EXPECT_EQ(value2.encodingType, EncodingType::Base64);
+ EXPECT_EQ(value2.compressionType, CompressionType::Raw);
EXPECT_THAT(std::span(buffer.data(), out),
ElementsAre('t', 'e', 's', 't', 's', 't', 'r', 'i', 'n', 'g'));
@@ -92,7 +94,7 @@
TEST(HttpHttpBodyValueType, MoveOperatorFile)
{
- HttpBody::value_type value(EncodingType::Base64);
+ HttpBody::value_type value(EncodingType::Base64, CompressionType::Raw);
TemporaryFileHandle temporaryFile("teststring");
boost::system::error_code ec;
value.open(temporaryFile.stringPath.c_str(), boost::beast::file_mode::read,
@@ -104,6 +106,7 @@
size_t out = value2.file().read(buffer.data(), buffer.size(), ec);
ASSERT_FALSE(ec);
EXPECT_EQ(value2.encodingType, EncodingType::Base64);
+ EXPECT_EQ(value2.compressionType, CompressionType::Raw);
EXPECT_THAT(std::span(buffer.data(), out),
ElementsAre('t', 'e', 's', 't', 's', 't', 'r', 'i', 'n', 'g'));
@@ -115,7 +118,7 @@
TEST(HttpFileBodyValueType, SetFd)
{
- HttpBody::value_type value(EncodingType::Base64);
+ HttpBody::value_type value(EncodingType::Base64, CompressionType::Raw);
TemporaryFileHandle temporaryFile("teststring");
boost::system::error_code ec;
FILE* r = fopen(temporaryFile.stringPath.c_str(), "r");
diff --git a/test/http/zstd_decompressor_test.cpp b/test/http/zstd_decompressor_test.cpp
new file mode 100644
index 0000000..e765bd9
--- /dev/null
+++ b/test/http/zstd_decompressor_test.cpp
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright OpenBMC Authors
+#ifdef HAVE_ZSTD
+#include "zstd_decompressor.hpp"
+
+#include <boost/asio/buffer.hpp>
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <optional>
+#include <span>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+using ::testing::Each;
+using ::testing::Eq;
+
+namespace zstd
+{
+namespace
+{
+
+TEST(Zstd, EmptyFile)
+{
+ std::array<unsigned char, 13> empty{0x28, 0xb5, 0x2f, 0xfd, 0x24,
+ 0x00, 0x01, 0x00, 0x00, 0x99,
+ 0xe9, 0xd8, 0x51};
+
+ ZstdDecompressor comp;
+ std::optional<boost::asio::const_buffer> out =
+ comp.decompress(boost::asio::buffer(empty));
+ ASSERT_TRUE(out);
+ if (!out)
+ {
+ return;
+ }
+ EXPECT_TRUE(out->size() == 0);
+}
+
+TEST(Zstd, ZerosFile)
+{
+ // A 1MB file of all zeros created using
+ // dd if=/dev/zero of=zeros-file bs=1024 count=1024
+ // zstd -c zeros-file | xxd -i
+ std::array<unsigned char, 54> zeros = {
+ 0x28, 0xb5, 0x2f, 0xfd, 0xa4, 0x00, 0x00, 0x10, 0x00, 0x54, 0x00,
+ 0x00, 0x10, 0x00, 0x00, 0x01, 0x00, 0xfb, 0xff, 0x39, 0xc0, 0x02,
+ 0x02, 0x00, 0x10, 0x00, 0x02, 0x00, 0x10, 0x00, 0x02, 0x00, 0x10,
+ 0x00, 0x02, 0x00, 0x10, 0x00, 0x02, 0x00, 0x10, 0x00, 0x02, 0x00,
+ 0x10, 0x00, 0x03, 0x00, 0x10, 0x00, 0xf1, 0x3e, 0x16, 0xe1};
+
+ for (size_t chunkSize :
+ std::to_array<size_t>({1U, 2U, 4U, 8U, 16U, zeros.size()}))
+ {
+ ZstdDecompressor comp;
+ std::span<unsigned char> data = std::span(zeros);
+ size_t read = 0;
+ while (!data.empty())
+ {
+ std::span<unsigned char> chunk =
+ data.subspan(0, std::min(chunkSize, data.size()));
+ std::optional<boost::asio::const_buffer> out = comp.decompress(
+ boost::asio::buffer(chunk.data(), chunk.size()));
+ ASSERT_TRUE(out);
+ if (out)
+ {
+ EXPECT_THAT(
+ std::span(static_cast<const unsigned char*>(out->data()),
+ out->size()),
+ Each(Eq(0)));
+ read += out->size();
+ }
+ data = data.subspan(chunk.size());
+ }
+
+ EXPECT_EQ(read, 1024 * 1024);
+ }
+}
+
+TEST(Zstd, OnesFile)
+{
+ // A 1MB file of all ones created using
+ // dd if=/dev/zero bs=1024 count=1024 | tr "\000" "\377" > ones.txt
+ // zstd -c ones-file | xxd -i
+ std::array<unsigned char, 54> ones = {
+ 0x28, 0xb5, 0x2f, 0xfd, 0xa4, 0x00, 0x00, 0x10, 0x00, 0x54, 0x00,
+ 0x00, 0x10, 0xff, 0xff, 0x01, 0x00, 0xfb, 0xff, 0x39, 0xc0, 0x02,
+ 0x02, 0x00, 0x10, 0xff, 0x02, 0x00, 0x10, 0xff, 0x02, 0x00, 0x10,
+ 0xff, 0x02, 0x00, 0x10, 0xff, 0x02, 0x00, 0x10, 0xff, 0x02, 0x00,
+ 0x10, 0xff, 0x03, 0x00, 0x10, 0xff, 0xb4, 0xc8, 0xba, 0x13};
+
+ for (size_t chunkSize :
+ std::to_array<size_t>({1U, 2U, 4U, 8U, 16U, ones.size()}))
+ {
+ ZstdDecompressor comp;
+ std::span<unsigned char> data = std::span(ones);
+ size_t read = 0;
+ while (!data.empty())
+ {
+ std::span<unsigned char> chunk =
+ data.subspan(0, std::min(chunkSize, data.size()));
+ std::optional<boost::asio::const_buffer> out = comp.decompress(
+ boost::asio::buffer(chunk.data(), chunk.size()));
+ ASSERT_TRUE(out);
+ if (out)
+ {
+ EXPECT_THAT(
+ std::span(static_cast<const unsigned char*>(out->data()),
+ out->size()),
+ Each(Eq(0xFF)));
+ read += out->size();
+ }
+ data = data.subspan(chunk.size());
+ }
+
+ EXPECT_EQ(read, 1024 * 1024);
+ }
+}
+
+} // namespace
+} // namespace zstd
+#endif