From 451a5042273516a8f501a38b43524420c10f6fa7 Mon Sep 17 00:00:00 2001 From: John R Patek Sr Date: Wed, 30 Jul 2025 19:51:08 -0500 Subject: [PATCH 1/6] added zstd compressor and decompressor --- include/maxzip/decompressor.hpp | 5 - src/internal.hpp | 14 +-- src/zstd.cpp | 162 +++++++++++++++++++++++++++++++- tests/CMakeLists.txt | 3 +- tests/unit.cpp | 15 +++ 5 files changed, 179 insertions(+), 20 deletions(-) diff --git a/include/maxzip/decompressor.hpp b/include/maxzip/decompressor.hpp index e8291e6..0944753 100644 --- a/include/maxzip/decompressor.hpp +++ b/include/maxzip/decompressor.hpp @@ -41,11 +41,6 @@ namespace maxzip struct zstd_decompressor_params { std::optional window_log_max; - std::optional decoder_format; - std::optional stable_output_buffer; - std::optional ignore_checksum; - std::optional multiple_dictionaries; - std::optional disable_huffman_assembly; }; decompressor *create_brotli_decompressor(const brotli_decompressor_params ¶ms = {}); diff --git a/src/internal.hpp b/src/internal.hpp index 19623c8..7ead873 100644 --- a/src/internal.hpp +++ b/src/internal.hpp @@ -32,6 +32,7 @@ #include +#include #include #include @@ -44,19 +45,6 @@ namespace maxzip return (value >= min && value <= max); } - class zstd_compressor : public compressor - { - public: - zstd_compressor(int level); - ~zstd_compressor(); - size_t compress( - const uint8_t* input, - size_t input_size, - uint8_t* output, - size_t& output_size) override; - private: - ZSTD_CCtx* _cctx; - }; } #endif \ No newline at end of file diff --git a/src/zstd.cpp b/src/zstd.cpp index f2c13af..81abaa6 100644 --- a/src/zstd.cpp +++ b/src/zstd.cpp @@ -1 +1,161 @@ -#include \ No newline at end of file +#include + +namespace maxzip +{ + template< + class ContextType, + class ParameterType, + class SetParameterFuncType, + SetParameterFuncType SetParameterFunc, + class DeleterType, + DeleterType DeleterFunc> + class zstd_context + { + public: + static void deleter(ContextType *ctx) + { + if (ctx != nullptr) + { + DeleterFunc(ctx); + } + } + + zstd_context(ContextType *ctx) : _ctx(ctx, deleter) + { + } + + void set_parameter(ParameterType key, int value) + { + const size_t ret = SetParameterFunc(_ctx.get(), key, value); + if (ZSTD_isError(ret)) + { + throw std::runtime_error("Failed to set Zstandard parameter: " + std::to_string(key)); + } + } + + void set_flag(ParameterType key, bool value) + { + static_cast(SetParameterFunc(_ctx.get(), key, value ? 1 : 0)); + } + + protected: + std::unique_ptr> _ctx; + }; + + class zstd_compressor : public compressor, public zstd_context + { + public: + zstd_compressor() : zstd_context(ZSTD_createCCtx()) + { + } + + size_t compress( + const uint8_t *input, + size_t input_size, + uint8_t *output, + size_t &output_size) override + { + size_t compressed_size(0); + if (output != nullptr) + { + static_cast(ZSTD_CCtx_reset(_ctx.get(), ZSTD_reset_session_only)); + compressed_size = ZSTD_compressCCtx( + _ctx.get(), + output, + output_size, + input, + input_size, + 0); + if(ZSTD_isError(compressed_size)) + { + throw std::runtime_error("Zstandard compression failed: " + std::string(ZSTD_getErrorName(compressed_size))); + } + } + else + { + output_size = ZSTD_compressBound(input_size); + } + return compressed_size; + } + }; + + class zstd_decompressor : public decompressor, public zstd_context + { + public: + zstd_decompressor() : zstd_context(ZSTD_createDCtx()) + { + } + + size_t decompress( + const uint8_t *input, + size_t input_size, + uint8_t *output, + size_t output_size) override + { + size_t decompressed_size = ZSTD_decompressDCtx( + _ctx.get(), + output, + output_size, + input, + input_size); + if (ZSTD_isError(decompressed_size)) + { + throw std::runtime_error("Zstandard decompression failed: " + std::string(ZSTD_getErrorName(decompressed_size))); + } + return decompressed_size; + } + }; + + compressor *create_zstd_compressor(const zstd_compressor_params ¶ms) + { + std::unique_ptr compressor = std::make_unique(); + + const std::unordered_map> param_map = { + {ZSTD_c_compressionLevel, params.level}, + {ZSTD_c_windowLog, params.window_log}, + {ZSTD_c_hashLog, params.hash_log}, + {ZSTD_c_chainLog, params.chain_log}, + {ZSTD_c_searchLog, params.search_log}, + {ZSTD_c_minMatch, params.min_match}, + {ZSTD_c_targetLength, params.target_length}, + {ZSTD_c_strategy, params.strategy} + }; + + const std::unordered_map> flag_map = { + {ZSTD_c_enableLongDistanceMatching, params.enable_long_distance_matching}, + {ZSTD_c_contentSizeFlag, params.enable_content_size}, + {ZSTD_c_checksumFlag, params.enable_checksum}, + {ZSTD_c_dictIDFlag, params.enable_dict_id} + }; + + for (const auto &[key, value] : param_map) + { + if (value.has_value()) + { + compressor->set_parameter(key, value.value()); + } + } + + for (const auto &[key, value] : flag_map) + { + if (value.has_value()) + { + compressor->set_flag(key, value.value()); + } + } + + return compressor.release(); + } + + decompressor *create_zstd_decompressor(const zstd_decompressor_params ¶ms) + { + std::unique_ptr decompressor = std::make_unique(); + + if(params.window_log_max.has_value()) + { + decompressor->set_parameter(ZSTD_d_windowLogMax, params.window_log_max.value()); + } + + return decompressor.release(); + } +} \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7354852..d59789e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -37,4 +37,5 @@ if(MAXZIP_COVER) endif() maxtest_add_test(unit brotli::block) -maxtest_add_test(unit zlib::block) \ No newline at end of file +maxtest_add_test(unit zlib::block) +maxtest_add_test(unit zstd::block) \ No newline at end of file diff --git a/tests/unit.cpp b/tests/unit.cpp index 9d9790f..0008fbb 100644 --- a/tests/unit.cpp +++ b/tests/unit.cpp @@ -161,4 +161,19 @@ MAXTEST_MAIN MAXTEST_ASSERT(decompressor_result.first && (decompressor_result.second != nullptr)); test_block_compression(compressor_result.second, decompressor_result.second); }; + + MAXTEST_TEST_CASE(zstd::block) + { + maxzip::zstd_compressor_params compress_params; + maxzip::zstd_decompressor_params decompress_params; + compress_params.window_log = -100; + auto compressor_result = try_create_compressor(maxzip::create_zstd_compressor, compress_params); + MAXTEST_ASSERT(!compressor_result.first && (compressor_result.second == nullptr)); + compress_params.window_log.reset(); + compressor_result = try_create_compressor(maxzip::create_zstd_compressor, compress_params); + MAXTEST_ASSERT(compressor_result.first && (compressor_result.second != nullptr)); + auto decompressor_result = try_create_decompressor(maxzip::create_zstd_decompressor, decompress_params); + MAXTEST_ASSERT(decompressor_result.first && (decompressor_result.second != nullptr)); + test_block_compression(compressor_result.second, decompressor_result.second); + }; } \ No newline at end of file From 0dfb4a1e0403df1d2fee513e39b28d092b8a004c Mon Sep 17 00:00:00 2001 From: John R Patek Sr Date: Wed, 30 Jul 2025 19:56:44 -0500 Subject: [PATCH 2/6] trying to improve coverage --- src/zstd.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/zstd.cpp b/src/zstd.cpp index 81abaa6..f14c7d7 100644 --- a/src/zstd.cpp +++ b/src/zstd.cpp @@ -14,10 +14,7 @@ namespace maxzip public: static void deleter(ContextType *ctx) { - if (ctx != nullptr) - { - DeleterFunc(ctx); - } + static_cast(DeleterFunc(ctx)); } zstd_context(ContextType *ctx) : _ctx(ctx, deleter) @@ -27,7 +24,8 @@ namespace maxzip void set_parameter(ParameterType key, int value) { const size_t ret = SetParameterFunc(_ctx.get(), key, value); - if (ZSTD_isError(ret)) + const bool is_error = (ZSTD_isError(ret) == 1); + if (is_error) { throw std::runtime_error("Failed to set Zstandard parameter: " + std::to_string(key)); } From ba2fbb402b3f75ee496d77fc3e007760897b1269 Mon Sep 17 00:00:00 2001 From: John R Patek Sr Date: Wed, 30 Jul 2025 20:03:05 -0500 Subject: [PATCH 3/6] trying to hit more branches --- tests/unit.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit.cpp b/tests/unit.cpp index 0008fbb..2e65c19 100644 --- a/tests/unit.cpp +++ b/tests/unit.cpp @@ -169,10 +169,15 @@ MAXTEST_MAIN compress_params.window_log = -100; auto compressor_result = try_create_compressor(maxzip::create_zstd_compressor, compress_params); MAXTEST_ASSERT(!compressor_result.first && (compressor_result.second == nullptr)); - compress_params.window_log.reset(); + compress_params.window_log = 0; + compress_params.enable_checksum = true; compressor_result = try_create_compressor(maxzip::create_zstd_compressor, compress_params); MAXTEST_ASSERT(compressor_result.first && (compressor_result.second != nullptr)); + decompress_params.window_log_max = -100; auto decompressor_result = try_create_decompressor(maxzip::create_zstd_decompressor, decompress_params); + MAXTEST_ASSERT(!decompressor_result.first && (decompressor_result.second == nullptr)); + decompress_params.window_log_max.reset(); + decompressor_result = try_create_decompressor(maxzip::create_zstd_decompressor, decompress_params); MAXTEST_ASSERT(decompressor_result.first && (decompressor_result.second != nullptr)); test_block_compression(compressor_result.second, decompressor_result.second); }; From 74621c0477f7eddf7f6d1fe94d83fd3887bb1eee Mon Sep 17 00:00:00 2001 From: John R Patek Sr Date: Wed, 30 Jul 2025 20:09:42 -0500 Subject: [PATCH 4/6] trying a different map strategy --- src/zstd.cpp | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/zstd.cpp b/src/zstd.cpp index f14c7d7..6ecdb00 100644 --- a/src/zstd.cpp +++ b/src/zstd.cpp @@ -108,23 +108,21 @@ namespace maxzip { std::unique_ptr compressor = std::make_unique(); - const std::unordered_map> param_map = { - {ZSTD_c_compressionLevel, params.level}, - {ZSTD_c_windowLog, params.window_log}, - {ZSTD_c_hashLog, params.hash_log}, - {ZSTD_c_chainLog, params.chain_log}, - {ZSTD_c_searchLog, params.search_log}, - {ZSTD_c_minMatch, params.min_match}, - {ZSTD_c_targetLength, params.target_length}, - {ZSTD_c_strategy, params.strategy} - }; - - const std::unordered_map> flag_map = { - {ZSTD_c_enableLongDistanceMatching, params.enable_long_distance_matching}, - {ZSTD_c_contentSizeFlag, params.enable_content_size}, - {ZSTD_c_checksumFlag, params.enable_checksum}, - {ZSTD_c_dictIDFlag, params.enable_dict_id} - }; + std::unordered_map> param_map; + param_map[ZSTD_c_compressionLevel] = params.level; + param_map[ZSTD_c_windowLog] = params.window_log; + param_map[ZSTD_c_hashLog] = params.hash_log; + param_map[ZSTD_c_chainLog] = params.chain_log; + param_map[ZSTD_c_searchLog] = params.search_log; + param_map[ZSTD_c_minMatch] = params.min_match; + param_map[ZSTD_c_targetLength] = params.target_length; + param_map[ZSTD_c_strategy] = params.strategy; + + std::unordered_map> flag_map; + flag_map[ZSTD_c_enableLongDistanceMatching] = params.enable_long_distance_matching; + flag_map[ZSTD_c_contentSizeFlag] = params.enable_content_size; + flag_map[ZSTD_c_checksumFlag] = params.enable_checksum; + flag_map[ZSTD_c_dictIDFlag] = params.enable_dict_id; for (const auto &[key, value] : param_map) { From 15885ca22d10c029e08c315f472aca256b4c0d4b Mon Sep 17 00:00:00 2001 From: John R Patek Sr Date: Wed, 30 Jul 2025 20:14:24 -0500 Subject: [PATCH 5/6] trying one more time with set_parameter --- tests/unit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit.cpp b/tests/unit.cpp index 2e65c19..2436ac5 100644 --- a/tests/unit.cpp +++ b/tests/unit.cpp @@ -176,7 +176,7 @@ MAXTEST_MAIN decompress_params.window_log_max = -100; auto decompressor_result = try_create_decompressor(maxzip::create_zstd_decompressor, decompress_params); MAXTEST_ASSERT(!decompressor_result.first && (decompressor_result.second == nullptr)); - decompress_params.window_log_max.reset(); + decompress_params.window_log_max = 0; decompressor_result = try_create_decompressor(maxzip::create_zstd_decompressor, decompress_params); MAXTEST_ASSERT(decompressor_result.first && (decompressor_result.second != nullptr)); test_block_compression(compressor_result.second, decompressor_result.second); From 03b1a4fee4008107b03d09f8c135d761ea4ad299 Mon Sep 17 00:00:00 2001 From: John R Patek Sr Date: Wed, 30 Jul 2025 20:18:11 -0500 Subject: [PATCH 6/6] trying to cover last zstd branch --- tests/unit.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit.cpp b/tests/unit.cpp index 2436ac5..cdbb178 100644 --- a/tests/unit.cpp +++ b/tests/unit.cpp @@ -179,6 +179,9 @@ MAXTEST_MAIN decompress_params.window_log_max = 0; decompressor_result = try_create_decompressor(maxzip::create_zstd_decompressor, decompress_params); MAXTEST_ASSERT(decompressor_result.first && (decompressor_result.second != nullptr)); + decompress_params.window_log_max.reset(); + decompressor_result = try_create_decompressor(maxzip::create_zstd_decompressor, decompress_params); + MAXTEST_ASSERT(decompressor_result.first && (decompressor_result.second != nullptr)); test_block_compression(compressor_result.second, decompressor_result.second); }; } \ No newline at end of file