From b7f646e8b285e0813820300a6d83865d131b27f5 Mon Sep 17 00:00:00 2001 From: Olasoji Date: Thu, 23 Oct 2025 14:46:58 -0700 Subject: [PATCH 01/18] Adds test to validate log_stats_samples config Signed-off-by: Olasoji --- tests/zlib_accel_test.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/zlib_accel_test.cpp b/tests/zlib_accel_test.cpp index 6d66f9c..33d024a 100644 --- a/tests/zlib_accel_test.cpp +++ b/tests/zlib_accel_test.cpp @@ -1231,6 +1231,7 @@ void CreateAndWriteTempConfigFile(const char* file_path) { temp_file << "use_zlib_compress=!0222\n"; temp_file << "use_zlib_uncompress=AB23\n"; temp_file << "log_level=10\n"; + temp_file << "log_stats_samples=4294967296\n"; temp_file.close(); } @@ -1243,6 +1244,7 @@ TEST_F(ConfigLoaderTest, LoadInvalidConfig) { uint32_t DEFAULT_ZLIB_COMPRESS = GetConfig(USE_ZLIB_COMPRESS); uint32_t DEFAULT_ZLIB_UNCOMPRESS = GetConfig(USE_ZLIB_UNCOMPRESS); uint32_t DEFAULT_LOG_LEVEL = GetConfig(LOG_LEVEL); + uint32_t DEFAULT_LOG_STATS_SAMPLES = GetConfig(LOG_STATS_SAMPLES); CreateAndWriteTempConfigFile("/tmp/invalid_config"); EXPECT_TRUE(LoadConfigFile(file_content, "/tmp/invalid_config")); @@ -1253,6 +1255,7 @@ TEST_F(ConfigLoaderTest, LoadInvalidConfig) { EXPECT_EQ(GetConfig(USE_ZLIB_COMPRESS), DEFAULT_ZLIB_COMPRESS); EXPECT_EQ(GetConfig(USE_ZLIB_UNCOMPRESS), DEFAULT_ZLIB_UNCOMPRESS); EXPECT_EQ(GetConfig(LOG_LEVEL), DEFAULT_LOG_LEVEL); + EXPECT_EQ(GetConfig(LOG_STATS_SAMPLES), DEFAULT_LOG_STATS_SAMPLES); std::remove("/tmp/invalid_config"); // Restore config from official config file LoadConfigFile(file_content); From db531ffb6db578628a824b90fbe685dcc6ad0e41 Mon Sep 17 00:00:00 2001 From: Olasoji Date: Wed, 17 Dec 2025 16:01:34 -0800 Subject: [PATCH 02/18] Reduces excessive error logging in QAT mode Signed-off-by: Olasoji --- qat.cpp | 1 + qat.h | 2 ++ zlib_accel.cpp | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/qat.cpp b/qat.cpp index 2ef4c0c..d5bc57e 100644 --- a/qat.cpp +++ b/qat.cpp @@ -100,6 +100,7 @@ void QATJob::Init(QzSessionPtr &qzSession, CompressedFormat format, } // Initialize QAT hardware + qzSetLogLevel(LOG_NONE); int status = qzInit(session.get(), 0); if (status != QZ_OK && status != QZ_DUPLICATE) { Log(LogLevel::LOG_ERROR, "qzInit() failure Line ", __LINE__, " session ", diff --git a/qat.h b/qat.h index 3a04045..f81285e 100644 --- a/qat.h +++ b/qat.h @@ -14,6 +14,8 @@ #define VISIBLE_FOR_TESTING __attribute__((visibility("default"))) +inline constexpr unsigned int QAT_DEST_BUFFER_MIN_SIZE = 512; + inline constexpr unsigned int QAT_HW_BUFF_SZ = QZ_HW_BUFF_MAX_SZ; class QATJob { diff --git a/zlib_accel.cpp b/zlib_accel.cpp index e260841..018f7e1 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -297,6 +297,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { #ifdef USE_QAT qat_available = configs[USE_QAT_COMPRESS] && + output_len >= QAT_DEST_BUFFER_MIN_SIZE && SupportedOptionsQAT(deflate_settings->window_bits, input_len); #endif @@ -1352,7 +1353,6 @@ int ZEXPORT gzeof(gzFile file) { GzipFile* gz = gzip_files.Get(file); return gz->reached_eof; } - #if defined(__clang__) #pragma clang attribute pop #endif From 5d513a4659e427d39a2d419dc645763abb0b36c7 Mon Sep 17 00:00:00 2001 From: Olasoji Date: Wed, 17 Dec 2025 16:05:22 -0800 Subject: [PATCH 03/18] Bugfix for the log stream Signed-off-by: Olasoji --- logging.h | 1 + 1 file changed, 1 insertion(+) diff --git a/logging.h b/logging.h index 7b74d9f..b5e8301 100644 --- a/logging.h +++ b/logging.h @@ -61,6 +61,7 @@ inline void Log(LogLevel level, Args&&... args) { } std::ostream& stream = GetLogStream(); + stream << std::dec; switch (level) { case LogLevel::LOG_ERROR: stream << "Error: "; From 9aff4f8c50af5b3bd3ae92c24af17848a8338ccd Mon Sep 17 00:00:00 2001 From: Olasoji Date: Wed, 17 Dec 2025 16:07:54 -0800 Subject: [PATCH 04/18] Treats QPL_STS_MORE_OUTPUT_NEEDED as a non error condition for IAA Signed-off-by: Olasoji --- iaa.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iaa.cpp b/iaa.cpp index 1ec6fe2..2e0ca66 100644 --- a/iaa.cpp +++ b/iaa.cpp @@ -218,7 +218,7 @@ int UncompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, job->dictionary = nullptr; qpl_status status = qpl_execute_job(job); - if (status != QPL_STS_OK) { + if (status != QPL_STS_OK && status != QPL_STS_MORE_OUTPUT_NEEDED) { Log(LogLevel::LOG_ERROR, "UncompressIAA() Line ", __LINE__, " qpl_execute_job status ", status, "\n"); return 1; From 806ce1dd91df3e071b1f1108a5b54a61754e70d0 Mon Sep 17 00:00:00 2001 From: Olasoji Date: Thu, 18 Dec 2025 11:11:30 -0800 Subject: [PATCH 05/18] Fixed formatting Signed-off-by: Olasoji --- zlib_accel.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 018f7e1..72c5105 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -296,8 +296,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { #endif #ifdef USE_QAT qat_available = - configs[USE_QAT_COMPRESS] && - output_len >= QAT_DEST_BUFFER_MIN_SIZE && + configs[USE_QAT_COMPRESS] && output_len >= QAT_DEST_BUFFER_MIN_SIZE && SupportedOptionsQAT(deflate_settings->window_bits, input_len); #endif From 1b66d120a42414404317c9362148966473e38538 Mon Sep 17 00:00:00 2001 From: Olasoji Date: Tue, 17 Feb 2026 11:35:57 -0800 Subject: [PATCH 06/18] Initial commit of igzip. WIP --- CMakeLists.txt | 2 +- common.cmake | 20 ++ config/config.cpp | 6 + config/config.h | 2 + config/default_config | 2 + igzip.cpp | 627 ++++++++++++++++++++++++++++++++++++++ igzip.h | 48 +++ tests/CMakeLists.txt | 2 +- tests/zlib_accel_test.cpp | 62 ++++ zlib_accel.cpp | 90 +++++- zlib_accel.h | 2 +- 11 files changed, 859 insertions(+), 4 deletions(-) create mode 100644 igzip.cpp create mode 100644 igzip.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b36c5e..445c160 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ if(ENABLE_STATISTICS) add_compile_definitions(ENABLE_STATISTICS) endif() -add_library(${PROJECT_NAME} SHARED config/config_reader.cpp config/config.cpp zlib_accel.cpp iaa.cpp qat.cpp utils.cpp statistics.cpp) +add_library(${PROJECT_NAME} SHARED config/config_reader.cpp config/config.cpp zlib_accel.cpp iaa.cpp qat.cpp utils.cpp statistics.cpp igzip.cpp) add_custom_target(format find .. -iname '*.h' -o -iname '*.cpp' | xargs clang-format -style=Google -i diff --git a/common.cmake b/common.cmake index 7c60c0f..a422cb5 100644 --- a/common.cmake +++ b/common.cmake @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 option(USE_IAA "Use IAA (requires QPL)" OFF) +option(USE_IGZIP "Use IGZIP (requires isa-l)" OFF) option(USE_QAT "Use QAT (requires QATzip)" OFF) option(DEBUG_LOG "for logging" ON) option(COVERAGE "for coverage" OFF) @@ -14,6 +15,10 @@ if(USE_IAA) add_compile_definitions(USE_IAA) endif() +if(USE_IGZIP) + add_compile_definitions(USE_IGZIP) +endif() + if(USE_QAT) add_compile_definitions(USE_QAT) endif() @@ -89,6 +94,21 @@ if(USE_IAA) endif() endif() +if(USE_IGZIP) + if(NOT DEFINED ISAL_PATH) + find_package(isal REQUIRED) + if(isal_FOUND) + message(STATUS "Found ISA-L: ${isal_DIR}") + link_libraries(isal) + endif() + else() + message(STATUS "Using ISAL_PATH: ${ISAL_PATH}") + include_directories(${ISAL_PATH}/include) + link_directories(PUBLIC ${ISAL_PATH}/.libs) + link_libraries(isal) + endif() +endif() + if(USE_QAT) if(DEFINED QATZIP_PATH) message(STATUS "Using QATZIP_PATH: ${QATZIP_PATH}") diff --git a/config/config.cpp b/config/config.cpp index 29f93cf..51a30af 100644 --- a/config/config.cpp +++ b/config/config.cpp @@ -21,6 +21,8 @@ uint32_t configs[CONFIG_MAX] = { 0, /*use_iaa_uncompress*/ 1, /*use_zlib_compress*/ 1, /*use_zlib_uncompress*/ + 0, /*use_igzip_compress*/ + 0, /*use_igzip_uncompress*/ 50, /*iaa_compress_percentage*/ 50, /*iaa_uncompress_percentage*/ 0, /*iaa_prepend_empty_block*/ @@ -45,6 +47,8 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { "use_iaa_uncompress", "use_zlib_compress", "use_zlib_uncompress", + "use_igzip_compress", + "use_igzip_uncompress", "iaa_compress_percentage", "iaa_uncompress_percentage", "iaa_prepend_empty_block", @@ -78,6 +82,8 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { trySetConfig(USE_IAA_UNCOMPRESS, 1, 0); trySetConfig(USE_ZLIB_COMPRESS, 1, 0); trySetConfig(USE_ZLIB_UNCOMPRESS, 1, 0); + trySetConfig(USE_IGZIP_COMPRESS, 1, 0); + trySetConfig(USE_IGZIP_UNCOMPRESS, 1, 0); trySetConfig(IAA_COMPRESS_PERCENTAGE, 100, 0); trySetConfig(IAA_UNCOMPRESS_PERCENTAGE, 100, 0); trySetConfig(IAA_PREPEND_EMPTY_BLOCK, 1, 0); diff --git a/config/config.h b/config/config.h index acc8055..12d3563 100644 --- a/config/config.h +++ b/config/config.h @@ -16,6 +16,8 @@ enum ConfigOption { USE_IAA_UNCOMPRESS, USE_ZLIB_COMPRESS, USE_ZLIB_UNCOMPRESS, + USE_IGZIP_COMPRESS, + USE_IGZIP_UNCOMPRESS, IAA_COMPRESS_PERCENTAGE, IAA_UNCOMPRESS_PERCENTAGE, IAA_PREPEND_EMPTY_BLOCK, diff --git a/config/default_config b/config/default_config index 6ae0cc8..be76f5f 100644 --- a/config/default_config +++ b/config/default_config @@ -4,6 +4,8 @@ use_iaa_compress = 0 use_iaa_uncompress = 0 use_zlib_compress = 1 use_zlib_uncompress = 1 +use_igzip_compress = 0 +use_igzip_uncompress = 0 iaa_compress_percentage = 50 iaa_uncompress_percentage = 50 iaa_prepend_empty_block = 0 diff --git a/igzip.cpp b/igzip.cpp new file mode 100644 index 0000000..52874bd --- /dev/null +++ b/igzip.cpp @@ -0,0 +1,627 @@ +// Copyright (C) 2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#ifdef USE_IGZIP + +#include "igzip.h" +#include "crc.h" + +#include +#include +#include +#include "logging.h" + +struct isal_zstream* +InitCompressIGZIP(int level, int windowBits) +{ +#ifdef DEBUG + fprintf(stderr, + "\nInitializing deflate with level: %d, method: %d, windowBits: %d, memLevel: %d, " + "strategy: %d\n", + level, method, windowBits, memLevel, strategy); +#endif + Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, " level ", + level, ", windowBits ",windowBits, " \n"); + + struct isal_zstream *isal_strm = + (struct isal_zstream *) malloc(sizeof(struct isal_zstream)); + if (!isal_strm) { + fprintf(stderr, "Error: Memory allocation for isal_zstream failed\n"); + return nullptr; + } + + /* Setup ISA-L compression context */ + isal_deflate_init(isal_strm); + + isal_strm->end_of_stream = 0; + isal_strm->flush = NO_FLUSH; + + // Map Zlib levels to ISA-L levels + if (level >= 1 && level <= 2) { + isal_strm->level = 1; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL1_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL1_DEFAULT; + } else if ((level >= 3 && level <= 6) || level == -1) { + isal_strm->level = 2; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL2_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL2_DEFAULT; + } else if (level >= 7 && level <= 9) { + isal_strm->level = 3; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL3_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL3_DEFAULT; + } else { + fprintf(stderr, "Error: Invalid compression level\n"); + return nullptr; + } + + if (!isal_strm->level_buf) { + free(isal_strm); + fprintf(stderr, "Error: Memory allocation for level_buf failed\n"); + return nullptr; + } + + // Set stream->gzip_flag and hist_bits + // Ensure hist_bits are non-negative + if (windowBits < 0) { + // Raw deflate mode - no headers/trailers + isal_strm->gzip_flag = IGZIP_DEFLATE; + isal_strm->hist_bits = -windowBits; + } else { + // Standard zlib format + isal_strm->gzip_flag = IGZIP_ZLIB; + isal_strm->hist_bits = windowBits; + } + + return isal_strm; +} + +/*int +deflateInit_(z_streamp strm, int level) +{ + if (!strm) { + fprintf(stderr, "Error: z_streamp is NULL\n"); + return -1; + } + + return deflateInit2_(strm, level, 0, 15, 8, 0); // hardcoded windowBits +}*/ + +int +CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, + uint32_t *input_length, uint8_t *output, uint32_t *output_length, + unsigned long *total_in, unsigned long *total_out) { + int ret; + + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " input_length ", + *input_length, " \n"); + if (!isal_strm) { + fprintf(stderr, "Error: deflate isal_strm is NULL\n"); + return -1; + } + + // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ + isal_strm->next_out = output; + isal_strm->avail_out = *output_length; + isal_strm->next_in = input; + isal_strm->avail_in = *input_length; + isal_strm->total_out = *total_out; + isal_strm->total_in = *total_in; + + // stream->flush mapping + switch (flush) { + case Z_NO_FLUSH: + isal_strm->flush = NO_FLUSH; + break; + case Z_SYNC_FLUSH: + case Z_PARTIAL_FLUSH: + case Z_BLOCK: + isal_strm->flush = SYNC_FLUSH; + break; + case Z_FULL_FLUSH: + isal_strm->flush = FULL_FLUSH; + break; + case Z_FINISH: + isal_strm->flush = FULL_FLUSH; + isal_strm->end_of_stream = 1; + break; + default: + fprintf(stderr, "Error: Invalid flush value\n"); + return -1; + } + +#ifdef DEBUG + fprintf(stderr, "Gzip flag: %d, Window bits: %d, Flush: %d, Level: %d\n", + isal_strm->gzip_flag, isal_strm->hist_bits, isal_strm->flush, isal_strm->level); + fprintf(stderr, "Before isal_deflate: avail_in=%u, next_in=%p, avail_out=%u, next_out=%p\n", + isal_strm->avail_in, isal_strm->next_in, isal_strm->avail_out, isal_strm->next_out); + fprintf(stderr, "Total out: %u, Total in %lu\n", isal_strm->total_out, strm->total_in); +#endif + + int comp = isal_deflate(isal_strm); + + *output_length = isal_strm->avail_out; + *input_length = isal_strm->avail_in; + input = isal_strm->next_in; + output = isal_strm->next_out; + *total_out = isal_strm->total_out; + *total_in = isal_strm->total_in; + +#ifdef DEBUG + fprintf(stderr, "After isal_deflate: avail_in=%u, next_in=%p, avail_out=%u, next_out=%p\n", + strm->avail_in, strm->next_in, strm->avail_out, strm->next_out); + fprintf(stderr, "Total out: %lu, Total in: %lu\n", strm->total_out, strm->total_in); +#endif + + if (comp == COMP_OK) { + if (isal_strm->end_of_stream && isal_strm->avail_out > 0) { + ret = Z_STREAM_END; // Compression is done + } else { + ret = Z_OK; + } + } else { + ret = Z_ERRNO; // Compression error + } + +#ifdef DEBUG + if (ret == Z_OK) { + fprintf(stderr, "Deflate finished successfully Z_OK\n"); + } else if (ret == Z_STREAM_END) { + fprintf(stderr, "Deflate finished successfully Z_STREAM_END\n"); + } else { + fprintf(stderr, "Deflate finished with error code: %d\n", ret); + switch (comp) { + case INVALID_FLUSH: + fprintf(stderr, "Error: Invalid flush\n"); + break; + case INVALID_PARAM: + fprintf(stderr, "Error: Invalid parameter\n"); + break; + case STATELESS_OVERFLOW: + fprintf(stderr, "Error: Stateless overflow\n"); + break; + case ISAL_INVALID_OPERATION: + fprintf(stderr, "Error: Invalid operation\n"); + break; + case ISAL_INVALID_STATE: + fprintf(stderr, "Error: Invalid state\n"); + break; + case ISAL_INVALID_LEVEL: + fprintf(stderr, "Error: Invalid level\n"); + break; + case ISAL_INVALID_LEVEL_BUF: + fprintf(stderr, "Error: Invalid level buffer\n"); + break; + } + } +#endif + + return ret; +} + +int +EndCompressIGZIP(struct isal_zstream* isal_strm) +{ + if (!isal_strm) { + fprintf(stderr, "Error: isal_stream is NULL\n"); + return -1; + } + + // Free allocated memory for level_buf and isal_strm + if (isal_strm->level_buf) { + free(isal_strm->level_buf); + } + free(isal_strm); + +#ifdef DEBUG + fprintf(stderr, "Deflate end\n"); +#endif + return Z_OK; +} + +/*int +deflateSetHeader(z_streamp strm, void *head) +{ + (void) head; // Suppress unused parameter warning + return Z_OK; +}*/ + +int +deflateSetDictionary(z_streamp strm, unsigned char *dict_data, unsigned int dict_len) +{ + if (!strm || !strm->state || !dict_data || dict_len == 0) + return Z_STREAM_ERROR; + + deflate_state *s = (deflate_state *) strm->state; + + if (!s || !s->isal_strm) + return Z_STREAM_ERROR; + + return isal_deflate_set_dict(s->isal_strm, dict_data, dict_len); +} + +/*int +compress2(uint8_t *dest, unsigned long *dest_len, const uint8_t *source, unsigned long source_len, + int level) +{ + z_stream strm; + int err; + const unsigned int max = (unsigned int) -1; + unsigned long left = *dest_len; + if (dest == NULL || dest_len == NULL || source == NULL) { + return Z_STREAM_ERROR; + } + *dest_len = 0; + + strm.zalloc = NULL; + strm.zfree = NULL; + strm.opaque = NULL; + + err = deflateInit_(&strm, level); + if (err != Z_OK) + return err; + + strm.next_out = dest; + strm.avail_out = 0; + strm.next_in = (uint8_t *) source; + strm.avail_in = 0; + + do { + if (strm.avail_out == 0) { + strm.avail_out = left > (unsigned long) max ? max : (unsigned int) left; + left -= strm.avail_out; + } + if (strm.avail_in == 0) { + strm.avail_in = + source_len > (unsigned long) max ? max : (unsigned int) source_len; + source_len -= strm.avail_in; + } + err = deflate(&strm, source_len ? Z_NO_FLUSH : Z_FINISH); + } while (err == Z_OK); + + *dest_len = strm.total_out; + deflateEnd(&strm); + + return err == Z_STREAM_END ? Z_OK : err; +} + +int +compress(uint8_t *dest, unsigned long *dest_len, const uint8_t *source, unsigned long source_len) +{ + return compress2(dest, dest_len, source, source_len, Z_DEFAULT_COMPRESSION); +} +*/ +unsigned long +crc32(unsigned long crc, const unsigned char *buf, unsigned int len) +{ + return crc32_gzip_refl(crc, buf, len); +} + +unsigned long +adler32(unsigned long adler, const unsigned char *buf, unsigned int len) +{ + return isal_adler32(adler, buf, len); +} + +//inflateInit2_(z_streamp strm, int windowBits) +struct inflate_state* +InitUncompressIGZIP(int windowBits) +{ + struct inflate_state *isal_strm_inflate = + (struct inflate_state *) malloc(sizeof(struct inflate_state)); + if (!isal_strm_inflate) { + fprintf(stderr, "Error: Memory allocation for inflate_state failed\n"); + return nullptr; + } + +#ifdef DEBUG + fprintf(stderr, "\nInitializing inflate with windowBits: %d", windowBits); +#endif + Log(LogLevel::LOG_INFO, "InitUncompressIGZIP() Line ", __LINE__, + ", windowBits ",windowBits, " \n"); + + /* Setup ISA-L decompression context */ + isal_inflate_init(isal_strm_inflate); + + isal_strm_inflate->avail_in = 0; + isal_strm_inflate->next_in = NULL; + //strm->total_out = 0; + //strm->total_in = 0; + + //s->trailer_overconsumption_fixed = 0; // Initialize the workaround flag + + if (windowBits < 0) { + // Raw deflate mode - no headers/trailers + isal_strm_inflate->crc_flag = IGZIP_DEFLATE; + isal_strm_inflate->hist_bits = -windowBits; + } else { + // Standard zlib format + isal_strm_inflate->crc_flag = IGZIP_ZLIB; + isal_strm_inflate->hist_bits = windowBits; + } + + return isal_strm_inflate; +} + +/*int +inflateInit_(z_streamp strm) +{ + if (!strm) { + fprintf(stderr, "Error: z_streamp is NULL\n"); + return Z_STREAM_ERROR; + } + + return inflateInit2_(strm, 15); // hardcoded windowBits +}*/ + +int +UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, + uint32_t *input_length, uint8_t *output, uint32_t *output_length, + int *tofixed, unsigned long *total_in, unsigned long *total_out) +{ + if (!isal_strm_inflate) { + fprintf(stderr, "Error: isal_strm_inflate is NULL\n"); + return Z_STREAM_ERROR; + } + + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + "input length ", *input_length,"\n"); + // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ + isal_strm_inflate->next_out = output; + isal_strm_inflate->avail_out = *output_length; + isal_strm_inflate->avail_in = *input_length; + isal_strm_inflate->next_in = input; + isal_strm_inflate->total_out = *total_out; + + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " CRC flag: ", (uint32_t)isal_strm_inflate->crc_flag, + " Before isal_inflate: avail_in ",isal_strm_inflate->avail_in, + //" next_in= ", isal_strm_inflate->next_in, + " avail_out= ",(uint32_t)isal_strm_inflate->avail_out, + //" next_out= ", isal_strm_inflate->next_out, + " Total out: ", (uint32_t)isal_strm_inflate->total_out, + " Total_in: ", (unsigned long)*total_in); + + const int decomp = isal_inflate(isal_strm_inflate); + + const unsigned long total_in_ = *total_in; + const unsigned int original_avail_in = *input_length; + //const unsigned int bytes_consumed = original_avail_in - isal_strm_inflate->avail_in; + + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " After isal_inflate: avail_in ",isal_strm_inflate->avail_in, + //" next_in= ", isal_strm_inflate->next_in, + " avail_out= ",(uint32_t)isal_strm_inflate->avail_out, + //" next_out= ", isal_strm_inflate->next_out, + " Total out: ", (uint32_t)isal_strm_inflate->total_out, + " Total_in: ", total_in_ + bytes_consumed, + " Bytes consumed this call: ", bytes_consumed, + " Block state: ", isal_strm_inflate->block_state, " (ISAL_BLOCK_FINISH=", + ISAL_BLOCK_FINISH, " ISA-L result: ", decomp, "\n"); + + if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH && + isal_strm_inflate->avail_in > 0) { + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " WARNING: BLOCK_FINISH reached but ",isal_strm_inflate->avail_in, "bytes remain in input:\n"); + for (unsigned int i = 0; i < isal_strm_inflate->avail_in && i < 16; i++) { + fprintf(stderr, " %02x", ((unsigned char *) isal_strm_inflate->next_in)[i]); + } + fprintf(stderr, "\n"); + } + + // WORKAROUND: ISA-L over-consumption fix for raw deflate mode + if ((isal_strm_inflate->block_state == ISAL_BLOCK_FINISH || + isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE) && + (isal_strm_inflate->crc_flag == 0) && // raw deflate + *tofixed == 0 && // hasn't been applied yet + decomp == ISAL_DECOMP_OK && // successful decompression + isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) { + + // Calculate how many bytes were likely over-consumed + const unsigned int expected_trailer_bytes = 8; + const unsigned int over_consumed = + expected_trailer_bytes - isal_strm_inflate->avail_in; + + // Only apply fix if the over-consumption is reasonable (1-7 bytes) + if (over_consumed >= 1 && over_consumed <= 7) { +#ifdef DEBUG + fprintf(stderr, + "APPLYING WORKAROUND: Detected ISA-L over-consumption of %u " + "bytes\n", + over_consumed); + fprintf(stderr, "Adjusting next_in from %p to %p, avail_in from %u to %u\n", + isal_strm_inflate->next_in, + (unsigned char *) isal_strm_inflate->next_in - over_consumed, + isal_strm_inflate->avail_in, + isal_strm_inflate->avail_in + over_consumed); +#endif + // Rewind the input pointer to restore over-consumed bytes + isal_strm_inflate->next_in = + (unsigned char *) isal_strm_inflate->next_in - over_consumed; + isal_strm_inflate->avail_in += over_consumed; + + // Mark that the workaround has been applied + *tofixed = 1; + + // Also adjust the byte consumption count to reflect the actual deflate data + // consumed Note: bytes_consumed is calculated later, so we'll need to + // adjust it after the calculation + } + } + + // Update stream state - handle byte accounting correctly + *output_length = isal_strm_inflate->avail_out; + *input_length = isal_strm_inflate->avail_in; + input = isal_strm_inflate->next_in; + output = isal_strm_inflate->next_out; + *total_out = isal_strm_inflate->total_out; + + // Calculate bytes consumed by ISA-L from the original input + const unsigned int bytes_consumed_by_isal = original_avail_in - isal_strm_inflate->avail_in; + *total_in = total_in_ + bytes_consumed_by_isal; + + int ret; + + if (decomp == ISAL_DECOMP_OK) { + if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) { + // ISA-L has finished processing the deflate stream including trailer + // validation + ret = Z_STREAM_END; + //strm->msg = "ok"; + } else { + // Still processing, continue + ret = Z_OK; + } + } else if (decomp == ISAL_END_INPUT) { + ret = Z_OK; + } else { + ret = Z_DATA_ERROR; + } + +#ifdef DEBUG + if (ret == Z_OK) { + fprintf(stderr, "Inflate finished successfully Z_OK\n"); + } else if (ret == Z_STREAM_END) { + fprintf(stderr, "Inflate finished with Z_STREAM_END\n"); + } else { + fprintf(stderr, "Inflate finished with error code: %d\n", ret); + switch (decomp) { + case ISAL_INVALID_BLOCK: + fprintf(stderr, "Error: ISA-L error - Invalid block\n"); + break; + case ISAL_INVALID_SYMBOL: + fprintf(stderr, "Error: ISA-L error - Invalid symbol\n"); + break; + case ISAL_INVALID_LOOKBACK: + fprintf(stderr, "Error: ISA-L error - Invalid lookback\n"); + break; + case ISAL_END_INPUT: + fprintf(stderr, "Error: ISA-L error - End of input reached unexpectedly\n"); + break; + case ISAL_UNSUPPORTED_METHOD: + fprintf(stderr, "Error: ISA-L error - Unsupported method\n"); + break; + case ISAL_NEED_DICT: + fprintf(stderr, "Error: ISA-L error - Need dictionary\n"); + break; + default: + fprintf(stderr, "Error: ISA-L error code: %d\n", decomp); + break; + } + } +#endif + + return ret; +} + +int +EndUncompressIGZIP(struct inflate_state *isal_strm_inflate) +{ + if (!isal_strm_inflate) { + fprintf(stderr, "Error: z_streamp is NULL\n"); + return Z_STREAM_ERROR; + } + + free(isal_strm_inflate); + + +#ifdef DEBUG + fprintf(stderr, "Inflate end\n"); +#endif + return Z_OK; +} + +int +inflateSetDictionary(z_streamp strm, unsigned char *dict_data, unsigned int dict_len) +{ + if (!strm || !strm->state || !dict_data || dict_len == 0) + return Z_STREAM_ERROR; + + const inflate_state2 *s = (inflate_state2 *) strm->state; + + if (!s || !s->isal_strm_inflate) + return Z_STREAM_ERROR; + + return isal_inflate_set_dict(s->isal_strm_inflate, dict_data, dict_len); +} + +/*int +uncompress2(uint8_t *dest, unsigned long *dest_len, const uint8_t *source, + unsigned long *source_len) +{ + z_stream strm; + int err; + const unsigned int max = (unsigned int) -1; + unsigned long len, left; + uint8_t buf[1] = { 0 }; // for detection of incomplete strm when *dest_len == 0 + + len = *source_len; + if (*dest_len) { + left = *dest_len; + *dest_len = 0; + } else { + left = 1; + dest = buf; + } + + strm.next_in = (uint8_t *) source; + strm.avail_in = 0; + strm.zalloc = NULL; + strm.zfree = NULL; + strm.opaque = NULL; + + err = inflateInit_(&strm); + if (err != Z_OK) + return err; + + strm.next_out = dest; + strm.avail_out = 0; + + do { + if (strm.avail_out == 0) { + strm.avail_out = left > (unsigned long) max ? max : (unsigned int) left; + left -= strm.avail_out; + } + if (strm.avail_in == 0) { + strm.avail_in = len > (unsigned long) max ? max : (unsigned int) len; + len -= strm.avail_in; + } + err = inflate(&strm, Z_NO_FLUSH); + } while (err == Z_OK); + + *source_len -= len + strm.avail_in; + if (dest != buf) + *dest_len = strm.total_out; + else if (strm.total_out && err == Z_BUF_ERROR) + left = 1; + + inflateEnd(&strm); + return err == Z_STREAM_END ? Z_OK + : err == Z_NEED_DICT ? Z_DATA_ERROR + : err == Z_BUF_ERROR && left + strm.avail_out ? Z_DATA_ERROR + : err; +} + +int +uncompress(uint8_t *dest, unsigned long *dest_len, const uint8_t *source, unsigned long source_len) +{ + return uncompress2(dest, dest_len, source, &source_len); +}*/ + +int +ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed) +{ + if (!isal_strm_inflate) { + fprintf(stderr, "Error: isal_strm_inflate is NULL\n"); + return Z_STREAM_ERROR; + } + + // Reset ISA-L inflate state + isal_inflate_reset(isal_strm_inflate); + + + // Reset workaround flag + *tofixed = 0; + + return Z_OK; +} +#endif diff --git a/igzip.h b/igzip.h new file mode 100644 index 0000000..7cda4b1 --- /dev/null +++ b/igzip.h @@ -0,0 +1,48 @@ +// Copyright (C) 2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#ifdef USE_IGZIP +#include +#include + +typedef struct internal_state2 { + z_streamp strm; + int level; + int w_bits; + struct inflate_state *isal_strm_inflate; + int trailer_overconsumption_fixed; /* Indicates if fix has been applied for gzip trailer + overconsumption issue */ +} inflate_state2; + +typedef struct internal_state { + z_streamp strm; + int level; + int w_bits; + struct isal_zstream *isal_strm; +} deflate_state; + +struct isal_zstream* +InitCompressIGZIP(int level, int windowBits); +int +CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, + uint32_t* input_length, uint8_t *output, uint32_t* output_length, + unsigned long *total_in, unsigned long *total_out); +int +EndCompressIGZIP(struct isal_zstream* isal_strm); +struct isal_zstream* +InitCompressIGZIP(int level, int windowBits); + +struct inflate_state* +InitUncompressIGZIP(int windowBits); +int +UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, + uint32_t *input_length, uint8_t *output, uint32_t *output_length, + int *tofixed, unsigned long *total_in, unsigned long *total_out); +int +EndUncompressIGZIP(struct inflate_state *isal_strm_inflate); +int +ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed); +//#define Z_DEFAULT_COMPRESSION 6 +#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b0c4564..0ac0465 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -24,7 +24,7 @@ add_executable(zlib_accel_test ) add_custom_target(run - COMMAND ./zlib_accel_test + COMMAND ./zlib_accel_test # --gtest_filter=ConfigLoaderTest.LoadValidConfig DEPENDS zlib_accel_test ) diff --git a/tests/zlib_accel_test.cpp b/tests/zlib_accel_test.cpp index c8ec0ce..c56d8d5 100644 --- a/tests/zlib_accel_test.cpp +++ b/tests/zlib_accel_test.cpp @@ -240,16 +240,25 @@ void SetCompressPath(ExecutionPath path, bool zlib_fallback, switch (path) { case ZLIB: SetConfig(USE_IAA_COMPRESS, 0); + SetConfig(USE_IGZIP_COMPRESS, 0); SetConfig(USE_QAT_COMPRESS, 0); SetConfig(USE_ZLIB_COMPRESS, 1); break; case QAT: SetConfig(USE_IAA_COMPRESS, 0); + SetConfig(USE_IGZIP_COMPRESS, 0); SetConfig(USE_QAT_COMPRESS, 1); SetConfig(USE_ZLIB_COMPRESS, zlib_fallback ? 1 : 0); break; case IAA: SetConfig(USE_IAA_COMPRESS, 1); + SetConfig(USE_IGZIP_COMPRESS, 0); + SetConfig(USE_QAT_COMPRESS, 0); + SetConfig(USE_ZLIB_COMPRESS, zlib_fallback ? 1 : 0); + break; + case IGZIP: + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(USE_IAA_COMPRESS, 0); SetConfig(USE_QAT_COMPRESS, 0); SetConfig(USE_ZLIB_COMPRESS, zlib_fallback ? 1 : 0); break; @@ -265,16 +274,25 @@ void SetUncompressPath(ExecutionPath path, bool zlib_fallback, switch (path) { case ZLIB: SetConfig(USE_IAA_UNCOMPRESS, 0); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); SetConfig(USE_QAT_UNCOMPRESS, 0); SetConfig(USE_ZLIB_UNCOMPRESS, 1); break; case QAT: SetConfig(USE_IAA_UNCOMPRESS, 0); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); SetConfig(USE_QAT_UNCOMPRESS, 1); SetConfig(USE_ZLIB_UNCOMPRESS, zlib_fallback ? 1 : 0); break; case IAA: SetConfig(USE_IAA_UNCOMPRESS, 1); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); + SetConfig(USE_QAT_UNCOMPRESS, 0); + SetConfig(USE_ZLIB_UNCOMPRESS, zlib_fallback ? 1 : 0); + break; + case IGZIP: + SetConfig(USE_IAA_UNCOMPRESS, 0); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); SetConfig(USE_QAT_UNCOMPRESS, 0); SetConfig(USE_ZLIB_UNCOMPRESS, zlib_fallback ? 1 : 0); break; @@ -331,6 +349,8 @@ struct TestParam { return "QAT"; case IAA: return "IAA"; + case IGZIP: + return "IGZIP"; } return ""; } @@ -729,6 +749,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -740,6 +764,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(-15, 15, 31), @@ -827,6 +855,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -838,6 +870,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(15), @@ -923,6 +959,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -934,6 +974,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(15), @@ -1122,6 +1166,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -1133,6 +1181,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(-15, 15, 31), @@ -1203,6 +1255,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -1214,6 +1270,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(31), @@ -1272,6 +1332,8 @@ TEST_F(ConfigLoaderTest, LoadValidConfig) { EXPECT_EQ(GetConfig(USE_QAT_UNCOMPRESS), 1); EXPECT_EQ(GetConfig(USE_IAA_COMPRESS), 0); EXPECT_EQ(GetConfig(USE_IAA_UNCOMPRESS), 0); + EXPECT_EQ(GetConfig(USE_IGZIP_COMPRESS), 0); + EXPECT_EQ(GetConfig(USE_IGZIP_UNCOMPRESS), 0); EXPECT_EQ(GetConfig(USE_ZLIB_COMPRESS), 1); EXPECT_EQ(GetConfig(USE_ZLIB_UNCOMPRESS), 1); EXPECT_EQ(GetConfig(LOG_LEVEL), 2); diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 72c5105..37b1101 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -20,6 +20,9 @@ #ifdef USE_IAA #include "iaa.h" #endif +#ifdef USE_IGZIP +#include "igzip.h" +#endif #ifdef USE_QAT #include "qat.h" #endif @@ -192,12 +195,15 @@ struct DeflateSettings { int mem_level; int strategy; ExecutionPath path = UNDEFINED; + struct isal_zstream *isal_strm = nullptr; }; struct InflateSettings { InflateSettings(int _window_bits) : window_bits(_window_bits) {} int window_bits; + int trailer_overconsumption_fixed; /* indicates if fix has been applied for overconsumption issue*/ ExecutionPath path = UNDEFINED; + struct inflate_state *isal_strm = nullptr; }; class DeflateStreamSettings { @@ -251,6 +257,12 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, static_cast(strm), ", level ", level, ", window_bits ", window_bits, " \n"); +/*#ifdef USE_IGZIP + if (configs[USE_IGZIP_COMPRESS] && configs[USE_IGZIP_UNCOMPRESS] && + !configs[USE_ZLIB_COMPRESS]) { + method = 0; + } +#endif*/ deflate_stream_settings.Set(strm, level, method, window_bits, mem_level, strategy); return orig_deflateInit2_(strm, level, method, window_bits, mem_level, @@ -285,6 +297,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { int ret = 1; bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; if (!in_call && flush == Z_FINISH && deflate_settings->path != ZLIB) { uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; @@ -299,6 +312,11 @@ int ZEXPORT deflate(z_streamp strm, int flush) { configs[USE_QAT_COMPRESS] && output_len >= QAT_DEST_BUFFER_MIN_SIZE && SupportedOptionsQAT(deflate_settings->window_bits, input_len); #endif +#ifdef USE_IGZIP + igzip_available = configs[USE_QAT_COMPRESS]; + //deflate_settings->window_bits == 15; + //SupportedOptionsQAT(deflate_settings->window_bits, input_len); +#endif // If both accelerators are enabled, send configured ratio of requests to // one or the other @@ -312,6 +330,8 @@ int ZEXPORT deflate(z_streamp strm, int flush) { } } else if (iaa_available) { path_selected = IAA; + } else if (igzip_available) { + path_selected = IGZIP; } else if (qat_available) { path_selected = QAT; } @@ -340,6 +360,24 @@ int ZEXPORT deflate(z_streamp strm, int flush) { INCREMENT_STAT(DEFLATE_QAT_COUNT); INCREMENT_STAT_COND(ret != 0, DEFLATE_QAT_ERROR_COUNT); #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + if (deflate_settings->isal_strm == nullptr) { + deflate_settings->method = 0; + deflate_settings->isal_strm = InitCompressIGZIP(deflate_settings->level, deflate_settings->window_bits); //hardcoded window bits + } + if (deflate_settings->isal_strm == nullptr) { + Log(LogLevel::LOG_ERROR, "deflate Line ", __LINE__, ", strm ", + static_cast(strm), ", failed to initialize igzip \n"); + } + in_call = true; + ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, &input_len, + strm->next_out, &output_len, &strm->total_in, &strm->total_out); + deflate_settings->path = IGZIP; + in_call = false; + //INCREMENT_STAT(DEFLATE_IGZIP_COUNT); + //INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); +#endif } if (ret == 0) { @@ -385,6 +423,12 @@ int ZEXPORT deflate(z_streamp strm, int flush) { int ZEXPORT deflateEnd(z_streamp strm) { Log(LogLevel::LOG_INFO, "deflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); + DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); + if (deflate_settings->isal_strm != nullptr) { +#ifdef USE_IGZIP + EndCompressIGZIP(deflate_settings->isal_strm); +#endif + } deflate_stream_settings.Unset(strm); return orig_deflateEnd(strm); } @@ -448,6 +492,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { bool end_of_stream = true; bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; if (!in_call && strm->avail_in > 0 && inflate_settings->path != ZLIB) { uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; @@ -465,6 +510,12 @@ int ZEXPORT inflate(z_streamp strm, int flush) { configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(inflate_settings->window_bits, input_len); #endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; //&& + // inflate_settings->window_bits == 15; + Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", + static_cast(strm), ", igzip_available? ", igzip_available, "\n"); +#endif // If both accelerators are enabled, send configured ratio of requests to // one or the other @@ -480,8 +531,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } - if (path_selected == IAA) { #ifdef USE_IAA in_call = true; @@ -509,6 +561,30 @@ int ZEXPORT inflate(z_streamp strm, int flush) { INCREMENT_STAT(INFLATE_QAT_COUNT); INCREMENT_STAT_COND(ret != 0, INFLATE_QAT_ERROR_COUNT); #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + if (inflate_settings->isal_strm == nullptr) { + Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", + static_cast(strm), ", about to initalize igzip\n"); + inflate_settings->isal_strm = InitUncompressIGZIP(inflate_settings->window_bits); //hardcoded window bits + fflush(stdout); + } + if (inflate_settings->isal_strm == nullptr) { + Log(LogLevel::LOG_ERROR, "inflate Line ", __LINE__, ", strm ", + static_cast(strm), ", failed to initalize igzip\n "); + return Z_DATA_ERROR; + } + in_call = true; + ret = + UncompressIGZIP(inflate_settings->isal_strm, strm->next_in, &input_len, strm->next_out, + &output_len, &inflate_settings->trailer_overconsumption_fixed, + &strm->total_in, &strm->total_out + /*inflate_settings->window_bits, &end_of_stream*/); + inflate_settings->path = IGZIP; + in_call = false; + //INCREMENT_STAT(INFLATE_IGZIP_COUNT); + //INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); +#endif } if (ret == 0) { @@ -559,6 +635,12 @@ int ZEXPORT inflate(z_streamp strm, int flush) { int ZEXPORT inflateEnd(z_streamp strm) { Log(LogLevel::LOG_INFO, "inflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); + InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); + if (inflate_settings->isal_strm != nullptr) { +#ifdef USE_IGZIP + EndUncompressIGZIP(inflate_settings->isal_strm); +#endif + } inflate_stream_settings.Unset(strm); return orig_inflateEnd(strm); } @@ -570,6 +652,12 @@ int ZEXPORT inflateReset(z_streamp strm) { if (inflate_settings != nullptr) { inflate_settings->path = UNDEFINED; } + if (inflate_settings->isal_strm != nullptr) { +#ifdef USE_IGZIP + ResetUncompressIGZIP(inflate_settings->isal_strm, + &inflate_settings->trailer_overconsumption_fixed); +#endif + } return orig_inflateReset(strm); } diff --git a/zlib_accel.h b/zlib_accel.h index c763ea9..2dd2c78 100644 --- a/zlib_accel.h +++ b/zlib_accel.h @@ -7,7 +7,7 @@ #include // Visible for testing -enum ExecutionPath { UNDEFINED, ZLIB, QAT, IAA }; +enum ExecutionPath { UNDEFINED, ZLIB, QAT, IAA, IGZIP }; ExecutionPath GetDeflateExecutionPath(z_streamp strm); ExecutionPath GetInflateExecutionPath(z_streamp strm); From 20369fd4c2892259211e3b8d9a3ed8a270142a40 Mon Sep 17 00:00:00 2001 From: Olasoji Date: Wed, 18 Feb 2026 16:11:14 -0800 Subject: [PATCH 07/18] - Standardized IGZIP wrapper behavior to accelerator-style semantics - Added explicit end-of-stream signaling for IGZIP inflate via out-param and wired it into the inflate dispatch path - Corrected windowBits handling to map wrapper type/history properly - Replaced raw magic-number crc_flag check with IGZIP_DEFLATE constant - Fixed DEBUG-only issues and cleanup edge cases - Debugging Signed-off-by: Olasoji --- igzip.cpp | 165 ++++++++++++++++++++++++++----------------------- igzip.h | 5 +- zlib_accel.cpp | 148 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 220 insertions(+), 98 deletions(-) diff --git a/igzip.cpp b/igzip.cpp index 52874bd..4a376a2 100644 --- a/igzip.cpp +++ b/igzip.cpp @@ -11,14 +11,60 @@ #include #include "logging.h" +static uint16_t ClampHistBits(int bits) +{ + if (bits < 0) { + return 0; + } + if (bits > ISAL_DEF_MAX_HIST_BITS) { + return ISAL_DEF_MAX_HIST_BITS; + } + return (uint16_t) bits; +} + +static void ConfigureDeflateWindow(struct isal_zstream *isal_strm, int windowBits) +{ + if (windowBits < 0) { + isal_strm->gzip_flag = IGZIP_DEFLATE; + isal_strm->hist_bits = ClampHistBits(-windowBits); + return; + } + + if (windowBits >= 24 && windowBits <= 31) { + isal_strm->gzip_flag = IGZIP_GZIP; + isal_strm->hist_bits = ClampHistBits(windowBits - 16); + return; + } + + isal_strm->gzip_flag = IGZIP_ZLIB; + isal_strm->hist_bits = ClampHistBits(windowBits); +} + +static void ConfigureInflateWindow(struct inflate_state *isal_strm_inflate, int windowBits) +{ + if (windowBits < 0) { + isal_strm_inflate->crc_flag = IGZIP_DEFLATE; + isal_strm_inflate->hist_bits = ClampHistBits(-windowBits); + return; + } + + if ((windowBits >= 24 && windowBits <= 31) || (windowBits >= 40 && windowBits <= 47)) { + isal_strm_inflate->crc_flag = IGZIP_GZIP; + isal_strm_inflate->hist_bits = ClampHistBits(windowBits > 31 ? windowBits - 32 : windowBits - 16); + return; + } + + isal_strm_inflate->crc_flag = IGZIP_ZLIB; + isal_strm_inflate->hist_bits = ClampHistBits(windowBits); +} + struct isal_zstream* InitCompressIGZIP(int level, int windowBits) { #ifdef DEBUG fprintf(stderr, - "\nInitializing deflate with level: %d, method: %d, windowBits: %d, memLevel: %d, " - "strategy: %d\n", - level, method, windowBits, memLevel, strategy); + "\nInitializing deflate with level: %d, windowBits: %d\n", level, + windowBits); #endif Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, " level ", level, ", windowBits ",windowBits, " \n"); @@ -51,6 +97,7 @@ InitCompressIGZIP(int level, int windowBits) isal_strm->level_buf_size = ISAL_DEF_LVL3_DEFAULT; } else { fprintf(stderr, "Error: Invalid compression level\n"); + free(isal_strm); return nullptr; } @@ -60,17 +107,7 @@ InitCompressIGZIP(int level, int windowBits) return nullptr; } - // Set stream->gzip_flag and hist_bits - // Ensure hist_bits are non-negative - if (windowBits < 0) { - // Raw deflate mode - no headers/trailers - isal_strm->gzip_flag = IGZIP_DEFLATE; - isal_strm->hist_bits = -windowBits; - } else { - // Standard zlib format - isal_strm->gzip_flag = IGZIP_ZLIB; - isal_strm->hist_bits = windowBits; - } + ConfigureDeflateWindow(isal_strm, windowBits); return isal_strm; } @@ -92,6 +129,9 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, unsigned long *total_in, unsigned long *total_out) { int ret; + (void) total_in; + (void) total_out; + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " input_length ", *input_length, " \n"); if (!isal_strm) { @@ -101,9 +141,11 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ isal_strm->next_out = output; - isal_strm->avail_out = *output_length; + const uint32_t original_avail_out = *output_length; + isal_strm->avail_out = original_avail_out; isal_strm->next_in = input; - isal_strm->avail_in = *input_length; + const uint32_t original_avail_in = *input_length; + isal_strm->avail_in = original_avail_in; isal_strm->total_out = *total_out; isal_strm->total_in = *total_in; @@ -134,33 +176,23 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, isal_strm->gzip_flag, isal_strm->hist_bits, isal_strm->flush, isal_strm->level); fprintf(stderr, "Before isal_deflate: avail_in=%u, next_in=%p, avail_out=%u, next_out=%p\n", isal_strm->avail_in, isal_strm->next_in, isal_strm->avail_out, isal_strm->next_out); - fprintf(stderr, "Total out: %u, Total in %lu\n", isal_strm->total_out, strm->total_in); + fprintf(stderr, "Total out: %u, Total in %u\n", isal_strm->total_out, isal_strm->total_in); #endif int comp = isal_deflate(isal_strm); - *output_length = isal_strm->avail_out; - *input_length = isal_strm->avail_in; + *output_length = original_avail_out - isal_strm->avail_out; + *input_length = original_avail_in - isal_strm->avail_in; input = isal_strm->next_in; output = isal_strm->next_out; - *total_out = isal_strm->total_out; - *total_in = isal_strm->total_in; #ifdef DEBUG fprintf(stderr, "After isal_deflate: avail_in=%u, next_in=%p, avail_out=%u, next_out=%p\n", - strm->avail_in, strm->next_in, strm->avail_out, strm->next_out); - fprintf(stderr, "Total out: %lu, Total in: %lu\n", strm->total_out, strm->total_in); + isal_strm->avail_in, isal_strm->next_in, isal_strm->avail_out, isal_strm->next_out); + fprintf(stderr, "Bytes consumed: %u, Bytes produced: %u\n", *input_length, *output_length); #endif - if (comp == COMP_OK) { - if (isal_strm->end_of_stream && isal_strm->avail_out > 0) { - ret = Z_STREAM_END; // Compression is done - } else { - ret = Z_OK; - } - } else { - ret = Z_ERRNO; // Compression error - } + ret = (comp == COMP_OK) ? 0 : 1; #ifdef DEBUG if (ret == Z_OK) { @@ -329,15 +361,7 @@ InitUncompressIGZIP(int windowBits) //s->trailer_overconsumption_fixed = 0; // Initialize the workaround flag - if (windowBits < 0) { - // Raw deflate mode - no headers/trailers - isal_strm_inflate->crc_flag = IGZIP_DEFLATE; - isal_strm_inflate->hist_bits = -windowBits; - } else { - // Standard zlib format - isal_strm_inflate->crc_flag = IGZIP_ZLIB; - isal_strm_inflate->hist_bits = windowBits; - } + ConfigureInflateWindow(isal_strm_inflate, windowBits); return isal_strm_inflate; } @@ -356,8 +380,11 @@ inflateInit_(z_streamp strm) int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t *input_length, uint8_t *output, uint32_t *output_length, - int *tofixed, unsigned long *total_in, unsigned long *total_out) + int *tofixed, unsigned long *total_in, unsigned long *total_out, + bool *end_of_stream) { + (void) total_in; + if (!isal_strm_inflate) { fprintf(stderr, "Error: isal_strm_inflate is NULL\n"); return Z_STREAM_ERROR; @@ -367,12 +394,14 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, "input length ", *input_length,"\n"); // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ isal_strm_inflate->next_out = output; - isal_strm_inflate->avail_out = *output_length; - isal_strm_inflate->avail_in = *input_length; + const uint32_t original_avail_out = *output_length; + isal_strm_inflate->avail_out = original_avail_out; + const uint32_t original_avail_in = *input_length; + isal_strm_inflate->avail_in = original_avail_in; isal_strm_inflate->next_in = input; isal_strm_inflate->total_out = *total_out; - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, " CRC flag: ", (uint32_t)isal_strm_inflate->crc_flag, " Before isal_inflate: avail_in ",isal_strm_inflate->avail_in, //" next_in= ", isal_strm_inflate->next_in, @@ -383,24 +412,22 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, const int decomp = isal_inflate(isal_strm_inflate); - const unsigned long total_in_ = *total_in; - const unsigned int original_avail_in = *input_length; - //const unsigned int bytes_consumed = original_avail_in - isal_strm_inflate->avail_in; - - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, " After isal_inflate: avail_in ",isal_strm_inflate->avail_in, //" next_in= ", isal_strm_inflate->next_in, " avail_out= ",(uint32_t)isal_strm_inflate->avail_out, //" next_out= ", isal_strm_inflate->next_out, " Total out: ", (uint32_t)isal_strm_inflate->total_out, - " Total_in: ", total_in_ + bytes_consumed, - " Bytes consumed this call: ", bytes_consumed, + " Total_in: ", (unsigned long)*total_in + + (original_avail_in - isal_strm_inflate->avail_in), + " Bytes consumed this call: ", + (original_avail_in - isal_strm_inflate->avail_in), " Block state: ", isal_strm_inflate->block_state, " (ISAL_BLOCK_FINISH=", ISAL_BLOCK_FINISH, " ISA-L result: ", decomp, "\n"); if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH && isal_strm_inflate->avail_in > 0) { - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, " WARNING: BLOCK_FINISH reached but ",isal_strm_inflate->avail_in, "bytes remain in input:\n"); for (unsigned int i = 0; i < isal_strm_inflate->avail_in && i < 16; i++) { fprintf(stderr, " %02x", ((unsigned char *) isal_strm_inflate->next_in)[i]); @@ -411,7 +438,7 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, // WORKAROUND: ISA-L over-consumption fix for raw deflate mode if ((isal_strm_inflate->block_state == ISAL_BLOCK_FINISH || isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE) && - (isal_strm_inflate->crc_flag == 0) && // raw deflate + (isal_strm_inflate->crc_flag == IGZIP_DEFLATE) && *tofixed == 0 && // hasn't been applied yet decomp == ISAL_DECOMP_OK && // successful decompression isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) { @@ -448,35 +475,17 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, } } - // Update stream state - handle byte accounting correctly - *output_length = isal_strm_inflate->avail_out; - *input_length = isal_strm_inflate->avail_in; + *output_length = original_avail_out - isal_strm_inflate->avail_out; + *input_length = original_avail_in - isal_strm_inflate->avail_in; input = isal_strm_inflate->next_in; output = isal_strm_inflate->next_out; - *total_out = isal_strm_inflate->total_out; - - // Calculate bytes consumed by ISA-L from the original input - const unsigned int bytes_consumed_by_isal = original_avail_in - isal_strm_inflate->avail_in; - *total_in = total_in_ + bytes_consumed_by_isal; - int ret; - - if (decomp == ISAL_DECOMP_OK) { - if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) { - // ISA-L has finished processing the deflate stream including trailer - // validation - ret = Z_STREAM_END; - //strm->msg = "ok"; - } else { - // Still processing, continue - ret = Z_OK; - } - } else if (decomp == ISAL_END_INPUT) { - ret = Z_OK; - } else { - ret = Z_DATA_ERROR; + if (end_of_stream != nullptr) { + *end_of_stream = (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH); } + int ret = (decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) ? 0 : 1; + #ifdef DEBUG if (ret == Z_OK) { fprintf(stderr, "Inflate finished successfully Z_OK\n"); diff --git a/igzip.h b/igzip.h index 7cda4b1..e260576 100644 --- a/igzip.h +++ b/igzip.h @@ -31,15 +31,14 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, unsigned long *total_in, unsigned long *total_out); int EndCompressIGZIP(struct isal_zstream* isal_strm); -struct isal_zstream* -InitCompressIGZIP(int level, int windowBits); struct inflate_state* InitUncompressIGZIP(int windowBits); int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t *input_length, uint8_t *output, uint32_t *output_length, - int *tofixed, unsigned long *total_in, unsigned long *total_out); + int *tofixed, unsigned long *total_in, unsigned long *total_out, + bool *end_of_stream); int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate); int diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 37b1101..19a8d48 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -298,24 +298,22 @@ int ZEXPORT deflate(z_streamp strm, int flush) { bool iaa_available = false; bool qat_available = false; bool igzip_available = false; - if (!in_call && flush == Z_FINISH && deflate_settings->path != ZLIB) { + if (!in_call && deflate_settings->path != ZLIB) { uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; #ifdef USE_IAA - iaa_available = configs[USE_IAA_COMPRESS] && + iaa_available = (flush == Z_FINISH) && configs[USE_IAA_COMPRESS] && SupportedOptionsIAA(deflate_settings->window_bits, input_len, output_len); #endif #ifdef USE_QAT - qat_available = - configs[USE_QAT_COMPRESS] && output_len >= QAT_DEST_BUFFER_MIN_SIZE && + qat_available = (flush == Z_FINISH) && configs[USE_QAT_COMPRESS] && + output_len >= QAT_DEST_BUFFER_MIN_SIZE && SupportedOptionsQAT(deflate_settings->window_bits, input_len); #endif #ifdef USE_IGZIP - igzip_available = configs[USE_QAT_COMPRESS]; - //deflate_settings->window_bits == 15; - //SupportedOptionsQAT(deflate_settings->window_bits, input_len); + igzip_available = configs[USE_IGZIP_COMPRESS]; #endif // If both accelerators are enabled, send configured ratio of requests to @@ -330,10 +328,10 @@ int ZEXPORT deflate(z_streamp strm, int flush) { } } else if (iaa_available) { path_selected = IAA; - } else if (igzip_available) { - path_selected = IGZIP; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -387,10 +385,22 @@ int ZEXPORT deflate(z_streamp strm, int flush) { strm->next_out += output_len; strm->avail_out -= output_len; strm->total_out += output_len; - if (strm->avail_in == 0) { - ret = Z_STREAM_END; + if (path_selected == IGZIP) { + if (input_len > 0 || output_len > 0) { + if (flush == Z_FINISH && strm->avail_in == 0) { + ret = Z_STREAM_END; + } else { + ret = Z_OK; + } + } else { + ret = Z_BUF_ERROR; + } } else { - ret = Z_BUF_ERROR; + if (strm->avail_in == 0) { + ret = Z_STREAM_END; + } else { + ret = Z_BUF_ERROR; + } } Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", @@ -578,7 +588,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ret = UncompressIGZIP(inflate_settings->isal_strm, strm->next_in, &input_len, strm->next_out, &output_len, &inflate_settings->trailer_overconsumption_fixed, - &strm->total_in, &strm->total_out + &strm->total_in, &strm->total_out, &end_of_stream /*inflate_settings->window_bits, &end_of_stream*/); inflate_settings->path = IGZIP; in_call = false; @@ -674,6 +684,7 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA iaa_available = configs[USE_IAA_COMPRESS] && SupportedOptionsIAA(15, input_len, output_len); @@ -682,12 +693,17 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, qat_available = configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(15, input_len); #endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_COMPRESS]; +#endif ExecutionPath path_selected = ZLIB; if (iaa_available) { path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -704,6 +720,25 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, &output_len, 15); in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct isal_zstream* isal_strm = InitCompressIGZIP(level, 15); + if (isal_strm == nullptr) { + ret = 1; + } else { + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = CompressIGZIP(isal_strm, Z_FINISH, const_cast(source), + &input_len, dest, &output_len, &total_in, + &total_out); + EndCompressIGZIP(isal_strm); + if (ret == 0 && input_len != sourceLen) { + ret = 1; + } + } + in_call = false; +#endif } if (ret == 0) { @@ -746,6 +781,7 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA iaa_available = configs[USE_IAA_UNCOMPRESS] && @@ -756,12 +792,17 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, qat_available = configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(15, input_len); #endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; +#endif ExecutionPath path_selected = ZLIB; if (iaa_available) { path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -778,6 +819,26 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, &output_len, 15, &end_of_stream); in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct inflate_state* isal_strm = InitUncompressIGZIP(15); + if (isal_strm == nullptr) { + ret = 1; + } else { + int tofixed = 0; + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = UncompressIGZIP(isal_strm, const_cast(source), + &input_len, dest, &output_len, &tofixed, + &total_in, &total_out, &end_of_stream); + EndUncompressIGZIP(isal_strm); + if (ret == 0 && !end_of_stream) { + ret = 1; + } + } + in_call = false; +#endif } if (ret == 0) { @@ -1018,6 +1079,7 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, int ret = 1; bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA iaa_available = configs[USE_IAA_COMPRESS] && @@ -1027,12 +1089,17 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, qat_available = configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(31, *input_length); #endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_COMPRESS]; +#endif ExecutionPath path_selected = ZLIB; if (qat_available) { path_selected = QAT; } else if (iaa_available) { path_selected = IAA; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -1050,6 +1117,22 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, gz->path = QAT; in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct isal_zstream* isal_strm = InitCompressIGZIP(Z_DEFAULT_COMPRESSION, 31); + if (isal_strm == nullptr) { + ret = 1; + } else { + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = CompressIGZIP(isal_strm, Z_FINISH, input, input_length, output, + output_length, &total_in, &total_out); + EndCompressIGZIP(isal_strm); + } + gz->path = IGZIP; + in_call = false; +#endif } return ret; } @@ -1068,6 +1151,7 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, int ret = 1; bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA iaa_available = configs[USE_IAA_UNCOMPRESS] && @@ -1078,12 +1162,17 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, qat_available = configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(31, *input_length); #endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; +#endif ExecutionPath path_selected = ZLIB; if (qat_available) { path_selected = QAT; } else if (iaa_available) { path_selected = IAA; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -1102,6 +1191,24 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, gz->path = QAT; in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct inflate_state* isal_strm = InitUncompressIGZIP(31); + if (isal_strm == nullptr) { + ret = 1; + } else { + int tofixed = 0; + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = UncompressIGZIP(isal_strm, input, input_length, output, + output_length, &tofixed, &total_in, &total_out, + end_of_stream); + EndUncompressIGZIP(isal_strm); + } + gz->path = IGZIP; + in_call = false; +#endif } return ret; } @@ -1186,7 +1293,8 @@ int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len) { unsigned int written_bytes = 0; bool accelerator_selected = - configs[USE_IAA_COMPRESS] || configs[USE_QAT_COMPRESS]; + configs[USE_IAA_COMPRESS] || configs[USE_QAT_COMPRESS] || + configs[USE_IGZIP_COMPRESS]; if (gz->path != ZLIB && accelerator_selected) { gz->AllocateBuffers(); gz->data_buf_size = 256 << 10; @@ -1245,7 +1353,8 @@ int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { int ret = 1; uint32_t read_bytes = 0; bool accelerator_selected = - configs[USE_IAA_UNCOMPRESS] || configs[USE_QAT_UNCOMPRESS]; + configs[USE_IAA_UNCOMPRESS] || configs[USE_QAT_UNCOMPRESS] || + configs[USE_IGZIP_UNCOMPRESS]; if (gz->path != ZLIB && accelerator_selected) { gz->AllocateBuffers(); gz->data_buf_size = 512 << 10; @@ -1420,10 +1529,15 @@ int ZEXPORT gzclose(gzFile file) { if (write_ret != 0) { ret = Z_STREAM_ERROR; - } else if (close_ret != Z_OK) { - ret = close_ret; } else if (truncate_ret != 0) { ret = Z_STREAM_ERROR; + } else if (close_ret != Z_OK) { + Log(LogLevel::LOG_INFO, "gzclose Line ", __LINE__, + ", ignoring zlib close return in accelerator path ", close_ret, + "\n"); + ret = Z_OK; + } else { + ret = Z_OK; } } else { ret = orig_gzclose(file); From bc7243a1df10901ae8aadbe5416025eaec70dd6f Mon Sep 17 00:00:00 2001 From: Olasoji Date: Tue, 24 Feb 2026 16:14:02 -0800 Subject: [PATCH 08/18] bug fixes and test fixes --- igzip.cpp | 156 +++++++++++++++++++++++--------------- logging.h | 7 +- tests/zlib_accel_test.cpp | 98 ++++++++++++++++++++---- zlib_accel.cpp | 6 +- 4 files changed, 189 insertions(+), 78 deletions(-) diff --git a/igzip.cpp b/igzip.cpp index 4a376a2..771235b 100644 --- a/igzip.cpp +++ b/igzip.cpp @@ -6,7 +6,6 @@ #include "igzip.h" #include "crc.h" -#include #include #include #include "logging.h" @@ -62,9 +61,9 @@ struct isal_zstream* InitCompressIGZIP(int level, int windowBits) { #ifdef DEBUG - fprintf(stderr, - "\nInitializing deflate with level: %d, windowBits: %d\n", level, - windowBits); + Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, + " initializing deflate with level ", level, ", windowBits ", + windowBits, "\n"); #endif Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, " level ", level, ", windowBits ",windowBits, " \n"); @@ -72,7 +71,8 @@ InitCompressIGZIP(int level, int windowBits) struct isal_zstream *isal_strm = (struct isal_zstream *) malloc(sizeof(struct isal_zstream)); if (!isal_strm) { - fprintf(stderr, "Error: Memory allocation for isal_zstream failed\n"); + Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, + " memory allocation for isal_zstream failed\n"); return nullptr; } @@ -96,14 +96,16 @@ InitCompressIGZIP(int level, int windowBits) isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL3_DEFAULT); isal_strm->level_buf_size = ISAL_DEF_LVL3_DEFAULT; } else { - fprintf(stderr, "Error: Invalid compression level\n"); + Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, + " invalid compression level\n"); free(isal_strm); return nullptr; } if (!isal_strm->level_buf) { free(isal_strm); - fprintf(stderr, "Error: Memory allocation for level_buf failed\n"); + Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, + " memory allocation for level_buf failed\n"); return nullptr; } @@ -116,7 +118,8 @@ InitCompressIGZIP(int level, int windowBits) deflateInit_(z_streamp strm, int level) { if (!strm) { - fprintf(stderr, "Error: z_streamp is NULL\n"); + Log(LogLevel::LOG_ERROR, "deflateInit_() Line ", __LINE__, + " z_streamp is NULL\n"); return -1; } @@ -135,7 +138,8 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " input_length ", *input_length, " \n"); if (!isal_strm) { - fprintf(stderr, "Error: deflate isal_strm is NULL\n"); + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " deflate isal_strm is NULL\n"); return -1; } @@ -167,16 +171,19 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, isal_strm->end_of_stream = 1; break; default: - fprintf(stderr, "Error: Invalid flush value\n"); + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid flush value\n"); return -1; } #ifdef DEBUG - fprintf(stderr, "Gzip flag: %d, Window bits: %d, Flush: %d, Level: %d\n", - isal_strm->gzip_flag, isal_strm->hist_bits, isal_strm->flush, isal_strm->level); - fprintf(stderr, "Before isal_deflate: avail_in=%u, next_in=%p, avail_out=%u, next_out=%p\n", - isal_strm->avail_in, isal_strm->next_in, isal_strm->avail_out, isal_strm->next_out); - fprintf(stderr, "Total out: %u, Total in %u\n", isal_strm->total_out, isal_strm->total_in); + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " gzip_flag ", + isal_strm->gzip_flag, ", hist_bits ", isal_strm->hist_bits, + ", flush ", isal_strm->flush, ", level ", isal_strm->level, + ", avail_in ", isal_strm->avail_in, ", avail_out ", + (uint32_t)isal_strm->avail_out, ", total_out ", + (uint32_t)isal_strm->total_out, ", total_in ", + (uint32_t)isal_strm->total_in, "\n"); #endif int comp = isal_deflate(isal_strm); @@ -187,41 +194,53 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, output = isal_strm->next_out; #ifdef DEBUG - fprintf(stderr, "After isal_deflate: avail_in=%u, next_in=%p, avail_out=%u, next_out=%p\n", - isal_strm->avail_in, isal_strm->next_in, isal_strm->avail_out, isal_strm->next_out); - fprintf(stderr, "Bytes consumed: %u, Bytes produced: %u\n", *input_length, *output_length); + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, + " after isal_deflate: avail_in ", isal_strm->avail_in, + ", avail_out ", (uint32_t)isal_strm->avail_out, + ", bytes_consumed ", *input_length, + ", bytes_produced ", *output_length, "\n"); #endif ret = (comp == COMP_OK) ? 0 : 1; #ifdef DEBUG if (ret == Z_OK) { - fprintf(stderr, "Deflate finished successfully Z_OK\n"); + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, + " deflate finished successfully Z_OK\n"); } else if (ret == Z_STREAM_END) { - fprintf(stderr, "Deflate finished successfully Z_STREAM_END\n"); + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, + " deflate finished successfully Z_STREAM_END\n"); } else { - fprintf(stderr, "Deflate finished with error code: %d\n", ret); + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " deflate finished with error code ", ret, "\n"); switch (comp) { case INVALID_FLUSH: - fprintf(stderr, "Error: Invalid flush\n"); + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", + __LINE__, " invalid flush\n"); break; case INVALID_PARAM: - fprintf(stderr, "Error: Invalid parameter\n"); + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", + __LINE__, " invalid parameter\n"); break; case STATELESS_OVERFLOW: - fprintf(stderr, "Error: Stateless overflow\n"); + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", + __LINE__, " stateless overflow\n"); break; case ISAL_INVALID_OPERATION: - fprintf(stderr, "Error: Invalid operation\n"); + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", + __LINE__, " invalid operation\n"); break; case ISAL_INVALID_STATE: - fprintf(stderr, "Error: Invalid state\n"); + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", + __LINE__, " invalid state\n"); break; case ISAL_INVALID_LEVEL: - fprintf(stderr, "Error: Invalid level\n"); + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", + __LINE__, " invalid level\n"); break; case ISAL_INVALID_LEVEL_BUF: - fprintf(stderr, "Error: Invalid level buffer\n"); + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", + __LINE__, " invalid level buffer\n"); break; } } @@ -234,7 +253,8 @@ int EndCompressIGZIP(struct isal_zstream* isal_strm) { if (!isal_strm) { - fprintf(stderr, "Error: isal_stream is NULL\n"); + Log(LogLevel::LOG_ERROR, "EndCompressIGZIP() Line ", __LINE__, + " isal_stream is NULL\n"); return -1; } @@ -245,7 +265,8 @@ EndCompressIGZIP(struct isal_zstream* isal_strm) free(isal_strm); #ifdef DEBUG - fprintf(stderr, "Deflate end\n"); + Log(LogLevel::LOG_INFO, "EndCompressIGZIP() Line ", __LINE__, + " deflate end\n"); #endif return Z_OK; } @@ -341,12 +362,14 @@ InitUncompressIGZIP(int windowBits) struct inflate_state *isal_strm_inflate = (struct inflate_state *) malloc(sizeof(struct inflate_state)); if (!isal_strm_inflate) { - fprintf(stderr, "Error: Memory allocation for inflate_state failed\n"); + Log(LogLevel::LOG_ERROR, "InitUncompressIGZIP() Line ", + __LINE__, " memory allocation for inflate_state failed\n"); return nullptr; } #ifdef DEBUG - fprintf(stderr, "\nInitializing inflate with windowBits: %d", windowBits); + Log(LogLevel::LOG_INFO, "InitUncompressIGZIP() Line ", __LINE__, + " initializing inflate with windowBits ", windowBits, "\n"); #endif Log(LogLevel::LOG_INFO, "InitUncompressIGZIP() Line ", __LINE__, ", windowBits ",windowBits, " \n"); @@ -370,7 +393,8 @@ InitUncompressIGZIP(int windowBits) inflateInit_(z_streamp strm) { if (!strm) { - fprintf(stderr, "Error: z_streamp is NULL\n"); + Log(LogLevel::LOG_ERROR, "inflateInit_() Line ", __LINE__, + " z_streamp is NULL\n"); return Z_STREAM_ERROR; } @@ -386,7 +410,8 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, (void) total_in; if (!isal_strm_inflate) { - fprintf(stderr, "Error: isal_strm_inflate is NULL\n"); + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " isal_strm_inflate is NULL\n"); return Z_STREAM_ERROR; } @@ -428,11 +453,8 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH && isal_strm_inflate->avail_in > 0) { Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " WARNING: BLOCK_FINISH reached but ",isal_strm_inflate->avail_in, "bytes remain in input:\n"); - for (unsigned int i = 0; i < isal_strm_inflate->avail_in && i < 16; i++) { - fprintf(stderr, " %02x", ((unsigned char *) isal_strm_inflate->next_in)[i]); - } - fprintf(stderr, "\n"); + " WARNING: BLOCK_FINISH reached but ",isal_strm_inflate->avail_in, + " bytes remain in input\n"); } // WORKAROUND: ISA-L over-consumption fix for raw deflate mode @@ -451,15 +473,14 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, // Only apply fix if the over-consumption is reasonable (1-7 bytes) if (over_consumed >= 1 && over_consumed <= 7) { #ifdef DEBUG - fprintf(stderr, - "APPLYING WORKAROUND: Detected ISA-L over-consumption of %u " - "bytes\n", - over_consumed); - fprintf(stderr, "Adjusting next_in from %p to %p, avail_in from %u to %u\n", - isal_strm_inflate->next_in, - (unsigned char *) isal_strm_inflate->next_in - over_consumed, - isal_strm_inflate->avail_in, - isal_strm_inflate->avail_in + over_consumed); + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", + __LINE__, " applying workaround: detected ISA-L over-consumption of ", + over_consumed, " bytes\n"); + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", + __LINE__, " adjusting avail_in from ", + isal_strm_inflate->avail_in, " to ", + isal_strm_inflate->avail_in + over_consumed, + "\n"); #endif // Rewind the input pointer to restore over-consumed bytes isal_strm_inflate->next_in = @@ -488,32 +509,42 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, #ifdef DEBUG if (ret == Z_OK) { - fprintf(stderr, "Inflate finished successfully Z_OK\n"); + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " inflate finished successfully Z_OK\n"); } else if (ret == Z_STREAM_END) { - fprintf(stderr, "Inflate finished with Z_STREAM_END\n"); + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " inflate finished with Z_STREAM_END\n"); } else { - fprintf(stderr, "Inflate finished with error code: %d\n", ret); + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " inflate finished with error code ", ret, "\n"); switch (decomp) { case ISAL_INVALID_BLOCK: - fprintf(stderr, "Error: ISA-L error - Invalid block\n"); + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", + __LINE__, " ISA-L error - Invalid block\n"); break; case ISAL_INVALID_SYMBOL: - fprintf(stderr, "Error: ISA-L error - Invalid symbol\n"); + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", + __LINE__, " ISA-L error - Invalid symbol\n"); break; case ISAL_INVALID_LOOKBACK: - fprintf(stderr, "Error: ISA-L error - Invalid lookback\n"); + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", + __LINE__, " ISA-L error - Invalid lookback\n"); break; case ISAL_END_INPUT: - fprintf(stderr, "Error: ISA-L error - End of input reached unexpectedly\n"); + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", + __LINE__, " ISA-L error - End of input reached unexpectedly\n"); break; case ISAL_UNSUPPORTED_METHOD: - fprintf(stderr, "Error: ISA-L error - Unsupported method\n"); + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", + __LINE__, " ISA-L error - Unsupported method\n"); break; case ISAL_NEED_DICT: - fprintf(stderr, "Error: ISA-L error - Need dictionary\n"); + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", + __LINE__, " ISA-L error - Need dictionary\n"); break; default: - fprintf(stderr, "Error: ISA-L error code: %d\n", decomp); + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", + __LINE__, " ISA-L error code ", decomp, "\n"); break; } } @@ -526,7 +557,8 @@ int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate) { if (!isal_strm_inflate) { - fprintf(stderr, "Error: z_streamp is NULL\n"); + Log(LogLevel::LOG_ERROR, "EndUncompressIGZIP() Line ", + __LINE__, " z_streamp is NULL\n"); return Z_STREAM_ERROR; } @@ -534,7 +566,8 @@ EndUncompressIGZIP(struct inflate_state *isal_strm_inflate) #ifdef DEBUG - fprintf(stderr, "Inflate end\n"); + Log(LogLevel::LOG_INFO, "EndUncompressIGZIP() Line ", __LINE__, + " inflate end\n"); #endif return Z_OK; } @@ -620,7 +653,8 @@ int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed) { if (!isal_strm_inflate) { - fprintf(stderr, "Error: isal_strm_inflate is NULL\n"); + Log(LogLevel::LOG_ERROR, "ResetUncompressIGZIP() Line ", + __LINE__, " isal_strm_inflate is NULL\n"); return Z_STREAM_ERROR; } diff --git a/logging.h b/logging.h index b5e8301..2f6dde3 100644 --- a/logging.h +++ b/logging.h @@ -24,9 +24,14 @@ inline std::unique_ptr& LogFileStream() { return stream; } -inline void CreateLogFile(const char* file_name) { +inline bool CreateLogFile(const char* file_name) { auto& s = LogFileStream(); s = std::make_unique(file_name, std::ios::app); + if (!s || !s->is_open()) { + s.reset(); + return false; + } + return true; } inline void CloseLogFile() { diff --git a/tests/zlib_accel_test.cpp b/tests/zlib_accel_test.cpp index c56d8d5..7bfa677 100644 --- a/tests/zlib_accel_test.cpp +++ b/tests/zlib_accel_test.cpp @@ -183,6 +183,41 @@ int ZlibUncompressUtility2(const char* input, size_t input_length, return st; } +int ZlibCompressWithLevel(const char* input, size_t input_length, + std::string* output, int level, int window_bits, + int flush, size_t* output_upper_bound, + ExecutionPath* execution_path) { + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + + int st = + deflateInit2(&stream, level, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY); + if (st != Z_OK) { + deflateEnd(&stream); + return st; + } + + stream.next_in = (Bytef*)input; + stream.avail_in = static_cast(input_length); + + *output_upper_bound = + deflateBound(&stream, static_cast(input_length)); + output->resize(*output_upper_bound); + stream.avail_out = static_cast(*output_upper_bound); + stream.next_out = reinterpret_cast(&(*output)[0]); + + st = deflate(&stream, flush); + *execution_path = GetDeflateExecutionPath(&stream); + if (st != Z_STREAM_END) { + deflateEnd(&stream); + return st; + } + output->resize(stream.total_out); + + deflateEnd(&stream); + return st; +} + int ZlibCompressGzipFile(const char* input, size_t input_length) { const char* filename = "file.gz"; remove(filename); @@ -610,6 +645,15 @@ TEST_P(ZlibTest, CompressDecompress) { test_param.iaa_prepend_empty_block, test_param.qat_compression_allow_chunking); + // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses + // ISA-L level 3 (stricter match selection), which makes long-history + // limitations consistently observable. + const int compression_level = + (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; + size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; char* input = GenerateBlock(input_length, block_type); @@ -618,9 +662,10 @@ TEST_P(ZlibTest, CompressDecompress) { std::string compressed; size_t output_upper_bound; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompress( - input, input_length, &compressed, test_param.window_bits_compress, - test_param.flush_compress, &output_upper_bound, &execution_path); + int ret = ZlibCompressWithLevel( + input, input_length, &compressed, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound, &execution_path); VerifyStatIncremented(Statistic::DEFLATE_COUNT); bool compress_fallback_expected = @@ -1004,6 +1049,15 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { test_param.iaa_prepend_empty_block, test_param.qat_compression_allow_chunking); + // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses + // ISA-L level 3 (stricter match selection), which makes long-history + // limitations consistently observable. + const int compression_level = + (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; + size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; char* input = GenerateBlock(input_length, block_type); @@ -1012,9 +1066,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { std::string compressed; size_t output_upper_bound; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompress( - input, input_length, &compressed, test_param.window_bits_compress, - test_param.flush_compress, &output_upper_bound, &execution_path); + int ret = ZlibCompressWithLevel( + input, input_length, &compressed, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound, &execution_path); bool error_expected = ZlibCompressExpectError(test_param, input_length, output_upper_bound); @@ -1042,8 +1097,9 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { window_bits_uncompress, test_param.flush_uncompress, test_param.input_chunks_uncompress, &execution_path); - // Only zlib decompression won't return an error + // zlib and igzip decompression may return partial progress instead of error if (test_param.execution_path_uncompress == ZLIB || + test_param.execution_path_uncompress == IGZIP || test_param.zlib_fallback_uncompress) { ASSERT_EQ(ret, Z_OK); ASSERT_TRUE(uncompressed_length < input_length); @@ -1070,6 +1126,15 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { test_param.iaa_prepend_empty_block, test_param.qat_compression_allow_chunking); + // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses + // ISA-L level 3 (stricter match selection), which makes long-history + // limitations consistently observable. + const int compression_level = + (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; + size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; char* input = GenerateBlock(input_length, block_type); @@ -1080,9 +1145,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { size_t input_length1 = input_length / 2; size_t output_upper_bound1; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompress( - input, input_length1, &compressed1, test_param.window_bits_compress, - test_param.flush_compress, &output_upper_bound1, &execution_path); + int ret = ZlibCompressWithLevel( + input, input_length1, &compressed1, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound1, &execution_path); bool error_expected = ZlibCompressExpectError(test_param, input_length1, output_upper_bound1); @@ -1098,9 +1164,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { size_t input_length2 = input_length - input_length / 2; size_t output_upper_bound2; execution_path = UNDEFINED; - ret = ZlibCompress(input + input_length1, input_length2, &compressed2, - test_param.window_bits_compress, test_param.flush_compress, - &output_upper_bound2, &execution_path); + ret = ZlibCompressWithLevel( + input + input_length1, input_length2, &compressed2, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound2, &execution_path); error_expected = ZlibCompressExpectError(test_param, input_length2, output_upper_bound2); @@ -1145,8 +1212,9 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { ASSERT_EQ(uncompressed_length, input_length1); ASSERT_TRUE(memcmp(uncompressed, input, uncompressed_length) == 0); - // IAA does not handle concatenated streams - if (test_param.execution_path_uncompress != IAA) { + // IAA/IGZIP may consume bytes beyond first-stream boundary + if (test_param.execution_path_uncompress != IAA && + test_param.execution_path_uncompress != IGZIP) { ASSERT_EQ(input_consumed, compressed1.length()); } } diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 19a8d48..5ef1f8b 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -162,7 +162,11 @@ static int init_zlib_accel(void) { #if defined(DEBUG_LOG) || defined(ENABLE_STATISTICS) if (!config::log_file.empty()) { - CreateLogFile(config::log_file.c_str()); + if (!CreateLogFile(config::log_file.c_str())) { + Log(LogLevel::LOG_ERROR, "init_zlib_accel Line ", __LINE__, + " failed to open log_file '", config::log_file.c_str(), + "', falling back to stdout\n"); + } } #endif From ac59785672064a4613f7ef2152756a178d65ce5b Mon Sep 17 00:00:00 2001 From: Olasoji Date: Mon, 9 Mar 2026 11:30:34 -0700 Subject: [PATCH 09/18] Initial commit of igzip engine and tests. WIP Signed-off-by: Olasoji --- igzip.cpp | 117 ++-- igzip.h | 7 +- logging.h | 33 +- tests/zlib_accel_test.cpp | 1215 +++++++++++++++++++++++++++++++++++++ zlib_accel.cpp | 772 +++++++++++++++++++++-- 5 files changed, 2062 insertions(+), 82 deletions(-) diff --git a/igzip.cpp b/igzip.cpp index 771235b..c64fa75 100644 --- a/igzip.cpp +++ b/igzip.cpp @@ -404,7 +404,9 @@ inflateInit_(z_streamp strm) int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t *input_length, uint8_t *output, uint32_t *output_length, - int *tofixed, unsigned long *total_in, unsigned long *total_out, + int window_bits, int *tofixed, + uint32_t *deferred_correction_bytes, + unsigned long *total_in, unsigned long *total_out, bool *end_of_stream) { (void) total_in; @@ -416,7 +418,7 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, } Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - "input length ", *input_length,"\n"); + " input length ", *input_length,"\n"); // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ isal_strm_inflate->next_out = output; const uint32_t original_avail_out = *output_length; @@ -433,7 +435,7 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, " avail_out= ",(uint32_t)isal_strm_inflate->avail_out, //" next_out= ", isal_strm_inflate->next_out, " Total out: ", (uint32_t)isal_strm_inflate->total_out, - " Total_in: ", (unsigned long)*total_in); + " Total_in: ", (unsigned long)*total_in, "\n"); const int decomp = isal_inflate(isal_strm_inflate); @@ -457,47 +459,81 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, " bytes remain in input\n"); } - // WORKAROUND: ISA-L over-consumption fix for raw deflate mode - if ((isal_strm_inflate->block_state == ISAL_BLOCK_FINISH || + uint32_t consumed_before_adjust = 0; + if (isal_strm_inflate->avail_in <= original_avail_in) { + consumed_before_adjust = + original_avail_in - isal_strm_inflate->avail_in; + } else { + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " invalid avail_in ", isal_strm_inflate->avail_in, + " greater than original_avail_in ", original_avail_in, + ", clamping consumed bytes to 0\n"); + consumed_before_adjust = 0; + } + + uint32_t rewind_adjust_bytes = 0; + + // WORKAROUND: ISA-L over-consumption fix for raw deflate mode. + // Keep original 8-byte trailer heuristic, but for streaming callers + // defer correction discovered at INPUT_DONE and apply only when EOS is + // confirmed at BLOCK_FINISH. + if (window_bits < 0 && decomp == ISAL_DECOMP_OK && *tofixed == 0 && + (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH || isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE) && - (isal_strm_inflate->crc_flag == IGZIP_DEFLATE) && - *tofixed == 0 && // hasn't been applied yet - decomp == ISAL_DECOMP_OK && // successful decompression isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) { - - // Calculate how many bytes were likely over-consumed - const unsigned int expected_trailer_bytes = 8; - const unsigned int over_consumed = + const uint32_t expected_trailer_bytes = 8; + const uint32_t over_consumed = expected_trailer_bytes - isal_strm_inflate->avail_in; - - // Only apply fix if the over-consumption is reasonable (1-7 bytes) if (over_consumed >= 1 && over_consumed <= 7) { -#ifdef DEBUG - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", - __LINE__, " applying workaround: detected ISA-L over-consumption of ", - over_consumed, " bytes\n"); - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", - __LINE__, " adjusting avail_in from ", - isal_strm_inflate->avail_in, " to ", - isal_strm_inflate->avail_in + over_consumed, - "\n"); -#endif - // Rewind the input pointer to restore over-consumed bytes - isal_strm_inflate->next_in = - (unsigned char *) isal_strm_inflate->next_in - over_consumed; - isal_strm_inflate->avail_in += over_consumed; + if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) { + rewind_adjust_bytes = + consumed_before_adjust < over_consumed + ? consumed_before_adjust + : over_consumed; + if (rewind_adjust_bytes > 0) { + *tofixed = 1; + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " raw EOS correction applied: over_consumed ", + over_consumed, ", rewind_bytes ", + rewind_adjust_bytes, + ", consumed_before_adjust ", + consumed_before_adjust, + ", consumed_after_adjust ", + (consumed_before_adjust - rewind_adjust_bytes), + "\n"); + } + } else if (deferred_correction_bytes != nullptr && + *deferred_correction_bytes == 0) { + *deferred_correction_bytes = over_consumed; + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " raw correction deferred until EOS: over_consumed ", + over_consumed, "\n"); + } + } + } - // Mark that the workaround has been applied + if (window_bits < 0 && decomp == ISAL_DECOMP_OK && *tofixed == 0 && + isal_strm_inflate->block_state == ISAL_BLOCK_FINISH && + deferred_correction_bytes != nullptr && + *deferred_correction_bytes > 0) { + const uint32_t deferred = *deferred_correction_bytes; + rewind_adjust_bytes = consumed_before_adjust < deferred + ? consumed_before_adjust + : deferred; + if (rewind_adjust_bytes > 0) { *tofixed = 1; - - // Also adjust the byte consumption count to reflect the actual deflate data - // consumed Note: bytes_consumed is calculated later, so we'll need to - // adjust it after the calculation + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " deferred raw EOS correction applied: deferred ", + deferred, ", rewind_bytes ", rewind_adjust_bytes, + ", consumed_before_adjust ", consumed_before_adjust, + ", consumed_after_adjust ", + (consumed_before_adjust - rewind_adjust_bytes), "\n"); } + *deferred_correction_bytes = 0; } *output_length = original_avail_out - isal_strm_inflate->avail_out; - *input_length = original_avail_in - isal_strm_inflate->avail_in; + *input_length = consumed_before_adjust - rewind_adjust_bytes; input = isal_strm_inflate->next_in; output = isal_strm_inflate->next_out; @@ -505,7 +541,12 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, *end_of_stream = (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH); } - int ret = (decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) ? 0 : 1; + int ret = 1; + if (decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) { + ret = 0; + } else if (decomp == ISAL_NEED_DICT) { + ret = Z_NEED_DICT; + } #ifdef DEBUG if (ret == Z_OK) { @@ -650,7 +691,8 @@ uncompress(uint8_t *dest, unsigned long *dest_len, const uint8_t *source, unsign }*/ int -ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed) +ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed, + uint32_t *deferred_correction_bytes) { if (!isal_strm_inflate) { Log(LogLevel::LOG_ERROR, "ResetUncompressIGZIP() Line ", @@ -664,6 +706,9 @@ ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed) // Reset workaround flag *tofixed = 0; + if (deferred_correction_bytes != nullptr) { + *deferred_correction_bytes = 0; + } return Z_OK; } diff --git a/igzip.h b/igzip.h index e260576..298860c 100644 --- a/igzip.h +++ b/igzip.h @@ -37,11 +37,14 @@ InitUncompressIGZIP(int windowBits); int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t *input_length, uint8_t *output, uint32_t *output_length, - int *tofixed, unsigned long *total_in, unsigned long *total_out, + int window_bits, int *tofixed, + uint32_t *deferred_correction_bytes, + unsigned long *total_in, unsigned long *total_out, bool *end_of_stream); int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate); int -ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed); +ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed, + uint32_t *deferred_correction_bytes); //#define Z_DEFAULT_COMPRESSION 6 #endif diff --git a/logging.h b/logging.h index 2f6dde3..5579d83 100644 --- a/logging.h +++ b/logging.h @@ -4,10 +4,15 @@ #pragma once #include +#include +#include #include +#include #include #include #include +#include +#include #include #include "config/config.h" @@ -52,6 +57,24 @@ inline std::ostream& GetLogStream() { #ifdef DEBUG_LOG static std::mutex log_mutex; + +inline std::string FormatLogTimestamp() { + using clock = std::chrono::system_clock; + const auto now = clock::now(); + const auto ms = + std::chrono::duration_cast(now.time_since_epoch()) % + 1000; + const std::time_t t = clock::to_time_t(now); + + std::tm tm{}; + localtime_r(&t, &tm); + + std::ostringstream oss; + oss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%S") << '.' << std::setw(3) + << std::setfill('0') << ms.count(); + return oss.str(); +} + template inline void Log(LogLevel level, Args&&... args) { std::lock_guard lock(log_mutex); @@ -67,6 +90,7 @@ inline void Log(LogLevel level, Args&&... args) { std::ostream& stream = GetLogStream(); stream << std::dec; + stream << '[' << FormatLogTimestamp() << "] "; switch (level) { case LogLevel::LOG_ERROR: stream << "Error: "; @@ -77,7 +101,14 @@ inline void Log(LogLevel level, Args&&... args) { case LogLevel::LOG_NONE: return; } - (..., (stream << args)); + + std::ostringstream line; + (..., (line << args)); + std::string message = line.str(); + if (message.empty() || message.back() != '\n') { + message.push_back('\n'); + } + stream << message; stream << std::flush; } diff --git a/tests/zlib_accel_test.cpp b/tests/zlib_accel_test.cpp index 7bfa677..b9e0dc0 100644 --- a/tests/zlib_accel_test.cpp +++ b/tests/zlib_accel_test.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,10 @@ using namespace config; +#ifdef USE_IGZIP +#include "../igzip.h" +#endif + enum BlockCompressibilityType { compressible_block, incompressible_block, @@ -1223,6 +1228,1102 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { DestroyBlock(input); } +#ifdef USE_IGZIP +TEST(IGZIPInflateRegressionTest, EmptyInputContinuationKeepsIGZIPPath) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 1 << 20; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output_chunk(8192); + stream.next_in = reinterpret_cast(compressed.data()); + stream.avail_in = static_cast(compressed.size()); + + int iter = 0; + int last_ret = Z_OK; + while (iter++ < 2048) { + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_NE(last_ret, Z_DATA_ERROR); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + if (last_ret == Z_STREAM_END) { + break; + } + + if (stream.avail_in == 0 && last_ret == Z_OK) { + break; + } + } + + ASSERT_NE(last_ret, Z_STREAM_END); + ASSERT_EQ(stream.avail_in, 0u); + + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + stream.next_in = nullptr; + stream.avail_in = 0; + + int continuation_ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_TRUE(continuation_ret == Z_BUF_ERROR || continuation_ret == Z_OK || + continuation_ret == Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, RawContinuationMustNotIncreaseAvailIn) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 16384; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output_chunk(8192); + stream.next_in = reinterpret_cast(compressed.data()); + stream.avail_in = static_cast(compressed.size()); + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + ASSERT_TRUE(ret == Z_OK || ret == Z_STREAM_END); + + while (ret == Z_OK && stream.avail_in > 0) { + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + ASSERT_TRUE(ret == Z_OK || ret == Z_STREAM_END); + } + + ASSERT_EQ(stream.avail_in, 0u); + + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + stream.next_in = nullptr; + stream.avail_in = 0; + ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_TRUE(ret == Z_BUF_ERROR || ret == Z_OK || ret == Z_STREAM_END); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + uint8_t one_byte = 0; + stream.next_in = &one_byte; + stream.avail_in = 1; + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_NE(ret, Z_DATA_ERROR); + EXPECT_LE(stream.avail_in, 1u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, RawTrailingByteMustNotIncreaseAvailIn) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 4096; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + std::string compressed_with_trailing = compressed; + compressed_with_trailing.push_back('\0'); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector uncompressed(input_length * 2); + stream.next_in = reinterpret_cast(compressed_with_trailing.data()); + stream.avail_in = static_cast(compressed_with_trailing.size()); + stream.next_out = reinterpret_cast(uncompressed.data()); + stream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_NE(ret, Z_DATA_ERROR); + EXPECT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + // For valid raw-deflate stream with one extra byte, inflate may leave that + // byte unconsumed, but avail_in must never increase. + EXPECT_LE(stream.avail_in, 1u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, + RawOneByteContinuationMustNotIncreaseAvailInAcrossSizes) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + std::vector input_sizes = { + 1, 2, 3, 7, 8, 15, 16, 31, 32, 63, 64, + 127, 128, 255, 256, 511, 512, 1023, 1024, 2047, 2048, 4095, + 4096, 8191, 8192, 16384, 32768}; + + for (size_t input_length : input_sizes) { + std::vector input(input_length); + for (size_t i = 0; i < input_length; ++i) { + input[i] = static_cast((i * 131u + 17u) & 0xFFu); + } + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input.data(), input_length, &compressed, -15, + Z_FINISH, &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output_chunk(1024); + stream.next_in = reinterpret_cast(compressed.data()); + stream.avail_in = static_cast(compressed.size()); + + int last_ret = Z_OK; + for (int iter = 0; iter < 4096; ++iter) { + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + ASSERT_NE(last_ret, Z_DATA_ERROR); + + if (stream.avail_in == 0 || last_ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(stream.avail_in, 0u); + + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + stream.next_in = nullptr; + stream.avail_in = 0; + int empty_ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_TRUE(empty_ret == Z_BUF_ERROR || empty_ret == Z_OK || + empty_ret == Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + uint8_t trailing_byte = 0; + stream.next_in = &trailing_byte; + stream.avail_in = 1; + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + + int one_byte_ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_NE(one_byte_ret, Z_DATA_ERROR); + EXPECT_LE(stream.avail_in, 1u) << "input_length=" << input_length; + + inflateEnd(&stream); + } +} + +TEST(IGZIPInflateRegressionTest, + TinyRawEntryMustNotOverconsumePastStreamBoundary) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + // Empty raw-deflate entries are often tiny (commonly 2 bytes). The + // decompressor must stop exactly at stream end and leave trailing bytes for + // the caller. + const std::string empty_payload; + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(empty_payload.data(), empty_payload.size(), &compressed, + -15, Z_FINISH, &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + ASSERT_GT(compressed.size(), 0u); + + const size_t trailing_len = 510; + std::string input = compressed; + input.append(trailing_len, static_cast(0xA5)); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output(512); + stream.next_in = reinterpret_cast(input.data()); + stream.avail_in = static_cast(input.size()); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + const ExecutionPath observed_path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(observed_path == IGZIP || observed_path == ZLIB); + ASSERT_NE(ret, Z_DATA_ERROR); + + // No output is expected for empty payload. Most importantly, all trailing + // bytes must remain unconsumed for the caller. + EXPECT_EQ(output.size() - stream.avail_out, 0u); + EXPECT_EQ(stream.avail_in, trailing_len) + << "compressed_size=" << compressed.size(); + + inflateEnd(&stream); +} + +TEST(IGZIPInflateRegressionTest, + RawStreamEndMustPreserveEightTrailingBytes) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 32768; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + std::string with_trailing = compressed; + with_trailing.append("ABCDEFGH", 8); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output(input_length * 2); + stream.next_in = reinterpret_cast(with_trailing.data()); + stream.avail_in = static_cast(with_trailing.size()); + + int last_ret = Z_OK; + for (int iter = 0; iter < 64; ++iter) { + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + const ExecutionPath path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(last_ret, Z_DATA_ERROR); + if (last_ret == Z_STREAM_END) { + break; + } + if (stream.avail_out > 0 && stream.avail_in == 0) { + break; + } + } + + EXPECT_EQ(last_ret, Z_STREAM_END); + EXPECT_EQ(stream.avail_in, 8u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, + RawSplitInputDefersCorrectionUntilStreamEnd) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 65536; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + ASSERT_GT(compressed.size(), 2u); + + std::string with_trailing = compressed; + with_trailing.append("ABCDEFGH", 8); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output(input_length * 2); + const size_t split_offset = compressed.size() - 1; + + stream.next_in = reinterpret_cast(with_trailing.data()); + stream.avail_in = static_cast(split_offset); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + { + const ExecutionPath path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + } + ASSERT_NE(ret, Z_DATA_ERROR); + + stream.next_in = + reinterpret_cast(with_trailing.data() + split_offset); + stream.avail_in = + static_cast(with_trailing.size() - split_offset); + + int last_ret = ret; + for (int iter = 0; iter < 64; ++iter) { + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + const ExecutionPath path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(last_ret, Z_DATA_ERROR); + if (last_ret == Z_STREAM_END) { + break; + } + if (stream.avail_out > 0 && stream.avail_in == 0) { + break; + } + } + + EXPECT_EQ(last_ret, Z_STREAM_END); + EXPECT_EQ(stream.avail_in, 8u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPDeflateRegressionTest, + RepeatedFinishWithEmptyInputMustReturnStreamEnd) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + const char* input = "igzip-finish-regression-payload"; + const size_t input_length = strlen(input); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(4096); + stream.next_in = reinterpret_cast(const_cast(input)); + stream.avail_in = static_cast(input_length); + + int ret = Z_OK; + for (int iter = 0; iter < 32; ++iter) { + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + + for (int iter = 0; iter < 64; ++iter) { + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + int finish_ret = deflate(&stream, Z_FINISH); + EXPECT_EQ(finish_ret, Z_STREAM_END) << "iter=" << iter; + EXPECT_EQ(GetDeflateExecutionPath(&stream), IGZIP); + } + + deflateEnd(&stream); +} + +TEST(IGZIPDeflateRegressionTest, ResetMustNotStallSyncFlushOnSameStream) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(4096); + std::vector input(512, 'A'); + + for (int cycle = 0; cycle < 20; ++cycle) { + stream.next_in = reinterpret_cast(input.data()); + stream.avail_in = static_cast(input.size()); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + int ret = deflate(&stream, Z_NO_FLUSH); + ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR) << "cycle=" << cycle; + ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = deflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(ret, Z_OK) << "cycle=" << cycle; + ASSERT_LT(stream.avail_out, output.size()) << "cycle=" << cycle; + ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END) << "cycle=" << cycle; + + ASSERT_EQ(deflateReset(&stream), Z_OK) << "cycle=" << cycle; + } + + deflateEnd(&stream); +} + +TEST(IGZIPDeflateRegressionTest, + RepeatedEmptySyncFlushMustEventuallyReportNoProgress) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(4096); + std::vector input(1024, 'B'); + + stream.next_in = reinterpret_cast(input.data()); + stream.avail_in = static_cast(input.size()); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + int ret = deflate(&stream, Z_NO_FLUSH); + ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR); + ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + + bool observed_buf_error = false; + for (int iter = 0; iter < 128; ++iter) { + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = deflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + ASSERT_NE(ret, Z_DATA_ERROR) << "iter=" << iter; + + if (ret == Z_BUF_ERROR) { + observed_buf_error = true; + break; + } + } + + EXPECT_TRUE(observed_buf_error) + << "Repeated empty Z_SYNC_FLUSH calls must eventually stop producing " + << "new bytes and return Z_BUF_ERROR"; + + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + ret = deflate(&stream, Z_FINISH); + EXPECT_EQ(ret, Z_STREAM_END); + + deflateEnd(&stream); +} + +TEST(IGZIPDeflateRegressionTest, + DictionaryStreamMustStayOnZlibAcrossReset) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + const char dict[] = "0123456789abcdef"; + ASSERT_EQ(deflateSetDictionary( + &stream, reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), + Z_OK); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + + std::vector out1(4096); + const char* msg1 = "dictionary-stream-first-message"; + stream.next_in = reinterpret_cast(const_cast(msg1)); + stream.avail_in = static_cast(strlen(msg1)); + int ret = Z_OK; + for (int iter = 0; iter < 16; ++iter) { + stream.next_out = reinterpret_cast(out1.data()); + stream.avail_out = static_cast(out1.size()); + ret = deflate(&stream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + const size_t out1_size = out1.size() - stream.avail_out; + + ASSERT_EQ(deflateReset(&stream), Z_OK); + + // Re-set dictionary after reset (standard zlib API usage). + ASSERT_EQ(deflateSetDictionary( + &stream, reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), + Z_OK); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + + std::vector out2(4096); + const char* msg2 = "dictionary-stream-second-message"; + stream.next_in = reinterpret_cast(const_cast(msg2)); + stream.avail_in = static_cast(strlen(msg2)); + ret = Z_OK; + for (int iter = 0; iter < 16; ++iter) { + stream.next_out = reinterpret_cast(out2.data()); + stream.avail_out = static_cast(out2.size()); + ret = deflate(&stream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + const size_t out2_size = out2.size() - stream.avail_out; + + z_stream verify; + memset(&verify, 0, sizeof(verify)); + ASSERT_EQ(inflateInit2(&verify, 15), Z_OK); + + std::vector verify_out(1024); + verify.next_in = reinterpret_cast(out2.data()); + verify.avail_in = static_cast(out2_size); + verify.next_out = reinterpret_cast(verify_out.data()); + verify.avail_out = static_cast(verify_out.size()); + + ret = inflate(&verify, Z_NO_FLUSH); + EXPECT_EQ(ret, Z_NEED_DICT); + + ASSERT_EQ(inflateSetDictionary(&verify, + reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), + Z_OK); + ret = inflate(&verify, Z_FINISH); + EXPECT_EQ(ret, Z_STREAM_END); + + inflateEnd(&verify); + + EXPECT_GT(out1_size, 0u); + EXPECT_GT(out2_size, 0u); + + deflateEnd(&stream); +} + +TEST(IGZIPDeflateRegressionTest, + ParallelDictionaryResetStreamsRemainRoundTripSafe) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + const std::string dictionary = + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const int thread_count = 4; + const int iterations_per_thread = 120; + + std::atomic failed{false}; + std::string failure_reason; + std::mutex failure_mutex; + + auto set_failure = [&](const std::string& message) { + bool expected = false; + if (failed.compare_exchange_strong(expected, true)) { + std::lock_guard lock(failure_mutex); + failure_reason = message; + } + }; + + auto worker = [&](int worker_id) { + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + int ret = deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { + set_failure("deflateInit2 failed"); + return; + } + + for (int i = 0; i < iterations_per_thread && !failed.load(); ++i) { + if (deflateReset(&cstream) != Z_OK) { + set_failure("deflateReset failed"); + break; + } + + ret = deflateSetDictionary( + &cstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())); + if (ret != Z_OK) { + set_failure("deflateSetDictionary failed"); + break; + } + + if (GetDeflateExecutionPath(&cstream) != ZLIB) { + set_failure("dictionary stream did not stay on zlib"); + break; + } + + std::string input; + input.reserve(64 * 1024); + for (int j = 0; j < 1024; ++j) { + input += "tid="; + input += std::to_string(worker_id); + input += " iter="; + input += std::to_string(i); + input += " row="; + input += std::to_string(j); + input += " payload="; + input += dictionary; + input += "\n"; + } + + std::vector compressed(compressBound(input.size())); + cstream.next_in = + reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + int def_ret = Z_OK; + for (int guard = 0; guard < 64; ++guard) { + def_ret = deflate(&cstream, Z_FINISH); + if (def_ret == Z_STREAM_END) { + break; + } + if (def_ret != Z_OK && def_ret != Z_BUF_ERROR) { + break; + } + } + if (def_ret != Z_STREAM_END) { + set_failure("deflate did not reach Z_STREAM_END"); + break; + } + + const size_t compressed_size = compressed.size() - cstream.avail_out; + if (compressed_size == 0) { + set_failure("compressed payload was empty"); + break; + } + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ret = inflateInit2(&dstream, 15); + if (ret != Z_OK) { + set_failure("inflateInit2 failed"); + break; + } + + std::vector output(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + ret = inflate(&dstream, Z_NO_FLUSH); + if (ret == Z_NEED_DICT) { + ret = inflateSetDictionary( + &dstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())); + if (ret == Z_OK) { + ret = inflate(&dstream, Z_FINISH); + } + } + + const size_t produced = output.size() - dstream.avail_out; + const bool output_matches = + (produced == input.size()) && + (memcmp(output.data(), input.data(), input.size()) == 0); + + inflateEnd(&dstream); + + if (ret != Z_STREAM_END) { + set_failure("inflate did not reach Z_STREAM_END"); + break; + } + if (!output_matches) { + set_failure("round-trip data mismatch"); + break; + } + } + + deflateEnd(&cstream); + }; + + std::vector workers; + workers.reserve(thread_count); + for (int worker_id = 0; worker_id < thread_count; ++worker_id) { + workers.emplace_back(worker, worker_id); + } + for (auto& worker_thread : workers) { + worker_thread.join(); + } + + ASSERT_FALSE(failed.load()) << failure_reason; +} + +TEST(IGZIPDeflateRegressionTest, + IGZIPDictionaryOutputMustBeZlibDictionaryCompatible) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + const std::string dictionary = + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + std::string input; + for (int i = 0; i < 512; ++i) { + input += "doc:"; + input += dictionary; + input += "|"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + ASSERT_EQ(deflateSetDictionary( + &cstream, + reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + + std::vector compressed(compressBound(input.size())); + cstream.next_in = reinterpret_cast( + const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + int ret = Z_OK; + for (int iter = 0; iter < 32; ++iter) { + ret = deflate(&cstream, Z_FINISH); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + const size_t compressed_size = compressed.size() - cstream.avail_out; + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector uncompressed(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = uncompressed.data(); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_NO_FLUSH); + if (ret == Z_NEED_DICT) { + ASSERT_EQ(inflateSetDictionary( + &dstream, + reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ret = inflate(&dstream, Z_FINISH); + } + + EXPECT_EQ(ret, Z_STREAM_END); + const size_t produced = uncompressed.size() - dstream.avail_out; + ASSERT_EQ(produced, input.size()); + EXPECT_EQ(memcmp(uncompressed.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +TEST(IGZIPDeflateRegressionTest, + FinishWithTinyOutputBufferMustNotTruncateStream) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + std::string input; + input.reserve(256 * 1024); + for (int i = 0; i < 4096; ++i) { + input += "{\"k\":"; + input += std::to_string(i); + input += ",\"msg\":\"abcdefghijklmnopqrstuvwxyz\"}"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + cstream.next_in = + reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + + std::vector compressed; + compressed.reserve(input.size()); + + int ret = Z_OK; + for (int iter = 0; iter < 20000; ++iter) { + unsigned char out_chunk[64]; + cstream.next_out = out_chunk; + cstream.avail_out = sizeof(out_chunk); + + ret = deflate(&cstream, Z_FINISH); + const ExecutionPath path = GetDeflateExecutionPath(&cstream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(ret, Z_DATA_ERROR) << "iter=" << iter; + + const size_t produced = sizeof(out_chunk) - cstream.avail_out; + compressed.insert(compressed.end(), out_chunk, out_chunk + produced); + + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector output(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + for (int iter = 0; iter < 1024; ++iter) { + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + if (ret == Z_BUF_ERROR && dstream.avail_in == 0) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + const size_t out_size = output.size() - dstream.avail_out; + ASSERT_EQ(out_size, input.size()); + EXPECT_EQ(memcmp(output.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +TEST(IGZIPDeflateRegressionTest, + SyncFlushWithInputMustFallbackToZlibForStreamSafety) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + std::string input; + input.reserve(64 * 1024); + for (int i = 0; i < 1024; ++i) { + input += "record-"; + input += std::to_string(i); + input += "-abcdefghijklmnopqrstuvwxyz"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector compressed; + compressed.reserve(input.size()); + + cstream.next_in = + reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + + unsigned char sync_chunk[256]; + cstream.next_out = sync_chunk; + cstream.avail_out = sizeof(sync_chunk); + + int ret = deflate(&cstream, Z_SYNC_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + const size_t sync_produced = sizeof(sync_chunk) - cstream.avail_out; + compressed.insert(compressed.end(), sync_chunk, sync_chunk + sync_produced); + + for (int iter = 0; iter < 8192; ++iter) { + unsigned char out_chunk[256]; + cstream.next_out = out_chunk; + cstream.avail_out = sizeof(out_chunk); + cstream.next_in = nullptr; + cstream.avail_in = 0; + + ret = deflate(&cstream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + + const size_t produced = sizeof(out_chunk) - cstream.avail_out; + compressed.insert(compressed.end(), out_chunk, out_chunk + produced); + + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector output(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + for (int iter = 0; iter < 1024; ++iter) { + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + const size_t out_size = output.size() - dstream.avail_out; + ASSERT_EQ(out_size, input.size()); + EXPECT_EQ(memcmp(output.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +TEST(IGZIPInflateRegressionTest, + NeedDictFromIGZIPMustFallbackToZlibOnFirstInflateCall) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + const std::string dictionary = + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + std::string input; + for (int i = 0; i < 512; ++i) { + input += "doc:"; + input += dictionary; + input += "|"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + ASSERT_EQ(deflateSetDictionary( + &cstream, + reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + + std::vector compressed(compressBound(input.size())); + cstream.next_in = + reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + int ret = Z_OK; + for (int iter = 0; iter < 64; ++iter) { + ret = deflate(&cstream, Z_FINISH); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + const size_t compressed_size = compressed.size() - cstream.avail_out; + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector uncompressed(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = uncompressed.data(); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&dstream), ZLIB); + ASSERT_EQ(ret, Z_NEED_DICT); + + ASSERT_EQ(inflateSetDictionary( + &dstream, + reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ret = inflate(&dstream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + const size_t out_size = uncompressed.size() - dstream.avail_out; + ASSERT_EQ(out_size, input.size()); + EXPECT_EQ(memcmp(uncompressed.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +#endif + INSTANTIATE_TEST_SUITE_P( CompressDecompress, ZlibPartialAndMultiStreamTest, testing::Combine( @@ -1354,6 +2455,120 @@ INSTANTIATE_TEST_SUITE_P( class ConfigLoaderTest : public ::testing::Test {}; +#if defined(USE_IGZIP) || defined(USE_QAT) || defined(USE_IAA) +class DictionaryMidstreamFallbackRegressionTest : public ::testing::Test {}; + +static void RunMidstreamSetDictionaryRegression(ExecutionPath accel_path) { + SetCompressPath(accel_path, true, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + std::string first_chunk; + std::string second_chunk; + for (int i = 0; i < 256; ++i) { + first_chunk += "first:"; + first_chunk += std::to_string(i); + first_chunk += ":abcdefghijklmnopqrstuvwxyz|"; + + second_chunk += "second:"; + second_chunk += std::to_string(i); + second_chunk += ":0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ|"; + } + const std::string expected = first_chunk + second_chunk; + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector compressed(compressBound(expected.size()) + 512); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + cstream.next_in = + reinterpret_cast(const_cast(first_chunk.data())); + cstream.avail_in = static_cast(first_chunk.size()); + int ret = deflate(&cstream, Z_NO_FLUSH); + ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR); + const ExecutionPath observed_before_dict = GetDeflateExecutionPath(&cstream); + EXPECT_TRUE(observed_before_dict == accel_path || + observed_before_dict == ZLIB); + + const unsigned char dict[] = "midstream-dictionary"; + const int dict_ret = deflateSetDictionary( + &cstream, dict, static_cast(sizeof(dict) - 1)); + + // zlib semantics require dictionary to be set before any deflate output. + // Accepting this call midstream can desynchronize accelerator and zlib state. + EXPECT_EQ(dict_ret, Z_STREAM_ERROR); + // Rejected dictionary update must not mutate active stream path. + EXPECT_EQ(GetDeflateExecutionPath(&cstream), observed_before_dict); + + cstream.next_in = + reinterpret_cast(const_cast(second_chunk.data())); + cstream.avail_in = static_cast(second_chunk.size()); + for (int guard = 0; guard < 128; ++guard) { + ret = deflate(&cstream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + + const size_t compressed_size = compressed.size() - cstream.avail_out; + ASSERT_GT(compressed_size, 0u); + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector uncompressed(expected.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = uncompressed.data(); + dstream.avail_out = static_cast(uncompressed.size()); + + for (int guard = 0; guard < 128; ++guard) { + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + const size_t produced = uncompressed.size() - dstream.avail_out; + ASSERT_EQ(produced, expected.size()); + EXPECT_EQ(memcmp(uncompressed.data(), expected.data(), expected.size()), 0); + + inflateEnd(&dstream); +} + +#ifdef USE_IGZIP +TEST_F(DictionaryMidstreamFallbackRegressionTest, + IGZIPRejectsMidstreamSetDictionary) { + RunMidstreamSetDictionaryRegression(IGZIP); +} +#endif + +#ifdef USE_QAT +TEST_F(DictionaryMidstreamFallbackRegressionTest, + QATRejectsMidstreamSetDictionary) { + RunMidstreamSetDictionaryRegression(QAT); +} +#endif + +#ifdef USE_IAA +TEST_F(DictionaryMidstreamFallbackRegressionTest, + IAARejectsMidstreamSetDictionary) { + RunMidstreamSetDictionaryRegression(IAA); +} +#endif +#endif + void CreateAndWriteTempConfigFile(const char* file_path) { std::ofstream temp_file(file_path); temp_file << "use_qat_compress=5000\n"; diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 5ef1f8b..57d5d26 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -5,13 +5,17 @@ #include #include +#include #include #include #include #include +#include #include +#include #include +#include #include #include "config/config.h" @@ -184,6 +188,12 @@ static void cleanup_zlib_accel(void) { // Avoid recursive call (e.g., if QATzip falls back to zlib internally) static thread_local bool in_call = false; +enum class IGZIPFinishPhase { + ACTIVE = 0, + DRAINING = 1, + FINISHED = 2, +}; + struct DeflateSettings { DeflateSettings(int _level, int _method, int _window_bits, int _mem_level, int _strategy) @@ -198,14 +208,38 @@ struct DeflateSettings { int window_bits; int mem_level; int strategy; + bool uses_dictionary = false; + std::vector dictionary; + uint32_t dictionary_crc32 = 0; + uInt dictionary_length = 0; + uint64_t dictionary_set_count = 0; + bool igzip_sync_flush_drained = false; + uint64_t igzip_calls = 0; + uint64_t igzip_finish_calls = 0; + uint64_t igzip_stream_end_count = 0; + uint64_t igzip_no_progress_count = 0; + uint64_t igzip_zbuf_error_count = 0; + uint64_t igzip_bytes_in = 0; + uint64_t igzip_bytes_out = 0; + bool igzip_seen_stream_end = false; + IGZIPFinishPhase igzip_finish_phase = IGZIPFinishPhase::ACTIVE; + uint64_t igzip_finish_drain_calls = 0; + uint64_t igzip_finish_no_progress_calls = 0; ExecutionPath path = UNDEFINED; struct isal_zstream *isal_strm = nullptr; }; struct InflateSettings { - InflateSettings(int _window_bits) : window_bits(_window_bits) {} + InflateSettings(int _window_bits) + : window_bits(_window_bits), + trailer_overconsumption_fixed(0), + deferred_trailer_correction_bytes(0) {} int window_bits; int trailer_overconsumption_fixed; /* indicates if fix has been applied for overconsumption issue*/ + uint32_t deferred_trailer_correction_bytes; + bool force_zlib_for_raw_boundary = false; + bool uses_dictionary = false; + std::vector dictionary; ExecutionPath path = UNDEFINED; struct inflate_state *isal_strm = nullptr; }; @@ -244,10 +278,112 @@ class InflateStreamSettings { }; InflateStreamSettings inflate_stream_settings; +#ifdef USE_IGZIP +static bool IsIGZIPDeflateFinished(const struct isal_zstream* stream) { + if (stream == nullptr) { + return false; + } + const enum isal_zstate_state state = stream->internal_state.state; + return state == ZSTATE_END || state == ZSTATE_TMP_END; +} +#endif + +static const char* ExecutionPathToString(ExecutionPath path) { + switch (path) { + case UNDEFINED: + return "UNDEFINED"; + case ZLIB: + return "ZLIB"; + case QAT: + return "QAT"; + case IAA: + return "IAA"; + case IGZIP: + return "IGZIP"; + default: + return "UNKNOWN"; + } +} + +static const char* IGZIPFinishPhaseToString(IGZIPFinishPhase phase) { + switch (phase) { + case IGZIPFinishPhase::ACTIVE: + return "ACTIVE"; + case IGZIPFinishPhase::DRAINING: + return "DRAINING"; + case IGZIPFinishPhase::FINISHED: + return "FINISHED"; + default: + return "UNKNOWN"; + } +} + +static unsigned long CurrentThreadId() { + return static_cast(reinterpret_cast(pthread_self())); +} + +static std::atomic g_next_stream_id{1}; +static std::mutex g_stream_id_mu; +static std::unordered_map g_stream_ids; + +static uint64_t GetOrCreateStreamId(z_streamp strm) { + if (strm == nullptr) { + return 0; + } + + std::lock_guard lock(g_stream_id_mu); + auto it = g_stream_ids.find(strm); + if (it != g_stream_ids.end()) { + return it->second; + } + + const uint64_t id = g_next_stream_id.fetch_add(1, std::memory_order_relaxed); + g_stream_ids.emplace(strm, id); + return id; +} + +static void ClearStreamId(z_streamp strm) { + if (strm == nullptr) { + return; + } + + std::lock_guard lock(g_stream_id_mu); + g_stream_ids.erase(strm); +} + +static void SetDeflatePath(DeflateSettings* settings, z_streamp strm, + ExecutionPath new_path, const char* reason) { + if (settings == nullptr || settings->path == new_path) { + return; + } + Log(LogLevel::LOG_INFO, "deflate path transition Line ", __LINE__, ", strm ", + static_cast(strm), ", tid ", CurrentThreadId(), ", old ", + ExecutionPathToString(settings->path), ", new ", + ExecutionPathToString(new_path), ", reason ", reason, "\n"); + settings->path = new_path; +} + +static void SetInflatePath(InflateSettings* settings, z_streamp strm, + ExecutionPath new_path, const char* reason) { + if (settings == nullptr || settings->path == new_path) { + return; + } + Log(LogLevel::LOG_INFO, "inflate path transition Line ", __LINE__, ", strm ", + static_cast(strm), ", tid ", CurrentThreadId(), ", old ", + ExecutionPathToString(settings->path), ", new ", + ExecutionPathToString(new_path), ", reason ", reason, "\n"); + settings->path = new_path; +} + int ZEXPORT deflateInit_(z_streamp strm, int level, const char* version, int stream_size) { + const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "deflateInit_ Line ", __LINE__, ", strm ", static_cast(strm), ", level ", level, "\n"); + Log(LogLevel::LOG_INFO, + "trace event=deflate_init stream_id=", stream_id, " strm=", + static_cast(strm), " level=", level, " tid=", CurrentThreadId(), + "\n"); deflate_stream_settings.Set(strm, level, Z_DEFLATED, 15, 8, Z_DEFAULT_STRATEGY); @@ -257,9 +393,15 @@ int ZEXPORT deflateInit_(z_streamp strm, int level, const char* version, int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, int window_bits, int mem_level, int strategy, const char* version, int stream_size) { + const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "deflateInit2_ Line ", __LINE__, ", strm ", static_cast(strm), ", level ", level, ", window_bits ", window_bits, " \n"); + Log(LogLevel::LOG_INFO, + "trace event=deflate_init2 stream_id=", stream_id, " strm=", + static_cast(strm), " level=", level, " method=", method, + " window_bits=", window_bits, " mem_level=", mem_level, + " strategy=", strategy, " tid=", CurrentThreadId(), "\n"); /*#ifdef USE_IGZIP if (configs[USE_IGZIP_COMPRESS] && configs[USE_IGZIP_UNCOMPRESS] && @@ -275,12 +417,51 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, uInt dictLength) { + const uint64_t stream_id = GetOrCreateStreamId(strm); if (!configs[IGNORE_ZLIB_DICTIONARY]) { Log(LogLevel::LOG_INFO, "deflateSetDictionary Line ", __LINE__, ", strm ", - static_cast(strm), ", dictLength ", dictLength, "\n"); + static_cast(strm), ", tid ", CurrentThreadId(), + ", dictLength ", dictLength, "\n"); + + // Match zlib API semantics even when acceleration bypassed orig_deflate: + // dictionary must be set right after init/reset and before any progress. + if (strm->total_in != 0 || strm->total_out != 0) { + Log(LogLevel::LOG_ERROR, "deflateSetDictionary Line ", __LINE__, + ", strm ", static_cast(strm), ", tid ", CurrentThreadId(), + ", rejecting midstream dictionary set after progress total_in ", + strm->total_in, ", total_out ", strm->total_out, "\n"); + Log(LogLevel::LOG_ERROR, + "trace event=reject_midstream_dict stream_id=", stream_id, + " strm=", static_cast(strm), " ret=", Z_STREAM_ERROR, + " total_in=", strm->total_in, " total_out=", strm->total_out, + " adler=", strm->adler, " dict_len=", dictLength, + " tid=", CurrentThreadId(), "\n"); + return Z_STREAM_ERROR; + } + DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); - deflate_settings->path = ZLIB; - return orig_deflateSetDictionary(strm, dictionary, dictLength); + const int ret = orig_deflateSetDictionary(strm, dictionary, dictLength); + Log(LogLevel::LOG_INFO, + "trace event=deflate_set_dict stream_id=", stream_id, " strm=", + static_cast(strm), " ret=", ret, " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, + " dict_len=", dictLength, " tid=", CurrentThreadId(), "\n"); + if (ret == Z_OK) { + deflate_settings->uses_dictionary = true; + deflate_settings->dictionary_crc32 = + static_cast(crc32(0L, dictionary, dictLength)); + deflate_settings->dictionary_length = dictLength; + deflate_settings->dictionary_set_count++; + SetDeflatePath(deflate_settings, strm, ZLIB, + "deflateSetDictionary called"); + Log(LogLevel::LOG_INFO, + "trace event=deflate_set_dict_fingerprint stream_id=", stream_id, + " strm=", static_cast(strm), " dict_len=", dictLength, + " dict_crc32=", deflate_settings->dictionary_crc32, + " dict_set_count=", deflate_settings->dictionary_set_count, + " tid=", CurrentThreadId(), "\n"); + } + return ret; } Log(LogLevel::LOG_INFO, "deflateSetDictionary Line ", __LINE__, " ignored because ignore_zlib_dictionary is set to ", @@ -290,13 +471,58 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, int ZEXPORT deflate(z_streamp strm, int flush) { DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); + const uint64_t stream_id = GetOrCreateStreamId(strm); INCREMENT_STAT(DEFLATE_COUNT); PrintStats(); + if (deflate_settings->uses_dictionary) { + SetDeflatePath(deflate_settings, strm, ZLIB, + "stream marked uses_dictionary"); + } + + if (flush != Z_FINISH && + deflate_settings->igzip_finish_phase == IGZIPFinishPhase::DRAINING) { + deflate_settings->igzip_finish_phase = IGZIPFinishPhase::ACTIVE; + deflate_settings->igzip_finish_no_progress_calls = 0; + } + + if (flush == Z_FINISH && deflate_settings->path == IGZIP && + deflate_settings->igzip_finish_phase == IGZIPFinishPhase::FINISHED) { + Log(LogLevel::LOG_INFO, + "trace event=igzip_finish_already_complete stream_id=", stream_id, + " strm=", static_cast(strm), " total_in=", strm->total_in, + " total_out=", strm->total_out, " tid=", CurrentThreadId(), "\n"); + return Z_STREAM_END; + } + + const bool is_streaming_flush_with_input = + (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || + flush == Z_FULL_FLUSH || flush == Z_BLOCK) && + (strm->avail_in > 0); + if (deflate_settings->path == UNDEFINED && is_streaming_flush_with_input) { + SetDeflatePath(deflate_settings, strm, ZLIB, + "streaming flush with input"); + } + + const bool is_sync_flush = + (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || flush == Z_BLOCK); + Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", - static_cast(deflate_settings->path), "\n"); + static_cast(deflate_settings->path), ", path_name ", + ExecutionPathToString(deflate_settings->path), ", window_bits ", + deflate_settings->window_bits, ", uses_dictionary ", + deflate_settings->uses_dictionary, ", stream_id ", stream_id, + ", total_in ", strm->total_in, ", total_out ", strm->total_out, + ", adler ", strm->adler, ", tid ", CurrentThreadId(), "\n"); + Log(LogLevel::LOG_INFO, + "trace event=deflate_enter stream_id=", stream_id, " strm=", + static_cast(strm), " flush=", flush, " path=", + ExecutionPathToString(deflate_settings->path), " avail_in=", strm->avail_in, + " avail_out=", strm->avail_out, " total_in=", strm->total_in, + " total_out=", strm->total_out, " uses_dict=", + deflate_settings->uses_dictionary, " tid=", CurrentThreadId(), "\n"); int ret = 1; bool iaa_available = false; @@ -317,7 +543,38 @@ int ZEXPORT deflate(z_streamp strm, int flush) { SupportedOptionsQAT(deflate_settings->window_bits, input_len); #endif #ifdef USE_IGZIP - igzip_available = configs[USE_IGZIP_COMPRESS]; + const bool igzip_finish_only_mode = (flush == Z_FINISH); + const bool igzip_finish_buffer_ok = + (flush != Z_FINISH || strm->avail_out >= 256); + igzip_available = configs[USE_IGZIP_COMPRESS] && igzip_finish_only_mode && + igzip_finish_buffer_ok; + if (configs[USE_IGZIP_COMPRESS] && !igzip_finish_only_mode) { + if (deflate_settings->path == UNDEFINED) { + SetDeflatePath(deflate_settings, strm, ZLIB, + "igzip unavailable for non-finish flush"); + } + Log(LogLevel::LOG_INFO, + "trace event=igzip_compress_unavailable stream_id=", stream_id, + " strm=", static_cast(strm), + " reason=flush_not_finish flush=", flush, + " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, + " total_in=", strm->total_in, " total_out=", strm->total_out, + " tid=", CurrentThreadId(), "\n"); + } + if (configs[USE_IGZIP_COMPRESS] && igzip_finish_only_mode && + !igzip_finish_buffer_ok) { + if (deflate_settings->path == UNDEFINED) { + SetDeflatePath(deflate_settings, strm, ZLIB, + "igzip unavailable for tiny finish buffer"); + } + Log(LogLevel::LOG_INFO, + "trace event=igzip_compress_unavailable stream_id=", stream_id, + " strm=", static_cast(strm), + " reason=finish_buffer_too_small flush=", flush, + " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, + " total_in=", strm->total_in, " total_out=", strm->total_out, + " tid=", CurrentThreadId(), "\n"); + } #endif // If both accelerators are enabled, send configured ratio of requests to @@ -347,7 +604,8 @@ int ZEXPORT deflate(z_streamp strm, int flush) { ret = CompressIAA(strm->next_in, &input_len, strm->next_out, &output_len, qpl_path_hardware, deflate_settings->window_bits, max_compressed_size); - deflate_settings->path = IAA; + SetDeflatePath(deflate_settings, strm, IAA, + "selected IAA accelerator"); in_call = false; INCREMENT_STAT(DEFLATE_IAA_COUNT); INCREMENT_STAT_COND(ret != 0, DEFLATE_IAA_ERROR_COUNT); @@ -357,13 +615,37 @@ int ZEXPORT deflate(z_streamp strm, int flush) { in_call = true; ret = CompressQAT(strm->next_in, &input_len, strm->next_out, &output_len, deflate_settings->window_bits); - deflate_settings->path = QAT; + SetDeflatePath(deflate_settings, strm, QAT, + "selected QAT accelerator"); in_call = false; INCREMENT_STAT(DEFLATE_QAT_COUNT); INCREMENT_STAT_COND(ret != 0, DEFLATE_QAT_ERROR_COUNT); #endif // USE_QAT } else if (path_selected == IGZIP) { #ifdef USE_IGZIP + const uInt pre_avail_in = strm->avail_in; + const uInt pre_avail_out = strm->avail_out; + const uLong pre_total_in = strm->total_in; + const uLong pre_total_out = strm->total_out; + + if (strm->avail_in > 0) { + deflate_settings->igzip_sync_flush_drained = false; + } + + if (flush == Z_FINISH && + deflate_settings->igzip_finish_phase == IGZIPFinishPhase::ACTIVE) { + deflate_settings->igzip_finish_phase = IGZIPFinishPhase::DRAINING; + deflate_settings->igzip_finish_no_progress_calls = 0; + } + + if (is_sync_flush && strm->avail_in == 0 && + deflate_settings->igzip_sync_flush_drained) { + Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", + static_cast(strm), + ", repeated empty sync flush on drained IGZIP stream; returning Z_BUF_ERROR\n"); + return Z_BUF_ERROR; + } + if (deflate_settings->isal_strm == nullptr) { deflate_settings->method = 0; deflate_settings->isal_strm = InitCompressIGZIP(deflate_settings->level, deflate_settings->window_bits); //hardcoded window bits @@ -375,8 +657,49 @@ int ZEXPORT deflate(z_streamp strm, int flush) { in_call = true; ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, &input_len, strm->next_out, &output_len, &strm->total_in, &strm->total_out); - deflate_settings->path = IGZIP; + SetDeflatePath(deflate_settings, strm, IGZIP, + "selected IGZIP accelerator"); in_call = false; + + deflate_settings->igzip_calls++; + if (flush == Z_FINISH) { + deflate_settings->igzip_finish_calls++; + } + deflate_settings->igzip_bytes_in += input_len; + deflate_settings->igzip_bytes_out += output_len; + + uint32_t output_crc32 = 0; + uint32_t out_head_b0 = 256; + uint32_t out_head_b1 = 256; + uint32_t out_tail_b0 = 256; + uint32_t out_tail_b1 = 256; + if (output_len > 0) { + output_crc32 = + static_cast(crc32(0L, strm->next_out, output_len)); + out_head_b0 = static_cast(strm->next_out[0]); + if (output_len > 1) { + out_head_b1 = static_cast(strm->next_out[1]); + } + out_tail_b0 = static_cast(strm->next_out[output_len - 1]); + if (output_len > 1) { + out_tail_b1 = static_cast(strm->next_out[output_len - 2]); + } + } + + Log(LogLevel::LOG_INFO, + "trace event=igzip_deflate_call stream_id=", stream_id, + " strm=", static_cast(strm), " flush=", flush, + " pre_avail_in=", pre_avail_in, " pre_avail_out=", pre_avail_out, + " pre_total_in=", pre_total_in, " pre_total_out=", pre_total_out, + " bytes_in=", input_len, " bytes_out=", output_len, + " ret_raw=", ret, + " isal_state=", + (deflate_settings->isal_strm != nullptr) + ? static_cast(deflate_settings->isal_strm->internal_state.state) + : -1, + " output_crc32=", output_crc32, " out_head_b0=", out_head_b0, + " out_head_b1=", out_head_b1, " out_tail_b0=", out_tail_b0, + " out_tail_b1=", out_tail_b1, " tid=", CurrentThreadId(), "\n"); //INCREMENT_STAT(DEFLATE_IGZIP_COUNT); //INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); #endif @@ -390,14 +713,95 @@ int ZEXPORT deflate(z_streamp strm, int flush) { strm->avail_out -= output_len; strm->total_out += output_len; if (path_selected == IGZIP) { - if (input_len > 0 || output_len > 0) { - if (flush == Z_FINISH && strm->avail_in == 0) { - ret = Z_STREAM_END; - } else { - ret = Z_OK; + if (is_sync_flush) { + if (input_len > 0) { + deflate_settings->igzip_sync_flush_drained = false; + } else if (output_len > 0) { + deflate_settings->igzip_sync_flush_drained = true; } } else { - ret = Z_BUF_ERROR; + deflate_settings->igzip_sync_flush_drained = false; + } + + const bool no_progress = (input_len == 0 && output_len == 0); + bool finish_done = false; + #ifdef USE_IGZIP + finish_done = (flush == Z_FINISH) && + IsIGZIPDeflateFinished(deflate_settings->isal_strm); + #endif + + if (flush == Z_FINISH && + deflate_settings->igzip_finish_phase == IGZIPFinishPhase::DRAINING) { + deflate_settings->igzip_finish_drain_calls++; + } + + if (flush == Z_FINISH && strm->avail_in == 0 && !finish_done) { + const bool need_more_output_room = (strm->avail_out == 0); + int isal_state_snapshot = -1; +#ifdef USE_IGZIP + if (deflate_settings->isal_strm != nullptr) { + isal_state_snapshot = static_cast( + deflate_settings->isal_strm->internal_state.state); + } +#endif + if (need_more_output_room) { + Log(LogLevel::LOG_INFO, + "trace event=igzip_finish_needs_more_output stream_id=", + stream_id, " strm=", static_cast(strm), + " finish_done=0 avail_in=", strm->avail_in, + " avail_out=", strm->avail_out, " total_in=", strm->total_in, + " total_out=", strm->total_out, + " isal_state=", isal_state_snapshot, + " tid=", CurrentThreadId(), "\n"); + } else if (output_len == 0) { + Log(LogLevel::LOG_ERROR, + "trace event=igzip_finish_invariant_violation stream_id=", + stream_id, " strm=", static_cast(strm), + " finish_done=0 avail_in=", strm->avail_in, + " avail_out=", strm->avail_out, + " output_len=", output_len, + " total_in=", strm->total_in, + " total_out=", strm->total_out, + " isal_state=", isal_state_snapshot, + " tid=", CurrentThreadId(), "\n"); + } + } + + if (finish_done) { + ret = Z_STREAM_END; + deflate_settings->igzip_seen_stream_end = true; + deflate_settings->igzip_stream_end_count++; + deflate_settings->igzip_finish_phase = IGZIPFinishPhase::FINISHED; + deflate_settings->igzip_finish_no_progress_calls = 0; + } else if (!no_progress) { + ret = Z_OK; + if (flush == Z_FINISH) { + deflate_settings->igzip_finish_phase = IGZIPFinishPhase::DRAINING; + } + deflate_settings->igzip_finish_no_progress_calls = 0; + } else { + if (flush == Z_FINISH) { + deflate_settings->igzip_finish_no_progress_calls++; + if (strm->avail_out > 0 && + deflate_settings->igzip_finish_no_progress_calls > 1) { + Log(LogLevel::LOG_ERROR, + "trace event=igzip_finish_stall stream_id=", stream_id, + " strm=", static_cast(strm), " avail_in=", + strm->avail_in, " avail_out=", strm->avail_out, + " total_in=", strm->total_in, " total_out=", + strm->total_out, " finish_phase=", + IGZIPFinishPhaseToString(deflate_settings->igzip_finish_phase), + " no_progress_calls=", + deflate_settings->igzip_finish_no_progress_calls, + " tid=", CurrentThreadId(), "\n"); + ret = Z_STREAM_ERROR; + } + } + if (ret != Z_STREAM_ERROR) { + ret = Z_BUF_ERROR; + } + deflate_settings->igzip_zbuf_error_count++; + deflate_settings->igzip_no_progress_count++; } } else { if (strm->avail_in == 0) { @@ -409,17 +813,30 @@ int ZEXPORT deflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", accelerator return code ", ret, + ", bytes_in ", input_len, ", bytes_out ", output_len, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, - ", path ", static_cast(deflate_settings->path), "\n"); + ", path ", static_cast(deflate_settings->path), ", path_name ", + ExecutionPathToString(deflate_settings->path), ", tid ", + CurrentThreadId(), "\n"); + Log(LogLevel::LOG_INFO, + "trace event=deflate_exit stream_id=", stream_id, " strm=", + static_cast(strm), " ret=", ret, " engine=accelerator", + " path=", ExecutionPathToString(deflate_settings->path), + " bytes_in=", input_len, " bytes_out=", output_len, + " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, + " total_in=", strm->total_in, " total_out=", strm->total_out, + " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); return ret; } } - if (in_call || configs[USE_ZLIB_COMPRESS]) { + if (in_call || configs[USE_ZLIB_COMPRESS] || + deflate_settings->path == ZLIB) { ret = orig_deflate(strm, flush); INCREMENT_STAT(DEFLATE_ZLIB_COUNT); if (!in_call) { - deflate_settings->path = ZLIB; + SetDeflatePath(deflate_settings, strm, ZLIB, + "zlib fallback or configured zlib"); } } else { ret = Z_DATA_ERROR; @@ -428,61 +845,149 @@ int ZEXPORT deflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", zlib return code ", ret, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", - static_cast(deflate_settings->path), "\n"); + static_cast(deflate_settings->path), ", path_name ", + ExecutionPathToString(deflate_settings->path), ", tid ", + CurrentThreadId(), "\n"); + Log(LogLevel::LOG_INFO, + "trace event=deflate_exit stream_id=", stream_id, " strm=", + static_cast(strm), " ret=", ret, " engine=zlib", + " path=", ExecutionPathToString(deflate_settings->path), + " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, + " total_in=", strm->total_in, " total_out=", strm->total_out, + " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); INCREMENT_STAT_COND(ret < 0, DEFLATE_ERROR_COUNT); return ret; } int ZEXPORT deflateEnd(z_streamp strm) { + const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "deflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); if (deflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP + Log(LogLevel::LOG_INFO, + "trace event=igzip_stream_summary stream_id=", stream_id, + " strm=", static_cast(strm), + " calls=", deflate_settings->igzip_calls, + " finish_calls=", deflate_settings->igzip_finish_calls, + " stream_end_count=", deflate_settings->igzip_stream_end_count, + " no_progress_count=", deflate_settings->igzip_no_progress_count, + " zbuf_error_count=", deflate_settings->igzip_zbuf_error_count, + " bytes_in=", deflate_settings->igzip_bytes_in, + " bytes_out=", deflate_settings->igzip_bytes_out, + " seen_stream_end=", deflate_settings->igzip_seen_stream_end, + " finish_phase=", + IGZIPFinishPhaseToString(deflate_settings->igzip_finish_phase), + " finish_drain_calls=", deflate_settings->igzip_finish_drain_calls, + " finish_no_progress_calls=", + deflate_settings->igzip_finish_no_progress_calls, + " dict_len=", deflate_settings->dictionary_length, + " dict_crc32=", deflate_settings->dictionary_crc32, + " dict_set_count=", deflate_settings->dictionary_set_count, + " tid=", CurrentThreadId(), "\n"); EndCompressIGZIP(deflate_settings->isal_strm); #endif } deflate_stream_settings.Unset(strm); + Log(LogLevel::LOG_INFO, + "trace event=deflate_end stream_id=", stream_id, " strm=", + static_cast(strm), " total_in=", (strm ? strm->total_in : 0), + " total_out=", (strm ? strm->total_out : 0), " adler=", + (strm ? strm->adler : 0), " tid=", CurrentThreadId(), "\n"); + ClearStreamId(strm); return orig_deflateEnd(strm); } int ZEXPORT deflateReset(z_streamp strm) { + const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "deflateReset Line ", __LINE__, ", strm ", - static_cast(strm), "\n"); + static_cast(strm), ", tid ", CurrentThreadId(), "\n"); + Log(LogLevel::LOG_INFO, + "trace event=deflate_reset stream_id=", stream_id, " strm=", + static_cast(strm), " total_in=", (strm ? strm->total_in : 0), + " total_out=", (strm ? strm->total_out : 0), " adler=", + (strm ? strm->adler : 0), " tid=", CurrentThreadId(), "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); + int ret = orig_deflateReset(strm); if (deflate_settings != nullptr) { - deflate_settings->path = UNDEFINED; + if (deflate_settings->path == IGZIP && strm != nullptr && + strm->total_in > 0 && !deflate_settings->igzip_seen_stream_end) { + Log(LogLevel::LOG_ERROR, + "trace event=igzip_reset_without_stream_end stream_id=", stream_id, + " strm=", static_cast(strm), " total_in=", strm->total_in, + " total_out=", strm->total_out, + " calls=", deflate_settings->igzip_calls, + " finish_calls=", deflate_settings->igzip_finish_calls, + " tid=", CurrentThreadId(), "\n"); + } + + SetDeflatePath(deflate_settings, strm, + deflate_settings->uses_dictionary ? ZLIB : UNDEFINED, + "deflateReset"); + deflate_settings->igzip_sync_flush_drained = false; + deflate_settings->igzip_seen_stream_end = false; + deflate_settings->igzip_finish_phase = IGZIPFinishPhase::ACTIVE; + deflate_settings->igzip_finish_drain_calls = 0; + deflate_settings->igzip_finish_no_progress_calls = 0; + +#ifdef USE_IGZIP + if (deflate_settings->isal_strm != nullptr) { + isal_deflate_reset(deflate_settings->isal_strm); + deflate_settings->isal_strm->end_of_stream = 0; + deflate_settings->isal_strm->flush = NO_FLUSH; + } +#endif } - return orig_deflateReset(strm); + return ret; } int ZEXPORT inflateInit_(z_streamp strm, const char* version, int stream_size) { + const uint64_t stream_id = GetOrCreateStreamId(strm); inflate_stream_settings.Set(strm, 15); Log(LogLevel::LOG_INFO, "inflateInit_ Line ", __LINE__, ", strm ", static_cast(strm), "\n"); + Log(LogLevel::LOG_INFO, + "trace event=inflate_init stream_id=", stream_id, " strm=", + static_cast(strm), " tid=", CurrentThreadId(), "\n"); return orig_inflateInit_(strm, version, stream_size); } int ZEXPORT inflateInit2_(z_streamp strm, int window_bits, const char* version, int stream_size) { + const uint64_t stream_id = GetOrCreateStreamId(strm); inflate_stream_settings.Set(strm, window_bits); Log(LogLevel::LOG_INFO, "inflateInit2_ Line ", __LINE__, ", strm ", static_cast(strm), ", window_bits ", window_bits, "\n"); + Log(LogLevel::LOG_INFO, + "trace event=inflate_init2 stream_id=", stream_id, " strm=", + static_cast(strm), " window_bits=", window_bits, + " tid=", CurrentThreadId(), "\n"); return orig_inflateInit2_(strm, window_bits, version, stream_size); } int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, uInt dictLength) { + const uint64_t stream_id = GetOrCreateStreamId(strm); if (!configs[IGNORE_ZLIB_DICTIONARY]) { Log(LogLevel::LOG_INFO, "inflateSetDictionary Line ", __LINE__, ", strm ", - static_cast(strm), "dictLength ", dictLength, "\n"); + static_cast(strm), ", tid ", CurrentThreadId(), + ", dictLength ", dictLength, "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); - inflate_settings->path = ZLIB; - return orig_inflateSetDictionary(strm, dictionary, dictLength); + inflate_settings->uses_dictionary = true; + SetInflatePath(inflate_settings, strm, ZLIB, + "inflateSetDictionary called"); + const int ret = orig_inflateSetDictionary(strm, dictionary, dictLength); + Log(LogLevel::LOG_INFO, + "trace event=inflate_set_dict stream_id=", stream_id, " strm=", + static_cast(strm), " ret=", ret, " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, + " dict_len=", dictLength, " tid=", CurrentThreadId(), "\n"); + return ret; } Log(LogLevel::LOG_INFO, "inflateSetDictionary Line ", __LINE__, " ignored because ignore_zlib_dictionary is set to ", @@ -492,13 +997,27 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, int ZEXPORT inflate(z_streamp strm, int flush) { InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); + const uint64_t stream_id = GetOrCreateStreamId(strm); + INCREMENT_STAT(INFLATE_COUNT); PrintStats(); Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", - static_cast(inflate_settings->path), "\n"); + static_cast(inflate_settings->path), ", path_name ", + ExecutionPathToString(inflate_settings->path), ", window_bits ", + inflate_settings->window_bits, ", uses_dictionary ", + inflate_settings->uses_dictionary, ", stream_id ", stream_id, + ", total_in ", strm->total_in, ", total_out ", strm->total_out, + ", adler ", strm->adler, ", tid ", CurrentThreadId(), "\n"); + Log(LogLevel::LOG_INFO, + "trace event=inflate_enter stream_id=", stream_id, " strm=", + static_cast(strm), " flush=", flush, " path=", + ExecutionPathToString(inflate_settings->path), " avail_in=", strm->avail_in, + " avail_out=", strm->avail_out, " total_in=", strm->total_in, + " total_out=", strm->total_out, " uses_dict=", + inflate_settings->uses_dictionary, " tid=", CurrentThreadId(), "\n"); PrintDeflateBlockHeader(LogLevel::LOG_INFO, strm->next_in, strm->avail_in, inflate_settings->window_bits); @@ -507,7 +1026,79 @@ int ZEXPORT inflate(z_streamp strm, int flush) { bool iaa_available = false; bool qat_available = false; bool igzip_available = false; + + if (inflate_settings->force_zlib_for_raw_boundary && + inflate_settings->path != ZLIB) { + SetInflatePath(inflate_settings, strm, ZLIB, + "raw boundary guard pinned stream to zlib"); + } + + const bool igzip_stream_active = + (inflate_settings->path == IGZIP && inflate_settings->isal_strm != nullptr); + + // Keep stateful IGZIP stream handling on the same engine. + // For avail_in==0, let IGZIP process any buffered bits in its internal + // state before reporting Z_BUF_ERROR. +#ifdef USE_IGZIP + if (!in_call && igzip_stream_active && strm->avail_in == 0) { + uint32_t input_len = 0; + uint32_t output_len = strm->avail_out; + + in_call = true; + ret = UncompressIGZIP(inflate_settings->isal_strm, strm->next_in, &input_len, + strm->next_out, &output_len, + inflate_settings->window_bits, + &inflate_settings->trailer_overconsumption_fixed, + &inflate_settings->deferred_trailer_correction_bytes, + &strm->total_in, &strm->total_out, &end_of_stream); + in_call = false; + + if (ret == 0) { + strm->next_out += output_len; + strm->avail_out -= output_len; + strm->total_out += output_len; + + if (output_len > 0) { + ret = end_of_stream ? Z_STREAM_END : Z_OK; + } else { + ret = Z_BUF_ERROR; + } + + Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", + static_cast(strm), + ", active IGZIP stream with avail_in==0, return code ", ret, + ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, + ", bytes_in 0, bytes_out ", output_len, ", end_of_stream ", + end_of_stream, ", window_bits ", inflate_settings->window_bits, + ", tid ", CurrentThreadId(), "\n"); + return ret; + } + + Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", + static_cast(strm), + ", active IGZIP stream with avail_in==0 and no buffered progress, returning Z_BUF_ERROR\n"); + return Z_BUF_ERROR; + } +#endif + + // Early detection: if this is a zlib-format stream with the FDICT bit set + // in the header, pin to ZLIB immediately so dictionary streams never reach + // any accelerator (QAT/IAA/IGZIP don't support preset dictionaries). + if (!in_call && inflate_settings->path == UNDEFINED && + inflate_settings->window_bits >= 8 && inflate_settings->window_bits <= 15 && + strm->avail_in >= 2 && (strm->next_in[1] & 0x20)) { + Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", + static_cast(strm), + ", FDICT bit set in zlib header, cmf ", + static_cast(strm->next_in[0]), ", flg ", + static_cast(strm->next_in[1]), + ", pinning to ZLIB, tid ", CurrentThreadId(), "\n"); + SetInflatePath(inflate_settings, strm, ZLIB, + "FDICT bit detected in zlib header"); + } + if (!in_call && strm->avail_in > 0 && inflate_settings->path != ZLIB) { + const uInt pre_avail_in = strm->avail_in; uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; @@ -534,7 +1125,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { // If both accelerators are enabled, send configured ratio of requests to // one or the other ExecutionPath path_selected = ZLIB; - if (iaa_available && qat_available) { + if (igzip_stream_active) { + path_selected = IGZIP; + } else if (iaa_available && qat_available) { if (static_cast(std::rand()) % 100 < configs[IAA_UNCOMPRESS_PERCENTAGE]) { path_selected = IAA; @@ -554,7 +1147,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ret = UncompressIAA(strm->next_in, &input_len, strm->next_out, &output_len, qpl_path_hardware, inflate_settings->window_bits, &end_of_stream); - inflate_settings->path = IAA; + SetInflatePath(inflate_settings, strm, IAA, + "selected IAA accelerator"); in_call = false; INCREMENT_STAT(INFLATE_IAA_COUNT); INCREMENT_STAT_COND(ret != 0, INFLATE_IAA_ERROR_COUNT); @@ -565,7 +1159,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ret = UncompressQAT(strm->next_in, &input_len, strm->next_out, &output_len, inflate_settings->window_bits, &end_of_stream); - inflate_settings->path = QAT; + SetInflatePath(inflate_settings, strm, QAT, + "selected QAT accelerator"); // QATzip does not support stateful decompression // Fall back to zlib if end-of-stream not reached in one call if (!end_of_stream) { @@ -580,8 +1175,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { if (inflate_settings->isal_strm == nullptr) { Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", about to initalize igzip\n"); - inflate_settings->isal_strm = InitUncompressIGZIP(inflate_settings->window_bits); //hardcoded window bits - fflush(stdout); + inflate_settings->isal_strm = InitUncompressIGZIP(inflate_settings->window_bits); //hardcoded window bits } if (inflate_settings->isal_strm == nullptr) { Log(LogLevel::LOG_ERROR, "inflate Line ", __LINE__, ", strm ", @@ -591,10 +1185,43 @@ int ZEXPORT inflate(z_streamp strm, int flush) { in_call = true; ret = UncompressIGZIP(inflate_settings->isal_strm, strm->next_in, &input_len, strm->next_out, - &output_len, &inflate_settings->trailer_overconsumption_fixed, + &output_len, inflate_settings->window_bits, + &inflate_settings->trailer_overconsumption_fixed, + &inflate_settings->deferred_trailer_correction_bytes, &strm->total_in, &strm->total_out, &end_of_stream /*inflate_settings->window_bits, &end_of_stream*/); - inflate_settings->path = IGZIP; + + const uInt remaining_after_igzip = + (pre_avail_in >= input_len) ? (pre_avail_in - input_len) : 0; + if (ret == 0 && inflate_settings->window_bits < 0 && end_of_stream && + remaining_after_igzip > 0 && strm->total_in == 0 && + strm->total_out == 0) { + inflate_settings->force_zlib_for_raw_boundary = true; + Log(LogLevel::LOG_ERROR, + "trace event=igzip_raw_boundary_guard stream_id=", stream_id, + " strm=", static_cast(strm), " bytes_in=", input_len, + " bytes_out=", output_len, " pre_avail_in=", pre_avail_in, + " remaining_in=", remaining_after_igzip, + " action=pin_to_zlib tid=", CurrentThreadId(), "\n"); + ret = 1; + end_of_stream = false; + SetInflatePath(inflate_settings, strm, ZLIB, + "raw boundary guard fallback"); + } + + if (ret == Z_NEED_DICT) { + Log(LogLevel::LOG_ERROR, + "trace event=inflate_need_dict_signal stream_id=", stream_id, + " strm=", static_cast(strm), " source=igzip", + " uses_dict=", inflate_settings->uses_dictionary, + " total_in=", strm->total_in, " total_out=", strm->total_out, + " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); + SetInflatePath(inflate_settings, strm, ZLIB, + "IGZIP returned Z_NEED_DICT"); + } else if (inflate_settings->path != ZLIB) { + SetInflatePath(inflate_settings, strm, IGZIP, + "selected IGZIP accelerator"); + } in_call = false; //INCREMENT_STAT(INFLATE_IGZIP_COUNT); //INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); @@ -620,18 +1247,39 @@ int ZEXPORT inflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", accelerator return code ", ret, + ", bytes_in ", input_len, ", bytes_out ", output_len, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", end_of_stream ", end_of_stream, ", path ", - static_cast(inflate_settings->path), "\n"); + static_cast(inflate_settings->path), ", path_name ", + ExecutionPathToString(inflate_settings->path), ", window_bits ", + inflate_settings->window_bits, ", tid ", CurrentThreadId(), "\n"); + Log(LogLevel::LOG_INFO, + "trace event=inflate_exit stream_id=", stream_id, " strm=", + static_cast(strm), " ret=", ret, " engine=accelerator", + " path=", ExecutionPathToString(inflate_settings->path), + " bytes_in=", input_len, " bytes_out=", output_len, + " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, + " total_in=", strm->total_in, " total_out=", strm->total_out, + " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); return ret; } } - if (in_call || configs[USE_ZLIB_UNCOMPRESS]) { + if (in_call || configs[USE_ZLIB_UNCOMPRESS] || + inflate_settings->path == ZLIB) { ret = orig_inflate(strm, flush); + if (ret == Z_NEED_DICT && !inflate_settings->uses_dictionary) { + Log(LogLevel::LOG_ERROR, + "trace event=inflate_need_dict_signal stream_id=", stream_id, + " strm=", static_cast(strm), " source=zlib", + " uses_dict=", inflate_settings->uses_dictionary, + " total_in=", strm->total_in, " total_out=", strm->total_out, + " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); + } INCREMENT_STAT(INFLATE_ZLIB_COUNT); if (!in_call) { - inflate_settings->path = ZLIB; + SetInflatePath(inflate_settings, strm, ZLIB, + "zlib fallback or configured zlib"); } } else { ret = Z_DATA_ERROR; @@ -640,13 +1288,23 @@ int ZEXPORT inflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", zlib return code ", ret, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", - static_cast(inflate_settings->path), "\n"); + static_cast(inflate_settings->path), ", path_name ", + ExecutionPathToString(inflate_settings->path), ", window_bits ", + inflate_settings->window_bits, ", tid ", CurrentThreadId(), "\n"); + Log(LogLevel::LOG_INFO, + "trace event=inflate_exit stream_id=", stream_id, " strm=", + static_cast(strm), " ret=", ret, " engine=zlib", + " path=", ExecutionPathToString(inflate_settings->path), + " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, + " total_in=", strm->total_in, " total_out=", strm->total_out, + " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); INCREMENT_STAT_COND(ret < 0, INFLATE_ERROR_COUNT); return ret; } int ZEXPORT inflateEnd(z_streamp strm) { + const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "inflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); @@ -656,24 +1314,50 @@ int ZEXPORT inflateEnd(z_streamp strm) { #endif } inflate_stream_settings.Unset(strm); + Log(LogLevel::LOG_INFO, + "trace event=inflate_end stream_id=", stream_id, " strm=", + static_cast(strm), " total_in=", (strm ? strm->total_in : 0), + " total_out=", (strm ? strm->total_out : 0), " adler=", + (strm ? strm->adler : 0), " tid=", CurrentThreadId(), "\n"); + ClearStreamId(strm); return orig_inflateEnd(strm); } int ZEXPORT inflateReset(z_streamp strm) { + const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "inflateReset Line ", __LINE__, ", strm ", - static_cast(strm), "\n"); + static_cast(strm), ", tid ", CurrentThreadId(), "\n"); + Log(LogLevel::LOG_INFO, + "trace event=inflate_reset stream_id=", stream_id, " strm=", + static_cast(strm), " total_in=", (strm ? strm->total_in : 0), + " total_out=", (strm ? strm->total_out : 0), " adler=", + (strm ? strm->adler : 0), " tid=", CurrentThreadId(), "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); + const bool was_igzip_path = + (inflate_settings != nullptr && inflate_settings->path == IGZIP); + int ret = orig_inflateReset(strm); if (inflate_settings != nullptr) { - inflate_settings->path = UNDEFINED; + inflate_settings->force_zlib_for_raw_boundary = false; + SetInflatePath(inflate_settings, strm, + inflate_settings->uses_dictionary ? ZLIB : UNDEFINED, + "inflateReset"); } if (inflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP ResetUncompressIGZIP(inflate_settings->isal_strm, - &inflate_settings->trailer_overconsumption_fixed); + &inflate_settings->trailer_overconsumption_fixed, + &inflate_settings->deferred_trailer_correction_bytes); #endif + if (was_igzip_path) { + inflate_settings->trailer_overconsumption_fixed = 1; + inflate_settings->deferred_trailer_correction_bytes = 0; + Log(LogLevel::LOG_INFO, "inflateReset Line ", __LINE__, ", strm ", + static_cast(strm), ", tid ", CurrentThreadId(), + ", rewinds disabled after IGZIP reset\n"); + } } - return orig_inflateReset(strm); + return ret; } int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, @@ -834,7 +1518,8 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, unsigned long total_in = 0; unsigned long total_out = 0; ret = UncompressIGZIP(isal_strm, const_cast(source), - &input_len, dest, &output_len, &tofixed, + &input_len, dest, &output_len, 15, &tofixed, + nullptr, &total_in, &total_out, &end_of_stream); EndUncompressIGZIP(isal_strm); if (ret == 0 && !end_of_stream) { @@ -1206,7 +1891,8 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, unsigned long total_in = 0; unsigned long total_out = 0; ret = UncompressIGZIP(isal_strm, input, input_length, output, - output_length, &tofixed, &total_in, &total_out, + output_length, 31, &tofixed, nullptr, + &total_in, &total_out, end_of_stream); EndUncompressIGZIP(isal_strm); } From 5bdec2c37cfa6a34c9ae6d30cc66b6af2a8ad64b Mon Sep 17 00:00:00 2001 From: Olasoji Date: Tue, 10 Mar 2026 10:36:03 -0700 Subject: [PATCH 10/18] - Modifies iaa behavior so that when QPL_STS_MORE_OUTPUT_NEEDED is returned eos isnt erronously set. Signed-off-by: Olasoji --- iaa.cpp | 7 +++++-- zlib_accel.cpp | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/iaa.cpp b/iaa.cpp index 2e0ca66..8b80ade 100644 --- a/iaa.cpp +++ b/iaa.cpp @@ -230,9 +230,12 @@ int UncompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, if (gzip_ext) { *input_length = gzip_ext_dest_size + GZIP_EXT_HDRFTR_SIZE; } - *end_of_stream = true; + // IAA decompression is stateless in this wrapper; when more output is needed + // the caller must continue via zlib path. + *end_of_stream = (status == QPL_STS_OK); Log(LogLevel::LOG_INFO, "UncompressIAA() Line ", __LINE__, " output size ", - job->total_out, "\n"); + job->total_out, ", status ", status, ", end_of_stream ", + *end_of_stream, "\n"); return 0; } diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 72c5105..26188e7 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -489,6 +489,11 @@ int ZEXPORT inflate(z_streamp strm, int flush) { &output_len, qpl_path_hardware, inflate_settings->window_bits, &end_of_stream); inflate_settings->path = IAA; + // IAA inflate is stateless in this wrapper. If stream end was not + // reached, use zlib for stateful continuation. + if (!end_of_stream) { + ret = 1; + } in_call = false; INCREMENT_STAT(INFLATE_IAA_COUNT); INCREMENT_STAT_COND(ret != 0, INFLATE_IAA_ERROR_COUNT); @@ -681,6 +686,9 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, in_call = true; ret = UncompressIAA(const_cast(source), &input_len, dest, &output_len, qpl_path_hardware, 15, &end_of_stream); + if (!end_of_stream) { + ret = 1; + } in_call = false; #endif // USE_IAA } else if (path_selected == QAT) { From 25434615075690283eab3fc6e4d323fbdf3c542f Mon Sep 17 00:00:00 2001 From: Olasoji Date: Tue, 10 Mar 2026 10:47:52 -0700 Subject: [PATCH 11/18] -Adds proper formatting Signed-off-by: Olasoji --- iaa.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iaa.cpp b/iaa.cpp index 8b80ade..2bbcd27 100644 --- a/iaa.cpp +++ b/iaa.cpp @@ -234,8 +234,8 @@ int UncompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, // the caller must continue via zlib path. *end_of_stream = (status == QPL_STS_OK); Log(LogLevel::LOG_INFO, "UncompressIAA() Line ", __LINE__, " output size ", - job->total_out, ", status ", status, ", end_of_stream ", - *end_of_stream, "\n"); + job->total_out, ", status ", status, ", end_of_stream ", *end_of_stream, + "\n"); return 0; } From 65af227b0c591fe75a00b5d483a14d66ab6bc792 Mon Sep 17 00:00:00 2001 From: Olasoji Date: Tue, 10 Mar 2026 14:22:36 -0700 Subject: [PATCH 12/18] Remove PR-B/PR-D diagnostics changes from igzip branch --- logging.h | 40 ++-------------------------------------- zlib_accel.cpp | 15 +++------------ 2 files changed, 5 insertions(+), 50 deletions(-) diff --git a/logging.h b/logging.h index 5579d83..b5e8301 100644 --- a/logging.h +++ b/logging.h @@ -4,15 +4,10 @@ #pragma once #include -#include -#include #include -#include #include #include #include -#include -#include #include #include "config/config.h" @@ -29,14 +24,9 @@ inline std::unique_ptr& LogFileStream() { return stream; } -inline bool CreateLogFile(const char* file_name) { +inline void CreateLogFile(const char* file_name) { auto& s = LogFileStream(); s = std::make_unique(file_name, std::ios::app); - if (!s || !s->is_open()) { - s.reset(); - return false; - } - return true; } inline void CloseLogFile() { @@ -57,24 +47,6 @@ inline std::ostream& GetLogStream() { #ifdef DEBUG_LOG static std::mutex log_mutex; - -inline std::string FormatLogTimestamp() { - using clock = std::chrono::system_clock; - const auto now = clock::now(); - const auto ms = - std::chrono::duration_cast(now.time_since_epoch()) % - 1000; - const std::time_t t = clock::to_time_t(now); - - std::tm tm{}; - localtime_r(&t, &tm); - - std::ostringstream oss; - oss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%S") << '.' << std::setw(3) - << std::setfill('0') << ms.count(); - return oss.str(); -} - template inline void Log(LogLevel level, Args&&... args) { std::lock_guard lock(log_mutex); @@ -90,7 +62,6 @@ inline void Log(LogLevel level, Args&&... args) { std::ostream& stream = GetLogStream(); stream << std::dec; - stream << '[' << FormatLogTimestamp() << "] "; switch (level) { case LogLevel::LOG_ERROR: stream << "Error: "; @@ -101,14 +72,7 @@ inline void Log(LogLevel level, Args&&... args) { case LogLevel::LOG_NONE: return; } - - std::ostringstream line; - (..., (line << args)); - std::string message = line.str(); - if (message.empty() || message.back() != '\n') { - message.push_back('\n'); - } - stream << message; + (..., (stream << args)); stream << std::flush; } diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 57d5d26..f35661d 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -166,11 +166,7 @@ static int init_zlib_accel(void) { #if defined(DEBUG_LOG) || defined(ENABLE_STATISTICS) if (!config::log_file.empty()) { - if (!CreateLogFile(config::log_file.c_str())) { - Log(LogLevel::LOG_ERROR, "init_zlib_accel Line ", __LINE__, - " failed to open log_file '", config::log_file.c_str(), - "', falling back to stdout\n"); - } + CreateLogFile(config::log_file.c_str()); } #endif @@ -2219,15 +2215,10 @@ int ZEXPORT gzclose(gzFile file) { if (write_ret != 0) { ret = Z_STREAM_ERROR; + } else if (close_ret != Z_OK) { + ret = close_ret; } else if (truncate_ret != 0) { ret = Z_STREAM_ERROR; - } else if (close_ret != Z_OK) { - Log(LogLevel::LOG_INFO, "gzclose Line ", __LINE__, - ", ignoring zlib close return in accelerator path ", close_ret, - "\n"); - ret = Z_OK; - } else { - ret = Z_OK; } } else { ret = orig_gzclose(file); From d088b18fe552686785be8e532d341fab59a0720a Mon Sep 17 00:00:00 2001 From: Olasoji Date: Tue, 10 Mar 2026 18:47:42 -0700 Subject: [PATCH 13/18] Removes unused uses_dictionary flag and other code cleanup Signed-off-by: Olasoji --- zlib_accel.cpp | 59 ++++++++++++-------------------------------------- 1 file changed, 14 insertions(+), 45 deletions(-) diff --git a/zlib_accel.cpp b/zlib_accel.cpp index f35661d..fdcbcec 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -184,6 +184,8 @@ static void cleanup_zlib_accel(void) { // Avoid recursive call (e.g., if QATzip falls back to zlib internally) static thread_local bool in_call = false; +constexpr uint8_t ZLIB_FDICT_MASK = 0x20; + enum class IGZIPFinishPhase { ACTIVE = 0, DRAINING = 1, @@ -204,7 +206,6 @@ struct DeflateSettings { int window_bits; int mem_level; int strategy; - bool uses_dictionary = false; std::vector dictionary; uint32_t dictionary_crc32 = 0; uInt dictionary_length = 0; @@ -234,7 +235,6 @@ struct InflateSettings { int trailer_overconsumption_fixed; /* indicates if fix has been applied for overconsumption issue*/ uint32_t deferred_trailer_correction_bytes; bool force_zlib_for_raw_boundary = false; - bool uses_dictionary = false; std::vector dictionary; ExecutionPath path = UNDEFINED; struct inflate_state *isal_strm = nullptr; @@ -419,22 +419,6 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, static_cast(strm), ", tid ", CurrentThreadId(), ", dictLength ", dictLength, "\n"); - // Match zlib API semantics even when acceleration bypassed orig_deflate: - // dictionary must be set right after init/reset and before any progress. - if (strm->total_in != 0 || strm->total_out != 0) { - Log(LogLevel::LOG_ERROR, "deflateSetDictionary Line ", __LINE__, - ", strm ", static_cast(strm), ", tid ", CurrentThreadId(), - ", rejecting midstream dictionary set after progress total_in ", - strm->total_in, ", total_out ", strm->total_out, "\n"); - Log(LogLevel::LOG_ERROR, - "trace event=reject_midstream_dict stream_id=", stream_id, - " strm=", static_cast(strm), " ret=", Z_STREAM_ERROR, - " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, " dict_len=", dictLength, - " tid=", CurrentThreadId(), "\n"); - return Z_STREAM_ERROR; - } - DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); const int ret = orig_deflateSetDictionary(strm, dictionary, dictLength); Log(LogLevel::LOG_INFO, @@ -443,7 +427,6 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, " total_out=", strm->total_out, " adler=", strm->adler, " dict_len=", dictLength, " tid=", CurrentThreadId(), "\n"); if (ret == Z_OK) { - deflate_settings->uses_dictionary = true; deflate_settings->dictionary_crc32 = static_cast(crc32(0L, dictionary, dictLength)); deflate_settings->dictionary_length = dictLength; @@ -471,11 +454,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { INCREMENT_STAT(DEFLATE_COUNT); PrintStats(); - if (deflate_settings->uses_dictionary) { - SetDeflatePath(deflate_settings, strm, ZLIB, - "stream marked uses_dictionary"); - } - if (flush != Z_FINISH && deflate_settings->igzip_finish_phase == IGZIPFinishPhase::DRAINING) { deflate_settings->igzip_finish_phase = IGZIPFinishPhase::ACTIVE; @@ -508,8 +486,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", static_cast(deflate_settings->path), ", path_name ", ExecutionPathToString(deflate_settings->path), ", window_bits ", - deflate_settings->window_bits, ", uses_dictionary ", - deflate_settings->uses_dictionary, ", stream_id ", stream_id, + deflate_settings->window_bits, ", stream_id ", stream_id, ", total_in ", strm->total_in, ", total_out ", strm->total_out, ", adler ", strm->adler, ", tid ", CurrentThreadId(), "\n"); Log(LogLevel::LOG_INFO, @@ -517,8 +494,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { static_cast(strm), " flush=", flush, " path=", ExecutionPathToString(deflate_settings->path), " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " total_in=", strm->total_in, - " total_out=", strm->total_out, " uses_dict=", - deflate_settings->uses_dictionary, " tid=", CurrentThreadId(), "\n"); + " total_out=", strm->total_out, " tid=", CurrentThreadId(), "\n"); int ret = 1; bool iaa_available = false; @@ -919,9 +895,7 @@ int ZEXPORT deflateReset(z_streamp strm) { " tid=", CurrentThreadId(), "\n"); } - SetDeflatePath(deflate_settings, strm, - deflate_settings->uses_dictionary ? ZLIB : UNDEFINED, - "deflateReset"); + SetDeflatePath(deflate_settings, strm, UNDEFINED, "deflateReset"); deflate_settings->igzip_sync_flush_drained = false; deflate_settings->igzip_seen_stream_end = false; deflate_settings->igzip_finish_phase = IGZIPFinishPhase::ACTIVE; @@ -974,10 +948,11 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, static_cast(strm), ", tid ", CurrentThreadId(), ", dictLength ", dictLength, "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); - inflate_settings->uses_dictionary = true; - SetInflatePath(inflate_settings, strm, ZLIB, - "inflateSetDictionary called"); const int ret = orig_inflateSetDictionary(strm, dictionary, dictLength); + if (ret == Z_OK) { + SetInflatePath(inflate_settings, strm, ZLIB, + "inflateSetDictionary called"); + } Log(LogLevel::LOG_INFO, "trace event=inflate_set_dict stream_id=", stream_id, " strm=", static_cast(strm), " ret=", ret, " total_in=", strm->total_in, @@ -1003,8 +978,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", static_cast(inflate_settings->path), ", path_name ", ExecutionPathToString(inflate_settings->path), ", window_bits ", - inflate_settings->window_bits, ", uses_dictionary ", - inflate_settings->uses_dictionary, ", stream_id ", stream_id, + inflate_settings->window_bits, ", stream_id ", stream_id, ", total_in ", strm->total_in, ", total_out ", strm->total_out, ", adler ", strm->adler, ", tid ", CurrentThreadId(), "\n"); Log(LogLevel::LOG_INFO, @@ -1012,8 +986,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { static_cast(strm), " flush=", flush, " path=", ExecutionPathToString(inflate_settings->path), " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " total_in=", strm->total_in, - " total_out=", strm->total_out, " uses_dict=", - inflate_settings->uses_dictionary, " tid=", CurrentThreadId(), "\n"); + " total_out=", strm->total_out, " tid=", CurrentThreadId(), "\n"); PrintDeflateBlockHeader(LogLevel::LOG_INFO, strm->next_in, strm->avail_in, inflate_settings->window_bits); @@ -1082,7 +1055,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { // any accelerator (QAT/IAA/IGZIP don't support preset dictionaries). if (!in_call && inflate_settings->path == UNDEFINED && inflate_settings->window_bits >= 8 && inflate_settings->window_bits <= 15 && - strm->avail_in >= 2 && (strm->next_in[1] & 0x20)) { + strm->avail_in >= 2 && (strm->next_in[1] & ZLIB_FDICT_MASK)) { Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", FDICT bit set in zlib header, cmf ", @@ -1209,7 +1182,6 @@ int ZEXPORT inflate(z_streamp strm, int flush) { Log(LogLevel::LOG_ERROR, "trace event=inflate_need_dict_signal stream_id=", stream_id, " strm=", static_cast(strm), " source=igzip", - " uses_dict=", inflate_settings->uses_dictionary, " total_in=", strm->total_in, " total_out=", strm->total_out, " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); SetInflatePath(inflate_settings, strm, ZLIB, @@ -1264,11 +1236,10 @@ int ZEXPORT inflate(z_streamp strm, int flush) { if (in_call || configs[USE_ZLIB_UNCOMPRESS] || inflate_settings->path == ZLIB) { ret = orig_inflate(strm, flush); - if (ret == Z_NEED_DICT && !inflate_settings->uses_dictionary) { + if (ret == Z_NEED_DICT) { Log(LogLevel::LOG_ERROR, "trace event=inflate_need_dict_signal stream_id=", stream_id, " strm=", static_cast(strm), " source=zlib", - " uses_dict=", inflate_settings->uses_dictionary, " total_in=", strm->total_in, " total_out=", strm->total_out, " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); } @@ -1334,9 +1305,7 @@ int ZEXPORT inflateReset(z_streamp strm) { int ret = orig_inflateReset(strm); if (inflate_settings != nullptr) { inflate_settings->force_zlib_for_raw_boundary = false; - SetInflatePath(inflate_settings, strm, - inflate_settings->uses_dictionary ? ZLIB : UNDEFINED, - "inflateReset"); + SetInflatePath(inflate_settings, strm, UNDEFINED, "inflateReset"); } if (inflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP From 9ab87c1790a9dbbfc36def308e2a6472df85eaf3 Mon Sep 17 00:00:00 2001 From: Olasoji Date: Wed, 11 Mar 2026 20:16:04 -0700 Subject: [PATCH 14/18] igzip: replace stateful guards with early zlib fallback --- zlib_accel.cpp | 184 +++++-------------------------------------------- 1 file changed, 18 insertions(+), 166 deletions(-) diff --git a/zlib_accel.cpp b/zlib_accel.cpp index fdcbcec..156540e 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -186,12 +186,6 @@ static thread_local bool in_call = false; constexpr uint8_t ZLIB_FDICT_MASK = 0x20; -enum class IGZIPFinishPhase { - ACTIVE = 0, - DRAINING = 1, - FINISHED = 2, -}; - struct DeflateSettings { DeflateSettings(int _level, int _method, int _window_bits, int _mem_level, int _strategy) @@ -206,22 +200,6 @@ struct DeflateSettings { int window_bits; int mem_level; int strategy; - std::vector dictionary; - uint32_t dictionary_crc32 = 0; - uInt dictionary_length = 0; - uint64_t dictionary_set_count = 0; - bool igzip_sync_flush_drained = false; - uint64_t igzip_calls = 0; - uint64_t igzip_finish_calls = 0; - uint64_t igzip_stream_end_count = 0; - uint64_t igzip_no_progress_count = 0; - uint64_t igzip_zbuf_error_count = 0; - uint64_t igzip_bytes_in = 0; - uint64_t igzip_bytes_out = 0; - bool igzip_seen_stream_end = false; - IGZIPFinishPhase igzip_finish_phase = IGZIPFinishPhase::ACTIVE; - uint64_t igzip_finish_drain_calls = 0; - uint64_t igzip_finish_no_progress_calls = 0; ExecutionPath path = UNDEFINED; struct isal_zstream *isal_strm = nullptr; }; @@ -235,7 +213,6 @@ struct InflateSettings { int trailer_overconsumption_fixed; /* indicates if fix has been applied for overconsumption issue*/ uint32_t deferred_trailer_correction_bytes; bool force_zlib_for_raw_boundary = false; - std::vector dictionary; ExecutionPath path = UNDEFINED; struct inflate_state *isal_strm = nullptr; }; @@ -301,19 +278,6 @@ static const char* ExecutionPathToString(ExecutionPath path) { } } -static const char* IGZIPFinishPhaseToString(IGZIPFinishPhase phase) { - switch (phase) { - case IGZIPFinishPhase::ACTIVE: - return "ACTIVE"; - case IGZIPFinishPhase::DRAINING: - return "DRAINING"; - case IGZIPFinishPhase::FINISHED: - return "FINISHED"; - default: - return "UNKNOWN"; - } -} - static unsigned long CurrentThreadId() { return static_cast(reinterpret_cast(pthread_self())); } @@ -427,18 +391,8 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, " total_out=", strm->total_out, " adler=", strm->adler, " dict_len=", dictLength, " tid=", CurrentThreadId(), "\n"); if (ret == Z_OK) { - deflate_settings->dictionary_crc32 = - static_cast(crc32(0L, dictionary, dictLength)); - deflate_settings->dictionary_length = dictLength; - deflate_settings->dictionary_set_count++; SetDeflatePath(deflate_settings, strm, ZLIB, "deflateSetDictionary called"); - Log(LogLevel::LOG_INFO, - "trace event=deflate_set_dict_fingerprint stream_id=", stream_id, - " strm=", static_cast(strm), " dict_len=", dictLength, - " dict_crc32=", deflate_settings->dictionary_crc32, - " dict_set_count=", deflate_settings->dictionary_set_count, - " tid=", CurrentThreadId(), "\n"); } return ret; } @@ -454,21 +408,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { INCREMENT_STAT(DEFLATE_COUNT); PrintStats(); - if (flush != Z_FINISH && - deflate_settings->igzip_finish_phase == IGZIPFinishPhase::DRAINING) { - deflate_settings->igzip_finish_phase = IGZIPFinishPhase::ACTIVE; - deflate_settings->igzip_finish_no_progress_calls = 0; - } - - if (flush == Z_FINISH && deflate_settings->path == IGZIP && - deflate_settings->igzip_finish_phase == IGZIPFinishPhase::FINISHED) { - Log(LogLevel::LOG_INFO, - "trace event=igzip_finish_already_complete stream_id=", stream_id, - " strm=", static_cast(strm), " total_in=", strm->total_in, - " total_out=", strm->total_out, " tid=", CurrentThreadId(), "\n"); - return Z_STREAM_END; - } - const bool is_streaming_flush_with_input = (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || flush == Z_FULL_FLUSH || flush == Z_BLOCK) && @@ -481,6 +420,24 @@ int ZEXPORT deflate(z_streamp strm, int flush) { const bool is_sync_flush = (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || flush == Z_BLOCK); + const bool igzip_sync_flush_empty_reentry = + (deflate_settings->path == IGZIP && is_sync_flush && strm->avail_in == 0); + const bool igzip_finish_reentry_without_input = + (deflate_settings->path == IGZIP && flush == Z_FINISH && + strm->avail_in == 0); + if (igzip_sync_flush_empty_reentry || igzip_finish_reentry_without_input) { + const char* reason = igzip_sync_flush_empty_reentry + ? "igzip risky empty sync flush; forcing zlib" + : "igzip risky finish reentry without input; forcing zlib"; + SetDeflatePath(deflate_settings, strm, ZLIB, reason); + Log(LogLevel::LOG_INFO, + "trace event=igzip_compress_unavailable stream_id=", stream_id, + " strm=", static_cast(strm), " reason=", reason, + " flush=", flush, " avail_in=", strm->avail_in, + " avail_out=", strm->avail_out, " total_in=", strm->total_in, + " total_out=", strm->total_out, " tid=", CurrentThreadId(), "\n"); + } + Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", @@ -600,24 +557,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { const uLong pre_total_in = strm->total_in; const uLong pre_total_out = strm->total_out; - if (strm->avail_in > 0) { - deflate_settings->igzip_sync_flush_drained = false; - } - - if (flush == Z_FINISH && - deflate_settings->igzip_finish_phase == IGZIPFinishPhase::ACTIVE) { - deflate_settings->igzip_finish_phase = IGZIPFinishPhase::DRAINING; - deflate_settings->igzip_finish_no_progress_calls = 0; - } - - if (is_sync_flush && strm->avail_in == 0 && - deflate_settings->igzip_sync_flush_drained) { - Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", - static_cast(strm), - ", repeated empty sync flush on drained IGZIP stream; returning Z_BUF_ERROR\n"); - return Z_BUF_ERROR; - } - if (deflate_settings->isal_strm == nullptr) { deflate_settings->method = 0; deflate_settings->isal_strm = InitCompressIGZIP(deflate_settings->level, deflate_settings->window_bits); //hardcoded window bits @@ -633,13 +572,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { "selected IGZIP accelerator"); in_call = false; - deflate_settings->igzip_calls++; - if (flush == Z_FINISH) { - deflate_settings->igzip_finish_calls++; - } - deflate_settings->igzip_bytes_in += input_len; - deflate_settings->igzip_bytes_out += output_len; - uint32_t output_crc32 = 0; uint32_t out_head_b0 = 256; uint32_t out_head_b1 = 256; @@ -685,16 +617,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { strm->avail_out -= output_len; strm->total_out += output_len; if (path_selected == IGZIP) { - if (is_sync_flush) { - if (input_len > 0) { - deflate_settings->igzip_sync_flush_drained = false; - } else if (output_len > 0) { - deflate_settings->igzip_sync_flush_drained = true; - } - } else { - deflate_settings->igzip_sync_flush_drained = false; - } - const bool no_progress = (input_len == 0 && output_len == 0); bool finish_done = false; #ifdef USE_IGZIP @@ -702,11 +624,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { IsIGZIPDeflateFinished(deflate_settings->isal_strm); #endif - if (flush == Z_FINISH && - deflate_settings->igzip_finish_phase == IGZIPFinishPhase::DRAINING) { - deflate_settings->igzip_finish_drain_calls++; - } - if (flush == Z_FINISH && strm->avail_in == 0 && !finish_done) { const bool need_more_output_room = (strm->avail_out == 0); int isal_state_snapshot = -1; @@ -741,39 +658,10 @@ int ZEXPORT deflate(z_streamp strm, int flush) { if (finish_done) { ret = Z_STREAM_END; - deflate_settings->igzip_seen_stream_end = true; - deflate_settings->igzip_stream_end_count++; - deflate_settings->igzip_finish_phase = IGZIPFinishPhase::FINISHED; - deflate_settings->igzip_finish_no_progress_calls = 0; } else if (!no_progress) { ret = Z_OK; - if (flush == Z_FINISH) { - deflate_settings->igzip_finish_phase = IGZIPFinishPhase::DRAINING; - } - deflate_settings->igzip_finish_no_progress_calls = 0; } else { - if (flush == Z_FINISH) { - deflate_settings->igzip_finish_no_progress_calls++; - if (strm->avail_out > 0 && - deflate_settings->igzip_finish_no_progress_calls > 1) { - Log(LogLevel::LOG_ERROR, - "trace event=igzip_finish_stall stream_id=", stream_id, - " strm=", static_cast(strm), " avail_in=", - strm->avail_in, " avail_out=", strm->avail_out, - " total_in=", strm->total_in, " total_out=", - strm->total_out, " finish_phase=", - IGZIPFinishPhaseToString(deflate_settings->igzip_finish_phase), - " no_progress_calls=", - deflate_settings->igzip_finish_no_progress_calls, - " tid=", CurrentThreadId(), "\n"); - ret = Z_STREAM_ERROR; - } - } - if (ret != Z_STREAM_ERROR) { ret = Z_BUF_ERROR; - } - deflate_settings->igzip_zbuf_error_count++; - deflate_settings->igzip_no_progress_count++; } } else { if (strm->avail_in == 0) { @@ -839,26 +727,6 @@ int ZEXPORT deflateEnd(z_streamp strm) { DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); if (deflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP - Log(LogLevel::LOG_INFO, - "trace event=igzip_stream_summary stream_id=", stream_id, - " strm=", static_cast(strm), - " calls=", deflate_settings->igzip_calls, - " finish_calls=", deflate_settings->igzip_finish_calls, - " stream_end_count=", deflate_settings->igzip_stream_end_count, - " no_progress_count=", deflate_settings->igzip_no_progress_count, - " zbuf_error_count=", deflate_settings->igzip_zbuf_error_count, - " bytes_in=", deflate_settings->igzip_bytes_in, - " bytes_out=", deflate_settings->igzip_bytes_out, - " seen_stream_end=", deflate_settings->igzip_seen_stream_end, - " finish_phase=", - IGZIPFinishPhaseToString(deflate_settings->igzip_finish_phase), - " finish_drain_calls=", deflate_settings->igzip_finish_drain_calls, - " finish_no_progress_calls=", - deflate_settings->igzip_finish_no_progress_calls, - " dict_len=", deflate_settings->dictionary_length, - " dict_crc32=", deflate_settings->dictionary_crc32, - " dict_set_count=", deflate_settings->dictionary_set_count, - " tid=", CurrentThreadId(), "\n"); EndCompressIGZIP(deflate_settings->isal_strm); #endif } @@ -884,23 +752,7 @@ int ZEXPORT deflateReset(z_streamp strm) { DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); int ret = orig_deflateReset(strm); if (deflate_settings != nullptr) { - if (deflate_settings->path == IGZIP && strm != nullptr && - strm->total_in > 0 && !deflate_settings->igzip_seen_stream_end) { - Log(LogLevel::LOG_ERROR, - "trace event=igzip_reset_without_stream_end stream_id=", stream_id, - " strm=", static_cast(strm), " total_in=", strm->total_in, - " total_out=", strm->total_out, - " calls=", deflate_settings->igzip_calls, - " finish_calls=", deflate_settings->igzip_finish_calls, - " tid=", CurrentThreadId(), "\n"); - } - SetDeflatePath(deflate_settings, strm, UNDEFINED, "deflateReset"); - deflate_settings->igzip_sync_flush_drained = false; - deflate_settings->igzip_seen_stream_end = false; - deflate_settings->igzip_finish_phase = IGZIPFinishPhase::ACTIVE; - deflate_settings->igzip_finish_drain_calls = 0; - deflate_settings->igzip_finish_no_progress_calls = 0; #ifdef USE_IGZIP if (deflate_settings->isal_strm != nullptr) { From 001876e30478e7f2f996aaf8f71741b7ece6579d Mon Sep 17 00:00:00 2001 From: Olasoji Date: Thu, 12 Mar 2026 10:21:41 -0700 Subject: [PATCH 15/18] logging: drop thread/stream id helper plumbing --- zlib_accel.cpp | 207 ++++++++++++++----------------------------------- 1 file changed, 60 insertions(+), 147 deletions(-) diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 156540e..d70d7c6 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -11,11 +11,8 @@ #include #include -#include #include -#include #include -#include #include #include "config/config.h" @@ -278,46 +275,13 @@ static const char* ExecutionPathToString(ExecutionPath path) { } } -static unsigned long CurrentThreadId() { - return static_cast(reinterpret_cast(pthread_self())); -} - -static std::atomic g_next_stream_id{1}; -static std::mutex g_stream_id_mu; -static std::unordered_map g_stream_ids; - -static uint64_t GetOrCreateStreamId(z_streamp strm) { - if (strm == nullptr) { - return 0; - } - - std::lock_guard lock(g_stream_id_mu); - auto it = g_stream_ids.find(strm); - if (it != g_stream_ids.end()) { - return it->second; - } - - const uint64_t id = g_next_stream_id.fetch_add(1, std::memory_order_relaxed); - g_stream_ids.emplace(strm, id); - return id; -} - -static void ClearStreamId(z_streamp strm) { - if (strm == nullptr) { - return; - } - - std::lock_guard lock(g_stream_id_mu); - g_stream_ids.erase(strm); -} - static void SetDeflatePath(DeflateSettings* settings, z_streamp strm, ExecutionPath new_path, const char* reason) { if (settings == nullptr || settings->path == new_path) { return; } Log(LogLevel::LOG_INFO, "deflate path transition Line ", __LINE__, ", strm ", - static_cast(strm), ", tid ", CurrentThreadId(), ", old ", + static_cast(strm), ", old ", ExecutionPathToString(settings->path), ", new ", ExecutionPathToString(new_path), ", reason ", reason, "\n"); settings->path = new_path; @@ -329,7 +293,7 @@ static void SetInflatePath(InflateSettings* settings, z_streamp strm, return; } Log(LogLevel::LOG_INFO, "inflate path transition Line ", __LINE__, ", strm ", - static_cast(strm), ", tid ", CurrentThreadId(), ", old ", + static_cast(strm), ", old ", ExecutionPathToString(settings->path), ", new ", ExecutionPathToString(new_path), ", reason ", reason, "\n"); settings->path = new_path; @@ -337,12 +301,10 @@ static void SetInflatePath(InflateSettings* settings, z_streamp strm, int ZEXPORT deflateInit_(z_streamp strm, int level, const char* version, int stream_size) { - const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "deflateInit_ Line ", __LINE__, ", strm ", static_cast(strm), ", level ", level, "\n"); - Log(LogLevel::LOG_INFO, - "trace event=deflate_init stream_id=", stream_id, " strm=", - static_cast(strm), " level=", level, " tid=", CurrentThreadId(), + Log(LogLevel::LOG_INFO, " strm=", + static_cast(strm), " level=", level, "\n"); deflate_stream_settings.Set(strm, level, Z_DEFLATED, 15, 8, @@ -353,15 +315,13 @@ int ZEXPORT deflateInit_(z_streamp strm, int level, const char* version, int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, int window_bits, int mem_level, int strategy, const char* version, int stream_size) { - const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "deflateInit2_ Line ", __LINE__, ", strm ", static_cast(strm), ", level ", level, ", window_bits ", window_bits, " \n"); - Log(LogLevel::LOG_INFO, - "trace event=deflate_init2 stream_id=", stream_id, " strm=", + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " level=", level, " method=", method, " window_bits=", window_bits, " mem_level=", mem_level, - " strategy=", strategy, " tid=", CurrentThreadId(), "\n"); + " strategy=", strategy, "\n"); /*#ifdef USE_IGZIP if (configs[USE_IGZIP_COMPRESS] && configs[USE_IGZIP_UNCOMPRESS] && @@ -377,19 +337,17 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, uInt dictLength) { - const uint64_t stream_id = GetOrCreateStreamId(strm); if (!configs[IGNORE_ZLIB_DICTIONARY]) { Log(LogLevel::LOG_INFO, "deflateSetDictionary Line ", __LINE__, ", strm ", - static_cast(strm), ", tid ", CurrentThreadId(), + static_cast(strm), ", dictLength ", dictLength, "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); const int ret = orig_deflateSetDictionary(strm, dictionary, dictLength); - Log(LogLevel::LOG_INFO, - "trace event=deflate_set_dict stream_id=", stream_id, " strm=", + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " ret=", ret, " total_in=", strm->total_in, " total_out=", strm->total_out, " adler=", strm->adler, - " dict_len=", dictLength, " tid=", CurrentThreadId(), "\n"); + " dict_len=", dictLength, "\n"); if (ret == Z_OK) { SetDeflatePath(deflate_settings, strm, ZLIB, "deflateSetDictionary called"); @@ -404,7 +362,6 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, int ZEXPORT deflate(z_streamp strm, int flush) { DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); - const uint64_t stream_id = GetOrCreateStreamId(strm); INCREMENT_STAT(DEFLATE_COUNT); PrintStats(); @@ -431,11 +388,10 @@ int ZEXPORT deflate(z_streamp strm, int flush) { : "igzip risky finish reentry without input; forcing zlib"; SetDeflatePath(deflate_settings, strm, ZLIB, reason); Log(LogLevel::LOG_INFO, - "trace event=igzip_compress_unavailable stream_id=", stream_id, " strm=", static_cast(strm), " reason=", reason, " flush=", flush, " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " total_in=", strm->total_in, - " total_out=", strm->total_out, " tid=", CurrentThreadId(), "\n"); + " total_out=", strm->total_out, "\n"); } Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", @@ -443,15 +399,14 @@ int ZEXPORT deflate(z_streamp strm, int flush) { strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", static_cast(deflate_settings->path), ", path_name ", ExecutionPathToString(deflate_settings->path), ", window_bits ", - deflate_settings->window_bits, ", stream_id ", stream_id, + deflate_settings->window_bits, ", total_in ", strm->total_in, ", total_out ", strm->total_out, - ", adler ", strm->adler, ", tid ", CurrentThreadId(), "\n"); - Log(LogLevel::LOG_INFO, - "trace event=deflate_enter stream_id=", stream_id, " strm=", + ", adler ", strm->adler, "\n"); + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " flush=", flush, " path=", ExecutionPathToString(deflate_settings->path), " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " total_in=", strm->total_in, - " total_out=", strm->total_out, " tid=", CurrentThreadId(), "\n"); + " total_out=", strm->total_out, "\n"); int ret = 1; bool iaa_available = false; @@ -483,12 +438,10 @@ int ZEXPORT deflate(z_streamp strm, int flush) { "igzip unavailable for non-finish flush"); } Log(LogLevel::LOG_INFO, - "trace event=igzip_compress_unavailable stream_id=", stream_id, " strm=", static_cast(strm), " reason=flush_not_finish flush=", flush, " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, - " total_in=", strm->total_in, " total_out=", strm->total_out, - " tid=", CurrentThreadId(), "\n"); + " total_in=", strm->total_in, " total_out=", strm->total_out, "\n"); } if (configs[USE_IGZIP_COMPRESS] && igzip_finish_only_mode && !igzip_finish_buffer_ok) { @@ -497,12 +450,10 @@ int ZEXPORT deflate(z_streamp strm, int flush) { "igzip unavailable for tiny finish buffer"); } Log(LogLevel::LOG_INFO, - "trace event=igzip_compress_unavailable stream_id=", stream_id, " strm=", static_cast(strm), " reason=finish_buffer_too_small flush=", flush, " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, - " total_in=", strm->total_in, " total_out=", strm->total_out, - " tid=", CurrentThreadId(), "\n"); + " total_in=", strm->total_in, " total_out=", strm->total_out, "\n"); } #endif @@ -591,7 +542,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { } Log(LogLevel::LOG_INFO, - "trace event=igzip_deflate_call stream_id=", stream_id, " strm=", static_cast(strm), " flush=", flush, " pre_avail_in=", pre_avail_in, " pre_avail_out=", pre_avail_out, " pre_total_in=", pre_total_in, " pre_total_out=", pre_total_out, @@ -603,7 +553,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { : -1, " output_crc32=", output_crc32, " out_head_b0=", out_head_b0, " out_head_b1=", out_head_b1, " out_tail_b0=", out_tail_b0, - " out_tail_b1=", out_tail_b1, " tid=", CurrentThreadId(), "\n"); + " out_tail_b1=", out_tail_b1, "\n"); //INCREMENT_STAT(DEFLATE_IGZIP_COUNT); //INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); #endif @@ -634,25 +584,19 @@ int ZEXPORT deflate(z_streamp strm, int flush) { } #endif if (need_more_output_room) { - Log(LogLevel::LOG_INFO, - "trace event=igzip_finish_needs_more_output stream_id=", - stream_id, " strm=", static_cast(strm), + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " finish_done=0 avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " total_in=", strm->total_in, " total_out=", strm->total_out, - " isal_state=", isal_state_snapshot, - " tid=", CurrentThreadId(), "\n"); + " isal_state=", isal_state_snapshot, "\n"); } else if (output_len == 0) { - Log(LogLevel::LOG_ERROR, - "trace event=igzip_finish_invariant_violation stream_id=", - stream_id, " strm=", static_cast(strm), + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), " finish_done=0 avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " output_len=", output_len, " total_in=", strm->total_in, " total_out=", strm->total_out, - " isal_state=", isal_state_snapshot, - " tid=", CurrentThreadId(), "\n"); + " isal_state=", isal_state_snapshot, "\n"); } } @@ -676,16 +620,14 @@ int ZEXPORT deflate(z_streamp strm, int flush) { ", bytes_in ", input_len, ", bytes_out ", output_len, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", static_cast(deflate_settings->path), ", path_name ", - ExecutionPathToString(deflate_settings->path), ", tid ", - CurrentThreadId(), "\n"); - Log(LogLevel::LOG_INFO, - "trace event=deflate_exit stream_id=", stream_id, " strm=", + ExecutionPathToString(deflate_settings->path), "\n"); + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " ret=", ret, " engine=accelerator", " path=", ExecutionPathToString(deflate_settings->path), " bytes_in=", input_len, " bytes_out=", output_len, " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); + " adler=", strm->adler, "\n"); return ret; } } @@ -706,22 +648,19 @@ int ZEXPORT deflate(z_streamp strm, int flush) { static_cast(strm), ", zlib return code ", ret, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", static_cast(deflate_settings->path), ", path_name ", - ExecutionPathToString(deflate_settings->path), ", tid ", - CurrentThreadId(), "\n"); - Log(LogLevel::LOG_INFO, - "trace event=deflate_exit stream_id=", stream_id, " strm=", + ExecutionPathToString(deflate_settings->path), "\n"); + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " ret=", ret, " engine=zlib", " path=", ExecutionPathToString(deflate_settings->path), " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); + " adler=", strm->adler, "\n"); INCREMENT_STAT_COND(ret < 0, DEFLATE_ERROR_COUNT); return ret; } int ZEXPORT deflateEnd(z_streamp strm) { - const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "deflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); @@ -731,24 +670,20 @@ int ZEXPORT deflateEnd(z_streamp strm) { #endif } deflate_stream_settings.Unset(strm); - Log(LogLevel::LOG_INFO, - "trace event=deflate_end stream_id=", stream_id, " strm=", + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " total_in=", (strm ? strm->total_in : 0), " total_out=", (strm ? strm->total_out : 0), " adler=", - (strm ? strm->adler : 0), " tid=", CurrentThreadId(), "\n"); - ClearStreamId(strm); + (strm ? strm->adler : 0), "\n"); return orig_deflateEnd(strm); } int ZEXPORT deflateReset(z_streamp strm) { - const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "deflateReset Line ", __LINE__, ", strm ", - static_cast(strm), ", tid ", CurrentThreadId(), "\n"); - Log(LogLevel::LOG_INFO, - "trace event=deflate_reset stream_id=", stream_id, " strm=", + static_cast(strm), "\n"); + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " total_in=", (strm ? strm->total_in : 0), " total_out=", (strm ? strm->total_out : 0), " adler=", - (strm ? strm->adler : 0), " tid=", CurrentThreadId(), "\n"); + (strm ? strm->adler : 0), "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); int ret = orig_deflateReset(strm); if (deflate_settings != nullptr) { @@ -767,37 +702,31 @@ int ZEXPORT deflateReset(z_streamp strm) { } int ZEXPORT inflateInit_(z_streamp strm, const char* version, int stream_size) { - const uint64_t stream_id = GetOrCreateStreamId(strm); inflate_stream_settings.Set(strm, 15); Log(LogLevel::LOG_INFO, "inflateInit_ Line ", __LINE__, ", strm ", static_cast(strm), "\n"); - Log(LogLevel::LOG_INFO, - "trace event=inflate_init stream_id=", stream_id, " strm=", - static_cast(strm), " tid=", CurrentThreadId(), "\n"); + Log(LogLevel::LOG_INFO, " strm=", + static_cast(strm), "\n"); return orig_inflateInit_(strm, version, stream_size); } int ZEXPORT inflateInit2_(z_streamp strm, int window_bits, const char* version, int stream_size) { - const uint64_t stream_id = GetOrCreateStreamId(strm); inflate_stream_settings.Set(strm, window_bits); Log(LogLevel::LOG_INFO, "inflateInit2_ Line ", __LINE__, ", strm ", static_cast(strm), ", window_bits ", window_bits, "\n"); - Log(LogLevel::LOG_INFO, - "trace event=inflate_init2 stream_id=", stream_id, " strm=", - static_cast(strm), " window_bits=", window_bits, - " tid=", CurrentThreadId(), "\n"); + Log(LogLevel::LOG_INFO, " strm=", + static_cast(strm), " window_bits=", window_bits, "\n"); return orig_inflateInit2_(strm, window_bits, version, stream_size); } int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, uInt dictLength) { - const uint64_t stream_id = GetOrCreateStreamId(strm); if (!configs[IGNORE_ZLIB_DICTIONARY]) { Log(LogLevel::LOG_INFO, "inflateSetDictionary Line ", __LINE__, ", strm ", - static_cast(strm), ", tid ", CurrentThreadId(), + static_cast(strm), ", dictLength ", dictLength, "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); const int ret = orig_inflateSetDictionary(strm, dictionary, dictLength); @@ -805,11 +734,10 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, SetInflatePath(inflate_settings, strm, ZLIB, "inflateSetDictionary called"); } - Log(LogLevel::LOG_INFO, - "trace event=inflate_set_dict stream_id=", stream_id, " strm=", + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " ret=", ret, " total_in=", strm->total_in, " total_out=", strm->total_out, " adler=", strm->adler, - " dict_len=", dictLength, " tid=", CurrentThreadId(), "\n"); + " dict_len=", dictLength, "\n"); return ret; } Log(LogLevel::LOG_INFO, "inflateSetDictionary Line ", __LINE__, @@ -820,7 +748,6 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, int ZEXPORT inflate(z_streamp strm, int flush) { InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); - const uint64_t stream_id = GetOrCreateStreamId(strm); INCREMENT_STAT(INFLATE_COUNT); PrintStats(); @@ -830,15 +757,14 @@ int ZEXPORT inflate(z_streamp strm, int flush) { strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", static_cast(inflate_settings->path), ", path_name ", ExecutionPathToString(inflate_settings->path), ", window_bits ", - inflate_settings->window_bits, ", stream_id ", stream_id, + inflate_settings->window_bits, ", total_in ", strm->total_in, ", total_out ", strm->total_out, - ", adler ", strm->adler, ", tid ", CurrentThreadId(), "\n"); - Log(LogLevel::LOG_INFO, - "trace event=inflate_enter stream_id=", stream_id, " strm=", + ", adler ", strm->adler, "\n"); + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " flush=", flush, " path=", ExecutionPathToString(inflate_settings->path), " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " total_in=", strm->total_in, - " total_out=", strm->total_out, " tid=", CurrentThreadId(), "\n"); + " total_out=", strm->total_out, "\n"); PrintDeflateBlockHeader(LogLevel::LOG_INFO, strm->next_in, strm->avail_in, inflate_settings->window_bits); @@ -890,8 +816,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ", active IGZIP stream with avail_in==0, return code ", ret, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", bytes_in 0, bytes_out ", output_len, ", end_of_stream ", - end_of_stream, ", window_bits ", inflate_settings->window_bits, - ", tid ", CurrentThreadId(), "\n"); + end_of_stream, ", window_bits ", inflate_settings->window_bits, "\n"); return ret; } @@ -912,8 +837,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { static_cast(strm), ", FDICT bit set in zlib header, cmf ", static_cast(strm->next_in[0]), ", flg ", - static_cast(strm->next_in[1]), - ", pinning to ZLIB, tid ", CurrentThreadId(), "\n"); + static_cast(strm->next_in[1]), "\n"); SetInflatePath(inflate_settings, strm, ZLIB, "FDICT bit detected in zlib header"); } @@ -1019,11 +943,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { strm->total_out == 0) { inflate_settings->force_zlib_for_raw_boundary = true; Log(LogLevel::LOG_ERROR, - "trace event=igzip_raw_boundary_guard stream_id=", stream_id, " strm=", static_cast(strm), " bytes_in=", input_len, " bytes_out=", output_len, " pre_avail_in=", pre_avail_in, - " remaining_in=", remaining_after_igzip, - " action=pin_to_zlib tid=", CurrentThreadId(), "\n"); + " remaining_in=", remaining_after_igzip, "\n"); ret = 1; end_of_stream = false; SetInflatePath(inflate_settings, strm, ZLIB, @@ -1032,10 +954,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { if (ret == Z_NEED_DICT) { Log(LogLevel::LOG_ERROR, - "trace event=inflate_need_dict_signal stream_id=", stream_id, " strm=", static_cast(strm), " source=igzip", " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); + " adler=", strm->adler, "\n"); SetInflatePath(inflate_settings, strm, ZLIB, "IGZIP returned Z_NEED_DICT"); } else if (inflate_settings->path != ZLIB) { @@ -1072,15 +993,14 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ", end_of_stream ", end_of_stream, ", path ", static_cast(inflate_settings->path), ", path_name ", ExecutionPathToString(inflate_settings->path), ", window_bits ", - inflate_settings->window_bits, ", tid ", CurrentThreadId(), "\n"); - Log(LogLevel::LOG_INFO, - "trace event=inflate_exit stream_id=", stream_id, " strm=", + inflate_settings->window_bits, "\n"); + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " ret=", ret, " engine=accelerator", " path=", ExecutionPathToString(inflate_settings->path), " bytes_in=", input_len, " bytes_out=", output_len, " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); + " adler=", strm->adler, "\n"); return ret; } } @@ -1090,10 +1010,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ret = orig_inflate(strm, flush); if (ret == Z_NEED_DICT) { Log(LogLevel::LOG_ERROR, - "trace event=inflate_need_dict_signal stream_id=", stream_id, " strm=", static_cast(strm), " source=zlib", " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); + " adler=", strm->adler, "\n"); } INCREMENT_STAT(INFLATE_ZLIB_COUNT); if (!in_call) { @@ -1109,21 +1028,19 @@ int ZEXPORT inflate(z_streamp strm, int flush) { strm->avail_in, ", avail_out ", strm->avail_out, ", path ", static_cast(inflate_settings->path), ", path_name ", ExecutionPathToString(inflate_settings->path), ", window_bits ", - inflate_settings->window_bits, ", tid ", CurrentThreadId(), "\n"); - Log(LogLevel::LOG_INFO, - "trace event=inflate_exit stream_id=", stream_id, " strm=", + inflate_settings->window_bits, "\n"); + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " ret=", ret, " engine=zlib", " path=", ExecutionPathToString(inflate_settings->path), " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, " tid=", CurrentThreadId(), "\n"); + " adler=", strm->adler, "\n"); INCREMENT_STAT_COND(ret < 0, INFLATE_ERROR_COUNT); return ret; } int ZEXPORT inflateEnd(z_streamp strm) { - const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "inflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); @@ -1133,24 +1050,20 @@ int ZEXPORT inflateEnd(z_streamp strm) { #endif } inflate_stream_settings.Unset(strm); - Log(LogLevel::LOG_INFO, - "trace event=inflate_end stream_id=", stream_id, " strm=", + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " total_in=", (strm ? strm->total_in : 0), " total_out=", (strm ? strm->total_out : 0), " adler=", - (strm ? strm->adler : 0), " tid=", CurrentThreadId(), "\n"); - ClearStreamId(strm); + (strm ? strm->adler : 0), "\n"); return orig_inflateEnd(strm); } int ZEXPORT inflateReset(z_streamp strm) { - const uint64_t stream_id = GetOrCreateStreamId(strm); Log(LogLevel::LOG_INFO, "inflateReset Line ", __LINE__, ", strm ", - static_cast(strm), ", tid ", CurrentThreadId(), "\n"); - Log(LogLevel::LOG_INFO, - "trace event=inflate_reset stream_id=", stream_id, " strm=", + static_cast(strm), "\n"); + Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), " total_in=", (strm ? strm->total_in : 0), " total_out=", (strm ? strm->total_out : 0), " adler=", - (strm ? strm->adler : 0), " tid=", CurrentThreadId(), "\n"); + (strm ? strm->adler : 0), "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); const bool was_igzip_path = (inflate_settings != nullptr && inflate_settings->path == IGZIP); @@ -1169,7 +1082,7 @@ int ZEXPORT inflateReset(z_streamp strm) { inflate_settings->trailer_overconsumption_fixed = 1; inflate_settings->deferred_trailer_correction_bytes = 0; Log(LogLevel::LOG_INFO, "inflateReset Line ", __LINE__, ", strm ", - static_cast(strm), ", tid ", CurrentThreadId(), + static_cast(strm), ", rewinds disabled after IGZIP reset\n"); } } From 9facd1cecc1194bc6388fc484fed9bff4e74f16a Mon Sep 17 00:00:00 2001 From: Olasoji Date: Thu, 12 Mar 2026 23:16:27 -0700 Subject: [PATCH 16/18] Fix IGZIP finish state handling and stream path stability --- igzip.cpp | 443 ++++++++++++++++-------------------- igzip.h | 44 +++- tests/zlib_accel_test.cpp | 26 ++- zlib_accel.cpp | 468 +++++++++----------------------------- 4 files changed, 369 insertions(+), 612 deletions(-) diff --git a/igzip.cpp b/igzip.cpp index c64fa75..b9ad6fb 100644 --- a/igzip.cpp +++ b/igzip.cpp @@ -21,6 +21,8 @@ static uint16_t ClampHistBits(int bits) return (uint16_t) bits; } +static constexpr uint32_t kIGZIPMinFinishOutputSize = 256; + static void ConfigureDeflateWindow(struct isal_zstream *isal_strm, int windowBits) { if (windowBits < 0) { @@ -57,16 +59,102 @@ static void ConfigureInflateWindow(struct inflate_state *isal_strm_inflate, int isal_strm_inflate->hist_bits = ClampHistBits(windowBits); } +bool +IsIGZIPDeflateFinished(const struct isal_zstream* stream) +{ + if (stream == nullptr) { + return false; + } + const enum isal_zstate_state state = stream->internal_state.state; + // ZSTATE_TMP_END is a temporary state and may require reentry to + // flush remaining output; only ZSTATE_END is terminal. + return state == ZSTATE_END; +} + +bool +SupportedOptionsIGZIPCompress(int flush, uint32_t output_length, + bool stream_on_igzip_path) +{ + if (flush != Z_FINISH) { + Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPCompress() Line ", + __LINE__, " flush ", flush, + " is not Z_FINISH; IGZIP deflate path disabled\n"); + return false; + } + if (!stream_on_igzip_path && output_length < kIGZIPMinFinishOutputSize) { + Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPCompress() Line ", + __LINE__, " output length ", output_length, + " is less than minimum finish buffer ", + kIGZIPMinFinishOutputSize, "\n"); + return false; + } + return true; +} + +bool +SupportedOptionsIGZIPUncompress(int window_bits, uint32_t input_length, + uint32_t output_length, + bool stream_on_igzip_path) +{ + (void) window_bits; + (void) output_length; + + if (!stream_on_igzip_path && input_length == 0) { + Log(LogLevel::LOG_INFO, + "SupportedOptionsIGZIPUncompress() Line ", __LINE__, + " fallback reason=no_input_on_new_stream input_length ", + input_length, " output_length ", output_length, "\n"); + return false; + } + + return true; +} + +static bool IsIGZIPSyncFlush(int flush) +{ + return flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || + flush == Z_BLOCK; +} + +bool +IGZIPShouldFallbackDeflate(bool stream_on_igzip_path, int flush, + uint32_t avail_in) +{ + const bool is_streaming_flush = + (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || + flush == Z_FULL_FLUSH || flush == Z_BLOCK); + + if (!stream_on_igzip_path && is_streaming_flush && avail_in > 0) { + Log(LogLevel::LOG_INFO, + "IGZIPShouldFallbackDeflate() Line ", __LINE__, + " fallback reason=streaming_flush_with_input flush ", flush, + " avail_in ", avail_in, "\n"); + return true; + } + + if (!stream_on_igzip_path || avail_in != 0) { + return false; + } + + if (IsIGZIPSyncFlush(flush)) { + Log(LogLevel::LOG_INFO, + "IGZIPShouldFallbackDeflate() Line ", __LINE__, + " fallback reason=empty_sync_flush_reentry flush ", flush, + " avail_in ", avail_in, "\n"); + return true; + } + if (flush == Z_FINISH) { + return false; + } + return false; +} + struct isal_zstream* InitCompressIGZIP(int level, int windowBits) { -#ifdef DEBUG Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, " initializing deflate with level ", level, ", windowBits ", windowBits, "\n"); -#endif - Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, " level ", - level, ", windowBits ",windowBits, " \n"); struct isal_zstream *isal_strm = (struct isal_zstream *) malloc(sizeof(struct isal_zstream)); @@ -114,18 +202,6 @@ InitCompressIGZIP(int level, int windowBits) return isal_strm; } -/*int -deflateInit_(z_streamp strm, int level) -{ - if (!strm) { - Log(LogLevel::LOG_ERROR, "deflateInit_() Line ", __LINE__, - " z_streamp is NULL\n"); - return -1; - } - - return deflateInit2_(strm, level, 0, 15, 8, 0); // hardcoded windowBits -}*/ - int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, uint32_t *input_length, uint8_t *output, uint32_t *output_length, @@ -134,9 +210,6 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, (void) total_in; (void) total_out; - - Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " input_length ", - *input_length, " \n"); if (!isal_strm) { Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, " deflate isal_strm is NULL\n"); @@ -176,7 +249,6 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, return -1; } -#ifdef DEBUG Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " gzip_flag ", isal_strm->gzip_flag, ", hist_bits ", isal_strm->hist_bits, ", flush ", isal_strm->flush, ", level ", isal_strm->level, @@ -184,7 +256,6 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, (uint32_t)isal_strm->avail_out, ", total_out ", (uint32_t)isal_strm->total_out, ", total_in ", (uint32_t)isal_strm->total_in, "\n"); -#endif int comp = isal_deflate(isal_strm); @@ -193,17 +264,14 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, input = isal_strm->next_in; output = isal_strm->next_out; -#ifdef DEBUG Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " after isal_deflate: avail_in ", isal_strm->avail_in, ", avail_out ", (uint32_t)isal_strm->avail_out, ", bytes_consumed ", *input_length, ", bytes_produced ", *output_length, "\n"); -#endif ret = (comp == COMP_OK) ? 0 : 1; -#ifdef DEBUG if (ret == Z_OK) { Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " deflate finished successfully Z_OK\n"); @@ -244,7 +312,6 @@ CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, break; } } -#endif return ret; } @@ -264,20 +331,11 @@ EndCompressIGZIP(struct isal_zstream* isal_strm) } free(isal_strm); -#ifdef DEBUG Log(LogLevel::LOG_INFO, "EndCompressIGZIP() Line ", __LINE__, " deflate end\n"); -#endif return Z_OK; } -/*int -deflateSetHeader(z_streamp strm, void *head) -{ - (void) head; // Suppress unused parameter warning - return Z_OK; -}*/ - int deflateSetDictionary(z_streamp strm, unsigned char *dict_data, unsigned int dict_len) { @@ -292,57 +350,6 @@ deflateSetDictionary(z_streamp strm, unsigned char *dict_data, unsigned int dict return isal_deflate_set_dict(s->isal_strm, dict_data, dict_len); } -/*int -compress2(uint8_t *dest, unsigned long *dest_len, const uint8_t *source, unsigned long source_len, - int level) -{ - z_stream strm; - int err; - const unsigned int max = (unsigned int) -1; - unsigned long left = *dest_len; - if (dest == NULL || dest_len == NULL || source == NULL) { - return Z_STREAM_ERROR; - } - *dest_len = 0; - - strm.zalloc = NULL; - strm.zfree = NULL; - strm.opaque = NULL; - - err = deflateInit_(&strm, level); - if (err != Z_OK) - return err; - - strm.next_out = dest; - strm.avail_out = 0; - strm.next_in = (uint8_t *) source; - strm.avail_in = 0; - - do { - if (strm.avail_out == 0) { - strm.avail_out = left > (unsigned long) max ? max : (unsigned int) left; - left -= strm.avail_out; - } - if (strm.avail_in == 0) { - strm.avail_in = - source_len > (unsigned long) max ? max : (unsigned int) source_len; - source_len -= strm.avail_in; - } - err = deflate(&strm, source_len ? Z_NO_FLUSH : Z_FINISH); - } while (err == Z_OK); - - *dest_len = strm.total_out; - deflateEnd(&strm); - - return err == Z_STREAM_END ? Z_OK : err; -} - -int -compress(uint8_t *dest, unsigned long *dest_len, const uint8_t *source, unsigned long source_len) -{ - return compress2(dest, dest_len, source, source_len, Z_DEFAULT_COMPRESSION); -} -*/ unsigned long crc32(unsigned long crc, const unsigned char *buf, unsigned int len) { @@ -355,7 +362,6 @@ adler32(unsigned long adler, const unsigned char *buf, unsigned int len) return isal_adler32(adler, buf, len); } -//inflateInit2_(z_streamp strm, int windowBits) struct inflate_state* InitUncompressIGZIP(int windowBits) { @@ -367,12 +373,8 @@ InitUncompressIGZIP(int windowBits) return nullptr; } -#ifdef DEBUG Log(LogLevel::LOG_INFO, "InitUncompressIGZIP() Line ", __LINE__, " initializing inflate with windowBits ", windowBits, "\n"); -#endif - Log(LogLevel::LOG_INFO, "InitUncompressIGZIP() Line ", __LINE__, - ", windowBits ",windowBits, " \n"); /* Setup ISA-L decompression context */ isal_inflate_init(isal_strm_inflate); @@ -389,23 +391,115 @@ InitUncompressIGZIP(int windowBits) return isal_strm_inflate; } -/*int -inflateInit_(z_streamp strm) +IGZIPNoInputAction +IGZIPHandleActiveStreamNoInput(z_streamp strm, + struct inflate_state *isal_strm_inflate, + int window_bits, int *tofixed, int *ret) { - if (!strm) { - Log(LogLevel::LOG_ERROR, "inflateInit_() Line ", __LINE__, - " z_streamp is NULL\n"); - return Z_STREAM_ERROR; + if (strm == nullptr || isal_strm_inflate == nullptr || ret == nullptr || + tofixed == nullptr || strm->avail_in != 0) { + return IGZIP_NO_INPUT_NOT_HANDLED; + } + + uint32_t input_len = 0; + uint32_t output_len = strm->avail_out; + bool end_of_stream = true; + + *ret = UncompressIGZIP(isal_strm_inflate, strm->next_in, &input_len, + strm->next_out, &output_len, window_bits, + tofixed, &strm->total_in, &strm->total_out, + &end_of_stream); + + if (*ret == Z_DATA_ERROR) { + Log(LogLevel::LOG_INFO, "IGZIPHandleActiveStreamNoInput() Line ", + __LINE__, " requested zlib fallback for raw INPUT_DONE ambiguity\n"); + return IGZIP_NO_INPUT_FALLBACK_ZLIB; + } + + if (*ret == 0) { + strm->next_out += output_len; + strm->avail_out -= output_len; + strm->total_out += output_len; + if (output_len > 0) { + *ret = end_of_stream ? Z_STREAM_END : Z_OK; + } else { + *ret = Z_BUF_ERROR; + } + return IGZIP_NO_INPUT_RETURN; + } + + *ret = Z_BUF_ERROR; + return IGZIP_NO_INPUT_RETURN; +} + +IGZIPInflatePathAction +IGZIPRunInflateAndSelectPathAction(z_streamp strm, + struct inflate_state **isal_strm_inflate, + int window_bits, int *tofixed, + uint32_t *input_length, + uint32_t *output_length, int *ret, + bool *end_of_stream, + uint32_t pre_avail_in) +{ + if (strm == nullptr || isal_strm_inflate == nullptr || input_length == nullptr || + output_length == nullptr || ret == nullptr || end_of_stream == nullptr || + tofixed == nullptr) { + if (ret != nullptr) { + *ret = Z_DATA_ERROR; + } + return IGZIP_INFLATE_PATH_NONE; + } + + if (*isal_strm_inflate == nullptr) { + *isal_strm_inflate = InitUncompressIGZIP(window_bits); + if (*isal_strm_inflate == nullptr) { + Log(LogLevel::LOG_ERROR, + "IGZIPRunInflateAndSelectPathAction() Line ", + __LINE__, " failed to initialize igzip inflate stream\n"); + *ret = Z_DATA_ERROR; + return IGZIP_INFLATE_PATH_NONE; + } + } + + *ret = UncompressIGZIP(*isal_strm_inflate, strm->next_in, input_length, + strm->next_out, output_length, window_bits, + tofixed, &strm->total_in, &strm->total_out, + end_of_stream); + + const uint32_t remaining_after_igzip = + (pre_avail_in >= *input_length) ? (pre_avail_in - *input_length) : 0; + + if (*ret == 0 && window_bits < 0 && *end_of_stream && + remaining_after_igzip > 0 && strm->total_in == 0 && + strm->total_out == 0) { + Log(LogLevel::LOG_ERROR, + "IGZIPRunInflateAndSelectPathAction() raw boundary guard strm=", + static_cast(strm), " bytes_in=", *input_length, + " bytes_out=", *output_length, + " pre_avail_in=", pre_avail_in, + " remaining_in=", remaining_after_igzip, "\n"); + *ret = 1; + *end_of_stream = false; + return IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY; + } + + if (*ret == Z_NEED_DICT) { + return IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT; + } + if (*ret == Z_DATA_ERROR) { + return IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR; + } + if (*ret == 0) { + return IGZIP_INFLATE_PATH_SET_IGZIP; } - return inflateInit2_(strm, 15); // hardcoded windowBits -}*/ + return IGZIP_INFLATE_PATH_NONE; +} int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t *input_length, uint8_t *output, uint32_t *output_length, int window_bits, int *tofixed, - uint32_t *deferred_correction_bytes, unsigned long *total_in, unsigned long *total_out, bool *end_of_stream) { @@ -416,9 +510,6 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, " isal_strm_inflate is NULL\n"); return Z_STREAM_ERROR; } - - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " input length ", *input_length,"\n"); // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ isal_strm_inflate->next_out = output; const uint32_t original_avail_out = *output_length; @@ -428,37 +519,8 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, isal_strm_inflate->next_in = input; isal_strm_inflate->total_out = *total_out; - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " CRC flag: ", (uint32_t)isal_strm_inflate->crc_flag, - " Before isal_inflate: avail_in ",isal_strm_inflate->avail_in, - //" next_in= ", isal_strm_inflate->next_in, - " avail_out= ",(uint32_t)isal_strm_inflate->avail_out, - //" next_out= ", isal_strm_inflate->next_out, - " Total out: ", (uint32_t)isal_strm_inflate->total_out, - " Total_in: ", (unsigned long)*total_in, "\n"); - const int decomp = isal_inflate(isal_strm_inflate); - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " After isal_inflate: avail_in ",isal_strm_inflate->avail_in, - //" next_in= ", isal_strm_inflate->next_in, - " avail_out= ",(uint32_t)isal_strm_inflate->avail_out, - //" next_out= ", isal_strm_inflate->next_out, - " Total out: ", (uint32_t)isal_strm_inflate->total_out, - " Total_in: ", (unsigned long)*total_in + - (original_avail_in - isal_strm_inflate->avail_in), - " Bytes consumed this call: ", - (original_avail_in - isal_strm_inflate->avail_in), - " Block state: ", isal_strm_inflate->block_state, " (ISAL_BLOCK_FINISH=", - ISAL_BLOCK_FINISH, " ISA-L result: ", decomp, "\n"); - - if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH && - isal_strm_inflate->avail_in > 0) { - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " WARNING: BLOCK_FINISH reached but ",isal_strm_inflate->avail_in, - " bytes remain in input\n"); - } - uint32_t consumed_before_adjust = 0; if (isal_strm_inflate->avail_in <= original_avail_in) { consumed_before_adjust = @@ -473,10 +535,9 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t rewind_adjust_bytes = 0; - // WORKAROUND: ISA-L over-consumption fix for raw deflate mode. - // Keep original 8-byte trailer heuristic, but for streaming callers - // defer correction discovered at INPUT_DONE and apply only when EOS is - // confirmed at BLOCK_FINISH. + // WORKAROUND: ISA-L over-consumption fix for raw deflate mode. + // Option 2 behavior: if ambiguity appears at INPUT_DONE, request caller + // fallback to zlib instead of carrying deferred state. if (window_bits < 0 && decomp == ISAL_DECOMP_OK && *tofixed == 0 && (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH || isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE) && @@ -492,46 +553,17 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, : over_consumed; if (rewind_adjust_bytes > 0) { *tofixed = 1; - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " raw EOS correction applied: over_consumed ", - over_consumed, ", rewind_bytes ", - rewind_adjust_bytes, - ", consumed_before_adjust ", - consumed_before_adjust, - ", consumed_after_adjust ", - (consumed_before_adjust - rewind_adjust_bytes), - "\n"); } - } else if (deferred_correction_bytes != nullptr && - *deferred_correction_bytes == 0) { - *deferred_correction_bytes = over_consumed; + } else { Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " raw correction deferred until EOS: over_consumed ", - over_consumed, "\n"); + " raw INPUT_DONE ambiguity detected: over_consumed ", + over_consumed, + ", requesting zlib fallback\n"); + return Z_DATA_ERROR; } } } - if (window_bits < 0 && decomp == ISAL_DECOMP_OK && *tofixed == 0 && - isal_strm_inflate->block_state == ISAL_BLOCK_FINISH && - deferred_correction_bytes != nullptr && - *deferred_correction_bytes > 0) { - const uint32_t deferred = *deferred_correction_bytes; - rewind_adjust_bytes = consumed_before_adjust < deferred - ? consumed_before_adjust - : deferred; - if (rewind_adjust_bytes > 0) { - *tofixed = 1; - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " deferred raw EOS correction applied: deferred ", - deferred, ", rewind_bytes ", rewind_adjust_bytes, - ", consumed_before_adjust ", consumed_before_adjust, - ", consumed_after_adjust ", - (consumed_before_adjust - rewind_adjust_bytes), "\n"); - } - *deferred_correction_bytes = 0; - } - *output_length = original_avail_out - isal_strm_inflate->avail_out; *input_length = consumed_before_adjust - rewind_adjust_bytes; input = isal_strm_inflate->next_in; @@ -548,7 +580,6 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, ret = Z_NEED_DICT; } -#ifdef DEBUG if (ret == Z_OK) { Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, " inflate finished successfully Z_OK\n"); @@ -589,7 +620,6 @@ UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, break; } } -#endif return ret; } @@ -606,10 +636,8 @@ EndUncompressIGZIP(struct inflate_state *isal_strm_inflate) free(isal_strm_inflate); -#ifdef DEBUG Log(LogLevel::LOG_INFO, "EndUncompressIGZIP() Line ", __LINE__, " inflate end\n"); -#endif return Z_OK; } @@ -627,72 +655,8 @@ inflateSetDictionary(z_streamp strm, unsigned char *dict_data, unsigned int dict return isal_inflate_set_dict(s->isal_strm_inflate, dict_data, dict_len); } -/*int -uncompress2(uint8_t *dest, unsigned long *dest_len, const uint8_t *source, - unsigned long *source_len) -{ - z_stream strm; - int err; - const unsigned int max = (unsigned int) -1; - unsigned long len, left; - uint8_t buf[1] = { 0 }; // for detection of incomplete strm when *dest_len == 0 - - len = *source_len; - if (*dest_len) { - left = *dest_len; - *dest_len = 0; - } else { - left = 1; - dest = buf; - } - - strm.next_in = (uint8_t *) source; - strm.avail_in = 0; - strm.zalloc = NULL; - strm.zfree = NULL; - strm.opaque = NULL; - - err = inflateInit_(&strm); - if (err != Z_OK) - return err; - - strm.next_out = dest; - strm.avail_out = 0; - - do { - if (strm.avail_out == 0) { - strm.avail_out = left > (unsigned long) max ? max : (unsigned int) left; - left -= strm.avail_out; - } - if (strm.avail_in == 0) { - strm.avail_in = len > (unsigned long) max ? max : (unsigned int) len; - len -= strm.avail_in; - } - err = inflate(&strm, Z_NO_FLUSH); - } while (err == Z_OK); - - *source_len -= len + strm.avail_in; - if (dest != buf) - *dest_len = strm.total_out; - else if (strm.total_out && err == Z_BUF_ERROR) - left = 1; - - inflateEnd(&strm); - return err == Z_STREAM_END ? Z_OK - : err == Z_NEED_DICT ? Z_DATA_ERROR - : err == Z_BUF_ERROR && left + strm.avail_out ? Z_DATA_ERROR - : err; -} - -int -uncompress(uint8_t *dest, unsigned long *dest_len, const uint8_t *source, unsigned long source_len) -{ - return uncompress2(dest, dest_len, source, &source_len); -}*/ - int -ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed, - uint32_t *deferred_correction_bytes) +ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed) { if (!isal_strm_inflate) { Log(LogLevel::LOG_ERROR, "ResetUncompressIGZIP() Line ", @@ -706,9 +670,6 @@ ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed, // Reset workaround flag *tofixed = 0; - if (deferred_correction_bytes != nullptr) { - *deferred_correction_bytes = 0; - } return Z_OK; } diff --git a/igzip.h b/igzip.h index 298860c..5bbdf79 100644 --- a/igzip.h +++ b/igzip.h @@ -29,6 +29,46 @@ int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, uint32_t* input_length, uint8_t *output, uint32_t* output_length, unsigned long *total_in, unsigned long *total_out); +bool +IsIGZIPDeflateFinished(const struct isal_zstream* stream); +bool +SupportedOptionsIGZIPCompress(int flush, uint32_t output_length, + bool stream_on_igzip_path); +bool +SupportedOptionsIGZIPUncompress(int window_bits, uint32_t input_length, + uint32_t output_length, + bool stream_on_igzip_path); +enum IGZIPNoInputAction { + IGZIP_NO_INPUT_NOT_HANDLED, + IGZIP_NO_INPUT_RETURN, + IGZIP_NO_INPUT_FALLBACK_ZLIB, +}; + +enum IGZIPInflatePathAction { + IGZIP_INFLATE_PATH_NONE, + IGZIP_INFLATE_PATH_SET_IGZIP, + IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT, + IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR, + IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY, +}; + +IGZIPNoInputAction +IGZIPHandleActiveStreamNoInput(z_streamp strm, + struct inflate_state *isal_strm_inflate, + int window_bits, int *tofixed, int *ret); + +IGZIPInflatePathAction +IGZIPRunInflateAndSelectPathAction(z_streamp strm, + struct inflate_state **isal_strm_inflate, + int window_bits, int *tofixed, + uint32_t *input_length, + uint32_t *output_length, int *ret, + bool *end_of_stream, + uint32_t pre_avail_in); + +bool +IGZIPShouldFallbackDeflate(bool stream_on_igzip_path, int flush, + uint32_t avail_in); int EndCompressIGZIP(struct isal_zstream* isal_strm); @@ -38,13 +78,11 @@ int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t *input_length, uint8_t *output, uint32_t *output_length, int window_bits, int *tofixed, - uint32_t *deferred_correction_bytes, unsigned long *total_in, unsigned long *total_out, bool *end_of_stream); int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate); int -ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed, - uint32_t *deferred_correction_bytes); +ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed); //#define Z_DEFAULT_COMPRESSION 6 #endif diff --git a/tests/zlib_accel_test.cpp b/tests/zlib_accel_test.cpp index b9e0dc0..3e81451 100644 --- a/tests/zlib_accel_test.cpp +++ b/tests/zlib_accel_test.cpp @@ -1628,8 +1628,11 @@ TEST(IGZIPInflateRegressionTest, TEST(IGZIPDeflateRegressionTest, RepeatedFinishWithEmptyInputMustReturnStreamEnd) { - SetCompressPath(IGZIP, false, false, false); + SetCompressPath(IGZIP, true, false, false); SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } const char* input = "igzip-finish-regression-payload"; const size_t input_length = strlen(input); @@ -1649,7 +1652,8 @@ TEST(IGZIPDeflateRegressionTest, stream.next_out = reinterpret_cast(output.data()); stream.avail_out = static_cast(output.size()); ret = deflate(&stream, Z_FINISH); - ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP); + const ExecutionPath path = GetDeflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); ASSERT_NE(ret, Z_DATA_ERROR); if (ret == Z_STREAM_END) { break; @@ -1666,15 +1670,19 @@ TEST(IGZIPDeflateRegressionTest, int finish_ret = deflate(&stream, Z_FINISH); EXPECT_EQ(finish_ret, Z_STREAM_END) << "iter=" << iter; - EXPECT_EQ(GetDeflateExecutionPath(&stream), IGZIP); + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_TRUE(path == IGZIP || path == ZLIB) << "iter=" << iter; } deflateEnd(&stream); } TEST(IGZIPDeflateRegressionTest, ResetMustNotStallSyncFlushOnSameStream) { - SetCompressPath(IGZIP, false, false, false); + SetCompressPath(IGZIP, true, false, false); SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } z_stream stream; memset(&stream, 0, sizeof(z_stream)); @@ -1721,8 +1729,11 @@ TEST(IGZIPDeflateRegressionTest, ResetMustNotStallSyncFlushOnSameStream) { TEST(IGZIPDeflateRegressionTest, RepeatedEmptySyncFlushMustEventuallyReportNoProgress) { - SetCompressPath(IGZIP, false, false, false); + SetCompressPath(IGZIP, true, false, false); SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } z_stream stream; memset(&stream, 0, sizeof(z_stream)); @@ -2089,8 +2100,11 @@ TEST(IGZIPDeflateRegressionTest, TEST(IGZIPDeflateRegressionTest, FinishWithTinyOutputBufferMustNotTruncateStream) { - SetCompressPath(IGZIP, false, false, false); + SetCompressPath(IGZIP, true, false, false); SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } std::string input; input.reserve(256 * 1024); diff --git a/zlib_accel.cpp b/zlib_accel.cpp index d70d7c6..a724795 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -203,13 +203,9 @@ struct DeflateSettings { struct InflateSettings { InflateSettings(int _window_bits) - : window_bits(_window_bits), - trailer_overconsumption_fixed(0), - deferred_trailer_correction_bytes(0) {} + : window_bits(_window_bits), trailer_overconsumption_fixed(0) {} int window_bits; int trailer_overconsumption_fixed; /* indicates if fix has been applied for overconsumption issue*/ - uint32_t deferred_trailer_correction_bytes; - bool force_zlib_for_raw_boundary = false; ExecutionPath path = UNDEFINED; struct inflate_state *isal_strm = nullptr; }; @@ -248,42 +244,13 @@ class InflateStreamSettings { }; InflateStreamSettings inflate_stream_settings; -#ifdef USE_IGZIP -static bool IsIGZIPDeflateFinished(const struct isal_zstream* stream) { - if (stream == nullptr) { - return false; - } - const enum isal_zstate_state state = stream->internal_state.state; - return state == ZSTATE_END || state == ZSTATE_TMP_END; -} -#endif - -static const char* ExecutionPathToString(ExecutionPath path) { - switch (path) { - case UNDEFINED: - return "UNDEFINED"; - case ZLIB: - return "ZLIB"; - case QAT: - return "QAT"; - case IAA: - return "IAA"; - case IGZIP: - return "IGZIP"; - default: - return "UNKNOWN"; - } -} - static void SetDeflatePath(DeflateSettings* settings, z_streamp strm, ExecutionPath new_path, const char* reason) { if (settings == nullptr || settings->path == new_path) { return; } - Log(LogLevel::LOG_INFO, "deflate path transition Line ", __LINE__, ", strm ", - static_cast(strm), ", old ", - ExecutionPathToString(settings->path), ", new ", - ExecutionPathToString(new_path), ", reason ", reason, "\n"); + (void)strm; + (void)reason; settings->path = new_path; } @@ -292,10 +259,8 @@ static void SetInflatePath(InflateSettings* settings, z_streamp strm, if (settings == nullptr || settings->path == new_path) { return; } - Log(LogLevel::LOG_INFO, "inflate path transition Line ", __LINE__, ", strm ", - static_cast(strm), ", old ", - ExecutionPathToString(settings->path), ", new ", - ExecutionPathToString(new_path), ", reason ", reason, "\n"); + (void)strm; + (void)reason; settings->path = new_path; } @@ -303,9 +268,6 @@ int ZEXPORT deflateInit_(z_streamp strm, int level, const char* version, int stream_size) { Log(LogLevel::LOG_INFO, "deflateInit_ Line ", __LINE__, ", strm ", static_cast(strm), ", level ", level, "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " level=", level, - "\n"); deflate_stream_settings.Set(strm, level, Z_DEFLATED, 15, 8, Z_DEFAULT_STRATEGY); @@ -318,17 +280,7 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, Log(LogLevel::LOG_INFO, "deflateInit2_ Line ", __LINE__, ", strm ", static_cast(strm), ", level ", level, ", window_bits ", window_bits, " \n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " level=", level, " method=", method, - " window_bits=", window_bits, " mem_level=", mem_level, - " strategy=", strategy, "\n"); - -/*#ifdef USE_IGZIP - if (configs[USE_IGZIP_COMPRESS] && configs[USE_IGZIP_UNCOMPRESS] && - !configs[USE_ZLIB_COMPRESS]) { - method = 0; - } -#endif*/ + deflate_stream_settings.Set(strm, level, method, window_bits, mem_level, strategy); return orig_deflateInit2_(strm, level, method, window_bits, mem_level, @@ -339,15 +291,9 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, uInt dictLength) { if (!configs[IGNORE_ZLIB_DICTIONARY]) { Log(LogLevel::LOG_INFO, "deflateSetDictionary Line ", __LINE__, ", strm ", - static_cast(strm), - ", dictLength ", dictLength, "\n"); - + static_cast(strm), ", dictLength ", dictLength, "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); const int ret = orig_deflateSetDictionary(strm, dictionary, dictLength); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " ret=", ret, " total_in=", strm->total_in, - " total_out=", strm->total_out, " adler=", strm->adler, - " dict_len=", dictLength, "\n"); if (ret == Z_OK) { SetDeflatePath(deflate_settings, strm, ZLIB, "deflateSetDictionary called"); @@ -365,48 +311,29 @@ int ZEXPORT deflate(z_streamp strm, int flush) { INCREMENT_STAT(DEFLATE_COUNT); PrintStats(); - const bool is_streaming_flush_with_input = - (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || - flush == Z_FULL_FLUSH || flush == Z_BLOCK) && - (strm->avail_in > 0); - if (deflate_settings->path == UNDEFINED && is_streaming_flush_with_input) { - SetDeflatePath(deflate_settings, strm, ZLIB, - "streaming flush with input"); - } - - const bool is_sync_flush = - (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || flush == Z_BLOCK); - - const bool igzip_sync_flush_empty_reentry = - (deflate_settings->path == IGZIP && is_sync_flush && strm->avail_in == 0); - const bool igzip_finish_reentry_without_input = - (deflate_settings->path == IGZIP && flush == Z_FINISH && - strm->avail_in == 0); - if (igzip_sync_flush_empty_reentry || igzip_finish_reentry_without_input) { - const char* reason = igzip_sync_flush_empty_reentry - ? "igzip risky empty sync flush; forcing zlib" - : "igzip risky finish reentry without input; forcing zlib"; - SetDeflatePath(deflate_settings, strm, ZLIB, reason); - Log(LogLevel::LOG_INFO, - " strm=", static_cast(strm), " reason=", reason, - " flush=", flush, " avail_in=", strm->avail_in, - " avail_out=", strm->avail_out, " total_in=", strm->total_in, - " total_out=", strm->total_out, "\n"); - } - Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", static_cast(deflate_settings->path), ", path_name ", - ExecutionPathToString(deflate_settings->path), ", window_bits ", + static_cast(deflate_settings->path), ", window_bits ", deflate_settings->window_bits, ", total_in ", strm->total_in, ", total_out ", strm->total_out, ", adler ", strm->adler, "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " flush=", flush, " path=", - ExecutionPathToString(deflate_settings->path), " avail_in=", strm->avail_in, - " avail_out=", strm->avail_out, " total_in=", strm->total_in, - " total_out=", strm->total_out, "\n"); + +#ifdef USE_IGZIP + if (configs[USE_IGZIP_COMPRESS] && + deflate_settings->path == UNDEFINED && + IGZIPShouldFallbackDeflate(false, flush, strm->avail_in)) { + SetDeflatePath(deflate_settings, strm, ZLIB, + "igzip fallback condition"); + } + if (configs[USE_IGZIP_COMPRESS] && + IGZIPShouldFallbackDeflate(deflate_settings->path == IGZIP, + flush, strm->avail_in)) { + SetDeflatePath(deflate_settings, strm, ZLIB, + "igzip fallback condition"); + } +#endif int ret = 1; bool iaa_available = false; @@ -415,6 +342,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { if (!in_call && deflate_settings->path != ZLIB) { uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; + bool igzip_stream_active = false; #ifdef USE_IAA iaa_available = (flush == Z_FINISH) && configs[USE_IAA_COMPRESS] && @@ -427,40 +355,20 @@ int ZEXPORT deflate(z_streamp strm, int flush) { SupportedOptionsQAT(deflate_settings->window_bits, input_len); #endif #ifdef USE_IGZIP - const bool igzip_finish_only_mode = (flush == Z_FINISH); - const bool igzip_finish_buffer_ok = - (flush != Z_FINISH || strm->avail_out >= 256); - igzip_available = configs[USE_IGZIP_COMPRESS] && igzip_finish_only_mode && - igzip_finish_buffer_ok; - if (configs[USE_IGZIP_COMPRESS] && !igzip_finish_only_mode) { - if (deflate_settings->path == UNDEFINED) { - SetDeflatePath(deflate_settings, strm, ZLIB, - "igzip unavailable for non-finish flush"); - } - Log(LogLevel::LOG_INFO, - " strm=", static_cast(strm), - " reason=flush_not_finish flush=", flush, - " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, - " total_in=", strm->total_in, " total_out=", strm->total_out, "\n"); - } - if (configs[USE_IGZIP_COMPRESS] && igzip_finish_only_mode && - !igzip_finish_buffer_ok) { - if (deflate_settings->path == UNDEFINED) { - SetDeflatePath(deflate_settings, strm, ZLIB, - "igzip unavailable for tiny finish buffer"); - } - Log(LogLevel::LOG_INFO, - " strm=", static_cast(strm), - " reason=finish_buffer_too_small flush=", flush, - " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, - " total_in=", strm->total_in, " total_out=", strm->total_out, "\n"); - } + igzip_stream_active = + (deflate_settings->path == IGZIP && + deflate_settings->isal_strm != nullptr); + igzip_available = + configs[USE_IGZIP_COMPRESS] && + SupportedOptionsIGZIPCompress(flush, output_len, igzip_stream_active); #endif // If both accelerators are enabled, send configured ratio of requests to // one or the other ExecutionPath path_selected = ZLIB; - if (iaa_available && qat_available) { + if (igzip_stream_active) { + path_selected = IGZIP; + } else if (iaa_available && qat_available) { if (static_cast(std::rand() % 100) < configs[IAA_COMPRESS_PERCENTAGE]) { path_selected = IAA; @@ -503,18 +411,9 @@ int ZEXPORT deflate(z_streamp strm, int flush) { #endif // USE_QAT } else if (path_selected == IGZIP) { #ifdef USE_IGZIP - const uInt pre_avail_in = strm->avail_in; - const uInt pre_avail_out = strm->avail_out; - const uLong pre_total_in = strm->total_in; - const uLong pre_total_out = strm->total_out; - if (deflate_settings->isal_strm == nullptr) { deflate_settings->method = 0; - deflate_settings->isal_strm = InitCompressIGZIP(deflate_settings->level, deflate_settings->window_bits); //hardcoded window bits - } - if (deflate_settings->isal_strm == nullptr) { - Log(LogLevel::LOG_ERROR, "deflate Line ", __LINE__, ", strm ", - static_cast(strm), ", failed to initialize igzip \n"); + deflate_settings->isal_strm = InitCompressIGZIP(deflate_settings->level, deflate_settings->window_bits); } in_call = true; ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, &input_len, @@ -523,37 +422,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { "selected IGZIP accelerator"); in_call = false; - uint32_t output_crc32 = 0; - uint32_t out_head_b0 = 256; - uint32_t out_head_b1 = 256; - uint32_t out_tail_b0 = 256; - uint32_t out_tail_b1 = 256; - if (output_len > 0) { - output_crc32 = - static_cast(crc32(0L, strm->next_out, output_len)); - out_head_b0 = static_cast(strm->next_out[0]); - if (output_len > 1) { - out_head_b1 = static_cast(strm->next_out[1]); - } - out_tail_b0 = static_cast(strm->next_out[output_len - 1]); - if (output_len > 1) { - out_tail_b1 = static_cast(strm->next_out[output_len - 2]); - } - } - - Log(LogLevel::LOG_INFO, - " strm=", static_cast(strm), " flush=", flush, - " pre_avail_in=", pre_avail_in, " pre_avail_out=", pre_avail_out, - " pre_total_in=", pre_total_in, " pre_total_out=", pre_total_out, - " bytes_in=", input_len, " bytes_out=", output_len, - " ret_raw=", ret, - " isal_state=", - (deflate_settings->isal_strm != nullptr) - ? static_cast(deflate_settings->isal_strm->internal_state.state) - : -1, - " output_crc32=", output_crc32, " out_head_b0=", out_head_b0, - " out_head_b1=", out_head_b1, " out_tail_b0=", out_tail_b0, - " out_tail_b1=", out_tail_b1, "\n"); //INCREMENT_STAT(DEFLATE_IGZIP_COUNT); //INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); #endif @@ -574,32 +442,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { IsIGZIPDeflateFinished(deflate_settings->isal_strm); #endif - if (flush == Z_FINISH && strm->avail_in == 0 && !finish_done) { - const bool need_more_output_room = (strm->avail_out == 0); - int isal_state_snapshot = -1; -#ifdef USE_IGZIP - if (deflate_settings->isal_strm != nullptr) { - isal_state_snapshot = static_cast( - deflate_settings->isal_strm->internal_state.state); - } -#endif - if (need_more_output_room) { - Log(LogLevel::LOG_INFO, " strm=", static_cast(strm), - " finish_done=0 avail_in=", strm->avail_in, - " avail_out=", strm->avail_out, " total_in=", strm->total_in, - " total_out=", strm->total_out, - " isal_state=", isal_state_snapshot, "\n"); - } else if (output_len == 0) { - Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), - " finish_done=0 avail_in=", strm->avail_in, - " avail_out=", strm->avail_out, - " output_len=", output_len, - " total_in=", strm->total_in, - " total_out=", strm->total_out, - " isal_state=", isal_state_snapshot, "\n"); - } - } - if (finish_done) { ret = Z_STREAM_END; } else if (!no_progress) { @@ -620,14 +462,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { ", bytes_in ", input_len, ", bytes_out ", output_len, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", static_cast(deflate_settings->path), ", path_name ", - ExecutionPathToString(deflate_settings->path), "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " ret=", ret, " engine=accelerator", - " path=", ExecutionPathToString(deflate_settings->path), - " bytes_in=", input_len, " bytes_out=", output_len, - " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, - " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, "\n"); + static_cast(deflate_settings->path), "\n"); return ret; } } @@ -648,13 +483,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { static_cast(strm), ", zlib return code ", ret, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", static_cast(deflate_settings->path), ", path_name ", - ExecutionPathToString(deflate_settings->path), "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " ret=", ret, " engine=zlib", - " path=", ExecutionPathToString(deflate_settings->path), - " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, - " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, "\n"); + static_cast(deflate_settings->path), "\n"); INCREMENT_STAT_COND(ret < 0, DEFLATE_ERROR_COUNT); return ret; @@ -670,20 +499,12 @@ int ZEXPORT deflateEnd(z_streamp strm) { #endif } deflate_stream_settings.Unset(strm); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " total_in=", (strm ? strm->total_in : 0), - " total_out=", (strm ? strm->total_out : 0), " adler=", - (strm ? strm->adler : 0), "\n"); return orig_deflateEnd(strm); } int ZEXPORT deflateReset(z_streamp strm) { Log(LogLevel::LOG_INFO, "deflateReset Line ", __LINE__, ", strm ", static_cast(strm), "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " total_in=", (strm ? strm->total_in : 0), - " total_out=", (strm ? strm->total_out : 0), " adler=", - (strm ? strm->adler : 0), "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); int ret = orig_deflateReset(strm); if (deflate_settings != nullptr) { @@ -705,9 +526,6 @@ int ZEXPORT inflateInit_(z_streamp strm, const char* version, int stream_size) { inflate_stream_settings.Set(strm, 15); Log(LogLevel::LOG_INFO, "inflateInit_ Line ", __LINE__, ", strm ", static_cast(strm), "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), "\n"); - return orig_inflateInit_(strm, version, stream_size); } @@ -716,9 +534,6 @@ int ZEXPORT inflateInit2_(z_streamp strm, int window_bits, const char* version, inflate_stream_settings.Set(strm, window_bits); Log(LogLevel::LOG_INFO, "inflateInit2_ Line ", __LINE__, ", strm ", static_cast(strm), ", window_bits ", window_bits, "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " window_bits=", window_bits, "\n"); - return orig_inflateInit2_(strm, window_bits, version, stream_size); } @@ -734,10 +549,6 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, SetInflatePath(inflate_settings, strm, ZLIB, "inflateSetDictionary called"); } - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " ret=", ret, " total_in=", strm->total_in, - " total_out=", strm->total_out, " adler=", strm->adler, - " dict_len=", dictLength, "\n"); return ret; } Log(LogLevel::LOG_INFO, "inflateSetDictionary Line ", __LINE__, @@ -756,15 +567,11 @@ int ZEXPORT inflate(z_streamp strm, int flush) { static_cast(strm), ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", static_cast(inflate_settings->path), ", path_name ", - ExecutionPathToString(inflate_settings->path), ", window_bits ", + static_cast(inflate_settings->path), ", window_bits ", inflate_settings->window_bits, ", total_in ", strm->total_in, ", total_out ", strm->total_out, ", adler ", strm->adler, "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " flush=", flush, " path=", - ExecutionPathToString(inflate_settings->path), " avail_in=", strm->avail_in, - " avail_out=", strm->avail_out, " total_in=", strm->total_in, - " total_out=", strm->total_out, "\n"); + PrintDeflateBlockHeader(LogLevel::LOG_INFO, strm->next_in, strm->avail_in, inflate_settings->window_bits); @@ -774,56 +581,38 @@ int ZEXPORT inflate(z_streamp strm, int flush) { bool qat_available = false; bool igzip_available = false; - if (inflate_settings->force_zlib_for_raw_boundary && - inflate_settings->path != ZLIB) { - SetInflatePath(inflate_settings, strm, ZLIB, - "raw boundary guard pinned stream to zlib"); - } - const bool igzip_stream_active = (inflate_settings->path == IGZIP && inflate_settings->isal_strm != nullptr); +#ifdef USE_IGZIP + const bool igzip_supported_options = + !in_call && configs[USE_IGZIP_UNCOMPRESS] && + SupportedOptionsIGZIPUncompress(inflate_settings->window_bits, + strm->avail_in, strm->avail_out, + igzip_stream_active); + // Keep stateful IGZIP stream handling on the same engine. // For avail_in==0, let IGZIP process any buffered bits in its internal // state before reporting Z_BUF_ERROR. -#ifdef USE_IGZIP if (!in_call && igzip_stream_active && strm->avail_in == 0) { - uint32_t input_len = 0; - uint32_t output_len = strm->avail_out; - - in_call = true; - ret = UncompressIGZIP(inflate_settings->isal_strm, strm->next_in, &input_len, - strm->next_out, &output_len, - inflate_settings->window_bits, - &inflate_settings->trailer_overconsumption_fixed, - &inflate_settings->deferred_trailer_correction_bytes, - &strm->total_in, &strm->total_out, &end_of_stream); - in_call = false; - - if (ret == 0) { - strm->next_out += output_len; - strm->avail_out -= output_len; - strm->total_out += output_len; + if (!igzip_supported_options) { + SetInflatePath(inflate_settings, strm, ZLIB, + "IGZIP unsupported options for no-input stream"); + } else { + in_call = true; + IGZIPNoInputAction action = IGZIPHandleActiveStreamNoInput( + strm, inflate_settings->isal_strm, inflate_settings->window_bits, + &inflate_settings->trailer_overconsumption_fixed, &ret); + in_call = false; - if (output_len > 0) { - ret = end_of_stream ? Z_STREAM_END : Z_OK; - } else { - ret = Z_BUF_ERROR; + if (action == IGZIP_NO_INPUT_FALLBACK_ZLIB) { + SetInflatePath(inflate_settings, strm, ZLIB, + "igzip raw input_done ambiguity fallback"); + // Continue through normal zlib fallback path. + } else if (action == IGZIP_NO_INPUT_RETURN) { + return ret; } - - Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", - static_cast(strm), - ", active IGZIP stream with avail_in==0, return code ", ret, - ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, - ", bytes_in 0, bytes_out ", output_len, ", end_of_stream ", - end_of_stream, ", window_bits ", inflate_settings->window_bits, "\n"); - return ret; } - - Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", - static_cast(strm), - ", active IGZIP stream with avail_in==0 and no buffered progress, returning Z_BUF_ERROR\n"); - return Z_BUF_ERROR; } #endif @@ -833,11 +622,6 @@ int ZEXPORT inflate(z_streamp strm, int flush) { if (!in_call && inflate_settings->path == UNDEFINED && inflate_settings->window_bits >= 8 && inflate_settings->window_bits <= 15 && strm->avail_in >= 2 && (strm->next_in[1] & ZLIB_FDICT_MASK)) { - Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", - static_cast(strm), - ", FDICT bit set in zlib header, cmf ", - static_cast(strm->next_in[0]), ", flg ", - static_cast(strm->next_in[1]), "\n"); SetInflatePath(inflate_settings, strm, ZLIB, "FDICT bit detected in zlib header"); } @@ -861,10 +645,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { SupportedOptionsQAT(inflate_settings->window_bits, input_len); #endif #ifdef USE_IGZIP - igzip_available = configs[USE_IGZIP_UNCOMPRESS]; //&& - // inflate_settings->window_bits == 15; - Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", - static_cast(strm), ", igzip_available? ", igzip_available, "\n"); + igzip_available = igzip_supported_options; #endif // If both accelerators are enabled, send configured ratio of requests to @@ -917,53 +698,36 @@ int ZEXPORT inflate(z_streamp strm, int flush) { #endif // USE_QAT } else if (path_selected == IGZIP) { #ifdef USE_IGZIP + in_call = true; + const IGZIPInflatePathAction path_action = + IGZIPRunInflateAndSelectPathAction( + strm, &inflate_settings->isal_strm, inflate_settings->window_bits, + &inflate_settings->trailer_overconsumption_fixed, &input_len, + &output_len, &ret, &end_of_stream, pre_avail_in); + in_call = false; + if (inflate_settings->isal_strm == nullptr) { - Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", - static_cast(strm), ", about to initalize igzip\n"); - inflate_settings->isal_strm = InitUncompressIGZIP(inflate_settings->window_bits); //hardcoded window bits - } - if (inflate_settings->isal_strm == nullptr) { - Log(LogLevel::LOG_ERROR, "inflate Line ", __LINE__, ", strm ", - static_cast(strm), ", failed to initalize igzip\n "); - return Z_DATA_ERROR; + return Z_DATA_ERROR; } - in_call = true; - ret = - UncompressIGZIP(inflate_settings->isal_strm, strm->next_in, &input_len, strm->next_out, - &output_len, inflate_settings->window_bits, - &inflate_settings->trailer_overconsumption_fixed, - &inflate_settings->deferred_trailer_correction_bytes, - &strm->total_in, &strm->total_out, &end_of_stream - /*inflate_settings->window_bits, &end_of_stream*/); - - const uInt remaining_after_igzip = - (pre_avail_in >= input_len) ? (pre_avail_in - input_len) : 0; - if (ret == 0 && inflate_settings->window_bits < 0 && end_of_stream && - remaining_after_igzip > 0 && strm->total_in == 0 && - strm->total_out == 0) { - inflate_settings->force_zlib_for_raw_boundary = true; - Log(LogLevel::LOG_ERROR, - " strm=", static_cast(strm), " bytes_in=", input_len, - " bytes_out=", output_len, " pre_avail_in=", pre_avail_in, - " remaining_in=", remaining_after_igzip, "\n"); - ret = 1; - end_of_stream = false; - SetInflatePath(inflate_settings, strm, ZLIB, - "raw boundary guard fallback"); - } - if (ret == Z_NEED_DICT) { + if (path_action == IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT) { Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), " source=igzip", " total_in=", strm->total_in, " total_out=", strm->total_out, " adler=", strm->adler, "\n"); SetInflatePath(inflate_settings, strm, ZLIB, "IGZIP returned Z_NEED_DICT"); - } else if (inflate_settings->path != ZLIB) { + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR) { + SetInflatePath(inflate_settings, strm, ZLIB, + "IGZIP requested raw trailer fallback"); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY) { + SetInflatePath(inflate_settings, strm, ZLIB, + "raw boundary guard fallback"); + } else if (path_action == IGZIP_INFLATE_PATH_SET_IGZIP && + inflate_settings->path != ZLIB) { SetInflatePath(inflate_settings, strm, IGZIP, "selected IGZIP accelerator"); } - in_call = false; //INCREMENT_STAT(INFLATE_IGZIP_COUNT); //INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); #endif @@ -992,15 +756,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", end_of_stream ", end_of_stream, ", path ", static_cast(inflate_settings->path), ", path_name ", - ExecutionPathToString(inflate_settings->path), ", window_bits ", + static_cast(inflate_settings->path), ", window_bits ", inflate_settings->window_bits, "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " ret=", ret, " engine=accelerator", - " path=", ExecutionPathToString(inflate_settings->path), - " bytes_in=", input_len, " bytes_out=", output_len, - " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, - " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, "\n"); return ret; } } @@ -1027,14 +784,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { static_cast(strm), ", zlib return code ", ret, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", static_cast(inflate_settings->path), ", path_name ", - ExecutionPathToString(inflate_settings->path), ", window_bits ", + static_cast(inflate_settings->path), ", window_bits ", inflate_settings->window_bits, "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " ret=", ret, " engine=zlib", - " path=", ExecutionPathToString(inflate_settings->path), - " avail_in=", strm->avail_in, " avail_out=", strm->avail_out, - " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, "\n"); INCREMENT_STAT_COND(ret < 0, INFLATE_ERROR_COUNT); return ret; @@ -1050,40 +801,26 @@ int ZEXPORT inflateEnd(z_streamp strm) { #endif } inflate_stream_settings.Unset(strm); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " total_in=", (strm ? strm->total_in : 0), - " total_out=", (strm ? strm->total_out : 0), " adler=", - (strm ? strm->adler : 0), "\n"); return orig_inflateEnd(strm); } int ZEXPORT inflateReset(z_streamp strm) { Log(LogLevel::LOG_INFO, "inflateReset Line ", __LINE__, ", strm ", static_cast(strm), "\n"); - Log(LogLevel::LOG_INFO, " strm=", - static_cast(strm), " total_in=", (strm ? strm->total_in : 0), - " total_out=", (strm ? strm->total_out : 0), " adler=", - (strm ? strm->adler : 0), "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); const bool was_igzip_path = (inflate_settings != nullptr && inflate_settings->path == IGZIP); int ret = orig_inflateReset(strm); if (inflate_settings != nullptr) { - inflate_settings->force_zlib_for_raw_boundary = false; SetInflatePath(inflate_settings, strm, UNDEFINED, "inflateReset"); } if (inflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP ResetUncompressIGZIP(inflate_settings->isal_strm, - &inflate_settings->trailer_overconsumption_fixed, - &inflate_settings->deferred_trailer_correction_bytes); + &inflate_settings->trailer_overconsumption_fixed); #endif if (was_igzip_path) { inflate_settings->trailer_overconsumption_fixed = 1; - inflate_settings->deferred_trailer_correction_bytes = 0; - Log(LogLevel::LOG_INFO, "inflateReset Line ", __LINE__, ", strm ", - static_cast(strm), - ", rewinds disabled after IGZIP reset\n"); } } @@ -1112,7 +849,9 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(15, input_len); #endif #ifdef USE_IGZIP - igzip_available = configs[USE_IGZIP_COMPRESS]; + igzip_available = + configs[USE_IGZIP_COMPRESS] && + SupportedOptionsIGZIPCompress(Z_FINISH, output_len, false); #endif ExecutionPath path_selected = ZLIB; @@ -1211,7 +950,9 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(15, input_len); #endif #ifdef USE_IGZIP - igzip_available = configs[USE_IGZIP_UNCOMPRESS]; + igzip_available = + configs[USE_IGZIP_UNCOMPRESS] && + SupportedOptionsIGZIPUncompress(15, input_len, output_len, false); #endif ExecutionPath path_selected = ZLIB; @@ -1249,7 +990,6 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, unsigned long total_out = 0; ret = UncompressIGZIP(isal_strm, const_cast(source), &input_len, dest, &output_len, 15, &tofixed, - nullptr, &total_in, &total_out, &end_of_stream); EndUncompressIGZIP(isal_strm); if (ret == 0 && !end_of_stream) { @@ -1509,7 +1249,9 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(31, *input_length); #endif #ifdef USE_IGZIP - igzip_available = configs[USE_IGZIP_COMPRESS]; + igzip_available = + configs[USE_IGZIP_COMPRESS] && + SupportedOptionsIGZIPCompress(Z_FINISH, *output_length, false); #endif ExecutionPath path_selected = ZLIB; @@ -1582,7 +1324,10 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(31, *input_length); #endif #ifdef USE_IGZIP - igzip_available = configs[USE_IGZIP_UNCOMPRESS]; + igzip_available = + configs[USE_IGZIP_UNCOMPRESS] && + SupportedOptionsIGZIPUncompress(31, *input_length, *output_length, + false); #endif ExecutionPath path_selected = ZLIB; @@ -1621,7 +1366,7 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, unsigned long total_in = 0; unsigned long total_out = 0; ret = UncompressIGZIP(isal_strm, input, input_length, output, - output_length, 31, &tofixed, nullptr, + output_length, 31, &tofixed, &total_in, &total_out, end_of_stream); EndUncompressIGZIP(isal_strm); @@ -1676,10 +1421,10 @@ static int CompressAndWrite(gzFile file, GzipFile* gz) { gz->deflate_stream.avail_out = static_cast(gz->io_buf_size); ret = orig_deflate(&gz->deflate_stream, Z_FINISH); Log(LogLevel::LOG_INFO, "CompressAndWrite Line ", __LINE__, ", file ", - static_cast(file), ", zlib return code ", ret, ", input ", - input_len, ", output ", output_len, ", avail_in ", - gz->deflate_stream.avail_in, ", avail_out ", - gz->deflate_stream.avail_out, "\n"); + static_cast(file), ", zlib return code ", ret, ", input ", + input_len, ", output ", output_len, ", avail_in ", + gz->deflate_stream.avail_in, ", avail_out ", + gz->deflate_stream.avail_out, "\n"); if (ret == Z_STREAM_END) { gz->data_buf_pos = gz->data_buf_content - gz->deflate_stream.avail_in; output_len = gz->io_buf_size - gz->deflate_stream.avail_out; @@ -1840,8 +1585,8 @@ int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { ret = GzreadAcceleratorUncompress(gz, input, &input_len, output, &output_len, &end_of_stream); Log(LogLevel::LOG_INFO, "gzread Line ", __LINE__, ", file ", - static_cast(file), ", accelerator return code ", ret, - ", input ", input_len, ", output ", output_len, "\n"); + static_cast(file), ", accelerator return code ", ret, + ", input ", input_len, ", output ", output_len, "\n"); // If we didn't reach end-of-stream, it means io_buf is not large // enough to hold the entire stream @@ -1865,10 +1610,10 @@ int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { static_cast(gz->data_buf_size); ret = orig_inflate(&gz->inflate_stream, Z_SYNC_FLUSH); Log(LogLevel::LOG_INFO, "gzread Line ", __LINE__, ", file ", - static_cast(file), ", zlib return code ", ret, - ", input ", input_len, ", output ", output_len, ", avail_in ", - gz->inflate_stream.avail_in, ", avail_out ", - gz->inflate_stream.avail_out, "\n"); + static_cast(file), ", zlib return code ", ret, + ", input ", input_len, ", output ", output_len, ", avail_in ", + gz->inflate_stream.avail_in, ", avail_out ", + gz->inflate_stream.avail_out, "\n"); if (ret == Z_STREAM_END || ret == Z_OK) { gz->io_buf_pos += (gz->io_buf_content - gz->inflate_stream.avail_in); @@ -1957,7 +1702,6 @@ int ZEXPORT gzclose(gzFile file) { } else { ret = orig_gzclose(file); } - Log(LogLevel::LOG_INFO, "gzclose Line ", __LINE__, ", file ", static_cast(file), ", return code ", ret, ", buffered processed ", gz->data_buf_pos, "\n"); From f8c8a1c3411d2b948c959111189f7fafcec767eb Mon Sep 17 00:00:00 2001 From: Olasoji Date: Thu, 12 Mar 2026 23:36:20 -0700 Subject: [PATCH 17/18] Formats code for PR Signed-off-by: Olasoji --- igzip.cpp | 1159 ++++++++++++++++++------------------- igzip.h | 111 ++-- tests/zlib_accel_test.cpp | 106 ++-- zlib_accel.cpp | 219 ++++--- 4 files changed, 752 insertions(+), 843 deletions(-) diff --git a/igzip.cpp b/igzip.cpp index b9ad6fb..954dbd7 100644 --- a/igzip.cpp +++ b/igzip.cpp @@ -4,673 +4,626 @@ #ifdef USE_IGZIP #include "igzip.h" -#include "crc.h" #include #include + +#include "crc.h" #include "logging.h" -static uint16_t ClampHistBits(int bits) -{ - if (bits < 0) { - return 0; - } - if (bits > ISAL_DEF_MAX_HIST_BITS) { - return ISAL_DEF_MAX_HIST_BITS; - } - return (uint16_t) bits; +static uint16_t ClampHistBits(int bits) { + if (bits < 0) { + return 0; + } + if (bits > ISAL_DEF_MAX_HIST_BITS) { + return ISAL_DEF_MAX_HIST_BITS; + } + return (uint16_t)bits; } static constexpr uint32_t kIGZIPMinFinishOutputSize = 256; -static void ConfigureDeflateWindow(struct isal_zstream *isal_strm, int windowBits) -{ - if (windowBits < 0) { - isal_strm->gzip_flag = IGZIP_DEFLATE; - isal_strm->hist_bits = ClampHistBits(-windowBits); - return; - } - - if (windowBits >= 24 && windowBits <= 31) { - isal_strm->gzip_flag = IGZIP_GZIP; - isal_strm->hist_bits = ClampHistBits(windowBits - 16); - return; - } - - isal_strm->gzip_flag = IGZIP_ZLIB; - isal_strm->hist_bits = ClampHistBits(windowBits); +static void ConfigureDeflateWindow(struct isal_zstream *isal_strm, + int windowBits) { + if (windowBits < 0) { + isal_strm->gzip_flag = IGZIP_DEFLATE; + isal_strm->hist_bits = ClampHistBits(-windowBits); + return; + } + + if (windowBits >= 24 && windowBits <= 31) { + isal_strm->gzip_flag = IGZIP_GZIP; + isal_strm->hist_bits = ClampHistBits(windowBits - 16); + return; + } + + isal_strm->gzip_flag = IGZIP_ZLIB; + isal_strm->hist_bits = ClampHistBits(windowBits); } -static void ConfigureInflateWindow(struct inflate_state *isal_strm_inflate, int windowBits) -{ - if (windowBits < 0) { - isal_strm_inflate->crc_flag = IGZIP_DEFLATE; - isal_strm_inflate->hist_bits = ClampHistBits(-windowBits); - return; - } - - if ((windowBits >= 24 && windowBits <= 31) || (windowBits >= 40 && windowBits <= 47)) { - isal_strm_inflate->crc_flag = IGZIP_GZIP; - isal_strm_inflate->hist_bits = ClampHistBits(windowBits > 31 ? windowBits - 32 : windowBits - 16); - return; - } - - isal_strm_inflate->crc_flag = IGZIP_ZLIB; - isal_strm_inflate->hist_bits = ClampHistBits(windowBits); +static void ConfigureInflateWindow(struct inflate_state *isal_strm_inflate, + int windowBits) { + if (windowBits < 0) { + isal_strm_inflate->crc_flag = IGZIP_DEFLATE; + isal_strm_inflate->hist_bits = ClampHistBits(-windowBits); + return; + } + + if ((windowBits >= 24 && windowBits <= 31) || + (windowBits >= 40 && windowBits <= 47)) { + isal_strm_inflate->crc_flag = IGZIP_GZIP; + isal_strm_inflate->hist_bits = + ClampHistBits(windowBits > 31 ? windowBits - 32 : windowBits - 16); + return; + } + + isal_strm_inflate->crc_flag = IGZIP_ZLIB; + isal_strm_inflate->hist_bits = ClampHistBits(windowBits); } -bool -IsIGZIPDeflateFinished(const struct isal_zstream* stream) -{ - if (stream == nullptr) { - return false; - } - const enum isal_zstate_state state = stream->internal_state.state; - // ZSTATE_TMP_END is a temporary state and may require reentry to - // flush remaining output; only ZSTATE_END is terminal. - return state == ZSTATE_END; +bool IsIGZIPDeflateFinished(const struct isal_zstream *stream) { + if (stream == nullptr) { + return false; + } + const enum isal_zstate_state state = stream->internal_state.state; + // ZSTATE_TMP_END is a temporary state and may require reentry to + // flush remaining output; only ZSTATE_END is terminal. + return state == ZSTATE_END; } -bool -SupportedOptionsIGZIPCompress(int flush, uint32_t output_length, - bool stream_on_igzip_path) -{ - if (flush != Z_FINISH) { - Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPCompress() Line ", - __LINE__, " flush ", flush, - " is not Z_FINISH; IGZIP deflate path disabled\n"); - return false; - } - if (!stream_on_igzip_path && output_length < kIGZIPMinFinishOutputSize) { - Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPCompress() Line ", - __LINE__, " output length ", output_length, - " is less than minimum finish buffer ", - kIGZIPMinFinishOutputSize, "\n"); - return false; - } - return true; +bool SupportedOptionsIGZIPCompress(int flush, uint32_t output_length, + bool stream_on_igzip_path) { + if (flush != Z_FINISH) { + Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPCompress() Line ", __LINE__, + " flush ", flush, " is not Z_FINISH; IGZIP deflate path disabled\n"); + return false; + } + if (!stream_on_igzip_path && output_length < kIGZIPMinFinishOutputSize) { + Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPCompress() Line ", __LINE__, + " output length ", output_length, + " is less than minimum finish buffer ", kIGZIPMinFinishOutputSize, + "\n"); + return false; + } + return true; } -bool -SupportedOptionsIGZIPUncompress(int window_bits, uint32_t input_length, - uint32_t output_length, - bool stream_on_igzip_path) -{ - (void) window_bits; - (void) output_length; - - if (!stream_on_igzip_path && input_length == 0) { - Log(LogLevel::LOG_INFO, - "SupportedOptionsIGZIPUncompress() Line ", __LINE__, - " fallback reason=no_input_on_new_stream input_length ", - input_length, " output_length ", output_length, "\n"); - return false; - } +bool SupportedOptionsIGZIPUncompress(int window_bits, uint32_t input_length, + uint32_t output_length, + bool stream_on_igzip_path) { + (void)window_bits; + (void)output_length; - return true; -} + if (!stream_on_igzip_path && input_length == 0) { + Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPUncompress() Line ", __LINE__, + " fallback reason=no_input_on_new_stream input_length ", input_length, + " output_length ", output_length, "\n"); + return false; + } -static bool IsIGZIPSyncFlush(int flush) -{ - return flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || - flush == Z_BLOCK; + return true; } -bool -IGZIPShouldFallbackDeflate(bool stream_on_igzip_path, int flush, - uint32_t avail_in) -{ - const bool is_streaming_flush = - (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || - flush == Z_FULL_FLUSH || flush == Z_BLOCK); - - if (!stream_on_igzip_path && is_streaming_flush && avail_in > 0) { - Log(LogLevel::LOG_INFO, - "IGZIPShouldFallbackDeflate() Line ", __LINE__, - " fallback reason=streaming_flush_with_input flush ", flush, - " avail_in ", avail_in, "\n"); - return true; - } - - if (!stream_on_igzip_path || avail_in != 0) { - return false; - } - - if (IsIGZIPSyncFlush(flush)) { - Log(LogLevel::LOG_INFO, - "IGZIPShouldFallbackDeflate() Line ", __LINE__, - " fallback reason=empty_sync_flush_reentry flush ", flush, - " avail_in ", avail_in, "\n"); - return true; - } - if (flush == Z_FINISH) { - return false; - } - return false; +static bool IsIGZIPSyncFlush(int flush) { + return flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || flush == Z_BLOCK; } -struct isal_zstream* -InitCompressIGZIP(int level, int windowBits) -{ - Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, - " initializing deflate with level ", level, ", windowBits ", - windowBits, "\n"); - - struct isal_zstream *isal_strm = - (struct isal_zstream *) malloc(sizeof(struct isal_zstream)); - if (!isal_strm) { - Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, - " memory allocation for isal_zstream failed\n"); - return nullptr; - } - - /* Setup ISA-L compression context */ - isal_deflate_init(isal_strm); - - isal_strm->end_of_stream = 0; - isal_strm->flush = NO_FLUSH; - - // Map Zlib levels to ISA-L levels - if (level >= 1 && level <= 2) { - isal_strm->level = 1; - isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL1_DEFAULT); - isal_strm->level_buf_size = ISAL_DEF_LVL1_DEFAULT; - } else if ((level >= 3 && level <= 6) || level == -1) { - isal_strm->level = 2; - isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL2_DEFAULT); - isal_strm->level_buf_size = ISAL_DEF_LVL2_DEFAULT; - } else if (level >= 7 && level <= 9) { - isal_strm->level = 3; - isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL3_DEFAULT); - isal_strm->level_buf_size = ISAL_DEF_LVL3_DEFAULT; - } else { - Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, - " invalid compression level\n"); - free(isal_strm); - return nullptr; - } - - if (!isal_strm->level_buf) { - free(isal_strm); - Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, - " memory allocation for level_buf failed\n"); - return nullptr; - } - - ConfigureDeflateWindow(isal_strm, windowBits); - - return isal_strm; +bool IGZIPShouldFallbackDeflate(bool stream_on_igzip_path, int flush, + uint32_t avail_in) { + const bool is_streaming_flush = + (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || + flush == Z_FULL_FLUSH || flush == Z_BLOCK); + + if (!stream_on_igzip_path && is_streaming_flush && avail_in > 0) { + Log(LogLevel::LOG_INFO, "IGZIPShouldFallbackDeflate() Line ", __LINE__, + " fallback reason=streaming_flush_with_input flush ", flush, + " avail_in ", avail_in, "\n"); + return true; + } + + if (!stream_on_igzip_path || avail_in != 0) { + return false; + } + + if (IsIGZIPSyncFlush(flush)) { + Log(LogLevel::LOG_INFO, "IGZIPShouldFallbackDeflate() Line ", __LINE__, + " fallback reason=empty_sync_flush_reentry flush ", flush, " avail_in ", + avail_in, "\n"); + return true; + } + if (flush == Z_FINISH) { + return false; + } + return false; } -int -CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, - uint32_t *input_length, uint8_t *output, uint32_t *output_length, - unsigned long *total_in, unsigned long *total_out) { - int ret; - - (void) total_in; - (void) total_out; - if (!isal_strm) { - Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, - " deflate isal_strm is NULL\n"); - return -1; - } - - // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ - isal_strm->next_out = output; - const uint32_t original_avail_out = *output_length; - isal_strm->avail_out = original_avail_out; - isal_strm->next_in = input; - const uint32_t original_avail_in = *input_length; - isal_strm->avail_in = original_avail_in; - isal_strm->total_out = *total_out; - isal_strm->total_in = *total_in; - - // stream->flush mapping - switch (flush) { - case Z_NO_FLUSH: - isal_strm->flush = NO_FLUSH; - break; - case Z_SYNC_FLUSH: - case Z_PARTIAL_FLUSH: - case Z_BLOCK: - isal_strm->flush = SYNC_FLUSH; - break; - case Z_FULL_FLUSH: - isal_strm->flush = FULL_FLUSH; - break; - case Z_FINISH: - isal_strm->flush = FULL_FLUSH; - isal_strm->end_of_stream = 1; - break; - default: - Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, - " invalid flush value\n"); - return -1; - } - - Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " gzip_flag ", - isal_strm->gzip_flag, ", hist_bits ", isal_strm->hist_bits, - ", flush ", isal_strm->flush, ", level ", isal_strm->level, - ", avail_in ", isal_strm->avail_in, ", avail_out ", - (uint32_t)isal_strm->avail_out, ", total_out ", - (uint32_t)isal_strm->total_out, ", total_in ", - (uint32_t)isal_strm->total_in, "\n"); - - int comp = isal_deflate(isal_strm); - - *output_length = original_avail_out - isal_strm->avail_out; - *input_length = original_avail_in - isal_strm->avail_in; - input = isal_strm->next_in; - output = isal_strm->next_out; - - Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, - " after isal_deflate: avail_in ", isal_strm->avail_in, - ", avail_out ", (uint32_t)isal_strm->avail_out, - ", bytes_consumed ", *input_length, - ", bytes_produced ", *output_length, "\n"); - - ret = (comp == COMP_OK) ? 0 : 1; - - if (ret == Z_OK) { - Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, - " deflate finished successfully Z_OK\n"); - } else if (ret == Z_STREAM_END) { - Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, - " deflate finished successfully Z_STREAM_END\n"); - } else { - Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, - " deflate finished with error code ", ret, "\n"); - switch (comp) { - case INVALID_FLUSH: - Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", - __LINE__, " invalid flush\n"); - break; - case INVALID_PARAM: - Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", - __LINE__, " invalid parameter\n"); - break; - case STATELESS_OVERFLOW: - Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", - __LINE__, " stateless overflow\n"); - break; - case ISAL_INVALID_OPERATION: - Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", - __LINE__, " invalid operation\n"); - break; - case ISAL_INVALID_STATE: - Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", - __LINE__, " invalid state\n"); - break; - case ISAL_INVALID_LEVEL: - Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", - __LINE__, " invalid level\n"); - break; - case ISAL_INVALID_LEVEL_BUF: - Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", - __LINE__, " invalid level buffer\n"); - break; - } - } - - return ret; +struct isal_zstream *InitCompressIGZIP(int level, int windowBits) { + Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, + " initializing deflate with level ", level, ", windowBits ", windowBits, + "\n"); + + struct isal_zstream *isal_strm = + (struct isal_zstream *)malloc(sizeof(struct isal_zstream)); + if (!isal_strm) { + Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, + " memory allocation for isal_zstream failed\n"); + return nullptr; + } + + /* Setup ISA-L compression context */ + isal_deflate_init(isal_strm); + + isal_strm->end_of_stream = 0; + isal_strm->flush = NO_FLUSH; + + // Map Zlib levels to ISA-L levels + if (level >= 1 && level <= 2) { + isal_strm->level = 1; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL1_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL1_DEFAULT; + } else if ((level >= 3 && level <= 6) || level == -1) { + isal_strm->level = 2; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL2_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL2_DEFAULT; + } else if (level >= 7 && level <= 9) { + isal_strm->level = 3; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL3_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL3_DEFAULT; + } else { + Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, + " invalid compression level\n"); + free(isal_strm); + return nullptr; + } + + if (!isal_strm->level_buf) { + free(isal_strm); + Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, + " memory allocation for level_buf failed\n"); + return nullptr; + } + + ConfigureDeflateWindow(isal_strm, windowBits); + + return isal_strm; } -int -EndCompressIGZIP(struct isal_zstream* isal_strm) -{ - if (!isal_strm) { - Log(LogLevel::LOG_ERROR, "EndCompressIGZIP() Line ", __LINE__, - " isal_stream is NULL\n"); - return -1; - } - - // Free allocated memory for level_buf and isal_strm - if (isal_strm->level_buf) { - free(isal_strm->level_buf); - } - free(isal_strm); +int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, + uint32_t *input_length, uint8_t *output, + uint32_t *output_length, unsigned long *total_in, + unsigned long *total_out) { + int ret; + + (void)total_in; + (void)total_out; + if (!isal_strm) { + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " deflate isal_strm is NULL\n"); + return -1; + } + + // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ + isal_strm->next_out = output; + const uint32_t original_avail_out = *output_length; + isal_strm->avail_out = original_avail_out; + isal_strm->next_in = input; + const uint32_t original_avail_in = *input_length; + isal_strm->avail_in = original_avail_in; + isal_strm->total_out = *total_out; + isal_strm->total_in = *total_in; + + // stream->flush mapping + switch (flush) { + case Z_NO_FLUSH: + isal_strm->flush = NO_FLUSH; + break; + case Z_SYNC_FLUSH: + case Z_PARTIAL_FLUSH: + case Z_BLOCK: + isal_strm->flush = SYNC_FLUSH; + break; + case Z_FULL_FLUSH: + isal_strm->flush = FULL_FLUSH; + break; + case Z_FINISH: + isal_strm->flush = FULL_FLUSH; + isal_strm->end_of_stream = 1; + break; + default: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid flush value\n"); + return -1; + } + + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " gzip_flag ", + isal_strm->gzip_flag, ", hist_bits ", isal_strm->hist_bits, ", flush ", + isal_strm->flush, ", level ", isal_strm->level, ", avail_in ", + isal_strm->avail_in, ", avail_out ", (uint32_t)isal_strm->avail_out, + ", total_out ", (uint32_t)isal_strm->total_out, ", total_in ", + (uint32_t)isal_strm->total_in, "\n"); + + int comp = isal_deflate(isal_strm); + + *output_length = original_avail_out - isal_strm->avail_out; + *input_length = original_avail_in - isal_strm->avail_in; + input = isal_strm->next_in; + output = isal_strm->next_out; + + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, + " after isal_deflate: avail_in ", isal_strm->avail_in, ", avail_out ", + (uint32_t)isal_strm->avail_out, ", bytes_consumed ", *input_length, + ", bytes_produced ", *output_length, "\n"); + + ret = (comp == COMP_OK) ? 0 : 1; + + if (ret == Z_OK) { + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, + " deflate finished successfully Z_OK\n"); + } else if (ret == Z_STREAM_END) { + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, + " deflate finished successfully Z_STREAM_END\n"); + } else { + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " deflate finished with error code ", ret, "\n"); + switch (comp) { + case INVALID_FLUSH: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid flush\n"); + break; + case INVALID_PARAM: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid parameter\n"); + break; + case STATELESS_OVERFLOW: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " stateless overflow\n"); + break; + case ISAL_INVALID_OPERATION: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid operation\n"); + break; + case ISAL_INVALID_STATE: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid state\n"); + break; + case ISAL_INVALID_LEVEL: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid level\n"); + break; + case ISAL_INVALID_LEVEL_BUF: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid level buffer\n"); + break; + } + } + + return ret; +} - Log(LogLevel::LOG_INFO, "EndCompressIGZIP() Line ", __LINE__, - " deflate end\n"); - return Z_OK; +int EndCompressIGZIP(struct isal_zstream *isal_strm) { + if (!isal_strm) { + Log(LogLevel::LOG_ERROR, "EndCompressIGZIP() Line ", __LINE__, + " isal_stream is NULL\n"); + return -1; + } + + // Free allocated memory for level_buf and isal_strm + if (isal_strm->level_buf) { + free(isal_strm->level_buf); + } + free(isal_strm); + + Log(LogLevel::LOG_INFO, "EndCompressIGZIP() Line ", __LINE__, + " deflate end\n"); + return Z_OK; } -int -deflateSetDictionary(z_streamp strm, unsigned char *dict_data, unsigned int dict_len) -{ - if (!strm || !strm->state || !dict_data || dict_len == 0) - return Z_STREAM_ERROR; +int deflateSetDictionary(z_streamp strm, unsigned char *dict_data, + unsigned int dict_len) { + if (!strm || !strm->state || !dict_data || dict_len == 0) + return Z_STREAM_ERROR; - deflate_state *s = (deflate_state *) strm->state; + deflate_state *s = (deflate_state *)strm->state; - if (!s || !s->isal_strm) - return Z_STREAM_ERROR; + if (!s || !s->isal_strm) return Z_STREAM_ERROR; - return isal_deflate_set_dict(s->isal_strm, dict_data, dict_len); + return isal_deflate_set_dict(s->isal_strm, dict_data, dict_len); } -unsigned long -crc32(unsigned long crc, const unsigned char *buf, unsigned int len) -{ - return crc32_gzip_refl(crc, buf, len); +unsigned long crc32(unsigned long crc, const unsigned char *buf, + unsigned int len) { + return crc32_gzip_refl(crc, buf, len); } -unsigned long -adler32(unsigned long adler, const unsigned char *buf, unsigned int len) -{ - return isal_adler32(adler, buf, len); +unsigned long adler32(unsigned long adler, const unsigned char *buf, + unsigned int len) { + return isal_adler32(adler, buf, len); } -struct inflate_state* -InitUncompressIGZIP(int windowBits) -{ - struct inflate_state *isal_strm_inflate = - (struct inflate_state *) malloc(sizeof(struct inflate_state)); - if (!isal_strm_inflate) { - Log(LogLevel::LOG_ERROR, "InitUncompressIGZIP() Line ", - __LINE__, " memory allocation for inflate_state failed\n"); - return nullptr; - } +struct inflate_state *InitUncompressIGZIP(int windowBits) { + struct inflate_state *isal_strm_inflate = + (struct inflate_state *)malloc(sizeof(struct inflate_state)); + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, "InitUncompressIGZIP() Line ", __LINE__, + " memory allocation for inflate_state failed\n"); + return nullptr; + } - Log(LogLevel::LOG_INFO, "InitUncompressIGZIP() Line ", __LINE__, - " initializing inflate with windowBits ", windowBits, "\n"); + Log(LogLevel::LOG_INFO, "InitUncompressIGZIP() Line ", __LINE__, + " initializing inflate with windowBits ", windowBits, "\n"); - /* Setup ISA-L decompression context */ - isal_inflate_init(isal_strm_inflate); + /* Setup ISA-L decompression context */ + isal_inflate_init(isal_strm_inflate); - isal_strm_inflate->avail_in = 0; - isal_strm_inflate->next_in = NULL; - //strm->total_out = 0; - //strm->total_in = 0; + isal_strm_inflate->avail_in = 0; + isal_strm_inflate->next_in = NULL; + // strm->total_out = 0; + // strm->total_in = 0; - //s->trailer_overconsumption_fixed = 0; // Initialize the workaround flag + // s->trailer_overconsumption_fixed = 0; // Initialize the workaround flag - ConfigureInflateWindow(isal_strm_inflate, windowBits); + ConfigureInflateWindow(isal_strm_inflate, windowBits); - return isal_strm_inflate; + return isal_strm_inflate; } -IGZIPNoInputAction -IGZIPHandleActiveStreamNoInput(z_streamp strm, - struct inflate_state *isal_strm_inflate, - int window_bits, int *tofixed, int *ret) -{ - if (strm == nullptr || isal_strm_inflate == nullptr || ret == nullptr || - tofixed == nullptr || strm->avail_in != 0) { - return IGZIP_NO_INPUT_NOT_HANDLED; - } - - uint32_t input_len = 0; - uint32_t output_len = strm->avail_out; - bool end_of_stream = true; - - *ret = UncompressIGZIP(isal_strm_inflate, strm->next_in, &input_len, - strm->next_out, &output_len, window_bits, - tofixed, &strm->total_in, &strm->total_out, - &end_of_stream); - - if (*ret == Z_DATA_ERROR) { - Log(LogLevel::LOG_INFO, "IGZIPHandleActiveStreamNoInput() Line ", - __LINE__, " requested zlib fallback for raw INPUT_DONE ambiguity\n"); - return IGZIP_NO_INPUT_FALLBACK_ZLIB; - } - - if (*ret == 0) { - strm->next_out += output_len; - strm->avail_out -= output_len; - strm->total_out += output_len; - if (output_len > 0) { - *ret = end_of_stream ? Z_STREAM_END : Z_OK; - } else { - *ret = Z_BUF_ERROR; - } - return IGZIP_NO_INPUT_RETURN; - } - - *ret = Z_BUF_ERROR; - return IGZIP_NO_INPUT_RETURN; +IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( + z_streamp strm, struct inflate_state *isal_strm_inflate, int window_bits, + int *tofixed, int *ret) { + if (strm == nullptr || isal_strm_inflate == nullptr || ret == nullptr || + tofixed == nullptr || strm->avail_in != 0) { + return IGZIP_NO_INPUT_NOT_HANDLED; + } + + uint32_t input_len = 0; + uint32_t output_len = strm->avail_out; + bool end_of_stream = true; + + *ret = UncompressIGZIP(isal_strm_inflate, strm->next_in, &input_len, + strm->next_out, &output_len, window_bits, tofixed, + &strm->total_in, &strm->total_out, &end_of_stream); + + if (*ret == Z_DATA_ERROR) { + Log(LogLevel::LOG_INFO, "IGZIPHandleActiveStreamNoInput() Line ", __LINE__, + " requested zlib fallback for raw INPUT_DONE ambiguity\n"); + return IGZIP_NO_INPUT_FALLBACK_ZLIB; + } + + if (*ret == 0) { + strm->next_out += output_len; + strm->avail_out -= output_len; + strm->total_out += output_len; + if (output_len > 0) { + *ret = end_of_stream ? Z_STREAM_END : Z_OK; + } else { + *ret = Z_BUF_ERROR; + } + return IGZIP_NO_INPUT_RETURN; + } + + *ret = Z_BUF_ERROR; + return IGZIP_NO_INPUT_RETURN; } -IGZIPInflatePathAction -IGZIPRunInflateAndSelectPathAction(z_streamp strm, - struct inflate_state **isal_strm_inflate, - int window_bits, int *tofixed, - uint32_t *input_length, - uint32_t *output_length, int *ret, - bool *end_of_stream, - uint32_t pre_avail_in) -{ - if (strm == nullptr || isal_strm_inflate == nullptr || input_length == nullptr || - output_length == nullptr || ret == nullptr || end_of_stream == nullptr || - tofixed == nullptr) { - if (ret != nullptr) { - *ret = Z_DATA_ERROR; - } - return IGZIP_INFLATE_PATH_NONE; - } - - if (*isal_strm_inflate == nullptr) { - *isal_strm_inflate = InitUncompressIGZIP(window_bits); - if (*isal_strm_inflate == nullptr) { - Log(LogLevel::LOG_ERROR, - "IGZIPRunInflateAndSelectPathAction() Line ", - __LINE__, " failed to initialize igzip inflate stream\n"); - *ret = Z_DATA_ERROR; - return IGZIP_INFLATE_PATH_NONE; - } - } - - *ret = UncompressIGZIP(*isal_strm_inflate, strm->next_in, input_length, - strm->next_out, output_length, window_bits, - tofixed, &strm->total_in, &strm->total_out, - end_of_stream); - - const uint32_t remaining_after_igzip = - (pre_avail_in >= *input_length) ? (pre_avail_in - *input_length) : 0; - - if (*ret == 0 && window_bits < 0 && *end_of_stream && - remaining_after_igzip > 0 && strm->total_in == 0 && - strm->total_out == 0) { - Log(LogLevel::LOG_ERROR, - "IGZIPRunInflateAndSelectPathAction() raw boundary guard strm=", - static_cast(strm), " bytes_in=", *input_length, - " bytes_out=", *output_length, - " pre_avail_in=", pre_avail_in, - " remaining_in=", remaining_after_igzip, "\n"); - *ret = 1; - *end_of_stream = false; - return IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY; - } - - if (*ret == Z_NEED_DICT) { - return IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT; - } - if (*ret == Z_DATA_ERROR) { - return IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR; - } - if (*ret == 0) { - return IGZIP_INFLATE_PATH_SET_IGZIP; - } - - return IGZIP_INFLATE_PATH_NONE; +IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( + z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits, + int *tofixed, uint32_t *input_length, uint32_t *output_length, int *ret, + bool *end_of_stream, uint32_t pre_avail_in) { + if (strm == nullptr || isal_strm_inflate == nullptr || + input_length == nullptr || output_length == nullptr || ret == nullptr || + end_of_stream == nullptr || tofixed == nullptr) { + if (ret != nullptr) { + *ret = Z_DATA_ERROR; + } + return IGZIP_INFLATE_PATH_NONE; + } + + if (*isal_strm_inflate == nullptr) { + *isal_strm_inflate = InitUncompressIGZIP(window_bits); + if (*isal_strm_inflate == nullptr) { + Log(LogLevel::LOG_ERROR, "IGZIPRunInflateAndSelectPathAction() Line ", + __LINE__, " failed to initialize igzip inflate stream\n"); + *ret = Z_DATA_ERROR; + return IGZIP_INFLATE_PATH_NONE; + } + } + + *ret = UncompressIGZIP(*isal_strm_inflate, strm->next_in, input_length, + strm->next_out, output_length, window_bits, tofixed, + &strm->total_in, &strm->total_out, end_of_stream); + + const uint32_t remaining_after_igzip = + (pre_avail_in >= *input_length) ? (pre_avail_in - *input_length) : 0; + + if (*ret == 0 && window_bits < 0 && *end_of_stream && + remaining_after_igzip > 0 && strm->total_in == 0 && + strm->total_out == 0) { + Log(LogLevel::LOG_ERROR, + "IGZIPRunInflateAndSelectPathAction() raw boundary guard strm=", + static_cast(strm), " bytes_in=", *input_length, + " bytes_out=", *output_length, " pre_avail_in=", pre_avail_in, + " remaining_in=", remaining_after_igzip, "\n"); + *ret = 1; + *end_of_stream = false; + return IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY; + } + + if (*ret == Z_NEED_DICT) { + return IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT; + } + if (*ret == Z_DATA_ERROR) { + return IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR; + } + if (*ret == 0) { + return IGZIP_INFLATE_PATH_SET_IGZIP; + } + + return IGZIP_INFLATE_PATH_NONE; } -int -UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, - uint32_t *input_length, uint8_t *output, uint32_t *output_length, - int window_bits, int *tofixed, - unsigned long *total_in, unsigned long *total_out, - bool *end_of_stream) -{ - (void) total_in; - - if (!isal_strm_inflate) { - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, - " isal_strm_inflate is NULL\n"); - return Z_STREAM_ERROR; - } - // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ - isal_strm_inflate->next_out = output; - const uint32_t original_avail_out = *output_length; - isal_strm_inflate->avail_out = original_avail_out; - const uint32_t original_avail_in = *input_length; - isal_strm_inflate->avail_in = original_avail_in; - isal_strm_inflate->next_in = input; - isal_strm_inflate->total_out = *total_out; - - const int decomp = isal_inflate(isal_strm_inflate); - - uint32_t consumed_before_adjust = 0; - if (isal_strm_inflate->avail_in <= original_avail_in) { - consumed_before_adjust = - original_avail_in - isal_strm_inflate->avail_in; - } else { - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, - " invalid avail_in ", isal_strm_inflate->avail_in, - " greater than original_avail_in ", original_avail_in, - ", clamping consumed bytes to 0\n"); - consumed_before_adjust = 0; - } - - uint32_t rewind_adjust_bytes = 0; - - // WORKAROUND: ISA-L over-consumption fix for raw deflate mode. - // Option 2 behavior: if ambiguity appears at INPUT_DONE, request caller - // fallback to zlib instead of carrying deferred state. - if (window_bits < 0 && decomp == ISAL_DECOMP_OK && *tofixed == 0 && - (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH || - isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE) && - isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) { - const uint32_t expected_trailer_bytes = 8; - const uint32_t over_consumed = - expected_trailer_bytes - isal_strm_inflate->avail_in; - if (over_consumed >= 1 && over_consumed <= 7) { - if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) { - rewind_adjust_bytes = - consumed_before_adjust < over_consumed - ? consumed_before_adjust - : over_consumed; - if (rewind_adjust_bytes > 0) { - *tofixed = 1; - } - } else { - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " raw INPUT_DONE ambiguity detected: over_consumed ", - over_consumed, - ", requesting zlib fallback\n"); - return Z_DATA_ERROR; - } - } - } - - *output_length = original_avail_out - isal_strm_inflate->avail_out; - *input_length = consumed_before_adjust - rewind_adjust_bytes; - input = isal_strm_inflate->next_in; - output = isal_strm_inflate->next_out; - - if (end_of_stream != nullptr) { - *end_of_stream = (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH); - } - - int ret = 1; - if (decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) { - ret = 0; - } else if (decomp == ISAL_NEED_DICT) { - ret = Z_NEED_DICT; - } - - if (ret == Z_OK) { - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " inflate finished successfully Z_OK\n"); - } else if (ret == Z_STREAM_END) { - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " inflate finished with Z_STREAM_END\n"); - } else { - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, - " inflate finished with error code ", ret, "\n"); - switch (decomp) { - case ISAL_INVALID_BLOCK: - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", - __LINE__, " ISA-L error - Invalid block\n"); - break; - case ISAL_INVALID_SYMBOL: - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", - __LINE__, " ISA-L error - Invalid symbol\n"); - break; - case ISAL_INVALID_LOOKBACK: - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", - __LINE__, " ISA-L error - Invalid lookback\n"); - break; - case ISAL_END_INPUT: - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", - __LINE__, " ISA-L error - End of input reached unexpectedly\n"); - break; - case ISAL_UNSUPPORTED_METHOD: - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", - __LINE__, " ISA-L error - Unsupported method\n"); - break; - case ISAL_NEED_DICT: - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", - __LINE__, " ISA-L error - Need dictionary\n"); - break; - default: - Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", - __LINE__, " ISA-L error code ", decomp, "\n"); - break; - } - } - - return ret; +int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, + uint32_t *input_length, uint8_t *output, + uint32_t *output_length, int window_bits, int *tofixed, + unsigned long *total_in, unsigned long *total_out, + bool *end_of_stream) { + (void)total_in; + + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " isal_strm_inflate is NULL\n"); + return Z_STREAM_ERROR; + } + // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ + isal_strm_inflate->next_out = output; + const uint32_t original_avail_out = *output_length; + isal_strm_inflate->avail_out = original_avail_out; + const uint32_t original_avail_in = *input_length; + isal_strm_inflate->avail_in = original_avail_in; + isal_strm_inflate->next_in = input; + isal_strm_inflate->total_out = *total_out; + + const int decomp = isal_inflate(isal_strm_inflate); + + uint32_t consumed_before_adjust = 0; + if (isal_strm_inflate->avail_in <= original_avail_in) { + consumed_before_adjust = original_avail_in - isal_strm_inflate->avail_in; + } else { + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " invalid avail_in ", isal_strm_inflate->avail_in, + " greater than original_avail_in ", original_avail_in, + ", clamping consumed bytes to 0\n"); + consumed_before_adjust = 0; + } + + uint32_t rewind_adjust_bytes = 0; + + // WORKAROUND: ISA-L over-consumption fix for raw deflate mode. + // Option 2 behavior: if ambiguity appears at INPUT_DONE, request caller + // fallback to zlib instead of carrying deferred state. + if (window_bits < 0 && decomp == ISAL_DECOMP_OK && *tofixed == 0 && + (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH || + isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE) && + isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) { + const uint32_t expected_trailer_bytes = 8; + const uint32_t over_consumed = + expected_trailer_bytes - isal_strm_inflate->avail_in; + if (over_consumed >= 1 && over_consumed <= 7) { + if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) { + rewind_adjust_bytes = consumed_before_adjust < over_consumed + ? consumed_before_adjust + : over_consumed; + if (rewind_adjust_bytes > 0) { + *tofixed = 1; + } + } else { + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " raw INPUT_DONE ambiguity detected: over_consumed ", over_consumed, + ", requesting zlib fallback\n"); + return Z_DATA_ERROR; + } + } + } + + *output_length = original_avail_out - isal_strm_inflate->avail_out; + *input_length = consumed_before_adjust - rewind_adjust_bytes; + input = isal_strm_inflate->next_in; + output = isal_strm_inflate->next_out; + + if (end_of_stream != nullptr) { + *end_of_stream = (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH); + } + + int ret = 1; + if (decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) { + ret = 0; + } else if (decomp == ISAL_NEED_DICT) { + ret = Z_NEED_DICT; + } + + if (ret == Z_OK) { + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " inflate finished successfully Z_OK\n"); + } else if (ret == Z_STREAM_END) { + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " inflate finished with Z_STREAM_END\n"); + } else { + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " inflate finished with error code ", ret, "\n"); + switch (decomp) { + case ISAL_INVALID_BLOCK: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - Invalid block\n"); + break; + case ISAL_INVALID_SYMBOL: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - Invalid symbol\n"); + break; + case ISAL_INVALID_LOOKBACK: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - Invalid lookback\n"); + break; + case ISAL_END_INPUT: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - End of input reached unexpectedly\n"); + break; + case ISAL_UNSUPPORTED_METHOD: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - Unsupported method\n"); + break; + case ISAL_NEED_DICT: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - Need dictionary\n"); + break; + default: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error code ", decomp, "\n"); + break; + } + } + + return ret; } -int -EndUncompressIGZIP(struct inflate_state *isal_strm_inflate) -{ - if (!isal_strm_inflate) { - Log(LogLevel::LOG_ERROR, "EndUncompressIGZIP() Line ", - __LINE__, " z_streamp is NULL\n"); - return Z_STREAM_ERROR; - } - - free(isal_strm_inflate); +int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate) { + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, "EndUncompressIGZIP() Line ", __LINE__, + " z_streamp is NULL\n"); + return Z_STREAM_ERROR; + } + free(isal_strm_inflate); - Log(LogLevel::LOG_INFO, "EndUncompressIGZIP() Line ", __LINE__, - " inflate end\n"); - return Z_OK; + Log(LogLevel::LOG_INFO, "EndUncompressIGZIP() Line ", __LINE__, + " inflate end\n"); + return Z_OK; } -int -inflateSetDictionary(z_streamp strm, unsigned char *dict_data, unsigned int dict_len) -{ - if (!strm || !strm->state || !dict_data || dict_len == 0) - return Z_STREAM_ERROR; +int inflateSetDictionary(z_streamp strm, unsigned char *dict_data, + unsigned int dict_len) { + if (!strm || !strm->state || !dict_data || dict_len == 0) + return Z_STREAM_ERROR; - const inflate_state2 *s = (inflate_state2 *) strm->state; + const inflate_state2 *s = (inflate_state2 *)strm->state; - if (!s || !s->isal_strm_inflate) - return Z_STREAM_ERROR; + if (!s || !s->isal_strm_inflate) return Z_STREAM_ERROR; - return isal_inflate_set_dict(s->isal_strm_inflate, dict_data, dict_len); + return isal_inflate_set_dict(s->isal_strm_inflate, dict_data, dict_len); } -int -ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed) -{ - if (!isal_strm_inflate) { - Log(LogLevel::LOG_ERROR, "ResetUncompressIGZIP() Line ", - __LINE__, " isal_strm_inflate is NULL\n"); - return Z_STREAM_ERROR; - } - - // Reset ISA-L inflate state - isal_inflate_reset(isal_strm_inflate); +int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, + int *tofixed) { + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, "ResetUncompressIGZIP() Line ", __LINE__, + " isal_strm_inflate is NULL\n"); + return Z_STREAM_ERROR; + } + // Reset ISA-L inflate state + isal_inflate_reset(isal_strm_inflate); - // Reset workaround flag - *tofixed = 0; + // Reset workaround flag + *tofixed = 0; - return Z_OK; + return Z_OK; } #endif diff --git a/igzip.h b/igzip.h index 5bbdf79..0317907 100644 --- a/igzip.h +++ b/igzip.h @@ -8,81 +8,66 @@ #include typedef struct internal_state2 { - z_streamp strm; - int level; - int w_bits; - struct inflate_state *isal_strm_inflate; - int trailer_overconsumption_fixed; /* Indicates if fix has been applied for gzip trailer - overconsumption issue */ + z_streamp strm; + int level; + int w_bits; + struct inflate_state *isal_strm_inflate; + int trailer_overconsumption_fixed; /* Indicates if fix has been applied for + gzip trailer overconsumption issue */ } inflate_state2; typedef struct internal_state { - z_streamp strm; - int level; - int w_bits; - struct isal_zstream *isal_strm; + z_streamp strm; + int level; + int w_bits; + struct isal_zstream *isal_strm; } deflate_state; -struct isal_zstream* -InitCompressIGZIP(int level, int windowBits); -int -CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, - uint32_t* input_length, uint8_t *output, uint32_t* output_length, - unsigned long *total_in, unsigned long *total_out); -bool -IsIGZIPDeflateFinished(const struct isal_zstream* stream); -bool -SupportedOptionsIGZIPCompress(int flush, uint32_t output_length, - bool stream_on_igzip_path); -bool -SupportedOptionsIGZIPUncompress(int window_bits, uint32_t input_length, - uint32_t output_length, - bool stream_on_igzip_path); +struct isal_zstream *InitCompressIGZIP(int level, int windowBits); +int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, + uint32_t *input_length, uint8_t *output, + uint32_t *output_length, unsigned long *total_in, + unsigned long *total_out); +bool IsIGZIPDeflateFinished(const struct isal_zstream *stream); +bool SupportedOptionsIGZIPCompress(int flush, uint32_t output_length, + bool stream_on_igzip_path); +bool SupportedOptionsIGZIPUncompress(int window_bits, uint32_t input_length, + uint32_t output_length, + bool stream_on_igzip_path); enum IGZIPNoInputAction { - IGZIP_NO_INPUT_NOT_HANDLED, - IGZIP_NO_INPUT_RETURN, - IGZIP_NO_INPUT_FALLBACK_ZLIB, + IGZIP_NO_INPUT_NOT_HANDLED, + IGZIP_NO_INPUT_RETURN, + IGZIP_NO_INPUT_FALLBACK_ZLIB, }; enum IGZIPInflatePathAction { - IGZIP_INFLATE_PATH_NONE, - IGZIP_INFLATE_PATH_SET_IGZIP, - IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT, - IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR, - IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY, + IGZIP_INFLATE_PATH_NONE, + IGZIP_INFLATE_PATH_SET_IGZIP, + IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT, + IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR, + IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY, }; -IGZIPNoInputAction -IGZIPHandleActiveStreamNoInput(z_streamp strm, - struct inflate_state *isal_strm_inflate, - int window_bits, int *tofixed, int *ret); +IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( + z_streamp strm, struct inflate_state *isal_strm_inflate, int window_bits, + int *tofixed, int *ret); -IGZIPInflatePathAction -IGZIPRunInflateAndSelectPathAction(z_streamp strm, - struct inflate_state **isal_strm_inflate, - int window_bits, int *tofixed, - uint32_t *input_length, - uint32_t *output_length, int *ret, - bool *end_of_stream, - uint32_t pre_avail_in); +IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( + z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits, + int *tofixed, uint32_t *input_length, uint32_t *output_length, int *ret, + bool *end_of_stream, uint32_t pre_avail_in); -bool -IGZIPShouldFallbackDeflate(bool stream_on_igzip_path, int flush, - uint32_t avail_in); -int -EndCompressIGZIP(struct isal_zstream* isal_strm); +bool IGZIPShouldFallbackDeflate(bool stream_on_igzip_path, int flush, + uint32_t avail_in); +int EndCompressIGZIP(struct isal_zstream *isal_strm); -struct inflate_state* -InitUncompressIGZIP(int windowBits); -int -UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, - uint32_t *input_length, uint8_t *output, uint32_t *output_length, - int window_bits, int *tofixed, - unsigned long *total_in, unsigned long *total_out, - bool *end_of_stream); -int -EndUncompressIGZIP(struct inflate_state *isal_strm_inflate); -int -ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed); -//#define Z_DEFAULT_COMPRESSION 6 +struct inflate_state *InitUncompressIGZIP(int windowBits); +int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, + uint32_t *input_length, uint8_t *output, + uint32_t *output_length, int window_bits, int *tofixed, + unsigned long *total_in, unsigned long *total_out, + bool *end_of_stream); +int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate); +int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed); +// #define Z_DEFAULT_COMPRESSION 6 #endif diff --git a/tests/zlib_accel_test.cpp b/tests/zlib_accel_test.cpp index 3e81451..d09bf44 100644 --- a/tests/zlib_accel_test.cpp +++ b/tests/zlib_accel_test.cpp @@ -6,10 +6,10 @@ #include #include +#include #include #include #include -#include #include #include #include @@ -195,8 +195,8 @@ int ZlibCompressWithLevel(const char* input, size_t input_length, z_stream stream; memset(&stream, 0, sizeof(z_stream)); - int st = - deflateInit2(&stream, level, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY); + int st = deflateInit2(&stream, level, Z_DEFLATED, window_bits, 8, + Z_DEFAULT_STRATEGY); if (st != Z_OK) { deflateEnd(&stream); return st; @@ -653,11 +653,10 @@ TEST_P(ZlibTest, CompressDecompress) { // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses // ISA-L level 3 (stricter match selection), which makes long-history // limitations consistently observable. - const int compression_level = - (test_param.execution_path_compress == IGZIP && - test_param.execution_path_uncompress == IAA) - ? 9 - : -1; + const int compression_level = (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; @@ -667,7 +666,7 @@ TEST_P(ZlibTest, CompressDecompress) { std::string compressed; size_t output_upper_bound; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompressWithLevel( + int ret = ZlibCompressWithLevel( input, input_length, &compressed, compression_level, test_param.window_bits_compress, test_param.flush_compress, &output_upper_bound, &execution_path); @@ -1057,11 +1056,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses // ISA-L level 3 (stricter match selection), which makes long-history // limitations consistently observable. - const int compression_level = - (test_param.execution_path_compress == IGZIP && - test_param.execution_path_uncompress == IAA) - ? 9 - : -1; + const int compression_level = (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; @@ -1071,7 +1069,7 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { std::string compressed; size_t output_upper_bound; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompressWithLevel( + int ret = ZlibCompressWithLevel( input, input_length, &compressed, compression_level, test_param.window_bits_compress, test_param.flush_compress, &output_upper_bound, &execution_path); @@ -1134,11 +1132,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses // ISA-L level 3 (stricter match selection), which makes long-history // limitations consistently observable. - const int compression_level = - (test_param.execution_path_compress == IGZIP && - test_param.execution_path_uncompress == IAA) - ? 9 - : -1; + const int compression_level = (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; @@ -1150,7 +1147,7 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { size_t input_length1 = input_length / 2; size_t output_upper_bound1; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompressWithLevel( + int ret = ZlibCompressWithLevel( input, input_length1, &compressed1, compression_level, test_param.window_bits_compress, test_param.flush_compress, &output_upper_bound1, &execution_path); @@ -1396,10 +1393,10 @@ TEST(IGZIPInflateRegressionTest, SetCompressPath(ZLIB, false, false, false); SetUncompressPath(IGZIP, false, false); - std::vector input_sizes = { - 1, 2, 3, 7, 8, 15, 16, 31, 32, 63, 64, - 127, 128, 255, 256, 511, 512, 1023, 1024, 2047, 2048, 4095, - 4096, 8191, 8192, 16384, 32768}; + std::vector input_sizes = {1, 2, 3, 7, 8, 15, 16, + 31, 32, 63, 64, 127, 128, 255, + 256, 511, 512, 1023, 1024, 2047, 2048, + 4095, 4096, 8191, 8192, 16384, 32768}; for (size_t input_length : input_sizes) { std::vector input(input_length); @@ -1444,7 +1441,7 @@ TEST(IGZIPInflateRegressionTest, stream.avail_in = 0; int empty_ret = inflate(&stream, Z_SYNC_FLUSH); EXPECT_TRUE(empty_ret == Z_BUF_ERROR || empty_ret == Z_OK || - empty_ret == Z_STREAM_END); + empty_ret == Z_STREAM_END); EXPECT_EQ(GetInflateExecutionPath(&stream), IGZIP); uint8_t trailing_byte = 0; @@ -1473,8 +1470,9 @@ TEST(IGZIPInflateRegressionTest, std::string compressed; size_t output_upper_bound; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompress(empty_payload.data(), empty_payload.size(), &compressed, - -15, Z_FINISH, &output_upper_bound, &execution_path); + int ret = + ZlibCompress(empty_payload.data(), empty_payload.size(), &compressed, -15, + Z_FINISH, &output_upper_bound, &execution_path); ASSERT_EQ(ret, Z_STREAM_END); ASSERT_EQ(execution_path, ZLIB); ASSERT_GT(compressed.size(), 0u); @@ -1507,8 +1505,7 @@ TEST(IGZIPInflateRegressionTest, inflateEnd(&stream); } -TEST(IGZIPInflateRegressionTest, - RawStreamEndMustPreserveEightTrailingBytes) { +TEST(IGZIPInflateRegressionTest, RawStreamEndMustPreserveEightTrailingBytes) { SetCompressPath(ZLIB, false, false, false); SetUncompressPath(IGZIP, false, false); @@ -1558,8 +1555,7 @@ TEST(IGZIPInflateRegressionTest, DestroyBlock(input); } -TEST(IGZIPInflateRegressionTest, - RawSplitInputDefersCorrectionUntilStreamEnd) { +TEST(IGZIPInflateRegressionTest, RawSplitInputDefersCorrectionUntilStreamEnd) { SetCompressPath(ZLIB, false, false, false); SetUncompressPath(IGZIP, false, false); @@ -1783,8 +1779,7 @@ TEST(IGZIPDeflateRegressionTest, deflateEnd(&stream); } -TEST(IGZIPDeflateRegressionTest, - DictionaryStreamMustStayOnZlibAcrossReset) { +TEST(IGZIPDeflateRegressionTest, DictionaryStreamMustStayOnZlibAcrossReset) { SetCompressPath(IGZIP, false, false, false); SetUncompressPath(ZLIB, false, false); SetConfig(IGNORE_ZLIB_DICTIONARY, 0); @@ -1796,9 +1791,8 @@ TEST(IGZIPDeflateRegressionTest, Z_OK); const char dict[] = "0123456789abcdef"; - ASSERT_EQ(deflateSetDictionary( - &stream, reinterpret_cast(dict), - static_cast(sizeof(dict) - 1)), + ASSERT_EQ(deflateSetDictionary(&stream, reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), Z_OK); EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); @@ -1823,9 +1817,8 @@ TEST(IGZIPDeflateRegressionTest, ASSERT_EQ(deflateReset(&stream), Z_OK); // Re-set dictionary after reset (standard zlib API usage). - ASSERT_EQ(deflateSetDictionary( - &stream, reinterpret_cast(dict), - static_cast(sizeof(dict) - 1)), + ASSERT_EQ(deflateSetDictionary(&stream, reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), Z_OK); EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); @@ -1860,8 +1853,7 @@ TEST(IGZIPDeflateRegressionTest, ret = inflate(&verify, Z_NO_FLUSH); EXPECT_EQ(ret, Z_NEED_DICT); - ASSERT_EQ(inflateSetDictionary(&verify, - reinterpret_cast(dict), + ASSERT_EQ(inflateSetDictionary(&verify, reinterpret_cast(dict), static_cast(sizeof(dict) - 1)), Z_OK); ret = inflate(&verify, Z_FINISH); @@ -2046,15 +2038,13 @@ TEST(IGZIPDeflateRegressionTest, Z_DEFAULT_STRATEGY), Z_OK); ASSERT_EQ(deflateSetDictionary( - &cstream, - reinterpret_cast(dictionary.data()), + &cstream, reinterpret_cast(dictionary.data()), static_cast(dictionary.size())), Z_OK); ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); std::vector compressed(compressBound(input.size())); - cstream.next_in = reinterpret_cast( - const_cast(input.data())); + cstream.next_in = reinterpret_cast(const_cast(input.data())); cstream.avail_in = static_cast(input.size()); cstream.next_out = compressed.data(); cstream.avail_out = static_cast(compressed.size()); @@ -2083,8 +2073,7 @@ TEST(IGZIPDeflateRegressionTest, ret = inflate(&dstream, Z_NO_FLUSH); if (ret == Z_NEED_DICT) { ASSERT_EQ(inflateSetDictionary( - &dstream, - reinterpret_cast(dictionary.data()), + &dstream, reinterpret_cast(dictionary.data()), static_cast(dictionary.size())), Z_OK); ret = inflate(&dstream, Z_FINISH); @@ -2120,8 +2109,7 @@ TEST(IGZIPDeflateRegressionTest, Z_DEFAULT_STRATEGY), Z_OK); - cstream.next_in = - reinterpret_cast(const_cast(input.data())); + cstream.next_in = reinterpret_cast(const_cast(input.data())); cstream.avail_in = static_cast(input.size()); std::vector compressed; @@ -2200,8 +2188,7 @@ TEST(IGZIPDeflateRegressionTest, std::vector compressed; compressed.reserve(input.size()); - cstream.next_in = - reinterpret_cast(const_cast(input.data())); + cstream.next_in = reinterpret_cast(const_cast(input.data())); cstream.avail_in = static_cast(input.size()); unsigned char sync_chunk[256]; @@ -2263,7 +2250,7 @@ TEST(IGZIPDeflateRegressionTest, } TEST(IGZIPInflateRegressionTest, - NeedDictFromIGZIPMustFallbackToZlibOnFirstInflateCall) { + NeedDictFromIGZIPMustFallbackToZlibOnFirstInflateCall) { SetCompressPath(ZLIB, false, false, false); SetUncompressPath(IGZIP, false, false); SetConfig(IGNORE_ZLIB_DICTIONARY, 0); @@ -2283,15 +2270,13 @@ TEST(IGZIPInflateRegressionTest, Z_DEFAULT_STRATEGY), Z_OK); ASSERT_EQ(deflateSetDictionary( - &cstream, - reinterpret_cast(dictionary.data()), + &cstream, reinterpret_cast(dictionary.data()), static_cast(dictionary.size())), Z_OK); ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); std::vector compressed(compressBound(input.size())); - cstream.next_in = - reinterpret_cast(const_cast(input.data())); + cstream.next_in = reinterpret_cast(const_cast(input.data())); cstream.avail_in = static_cast(input.size()); cstream.next_out = compressed.data(); cstream.avail_out = static_cast(compressed.size()); @@ -2322,8 +2307,7 @@ TEST(IGZIPInflateRegressionTest, ASSERT_EQ(ret, Z_NEED_DICT); ASSERT_EQ(inflateSetDictionary( - &dstream, - reinterpret_cast(dictionary.data()), + &dstream, reinterpret_cast(dictionary.data()), static_cast(dictionary.size())), Z_OK); ret = inflate(&dstream, Z_FINISH); @@ -2510,8 +2494,8 @@ static void RunMidstreamSetDictionaryRegression(ExecutionPath accel_path) { observed_before_dict == ZLIB); const unsigned char dict[] = "midstream-dictionary"; - const int dict_ret = deflateSetDictionary( - &cstream, dict, static_cast(sizeof(dict) - 1)); + const int dict_ret = + deflateSetDictionary(&cstream, dict, static_cast(sizeof(dict) - 1)); // zlib semantics require dictionary to be set before any deflate output. // Accepting this call midstream can desynchronize accelerator and zlib state. diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 109bf7f..82b55c5 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -198,16 +198,17 @@ struct DeflateSettings { int mem_level; int strategy; ExecutionPath path = UNDEFINED; - struct isal_zstream *isal_strm = nullptr; + struct isal_zstream* isal_strm = nullptr; }; struct InflateSettings { InflateSettings(int _window_bits) : window_bits(_window_bits), trailer_overconsumption_fixed(0) {} int window_bits; - int trailer_overconsumption_fixed; /* indicates if fix has been applied for overconsumption issue*/ + int trailer_overconsumption_fixed; /* indicates if fix has been applied for + overconsumption issue*/ ExecutionPath path = UNDEFINED; - struct inflate_state *isal_strm = nullptr; + struct inflate_state* isal_strm = nullptr; }; class DeflateStreamSettings { @@ -316,22 +317,18 @@ int ZEXPORT deflate(z_streamp strm, int flush) { strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", static_cast(deflate_settings->path), ", path_name ", static_cast(deflate_settings->path), ", window_bits ", - deflate_settings->window_bits, - ", total_in ", strm->total_in, ", total_out ", strm->total_out, - ", adler ", strm->adler, "\n"); + deflate_settings->window_bits, ", total_in ", strm->total_in, + ", total_out ", strm->total_out, ", adler ", strm->adler, "\n"); #ifdef USE_IGZIP - if (configs[USE_IGZIP_COMPRESS] && - deflate_settings->path == UNDEFINED && + if (configs[USE_IGZIP_COMPRESS] && deflate_settings->path == UNDEFINED && IGZIPShouldFallbackDeflate(false, flush, strm->avail_in)) { - SetDeflatePath(deflate_settings, strm, ZLIB, - "igzip fallback condition"); + SetDeflatePath(deflate_settings, strm, ZLIB, "igzip fallback condition"); } if (configs[USE_IGZIP_COMPRESS] && - IGZIPShouldFallbackDeflate(deflate_settings->path == IGZIP, - flush, strm->avail_in)) { - SetDeflatePath(deflate_settings, strm, ZLIB, - "igzip fallback condition"); + IGZIPShouldFallbackDeflate(deflate_settings->path == IGZIP, flush, + strm->avail_in)) { + SetDeflatePath(deflate_settings, strm, ZLIB, "igzip fallback condition"); } #endif @@ -350,17 +347,17 @@ int ZEXPORT deflate(z_streamp strm, int flush) { input_len, output_len); #endif #ifdef USE_QAT - qat_available = (flush == Z_FINISH) && configs[USE_QAT_COMPRESS] && + qat_available = + (flush == Z_FINISH) && configs[USE_QAT_COMPRESS] && output_len >= QAT_DEST_BUFFER_MIN_SIZE && SupportedOptionsQAT(deflate_settings->window_bits, input_len); #endif #ifdef USE_IGZIP - igzip_stream_active = - (deflate_settings->path == IGZIP && - deflate_settings->isal_strm != nullptr); + igzip_stream_active = (deflate_settings->path == IGZIP && + deflate_settings->isal_strm != nullptr); igzip_available = configs[USE_IGZIP_COMPRESS] && - SupportedOptionsIGZIPCompress(flush, output_len, igzip_stream_active); + SupportedOptionsIGZIPCompress(flush, output_len, igzip_stream_active); #endif // If both accelerators are enabled, send configured ratio of requests to @@ -392,8 +389,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { ret = CompressIAA(strm->next_in, &input_len, strm->next_out, &output_len, qpl_path_hardware, deflate_settings->window_bits, max_compressed_size); - SetDeflatePath(deflate_settings, strm, IAA, - "selected IAA accelerator"); + SetDeflatePath(deflate_settings, strm, IAA, "selected IAA accelerator"); in_call = false; INCREMENT_STAT(DEFLATE_IAA_COUNT); INCREMENT_STAT_COND(ret != 0, DEFLATE_IAA_ERROR_COUNT); @@ -403,8 +399,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { in_call = true; ret = CompressQAT(strm->next_in, &input_len, strm->next_out, &output_len, deflate_settings->window_bits); - SetDeflatePath(deflate_settings, strm, QAT, - "selected QAT accelerator"); + SetDeflatePath(deflate_settings, strm, QAT, "selected QAT accelerator"); in_call = false; INCREMENT_STAT(DEFLATE_QAT_COUNT); INCREMENT_STAT_COND(ret != 0, DEFLATE_QAT_ERROR_COUNT); @@ -412,18 +407,20 @@ int ZEXPORT deflate(z_streamp strm, int flush) { } else if (path_selected == IGZIP) { #ifdef USE_IGZIP if (deflate_settings->isal_strm == nullptr) { - deflate_settings->method = 0; - deflate_settings->isal_strm = InitCompressIGZIP(deflate_settings->level, deflate_settings->window_bits); + deflate_settings->method = 0; + deflate_settings->isal_strm = InitCompressIGZIP( + deflate_settings->level, deflate_settings->window_bits); } in_call = true; - ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, &input_len, - strm->next_out, &output_len, &strm->total_in, &strm->total_out); + ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, + &input_len, strm->next_out, &output_len, + &strm->total_in, &strm->total_out); SetDeflatePath(deflate_settings, strm, IGZIP, - "selected IGZIP accelerator"); + "selected IGZIP accelerator"); in_call = false; - //INCREMENT_STAT(DEFLATE_IGZIP_COUNT); - //INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); + // INCREMENT_STAT(DEFLATE_IGZIP_COUNT); + // INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); #endif } @@ -437,17 +434,17 @@ int ZEXPORT deflate(z_streamp strm, int flush) { if (path_selected == IGZIP) { const bool no_progress = (input_len == 0 && output_len == 0); bool finish_done = false; - #ifdef USE_IGZIP +#ifdef USE_IGZIP finish_done = (flush == Z_FINISH) && - IsIGZIPDeflateFinished(deflate_settings->isal_strm); - #endif + IsIGZIPDeflateFinished(deflate_settings->isal_strm); +#endif if (finish_done) { ret = Z_STREAM_END; } else if (!no_progress) { ret = Z_OK; } else { - ret = Z_BUF_ERROR; + ret = Z_BUF_ERROR; } } else { if (strm->avail_in == 0) { @@ -459,16 +456,15 @@ int ZEXPORT deflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", accelerator return code ", ret, - ", bytes_in ", input_len, ", bytes_out ", output_len, - ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, - ", path ", static_cast(deflate_settings->path), ", path_name ", + ", bytes_in ", input_len, ", bytes_out ", output_len, ", avail_in ", + strm->avail_in, ", avail_out ", strm->avail_out, ", path ", + static_cast(deflate_settings->path), ", path_name ", static_cast(deflate_settings->path), "\n"); return ret; } } - if (in_call || configs[USE_ZLIB_COMPRESS] || - deflate_settings->path == ZLIB) { + if (in_call || configs[USE_ZLIB_COMPRESS] || deflate_settings->path == ZLIB) { ret = orig_deflate(strm, flush); INCREMENT_STAT(DEFLATE_ZLIB_COUNT); if (!in_call) { @@ -492,12 +488,12 @@ int ZEXPORT deflate(z_streamp strm, int flush) { int ZEXPORT deflateEnd(z_streamp strm) { Log(LogLevel::LOG_INFO, "deflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); - DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); - if (deflate_settings->isal_strm != nullptr) { + DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); + if (deflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP - EndCompressIGZIP(deflate_settings->isal_strm); + EndCompressIGZIP(deflate_settings->isal_strm); #endif - } + } deflate_stream_settings.Unset(strm); return orig_deflateEnd(strm); } @@ -541,8 +537,7 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, uInt dictLength) { if (!configs[IGNORE_ZLIB_DICTIONARY]) { Log(LogLevel::LOG_INFO, "inflateSetDictionary Line ", __LINE__, ", strm ", - static_cast(strm), - ", dictLength ", dictLength, "\n"); + static_cast(strm), ", dictLength ", dictLength, "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); const int ret = orig_inflateSetDictionary(strm, dictionary, dictLength); if (ret == Z_OK) { @@ -568,9 +563,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", static_cast(inflate_settings->path), ", path_name ", static_cast(inflate_settings->path), ", window_bits ", - inflate_settings->window_bits, - ", total_in ", strm->total_in, ", total_out ", strm->total_out, - ", adler ", strm->adler, "\n"); + inflate_settings->window_bits, ", total_in ", strm->total_in, + ", total_out ", strm->total_out, ", adler ", strm->adler, "\n"); PrintDeflateBlockHeader(LogLevel::LOG_INFO, strm->next_in, strm->avail_in, inflate_settings->window_bits); @@ -581,8 +575,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { bool qat_available = false; bool igzip_available = false; - const bool igzip_stream_active = - (inflate_settings->path == IGZIP && inflate_settings->isal_strm != nullptr); + const bool igzip_stream_active = (inflate_settings->path == IGZIP && + inflate_settings->isal_strm != nullptr); #ifdef USE_IGZIP const bool igzip_supported_options = @@ -606,9 +600,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { in_call = false; if (action == IGZIP_NO_INPUT_FALLBACK_ZLIB) { - SetInflatePath(inflate_settings, strm, ZLIB, - "igzip raw input_done ambiguity fallback"); - // Continue through normal zlib fallback path. + SetInflatePath(inflate_settings, strm, ZLIB, + "igzip raw input_done ambiguity fallback"); + // Continue through normal zlib fallback path. } else if (action == IGZIP_NO_INPUT_RETURN) { return ret; } @@ -620,8 +614,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { // in the header, pin to ZLIB immediately so dictionary streams never reach // any accelerator (QAT/IAA/IGZIP don't support preset dictionaries). if (!in_call && inflate_settings->path == UNDEFINED && - inflate_settings->window_bits >= 8 && inflate_settings->window_bits <= 15 && - strm->avail_in >= 2 && (strm->next_in[1] & ZLIB_FDICT_MASK)) { + inflate_settings->window_bits >= 8 && + inflate_settings->window_bits <= 15 && strm->avail_in >= 2 && + (strm->next_in[1] & ZLIB_FDICT_MASK)) { SetInflatePath(inflate_settings, strm, ZLIB, "FDICT bit detected in zlib header"); } @@ -673,8 +668,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ret = UncompressIAA(strm->next_in, &input_len, strm->next_out, &output_len, qpl_path_hardware, inflate_settings->window_bits, &end_of_stream); - SetInflatePath(inflate_settings, strm, IAA, - "selected IAA accelerator"); + SetInflatePath(inflate_settings, strm, IAA, "selected IAA accelerator"); // IAA inflate is stateless in this wrapper. If stream end was not // reached, use zlib for stateful continuation. if (!end_of_stream) { @@ -690,8 +684,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ret = UncompressQAT(strm->next_in, &input_len, strm->next_out, &output_len, inflate_settings->window_bits, &end_of_stream); - SetInflatePath(inflate_settings, strm, QAT, - "selected QAT accelerator"); + SetInflatePath(inflate_settings, strm, QAT, "selected QAT accelerator"); // QATzip does not support stateful decompression // Fall back to zlib if end-of-stream not reached in one call if (!end_of_stream) { @@ -716,10 +709,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { } if (path_action == IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT) { - Log(LogLevel::LOG_ERROR, - " strm=", static_cast(strm), " source=igzip", - " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, "\n"); + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), + " source=igzip", " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, "\n"); SetInflatePath(inflate_settings, strm, ZLIB, "IGZIP returned Z_NEED_DICT"); } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR) { @@ -733,8 +725,8 @@ int ZEXPORT inflate(z_streamp strm, int flush) { SetInflatePath(inflate_settings, strm, IGZIP, "selected IGZIP accelerator"); } - //INCREMENT_STAT(INFLATE_IGZIP_COUNT); - //INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); + // INCREMENT_STAT(INFLATE_IGZIP_COUNT); + // INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); #endif } @@ -757,12 +749,11 @@ int ZEXPORT inflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", accelerator return code ", ret, - ", bytes_in ", input_len, ", bytes_out ", output_len, - ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, - ", end_of_stream ", end_of_stream, ", path ", - static_cast(inflate_settings->path), ", path_name ", - static_cast(inflate_settings->path), ", window_bits ", - inflate_settings->window_bits, "\n"); + ", bytes_in ", input_len, ", bytes_out ", output_len, ", avail_in ", + strm->avail_in, ", avail_out ", strm->avail_out, ", end_of_stream ", + end_of_stream, ", path ", static_cast(inflate_settings->path), + ", path_name ", static_cast(inflate_settings->path), + ", window_bits ", inflate_settings->window_bits, "\n"); return ret; } } @@ -771,10 +762,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { inflate_settings->path == ZLIB) { ret = orig_inflate(strm, flush); if (ret == Z_NEED_DICT) { - Log(LogLevel::LOG_ERROR, - " strm=", static_cast(strm), " source=zlib", - " total_in=", strm->total_in, " total_out=", strm->total_out, - " adler=", strm->adler, "\n"); + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), + " source=zlib", " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, "\n"); } INCREMENT_STAT(INFLATE_ZLIB_COUNT); if (!in_call) { @@ -799,12 +789,12 @@ int ZEXPORT inflate(z_streamp strm, int flush) { int ZEXPORT inflateEnd(z_streamp strm) { Log(LogLevel::LOG_INFO, "inflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); - InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); - if (inflate_settings->isal_strm != nullptr) { + InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); + if (inflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP - EndUncompressIGZIP(inflate_settings->isal_strm); + EndUncompressIGZIP(inflate_settings->isal_strm); #endif - } + } inflate_stream_settings.Unset(strm); return orig_inflateEnd(strm); } @@ -819,15 +809,15 @@ int ZEXPORT inflateReset(z_streamp strm) { if (inflate_settings != nullptr) { SetInflatePath(inflate_settings, strm, UNDEFINED, "inflateReset"); } - if (inflate_settings->isal_strm != nullptr) { + if (inflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP - ResetUncompressIGZIP(inflate_settings->isal_strm, - &inflate_settings->trailer_overconsumption_fixed); + ResetUncompressIGZIP(inflate_settings->isal_strm, + &inflate_settings->trailer_overconsumption_fixed); #endif if (was_igzip_path) { inflate_settings->trailer_overconsumption_fixed = 1; } - } + } return ret; } @@ -854,9 +844,8 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(15, input_len); #endif #ifdef USE_IGZIP - igzip_available = - configs[USE_IGZIP_COMPRESS] && - SupportedOptionsIGZIPCompress(Z_FINISH, output_len, false); + igzip_available = configs[USE_IGZIP_COMPRESS] && + SupportedOptionsIGZIPCompress(Z_FINISH, output_len, false); #endif ExecutionPath path_selected = ZLIB; @@ -892,8 +881,7 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, unsigned long total_in = 0; unsigned long total_out = 0; ret = CompressIGZIP(isal_strm, Z_FINISH, const_cast(source), - &input_len, dest, &output_len, &total_in, - &total_out); + &input_len, dest, &output_len, &total_in, &total_out); EndCompressIGZIP(isal_strm); if (ret == 0 && input_len != sourceLen) { ret = 1; @@ -957,7 +945,7 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, #ifdef USE_IGZIP igzip_available = configs[USE_IGZIP_UNCOMPRESS] && - SupportedOptionsIGZIPUncompress(15, input_len, output_len, false); + SupportedOptionsIGZIPUncompress(15, input_len, output_len, false); #endif ExecutionPath path_selected = ZLIB; @@ -996,9 +984,9 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, int tofixed = 0; unsigned long total_in = 0; unsigned long total_out = 0; - ret = UncompressIGZIP(isal_strm, const_cast(source), - &input_len, dest, &output_len, 15, &tofixed, - &total_in, &total_out, &end_of_stream); + ret = UncompressIGZIP(isal_strm, const_cast(source), &input_len, + dest, &output_len, 15, &tofixed, &total_in, + &total_out, &end_of_stream); EndUncompressIGZIP(isal_strm); if (ret == 0 && !end_of_stream) { ret = 1; @@ -1259,7 +1247,7 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, #ifdef USE_IGZIP igzip_available = configs[USE_IGZIP_COMPRESS] && - SupportedOptionsIGZIPCompress(Z_FINISH, *output_length, false); + SupportedOptionsIGZIPCompress(Z_FINISH, *output_length, false); #endif ExecutionPath path_selected = ZLIB; @@ -1289,7 +1277,8 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, } else if (path_selected == IGZIP) { #ifdef USE_IGZIP in_call = true; - struct isal_zstream* isal_strm = InitCompressIGZIP(Z_DEFAULT_COMPRESSION, 31); + struct isal_zstream* isal_strm = + InitCompressIGZIP(Z_DEFAULT_COMPRESSION, 31); if (isal_strm == nullptr) { ret = 1; } else { @@ -1334,8 +1323,7 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, #ifdef USE_IGZIP igzip_available = configs[USE_IGZIP_UNCOMPRESS] && - SupportedOptionsIGZIPUncompress(31, *input_length, *output_length, - false); + SupportedOptionsIGZIPUncompress(31, *input_length, *output_length, false); #endif ExecutionPath path_selected = ZLIB; @@ -1373,10 +1361,9 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, int tofixed = 0; unsigned long total_in = 0; unsigned long total_out = 0; - ret = UncompressIGZIP(isal_strm, input, input_length, output, - output_length, 31, &tofixed, - &total_in, &total_out, - end_of_stream); + ret = + UncompressIGZIP(isal_strm, input, input_length, output, output_length, + 31, &tofixed, &total_in, &total_out, end_of_stream); EndUncompressIGZIP(isal_strm); } gz->path = IGZIP; @@ -1429,10 +1416,10 @@ static int CompressAndWrite(gzFile file, GzipFile* gz) { gz->deflate_stream.avail_out = static_cast(gz->io_buf_size); ret = orig_deflate(&gz->deflate_stream, Z_FINISH); Log(LogLevel::LOG_INFO, "CompressAndWrite Line ", __LINE__, ", file ", - static_cast(file), ", zlib return code ", ret, ", input ", - input_len, ", output ", output_len, ", avail_in ", - gz->deflate_stream.avail_in, ", avail_out ", - gz->deflate_stream.avail_out, "\n"); + static_cast(file), ", zlib return code ", ret, ", input ", + input_len, ", output ", output_len, ", avail_in ", + gz->deflate_stream.avail_in, ", avail_out ", + gz->deflate_stream.avail_out, "\n"); if (ret == Z_STREAM_END) { gz->data_buf_pos = gz->data_buf_content - gz->deflate_stream.avail_in; output_len = gz->io_buf_size - gz->deflate_stream.avail_out; @@ -1465,9 +1452,9 @@ int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len) { static_cast(file), ", buf ", buf, ", len ", len, "\n"); unsigned int written_bytes = 0; - bool accelerator_selected = - configs[USE_IAA_COMPRESS] || configs[USE_QAT_COMPRESS] || - configs[USE_IGZIP_COMPRESS]; + bool accelerator_selected = configs[USE_IAA_COMPRESS] || + configs[USE_QAT_COMPRESS] || + configs[USE_IGZIP_COMPRESS]; if (gz->path != ZLIB && accelerator_selected) { gz->AllocateBuffers(); gz->data_buf_size = 256 << 10; @@ -1525,9 +1512,9 @@ int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { int ret = 1; uint32_t read_bytes = 0; - bool accelerator_selected = - configs[USE_IAA_UNCOMPRESS] || configs[USE_QAT_UNCOMPRESS] || - configs[USE_IGZIP_UNCOMPRESS]; + bool accelerator_selected = configs[USE_IAA_UNCOMPRESS] || + configs[USE_QAT_UNCOMPRESS] || + configs[USE_IGZIP_UNCOMPRESS]; if (gz->path != ZLIB && accelerator_selected) { gz->AllocateBuffers(); gz->data_buf_size = 512 << 10; @@ -1593,8 +1580,8 @@ int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { ret = GzreadAcceleratorUncompress(gz, input, &input_len, output, &output_len, &end_of_stream); Log(LogLevel::LOG_INFO, "gzread Line ", __LINE__, ", file ", - static_cast(file), ", accelerator return code ", ret, - ", input ", input_len, ", output ", output_len, "\n"); + static_cast(file), ", accelerator return code ", ret, + ", input ", input_len, ", output ", output_len, "\n"); // If we didn't reach end-of-stream, it means io_buf is not large // enough to hold the entire stream @@ -1618,10 +1605,10 @@ int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { static_cast(gz->data_buf_size); ret = orig_inflate(&gz->inflate_stream, Z_SYNC_FLUSH); Log(LogLevel::LOG_INFO, "gzread Line ", __LINE__, ", file ", - static_cast(file), ", zlib return code ", ret, - ", input ", input_len, ", output ", output_len, ", avail_in ", - gz->inflate_stream.avail_in, ", avail_out ", - gz->inflate_stream.avail_out, "\n"); + static_cast(file), ", zlib return code ", ret, + ", input ", input_len, ", output ", output_len, ", avail_in ", + gz->inflate_stream.avail_in, ", avail_out ", + gz->inflate_stream.avail_out, "\n"); if (ret == Z_STREAM_END || ret == Z_OK) { gz->io_buf_pos += (gz->io_buf_content - gz->inflate_stream.avail_in); From 0947c4c27bd3efb9d93aa2e688e0da21d50b7ed8 Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Mon, 1 Jun 2026 14:29:45 -0700 Subject: [PATCH 18/18] =?UTF-8?q?IGZIP=20inflate=20correctness,=20statisti?= =?UTF-8?q?cs,=20and=20IAA=E2=86=92IGZIP=20fallback=20(#54)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * igzip: fix raw deflate over-consumption via read_in_length ISAL's stateful inflate pre-loads input in 8-byte word chunks into a 64-bit shift register (read_in). After reaching ISAL_BLOCK_FINISH for raw deflate (window_bits < 0), read_in_length still holds the bit count of bytes fetched beyond the true stream end. Shifting right by 3 gives the exact over-consumed byte count for every avail_in scenario: avail_in in [1,7] : previous avail_in<8 heuristic approximated this avail_in == 0 : previous heuristic was blind; now handled avail_in == 8 : gap in both old heuristics; now handled avail_in > 8 : multi-frame continuation; now handled The corrected consumed count is reported back to the caller via *input_length so that avail_in / next_in on the zlib stream are advanced accurately, preventing downstream compressed-size and CRC mismatches (reproduces as ZipException in Java ZipInputStream). Boundary guard updated: skip the raw-boundary fallback when *tofixed==1 because the correction has already accounted for any remaining bytes, so non-zero remaining is legitimate trailing data rather than un-corrected over-consumption. BLOCK_INPUT_DONE path (output-buffer-limited mid-stream) retains the existing Z_DATA_ERROR->zlib-fallback behaviour unchanged. Previously attempted stateless-probe approach removed entirely; the read_in_length field is simpler, exact, and has no multi-frame failure mode. Validated: 8/8 IGZIPInflateRegressionTest pass; full make run passes with only the pre-existing ordering-sensitive case 37961; JAR repro (250 entries, scan + build-init) passes clean. * rename trailer_overconsumption_fixed -> read_in_correction_applied; remove stale post-reset pre-set The old name 'trailer_overconsumption_fixed' was a holdover from the avail_in < 8 heuristic era. Since EXP-6f the fix is driven entirely by isal_inflate.read_in_length >> 3; the flag only signals that a correction was applied *within the current inflate call*. Rename the field, its parameter aliases, and all local variables to read_in_correction_applied to match the actual semantics. Also remove the stale block in inflateReset: if (was_igzip_path) { inflate_settings->trailer_overconsumption_fixed = 1; } This block predated the read_in_length fix. It pre-armed the flag so that the old avail_in < 8 boundary guard would not fire on the first call after a reset. That logic was correct under the old heuristic but is actively wrong now: ResetUncompressIGZIP already zeros the flag; re-arming it to 1 would suppress the boundary guard on exactly the call where it is needed most (first call after reset on a raw stream). Remove was_igzip_path (now unused after the block removal) and inline the INPUT_DONE log expression to eliminate an unused-variable warning when DEBUG_LOG=OFF. No behaviour change for correct streams; test suite: 8/8 IGZIP inflate regressions PASS, only pre-existing 37961 flaky failure remains. * add IGZIP inflate/deflate counters to statistics; enable ENABLE_STATISTICS build flag Statistics previously tracked QAT and IAA paths but left IGZIP with commented-out placeholders. Two problems with those placeholders: the enum values DEFLATE_IGZIP_COUNT, DEFLATE_IGZIP_ERROR_COUNT, INFLATE_IGZIP_COUNT, INFLATE_IGZIP_ERROR_COUNT did not exist, and the stat_names array had no entries for them. Add the four missing Statistic enum values (after the IAA entries, before ZLIB, matching the QAT/IAA grouping pattern) and their corresponding stat_names strings. Uncomment the INCREMENT_STAT calls in the deflate and inflate IGZIP dispatch blocks. The error condition mirrors IAA/QAT: unconditional IGZIP_COUNT on every dispatch, and IGZIP_ERROR_COUNT when ret != 0 (fallback or hard error). Enable ENABLE_STATISTICS=ON in cmake.txt so the counters compile in by default. Validation (OpenSearch opensearch-core-3.3.1.jar, 250-entry JAR repro, use_igzip_uncompress=1, log_stats_samples=50): Thread (main): inflate_count=750 igzip=749 igzip_errors=0 zlib=0 — 99.9% of inflate calls served by IGZIP with zero errors or zlib fallbacks; one call is the initial probe before path selection. * add iaa_fallback_igzip: IAA can fall back to IGZIP before software zlib When IAA inflate or deflate fails (ret != 0), and use_igzip_uncompress / use_igzip_compress is enabled, the new iaa_fallback_igzip config flag (default 0) routes the retry to IGZIP before allowing the call to fall through to software zlib. Motivation: IAA is stateless and leaves input unconsumed on failure, so IGZIP can retry from the same position at no extra cost. This keeps hardware-accelerated decompression in play for inputs that IAA rejects but IGZIP can handle (e.g. streams that fail IsIAADecompressible or exceed IAA's single-call constraint). Implementation: - New ConfigOption IAA_FALLBACK_IGZIP added between IGNORE_ZLIB_DICTIONARY and LOG_LEVEL; default 0 (opt-in), range [0,1]. - inflate(): after IAA dispatch block, if path_selected==IAA && ret!=0 && configs[IAA_FALLBACK_IGZIP] && igzip_available, reset input_len/output_len (IAA may have modified them on failure), clear read_in_correction_applied, and run IGZIPRunInflateAndSelectPathAction with the same path-action handling as the normal IGZIP dispatch. Falls through to zlib if IGZIP also fails. - deflate(): identical pattern; initialises isal_strm via InitCompressIGZIP if needed (matching normal IGZIP deflate path). - Both fallback blocks are wrapped in #ifdef USE_IGZIP and guarded by igzip_available so they compile away and are unreachable without IGZIP. - Existing INFLATE/DEFLATE_IGZIP_COUNT stats capture fallback calls. Test suite: 44756/44757 PASS (37961 pre-existing flaky only). * cmake: set ENABLE_STATISTICS=OFF by default * - Reset statistics to off by default and formatting Signed-off-by: Olasoji * IGZIP: lift Z_FINISH-only restriction; enable full streaming compress - SupportedOptionsIGZIPCompress: always returns true (ISA-L natively supports NO_FLUSH/SYNC_FLUSH/FULL_FLUSH/FINISH) - SupportedOptionsIGZIPUncompress: always returns true (remove dead zero-input new-stream guard) - IGZIPShouldFallbackDeflate + IsIGZIPSyncFlush: removed entirely; both streaming_flush_with_input and empty_sync_flush_reentry cases were artificial constraints inherited from one-shot IAA/QAT model - CompressIGZIP: add ZSTATE_NEW_HDR guard for empty SYNC_FLUSH calls; ISA-L emits sync bytes unconditionally, guard returns 0 progress so caller sees Z_BUF_ERROR matching zlib semantics - deflateSetDictionary: reject mid-stream calls when an accelerator path is active; underlying zlib stream is unadvanced so orig_deflateSetDictionary would incorrectly return Z_OK - inflate active-stream no-input block: remove dead - tests: remove 6 stale path==ZLIB assertions from 3 regression tests that assumed IGZIP would fall back on streaming flushes * fix iaa_fallback_igzip compress: set path_selected=IGZIP after fallback When IAA compress fails and IGZIP is used as a fallback, path_selected remained IAA. The return-code block below then used the IAA/QAT one-shot logic (avail_in==0 -> Z_STREAM_END) instead of IGZIP streaming logic (IsIGZIPDeflateFinished). ISA-L fills its internal level buffer and returns after consuming all input but producing only partial output; avail_in reaches 0 while the stream is not yet at ZSTATE_END. The result was Z_STREAM_END returned with ~48KB of compressed data still buffered internally, corrupting every document written during indexing. Fix: set path_selected = IGZIP immediately after the fallback completes so that the return-code logic correctly calls IsIGZIPDeflateFinished and returns Z_OK for partial output, allowing the caller to drain the stream. * iaa: replace marker-based IsIAADecompressible with 512-byte threshold When iaa_prepend_empty_block=0 (the default), the old code returned true for all raw deflate and gzip inputs unconditionally, causing QPL's overconsumption bug to surface for Java ZipInputStream callers. JAR loading feeds inflate() with 512-byte chunks where avail_in > actual compressed size; QPL reports total_in == available_in, causing the Java-level ZipException that kills OpenSearch at launch. Fix (ported from exp-8): replace the IAA_PREPEND_EMPTY_BLOCK=0 branch with a 512-byte threshold guard. ZipInputStream always feeds <=512-byte chunks; Lucene stored-field reads always provide the exact compressed size which is typically well above 512 bytes. The few Lucene entries below the threshold fall back to IGZIP, which is also correct. Also removes PrependedEmptyBlockPresent() which is no longer called from the decompression path. The compress-side marker write in CompressIAA is unaffected (still controlled by iaa_prepend_empty_block). * README: document USE_IGZIP cmake option and ISA-L dependency Add USE_IGZIP (ON/OFF) and ISAL_PATH to the cmake options list, and add a Requirements for IGZIP section with a link to ISA-L. Signed-off-by: Olasoji * zlib_accel: guard pre_avail_in declaration with USE_IGZIP pre_avail_in is only referenced inside #ifdef USE_IGZIP blocks. Without this guard, builds with USE_IGZIP=OFF trigger an -Wunused-variable error under -Werror, breaking CI. Signed-off-by: Olasoji * Clang format Signed-off-by: Olasoji * remove cmake.txt; fix null deref in deflateSetDictionary cmake.txt contained a local build command with hardcoded absolute paths (/home/sdp/...). Remove it from the repo and add to .gitignore. deflateSetDictionary: deflate_stream_settings.Get(strm) can return nullptr if called without a prior deflateInit. Guard the mid-stream rejection check to avoid a null dereference in that case, returning Z_STREAM_ERROR consistent with zlib behaviour. Addresses review comments on PR #54 (matt-welch, Copilot). Signed-off-by: Olasoji * igzip: remove dead SupportedOptions/IGZIPShouldFallback stubs SupportedOptionsIGZIPCompress, SupportedOptionsIGZIPUncompress, and IGZIPShouldFallbackDeflate unconditionally returned fixed values with all parameters voided. Remove them entirely and inline the fixed values at all 8 call sites in zlib_accel.cpp. Addresses review comment on PR #54 (matt-welch). Signed-off-by: Olasoji * zlib_accel: fix read_in_correction_applied semantics and reset asymmetry The flag is per-stream-session (set once when the read_in_length over-consumption correction fires; cleared only on inflateReset), not per-call as the comment incorrectly stated. Fix the struct comment to reflect this, and remove the erroneous reset to 0 on the IAA->IGZIP fallback path which was inconsistent with the primary IGZIP path and the inflateReset behavior. Addresses review comment on PR #54 (Copilot, matt-welch). Signed-off-by: Olasoji * iaa: deprecate iaa_prepend_empty_block config option The empty stored-block marker approach was abandoned in favour of a 512-byte minimum input length threshold in IsIAADecompressible. The config option is retained for backward compatibility but has no effect on decompression and will be removed in a future release. Rationale for the threshold approach: QPL hardware always consumes all available_in bytes regardless of where the BFINAL=1 token falls (overconsumption bug). Java ZipInputStream feeds <=512-byte chunks when csize is unknown, triggering overconsumption errors. Lucene stored-field reads always supply the exact compressed size (>512 bytes), where consuming all input is correct. Document the deprecation in README and add a comment in iaa.cpp explaining the history and the replacement mechanism. Addresses review comment on PR #54 (matt-welch). Signed-off-by: $(git config user.name) <$(git config user.email)> * tests: add IAA->IGZIP fallback coverage Add IAAFallbackIGZIPTest with three tests exercising the fallback path for both deflate and inflate: - DeflateUsesIGZIPWhenIAAFailsAndFallbackEnabled: configures USE_IAA_COMPRESS+USE_IGZIP_COMPRESS+IAA_FALLBACK_IGZIP=1 and asserts the stream lands on IGZIP (or IAA if hardware present). - DeflateDoesNotUseIGZIPWhenFallbackDisabled: same IAA+IGZIP config but IAA_FALLBACK_IGZIP=0; asserts IGZIP is never selected. - InflateUsesIGZIPWhenIAAFailsAndFallbackEnabled: data compressed with IGZIP, decompressed via USE_IAA_UNCOMPRESS+IAA_FALLBACK_IGZIP=1; asserts round-trip correctness and IGZIP (or IAA) execution path. On machines without IAA hardware the QPL init fails (non-zero return), which naturally exercises the fallback branch without mocking. On machines that do have IAA hardware and succeed, the assertions accept either IAA or IGZIP so the tests remain valid in both environments. Tests are guarded by #ifdef USE_IAA and #ifdef USE_IGZIP (both are in scope here via the outer USE_IGZIP guard around the regression block). Addresses should-fix #6 from PR #54 review (matt-welch). Signed-off-by: Olasoji * tests: restore IGZIP path assertions for SYNC_FLUSH regression tests Three IGZIPDeflateRegressionTest tests originally asserted ASSERT_EQ(path, ZLIB) because IGZIPShouldFallbackDeflate redirected SYNC_FLUSH calls to the zlib path. Commit f7ee0ec (lift Z_FINISH-only restriction) removed those assertions when IGZIP gained native SYNC_FLUSH support, but left no replacement. Now that IGZIPShouldFallbackDeflate is removed entirely, restore the path assertions with the new expected behavior: IGZIP must remain on the IGZIP path through Z_NO_FLUSH, Z_SYNC_FLUSH, and Z_FINISH calls. Changes: - ResetMustNotStallSyncFlushOnSameStream: assert IGZIP after Z_NO_FLUSH and Z_SYNC_FLUSH on each cycle. - RepeatedEmptySyncFlushMustEventuallyReportNoProgress: assert IGZIP after Z_NO_FLUSH and on every Z_SYNC_FLUSH iteration. - SyncFlushWithInputMustFallbackToZlibForStreamSafety: renamed to SyncFlushWithInputMustStayOnIGZIPPath (the old name described the removed fallback behavior); assert IGZIP after Z_SYNC_FLUSH and on every Z_FINISH iteration. Addresses review comment #7 on PR #54 (matt-welch). Signed-off-by: Olasoji * igzip: document read_in_correction_applied reset and ZSTATE_NEW_HDR coupling #8 — Explain why read_in_correction_applied is cleared on inflateReset: isal_inflate_reset clears the internal read_in/read_in_length buffer, so any over-consumption correction from the previous stream session no longer applies. The flag must be reset so the new session can fire the correction fresh if needed. This is intentional; keeping it set across a reset would incorrectly suppress the correction on data that has not yet over-consumed. #9 — Document ISA-L version for ZSTATE_NEW_HDR coupling: The SYNC_FLUSH no-progress guard directly accesses isal_strm->internal_state.state == ZSTATE_NEW_HDR. Note that this was validated against ISA-L v2.32.0 (commit c196241) so future ISA-L updates can be audited for state machine changes. Addresses review comments #8 and #9 on PR #54 (matt-welch). Signed-off-by: Olasoji * address round-2 Copilot review: README doc, igzip.h comment, IGZIP stats tests, inflate fallback=0 test - README.md: document iaa_fallback_igzip config option - igzip.h: fix read_in_correction_applied comment to reflect per-stream-session semantics (not per-call) - tests/zlib_accel_test.cpp: add DEFLATE_IGZIP_COUNT/INFLATE_IGZIP_COUNT branches to CompressDecompress stats verification - tests/zlib_accel_test.cpp: add InflateDoesNotUseIGZIPWhenFallbackDisabled as companion to the existing inflate=1 test * fix: restore gzip_flag after deflateReset on IGZIP stream (Cassandra corruption) isal_deflate_reset preserves gzip_flag. After the first chunk, ISA-L internally changes gzip_flag from IGZIP_ZLIB (3) to IGZIP_ZLIB_NO_HDR (4) to suppress the header on continuation calls. Without a reset, the reused stream produced headerless chunks, causing Java's Inflater (nowrap=false) to report 'incorrect header check' / 'unknown compression method' on every subsequent SSTable chunk — resulting in CorruptSSTableException at read time. Fix: add ResetCompressIGZIP() which wraps isal_deflate_reset + calls ConfigureDeflateWindow to restore gzip_flag and hist_bits from window_bits. Replace the inline isal_deflate_reset call in deflateReset() with this. Regression test: ResetMustRestoreZlibHeaderForSubsequentChunks compresses 5 independent chunks via a reused IGZIP z_stream (deflateReset between each) and decompresses each with a fresh zlib inflater — would have failed before this fix on chunk 2 onward. Reproduced via: Cassandra 5 mix workload (80% reads, 20% updates) with IGZIP compress enabled, writing to a zlib-format SSTable. --------- Signed-off-by: Olasoji Signed-off-by: $(git config user.name) <$(git config user.email)> --- .gitignore | 5 +- README.md | 15 ++- config/config.cpp | 3 + config/config.h | 3 +- iaa.cpp | 66 +++++----- igzip.cpp | 200 +++++++++++++----------------- igzip.h | 29 ++--- statistics.cpp | 7 +- statistics.h | 4 + tests/zlib_accel_test.cpp | 249 ++++++++++++++++++++++++++++++++++++-- zlib_accel.cpp | 185 +++++++++++++++++----------- 11 files changed, 516 insertions(+), 250 deletions(-) diff --git a/.gitignore b/.gitignore index fc1dadd..aedc64f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ /build/ -/tests/build \ No newline at end of file +/tests/build +cmake.txt +/resources/ + diff --git a/README.md b/README.md index b36a23b..102d44d 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,10 @@ make CMake supports the following options: - USE_QAT (ON/OFF): include QAT acceleration - USE_IAA (ON/OFF): include IAA acceleration +- USE_IGZIP (ON/OFF): include IGZIP acceleration (requires ISA-L) - QPL_PATH: path to QPL for IAA acceleration (if not in a standard directory) - QATZIP_PATH: path to QATzip for QAT acceleration (if not in a standard directory) +- ISAL_PATH: path to ISA-L for IGZIP acceleration (if not in a standard directory) - DEBUG_LOG (ON/OFF): enable logging - ENABLE_STATISTICS (ON/OFF): enable statistics - COVERAGE (ON/OFF): enable test coverage (more details in a later section) @@ -84,6 +86,9 @@ Requirements for IAA - [accel-config](https://github.com/intel/idxd-config) - [Query Processing Library](https://github.com/intel/qpl) +Requirements for IGZIP +- [ISA-L (Intel Intelligent Storage Acceleration Library)](https://github.com/intel/isa-l) + A setup with both QAT and IAA enabled has been tested on an AWS m7i.metal-24xl instance (Ubuntu 22.04, kernel 6.8.0). Refer to the links above for instructions on how to install the dependencies. @@ -178,20 +183,22 @@ use_zlib_uncompress - Enable zlib for decompression - Setting to 1 is recommended, to allow fall back to zlib in case accelerators cannot be used or experience an error. +iaa_fallback_igzip +- Values: 0,1. Default: 0 +- If 1, and an IAA compression or decompression operation fails, the request is retried using IGZIP (if enabled) before falling back to software zlib. Useful on machines where IAA hardware is intermittently unavailable. + iaa_compress_percentage - Values: 0-100. Default: 50 - If both IAA and QAT are enabled, percentage of compression calls to offload to IAA. iaa_prepend_empty_block - Values: 0,1. Default: 0 -- Prepend an empty stored block to the compressed data to "mark" that the data was compressed by IAA. -- IAA has a 4kB history window limit and it is not able to decompress blocks that use a longer history window (up to 32kB per deflate standard). -- During decompression, this marker indicates that the data was compressed by IAA and is therefore guarateed decompressible by IAA. +- **Deprecated.** This option is retained for backward compatibility and will be removed in a future release. Setting it to 1 has no effect on decompression. +- Background: the original design prepended a 5-byte empty stored-block marker to IAA-compressed output so the decompressor could identify IAA-produced data (which uses a 4kB history window). This approach was abandoned because QPL hardware always consumes all `available_in` bytes regardless of where the stream boundary falls, making marker-based detection unreliable when the caller does not supply the exact compressed size. IAA decompression eligibility is now determined by a 512-byte minimum input length threshold: callers such as Java's `ZipInputStream` feed chunks of ≤512 bytes when the compressed size is unknown, while Lucene stored-field reads always supply the exact size (>512 bytes). iaa_uncompress_percentage - Values: 0-100. Default: 50 - If both IAA and QAT are enabled, percentage of decompression calls to offload to IAA. -- If iaa_prepend_empty_block = 1, this percentage is only applied to data with the empty block marker. qat_periodical_polling = 0 - Values: 0,1. Default: 0 diff --git a/config/config.cpp b/config/config.cpp index 51a30af..3fa6001 100644 --- a/config/config.cpp +++ b/config/config.cpp @@ -30,6 +30,7 @@ uint32_t configs[CONFIG_MAX] = { 1, /*qat_compression_level*/ 0, /*qat_compression_allow_chunking*/ 0, /*ignore_zlib_dictionary*/ + 0, /*iaa_fallback_igzip*/ 2, /*log_level*/ 1000 /*log_stats_samples*/ }; @@ -56,6 +57,7 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { "qat_compression_level", "qat_compression_allow_chunking", "ignore_zlib_dictionary", + "iaa_fallback_igzip", "log_level", "log_stats_samples" }; @@ -91,6 +93,7 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { trySetConfig(QAT_COMPRESSION_LEVEL, 9, 1); trySetConfig(QAT_COMPRESSION_ALLOW_CHUNKING, 1, 0); trySetConfig(IGNORE_ZLIB_DICTIONARY, 1, 0); + trySetConfig(IAA_FALLBACK_IGZIP, 1, 0); trySetConfig(LOG_LEVEL, 2, 0); trySetConfig(LOG_STATS_SAMPLES, UINT32_MAX, 0); diff --git a/config/config.h b/config/config.h index 12d3563..fff483c 100644 --- a/config/config.h +++ b/config/config.h @@ -20,11 +20,12 @@ enum ConfigOption { USE_IGZIP_UNCOMPRESS, IAA_COMPRESS_PERCENTAGE, IAA_UNCOMPRESS_PERCENTAGE, - IAA_PREPEND_EMPTY_BLOCK, + IAA_PREPEND_EMPTY_BLOCK, // DEPRECATED — see README for details QAT_PERIODICAL_POLLING, QAT_COMPRESSION_LEVEL, QAT_COMPRESSION_ALLOW_CHUNKING, IGNORE_ZLIB_DICTIONARY, + IAA_FALLBACK_IGZIP, LOG_LEVEL, LOG_STATS_SAMPLES, CONFIG_MAX diff --git a/iaa.cpp b/iaa.cpp index 2bbcd27..5a22f87 100644 --- a/iaa.cpp +++ b/iaa.cpp @@ -99,9 +99,19 @@ int CompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, output_shift += GZIP_EXT_XHDR_SIZE; } - // If prepending an empty block, leave space for it to be added - // For zlib format, we don't need an empty block as a marker, as the zlib - // header includes info about the window size + // DEPRECATED: iaa_prepend_empty_block is no longer used by the decompressor. + // The original design used a 5-byte empty stored-block marker written at the + // start of IAA-compressed output so that the decompressor could detect and + // trust IAA-compressed data (which uses a 4kB history window). This approach + // was abandoned because QPL hardware always consumes all available_in bytes + // regardless of where the BFINAL=1 token falls (overconsumption bug), so a + // caller-supplied exact boundary is required instead. IsIAADecompressible now + // uses a 512-byte minimum input length threshold to gate IAA decompression: + // Java ZipInputStream feeds <=512-byte chunks (csize unknown), triggering + // overconsumption; Lucene stored-field reads always provide the exact + // compressed size (>512 bytes), where consuming all input is correct. + // The config option is retained for backward compatibility and will be + // removed in a future release. bool prepend_empty_block = false; CompressedFormat format = GetCompressedFormat(window_bits); if (format != CompressedFormat::ZLIB && @@ -255,44 +265,32 @@ bool SupportedOptionsIAA(int window_bits, uint32_t input_length, return false; } -bool PrependedEmptyBlockPresent(uint8_t* input, uint32_t input_length, - CompressedFormat format) { - uint32_t header_length = GetHeaderLength(format); - if (header_length + PREPENDED_BLOCK_LENGTH > input_length) { - return false; - } - - if (input[header_length] == 0 && input[header_length + 1] == 0 && - input[header_length + 2] == 0 && input[header_length + 3] == 0xFF && - input[header_length + 4] == 0xFF) { - Log(LogLevel::LOG_INFO, "PrependedEmptyBlockPresent() Line ", __LINE__, - " Empty block detected\n"); - return true; - } - - return false; -} - bool IsIAADecompressible(uint8_t* input, uint32_t input_length, int window_bits) { CompressedFormat format = GetCompressedFormat(window_bits); if (format == CompressedFormat::ZLIB) { int window = GetWindowSizeFromZlibHeader(input, input_length); - Log(LogLevel::LOG_INFO, "IsIAADecompressible() Line ", __LINE__, " window ", - window, "\n"); return window <= 12; - } else { - // if no empty block markers selected, we cannot tell for sure it's - // IAA-decompression, but we assume it is. - if (configs[IAA_PREPEND_EMPTY_BLOCK] == 0) { - return true; - } else if (configs[IAA_PREPEND_EMPTY_BLOCK] == 1 && - PrependedEmptyBlockPresent(input, input_length, format)) { - return true; - } else { - return false; - } } + // For raw deflate and gzip formats, QPL always reports total_in == + // available_in regardless of where BFINAL=1 falls in the stream. This is + // safe only when the caller provides avail_in == actual_compressed_size + // (e.g. Lucene stored-field reads, where the exact compressed size is known + // from the .fdt file format). + // + // Callers that do not know the compressed size a priori — notably Java's + // ZipInputStream, which uses a fixed 512-byte internal buffer and feeds + // chunks of that size to inflate() — will have avail_in > actual_csize. + // QPL consuming all 512 bytes then reporting total_in=512 when actual_csize + // was 2 triggers ZipException at the Java level. + // + // Guard: only attempt IAA when input_length > 512. ZipInputStream always + // feeds chunks of at most 512 bytes, so any call above that threshold is + // guaranteed not to be a ZipInputStream-chunked read. Lucene stored-field + // entries are typically much larger than 512 bytes; the few that are smaller + // fall back to IGZIP which is also correct. + static constexpr uint32_t kZipInputStreamBufferSize = 512; + return input_length > kZipInputStreamBufferSize; } #endif // USE_IAA diff --git a/igzip.cpp b/igzip.cpp index 954dbd7..01f1a47 100644 --- a/igzip.cpp +++ b/igzip.cpp @@ -21,8 +21,6 @@ static uint16_t ClampHistBits(int bits) { return (uint16_t)bits; } -static constexpr uint32_t kIGZIPMinFinishOutputSize = 256; - static void ConfigureDeflateWindow(struct isal_zstream *isal_strm, int windowBits) { if (windowBits < 0) { @@ -71,72 +69,6 @@ bool IsIGZIPDeflateFinished(const struct isal_zstream *stream) { return state == ZSTATE_END; } -bool SupportedOptionsIGZIPCompress(int flush, uint32_t output_length, - bool stream_on_igzip_path) { - if (flush != Z_FINISH) { - Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPCompress() Line ", __LINE__, - " flush ", flush, " is not Z_FINISH; IGZIP deflate path disabled\n"); - return false; - } - if (!stream_on_igzip_path && output_length < kIGZIPMinFinishOutputSize) { - Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPCompress() Line ", __LINE__, - " output length ", output_length, - " is less than minimum finish buffer ", kIGZIPMinFinishOutputSize, - "\n"); - return false; - } - return true; -} - -bool SupportedOptionsIGZIPUncompress(int window_bits, uint32_t input_length, - uint32_t output_length, - bool stream_on_igzip_path) { - (void)window_bits; - (void)output_length; - - if (!stream_on_igzip_path && input_length == 0) { - Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPUncompress() Line ", __LINE__, - " fallback reason=no_input_on_new_stream input_length ", input_length, - " output_length ", output_length, "\n"); - return false; - } - - return true; -} - -static bool IsIGZIPSyncFlush(int flush) { - return flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || flush == Z_BLOCK; -} - -bool IGZIPShouldFallbackDeflate(bool stream_on_igzip_path, int flush, - uint32_t avail_in) { - const bool is_streaming_flush = - (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || - flush == Z_FULL_FLUSH || flush == Z_BLOCK); - - if (!stream_on_igzip_path && is_streaming_flush && avail_in > 0) { - Log(LogLevel::LOG_INFO, "IGZIPShouldFallbackDeflate() Line ", __LINE__, - " fallback reason=streaming_flush_with_input flush ", flush, - " avail_in ", avail_in, "\n"); - return true; - } - - if (!stream_on_igzip_path || avail_in != 0) { - return false; - } - - if (IsIGZIPSyncFlush(flush)) { - Log(LogLevel::LOG_INFO, "IGZIPShouldFallbackDeflate() Line ", __LINE__, - " fallback reason=empty_sync_flush_reentry flush ", flush, " avail_in ", - avail_in, "\n"); - return true; - } - if (flush == Z_FINISH) { - return false; - } - return false; -} - struct isal_zstream *InitCompressIGZIP(int level, int windowBits) { Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, " initializing deflate with level ", level, ", windowBits ", windowBits, @@ -242,6 +174,20 @@ int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, ", total_out ", (uint32_t)isal_strm->total_out, ", total_in ", (uint32_t)isal_strm->total_in, "\n"); + // ISA-L always emits sync bytes on SYNC_FLUSH regardless of pending data. + // When the stream is already byte-aligned (ZSTATE_NEW_HDR) and there is no + // new input, no real progress can be made — return 0 progress so the caller + // reports Z_BUF_ERROR, matching zlib's semantics for empty flush calls. + // ZSTATE_NEW_HDR is the idle/byte-aligned state in ISA-L's internal deflate + // state machine; validated against ISA-L v2.32.0 (commit c196241). + if (isal_strm->avail_in == 0 && isal_strm->flush == SYNC_FLUSH && + isal_strm->end_of_stream == 0 && + isal_strm->internal_state.state == ZSTATE_NEW_HDR) { + *output_length = 0; + *input_length = 0; + return 0; + } + int comp = isal_deflate(isal_strm); *output_length = original_avail_out - isal_strm->avail_out; @@ -360,7 +306,7 @@ struct inflate_state *InitUncompressIGZIP(int windowBits) { // strm->total_out = 0; // strm->total_in = 0; - // s->trailer_overconsumption_fixed = 0; // Initialize the workaround flag + // s->read_in_correction_applied = 0; ConfigureInflateWindow(isal_strm_inflate, windowBits); @@ -369,9 +315,9 @@ struct inflate_state *InitUncompressIGZIP(int windowBits) { IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( z_streamp strm, struct inflate_state *isal_strm_inflate, int window_bits, - int *tofixed, int *ret) { + int *read_in_correction_applied, int *ret) { if (strm == nullptr || isal_strm_inflate == nullptr || ret == nullptr || - tofixed == nullptr || strm->avail_in != 0) { + read_in_correction_applied == nullptr || strm->avail_in != 0) { return IGZIP_NO_INPUT_NOT_HANDLED; } @@ -380,8 +326,9 @@ IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( bool end_of_stream = true; *ret = UncompressIGZIP(isal_strm_inflate, strm->next_in, &input_len, - strm->next_out, &output_len, window_bits, tofixed, - &strm->total_in, &strm->total_out, &end_of_stream); + strm->next_out, &output_len, window_bits, + read_in_correction_applied, &strm->total_in, + &strm->total_out, &end_of_stream); if (*ret == Z_DATA_ERROR) { Log(LogLevel::LOG_INFO, "IGZIPHandleActiveStreamNoInput() Line ", __LINE__, @@ -407,11 +354,12 @@ IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits, - int *tofixed, uint32_t *input_length, uint32_t *output_length, int *ret, - bool *end_of_stream, uint32_t pre_avail_in) { + int *read_in_correction_applied, uint32_t *input_length, + uint32_t *output_length, int *ret, bool *end_of_stream, + uint32_t pre_avail_in) { if (strm == nullptr || isal_strm_inflate == nullptr || input_length == nullptr || output_length == nullptr || ret == nullptr || - end_of_stream == nullptr || tofixed == nullptr) { + end_of_stream == nullptr || read_in_correction_applied == nullptr) { if (ret != nullptr) { *ret = Z_DATA_ERROR; } @@ -429,17 +377,18 @@ IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( } *ret = UncompressIGZIP(*isal_strm_inflate, strm->next_in, input_length, - strm->next_out, output_length, window_bits, tofixed, - &strm->total_in, &strm->total_out, end_of_stream); + strm->next_out, output_length, window_bits, + read_in_correction_applied, &strm->total_in, + &strm->total_out, end_of_stream); const uint32_t remaining_after_igzip = (pre_avail_in >= *input_length) ? (pre_avail_in - *input_length) : 0; if (*ret == 0 && window_bits < 0 && *end_of_stream && - remaining_after_igzip > 0 && strm->total_in == 0 && - strm->total_out == 0) { + remaining_after_igzip > 0 && *read_in_correction_applied == 0 && + strm->total_in == 0 && strm->total_out == 0) { Log(LogLevel::LOG_ERROR, - "IGZIPRunInflateAndSelectPathAction() raw boundary guard strm=", + "IGZIPRunInflateAndSelectPathAction() raw boundary guard FIRED strm=", static_cast(strm), " bytes_in=", *input_length, " bytes_out=", *output_length, " pre_avail_in=", pre_avail_in, " remaining_in=", remaining_after_igzip, "\n"); @@ -463,9 +412,9 @@ IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t *input_length, uint8_t *output, - uint32_t *output_length, int window_bits, int *tofixed, - unsigned long *total_in, unsigned long *total_out, - bool *end_of_stream) { + uint32_t *output_length, int window_bits, + int *read_in_correction_applied, unsigned long *total_in, + unsigned long *total_out, bool *end_of_stream) { (void)total_in; if (!isal_strm_inflate) { @@ -497,33 +446,44 @@ int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t rewind_adjust_bytes = 0; - // WORKAROUND: ISA-L over-consumption fix for raw deflate mode. - // Option 2 behavior: if ambiguity appears at INPUT_DONE, request caller - // fallback to zlib instead of carrying deferred state. - if (window_bits < 0 && decomp == ISAL_DECOMP_OK && *tofixed == 0 && - (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH || - isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE) && - isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) { - const uint32_t expected_trailer_bytes = 8; - const uint32_t over_consumed = - expected_trailer_bytes - isal_strm_inflate->avail_in; - if (over_consumed >= 1 && over_consumed <= 7) { - if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) { - rewind_adjust_bytes = consumed_before_adjust < over_consumed - ? consumed_before_adjust - : over_consumed; - if (rewind_adjust_bytes > 0) { - *tofixed = 1; - } - } else { - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " raw INPUT_DONE ambiguity detected: over_consumed ", over_consumed, - ", requesting zlib fallback\n"); - return Z_DATA_ERROR; - } + // WORKAROUND: ISA-L raw-deflate over-consumption fix. + // ISAL pre-loads input in 8-byte word chunks into a 64-bit shift register + // (read_in). After BLOCK_FINISH, read_in_length >> 3 is the exact byte + // count over-consumed, covering all avail_in scenarios: [0], [1,7], [8], + // and >8 (multi-frame), where prior heuristics were blind or inaccurate. + if (window_bits < 0 && + (decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) && + isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) { + const uint32_t read_in_correction = + (isal_strm_inflate->read_in_length > 0) + ? static_cast(isal_strm_inflate->read_in_length >> 3) + : 0u; + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " raw_finish avail_in ", isal_strm_inflate->avail_in, + " read_in_length_bits ", isal_strm_inflate->read_in_length, + " read_in_correction_bytes ", read_in_correction, "\n"); + if (read_in_correction > 0) { + rewind_adjust_bytes = (read_in_correction <= consumed_before_adjust) + ? read_in_correction + : consumed_before_adjust; + *read_in_correction_applied = 1; } } + // WORKAROUND: BLOCK_INPUT_DONE — output-buffer-limited with ambiguous + // trailer bytes. read_in_length does not apply here (not yet at + // BLOCK_FINISH); request caller fallback to zlib. BLOCK_FINISH is fully + // handled above. + if (window_bits < 0 && decomp == ISAL_DECOMP_OK && + *read_in_correction_applied == 0 && + isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE && + isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) { + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " raw INPUT_DONE ambiguity detected: over_consumed ", + 8u - isal_strm_inflate->avail_in, ", requesting zlib fallback\n"); + return Z_DATA_ERROR; + } + *output_length = original_avail_out - isal_strm_inflate->avail_out; *input_length = consumed_before_adjust - rewind_adjust_bytes; input = isal_strm_inflate->next_in; @@ -610,19 +570,33 @@ int inflateSetDictionary(z_streamp strm, unsigned char *dict_data, return isal_inflate_set_dict(s->isal_strm_inflate, dict_data, dict_len); } +void ResetCompressIGZIP(struct isal_zstream *isal_strm, int windowBits) { + // isal_deflate_reset preserves gzip_flag, hist_bits, level, and level_buf. + // gzip_flag must be restored: after the first chunk ISA-L changes it from + // IGZIP_ZLIB (3) to IGZIP_ZLIB_NO_HDR (4) to suppress the header on + // continuation calls. Without this reset, the next stream reused via + // deflateReset would produce headerless output, causing decompressors + // (e.g. Java Inflater with nowrap=false) to reject every subsequent chunk. + isal_deflate_reset(isal_strm); + isal_strm->end_of_stream = 0; + isal_strm->flush = NO_FLUSH; + ConfigureDeflateWindow(isal_strm, windowBits); +} + int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, - int *tofixed) { + int *read_in_correction_applied) { if (!isal_strm_inflate) { Log(LogLevel::LOG_ERROR, "ResetUncompressIGZIP() Line ", __LINE__, " isal_strm_inflate is NULL\n"); return Z_STREAM_ERROR; } - // Reset ISA-L inflate state + // Reset ISA-L inflate state. isal_inflate_reset clears the internal + // read_in / read_in_length buffer, so any over-consumption correction + // applied during the previous stream session no longer applies. + // Clear the flag so the new session fires the correction fresh if needed. isal_inflate_reset(isal_strm_inflate); - - // Reset workaround flag - *tofixed = 0; + *read_in_correction_applied = 0; return Z_OK; } diff --git a/igzip.h b/igzip.h index 0317907..a52ef75 100644 --- a/igzip.h +++ b/igzip.h @@ -12,8 +12,9 @@ typedef struct internal_state2 { int level; int w_bits; struct inflate_state *isal_strm_inflate; - int trailer_overconsumption_fixed; /* Indicates if fix has been applied for - gzip trailer overconsumption issue */ + int read_in_correction_applied; /* Set once per stream session when the + read_in_length over-consumption correction + fires; cleared only on inflateReset */ } inflate_state2; typedef struct internal_state { @@ -29,11 +30,6 @@ int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, uint32_t *output_length, unsigned long *total_in, unsigned long *total_out); bool IsIGZIPDeflateFinished(const struct isal_zstream *stream); -bool SupportedOptionsIGZIPCompress(int flush, uint32_t output_length, - bool stream_on_igzip_path); -bool SupportedOptionsIGZIPUncompress(int window_bits, uint32_t input_length, - uint32_t output_length, - bool stream_on_igzip_path); enum IGZIPNoInputAction { IGZIP_NO_INPUT_NOT_HANDLED, IGZIP_NO_INPUT_RETURN, @@ -50,24 +46,25 @@ enum IGZIPInflatePathAction { IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( z_streamp strm, struct inflate_state *isal_strm_inflate, int window_bits, - int *tofixed, int *ret); + int *read_in_correction_applied, int *ret); IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits, - int *tofixed, uint32_t *input_length, uint32_t *output_length, int *ret, - bool *end_of_stream, uint32_t pre_avail_in); + int *read_in_correction_applied, uint32_t *input_length, + uint32_t *output_length, int *ret, bool *end_of_stream, + uint32_t pre_avail_in); -bool IGZIPShouldFallbackDeflate(bool stream_on_igzip_path, int flush, - uint32_t avail_in); int EndCompressIGZIP(struct isal_zstream *isal_strm); +void ResetCompressIGZIP(struct isal_zstream *isal_strm, int windowBits); struct inflate_state *InitUncompressIGZIP(int windowBits); int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t *input_length, uint8_t *output, - uint32_t *output_length, int window_bits, int *tofixed, - unsigned long *total_in, unsigned long *total_out, - bool *end_of_stream); + uint32_t *output_length, int window_bits, + int *read_in_correction_applied, unsigned long *total_in, + unsigned long *total_out, bool *end_of_stream); int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate); -int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed); +int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, + int *read_in_correction_applied); // #define Z_DEFAULT_COMPRESSION 6 #endif diff --git a/statistics.cpp b/statistics.cpp index 9dabeb6..0610678 100644 --- a/statistics.cpp +++ b/statistics.cpp @@ -18,9 +18,10 @@ using namespace config; const std::array stat_names{ {"deflate_count", "deflate_error_count", "deflate_qat_count", "deflate_qat_error_count", "deflate_iaa_count", "deflate_iaa_error_count", - "deflate_zlib_count", "inflate_count", "inflate_error_count", - "inflate_qat_count", "inflate_qat_error_count", "inflate_iaa_count", - "inflate_iaa_error_count", "inflate_zlib_count"}}; + "deflate_igzip_count", "deflate_igzip_error_count", "deflate_zlib_count", + "inflate_count", "inflate_error_count", "inflate_qat_count", + "inflate_qat_error_count", "inflate_iaa_count", "inflate_iaa_error_count", + "inflate_igzip_count", "inflate_igzip_error_count", "inflate_zlib_count"}}; thread_local std::array stats{}; diff --git a/statistics.h b/statistics.h index 8cee404..b3654ce 100644 --- a/statistics.h +++ b/statistics.h @@ -17,6 +17,8 @@ enum class Statistic : size_t { DEFLATE_QAT_ERROR_COUNT, DEFLATE_IAA_COUNT, DEFLATE_IAA_ERROR_COUNT, + DEFLATE_IGZIP_COUNT, + DEFLATE_IGZIP_ERROR_COUNT, DEFLATE_ZLIB_COUNT, INFLATE_COUNT, INFLATE_ERROR_COUNT, @@ -24,6 +26,8 @@ enum class Statistic : size_t { INFLATE_QAT_ERROR_COUNT, INFLATE_IAA_COUNT, INFLATE_IAA_ERROR_COUNT, + INFLATE_IGZIP_COUNT, + INFLATE_IGZIP_ERROR_COUNT, INFLATE_ZLIB_COUNT, STATS_COUNT }; diff --git a/tests/zlib_accel_test.cpp b/tests/zlib_accel_test.cpp index d09bf44..b526d0c 100644 --- a/tests/zlib_accel_test.cpp +++ b/tests/zlib_accel_test.cpp @@ -690,6 +690,8 @@ TEST_P(ZlibTest, CompressDecompress) { VerifyStatIncremented(Statistic::DEFLATE_QAT_COUNT); } else if (test_param.execution_path_compress == IAA) { VerifyStatIncremented(Statistic::DEFLATE_IAA_COUNT); + } else if (test_param.execution_path_compress == IGZIP) { + VerifyStatIncremented(Statistic::DEFLATE_IGZIP_COUNT); } else if (test_param.execution_path_compress == ZLIB) { VerifyStatIncremented(Statistic::DEFLATE_ZLIB_COUNT); } @@ -743,6 +745,9 @@ TEST_P(ZlibTest, CompressDecompress) { VerifyStatIncremented(Statistic::INFLATE_QAT_COUNT); } else if (test_param.execution_path_uncompress == IAA) { VerifyStatIncremented(Statistic::INFLATE_IAA_COUNT); + } else if (test_param.execution_path_uncompress == IGZIP) { + VerifyStatIncrementedUpTo(Statistic::INFLATE_IGZIP_COUNT, + test_param.input_chunks_uncompress); } else if (test_param.execution_path_uncompress == ZLIB) { VerifyStatIncrementedUpTo(Statistic::INFLATE_ZLIB_COUNT, test_param.input_chunks_uncompress); @@ -1697,7 +1702,7 @@ TEST(IGZIPDeflateRegressionTest, ResetMustNotStallSyncFlushOnSameStream) { int ret = deflate(&stream, Z_NO_FLUSH); ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR) << "cycle=" << cycle; - ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "cycle=" << cycle; stream.next_in = nullptr; stream.avail_in = 0; @@ -1707,7 +1712,7 @@ TEST(IGZIPDeflateRegressionTest, ResetMustNotStallSyncFlushOnSameStream) { ret = deflate(&stream, Z_SYNC_FLUSH); ASSERT_EQ(ret, Z_OK) << "cycle=" << cycle; ASSERT_LT(stream.avail_out, output.size()) << "cycle=" << cycle; - ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "cycle=" << cycle; stream.next_in = nullptr; stream.avail_in = 0; @@ -1746,7 +1751,7 @@ TEST(IGZIPDeflateRegressionTest, stream.avail_out = static_cast(output.size()); int ret = deflate(&stream, Z_NO_FLUSH); ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR); - ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP); bool observed_buf_error = false; for (int iter = 0; iter < 128; ++iter) { @@ -1756,7 +1761,7 @@ TEST(IGZIPDeflateRegressionTest, stream.avail_out = static_cast(output.size()); ret = deflate(&stream, Z_SYNC_FLUSH); - ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "iter=" << iter; ASSERT_NE(ret, Z_DATA_ERROR) << "iter=" << iter; if (ret == Z_BUF_ERROR) { @@ -1779,6 +1784,65 @@ TEST(IGZIPDeflateRegressionTest, deflateEnd(&stream); } +// Regression test for: deflateReset on a reused IGZIP stream must restore the +// zlib header (gzip_flag = IGZIP_ZLIB) so that each independent chunk is +// self-contained and decompressible by a fresh zlib inflater. Without the +// fix, isal_deflate_reset preserved gzip_flag = IGZIP_ZLIB_NO_HDR (4) and the +// second and subsequent chunks were emitted without a zlib header, causing +// Java's Inflater (nowrap=false) to report "incorrect header check". +TEST(IGZIPDeflateRegressionTest, + ResetMustRestoreZlibHeaderForSubsequentChunks) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + z_stream cstream; + memset(&cstream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, + /*windowBits=*/15, /*memLevel=*/8, Z_DEFAULT_STRATEGY), + Z_OK); + + const int kChunks = 5; + const int kChunkSize = 16384; + const int kCompBound = deflateBound(&cstream, kChunkSize); + + for (int chunk = 0; chunk < kChunks; ++chunk) { + // Each chunk is independent data compressed with a fresh IGZIP stream + // (via deflateReset). + std::vector input(kChunkSize, + static_cast('A' + chunk % 26)); + std::vector compressed(kCompBound); + + cstream.next_in = input.data(); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + ASSERT_EQ(deflate(&cstream, Z_FINISH), Z_STREAM_END) << "chunk=" << chunk; + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP) << "chunk=" << chunk; + + const size_t compressed_size = compressed.size() - cstream.avail_out; + + // Decompress with a fresh zlib inflater — requires a valid zlib header. + std::vector decompressed(kChunkSize); + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit(&dstream), Z_OK) << "chunk=" << chunk; + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = decompressed.data(); + dstream.avail_out = static_cast(decompressed.size()); + ASSERT_EQ(inflate(&dstream, Z_FINISH), Z_STREAM_END) + << "chunk=" << chunk + << ": decompression failed (missing zlib header after deflateReset?)"; + ASSERT_EQ(decompressed, input) << "chunk=" << chunk; + inflateEnd(&dstream); + + ASSERT_EQ(deflateReset(&cstream), Z_OK) << "chunk=" << chunk; + } + + deflateEnd(&cstream); +} + TEST(IGZIPDeflateRegressionTest, DictionaryStreamMustStayOnZlibAcrossReset) { SetCompressPath(IGZIP, false, false, false); SetUncompressPath(ZLIB, false, false); @@ -2166,8 +2230,10 @@ TEST(IGZIPDeflateRegressionTest, inflateEnd(&dstream); } -TEST(IGZIPDeflateRegressionTest, - SyncFlushWithInputMustFallbackToZlibForStreamSafety) { +TEST(IGZIPDeflateRegressionTest, SyncFlushWithInputMustStayOnIGZIPPath) { + // Originally this test asserted the path was ZLIB (IGZIPShouldFallbackDeflate + // redirected SYNC_FLUSH to zlib). That function is removed; IGZIP now handles + // SYNC_FLUSH natively and the stream must stay on IGZIP throughout. SetCompressPath(IGZIP, false, false, false); SetUncompressPath(ZLIB, false, false); @@ -2197,7 +2263,7 @@ TEST(IGZIPDeflateRegressionTest, int ret = deflate(&cstream, Z_SYNC_FLUSH); ASSERT_NE(ret, Z_DATA_ERROR); - ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP); const size_t sync_produced = sizeof(sync_chunk) - cstream.avail_out; compressed.insert(compressed.end(), sync_chunk, sync_chunk + sync_produced); @@ -2210,7 +2276,7 @@ TEST(IGZIPDeflateRegressionTest, ret = deflate(&cstream, Z_FINISH); ASSERT_NE(ret, Z_DATA_ERROR); - ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP); const size_t produced = sizeof(out_chunk) - cstream.avail_out; compressed.insert(compressed.end(), out_chunk, out_chunk + produced); @@ -2320,6 +2386,173 @@ TEST(IGZIPInflateRegressionTest, inflateEnd(&dstream); } +#ifdef USE_IAA +// IAA->IGZIP fallback tests. +// On machines without IAA hardware, CompressIAA/UncompressIAA return non-zero, +// which naturally triggers the fallback path. These tests verify: +// - When iaa_fallback_igzip=1: the stream lands on IGZIP after IAA fails. +// - When iaa_fallback_igzip=0: the stream falls through to zlib, not IGZIP. + +TEST(IAAFallbackIGZIPTest, DeflateUsesIGZIPWhenIAAFailsAndFallbackEnabled) { + SetCompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false, + /*qat_compression_allow_chunking=*/false); + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 1); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(deflateBound(&stream, input_length)); + stream.next_in = reinterpret_cast(input); + stream.avail_in = static_cast(input_length); + stream.next_out = output.data(); + stream.avail_out = static_cast(output.size()); + + int ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // If IAA hardware is absent, the fallback must have routed to IGZIP. + // If IAA hardware is present and succeeds, IAA path is also acceptable. + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_TRUE(path == IGZIP || path == IAA) + << "Expected IGZIP (fallback) or IAA (hardware success), got " + << static_cast(path); + + deflateEnd(&stream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, DeflateDoesNotUseIGZIPWhenFallbackDisabled) { + SetCompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false, + /*qat_compression_allow_chunking=*/false); + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 0); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(deflateBound(&stream, input_length)); + stream.next_in = reinterpret_cast(input); + stream.avail_in = static_cast(input_length); + stream.next_out = output.data(); + stream.avail_out = static_cast(output.size()); + + deflate(&stream, Z_FINISH); + + // With fallback disabled, IGZIP must not be selected for an IAA-configured + // stream; it should fall through to zlib. + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_NE(path, IGZIP) << "IGZIP must not be used when iaa_fallback_igzip=0"; + + deflateEnd(&stream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, InflateUsesIGZIPWhenIAAFailsAndFallbackEnabled) { + // Compress with IGZIP so the output is IGZIP-compatible (4kB window). + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 1); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + ASSERT_EQ(ret, Z_STREAM_END); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + + std::vector uncompressed(input_length); + dstream.next_in = reinterpret_cast(compressed.data()); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = reinterpret_cast(uncompressed.data()); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // If IAA hardware is absent, fallback must route to IGZIP. + // If IAA hardware is present and succeeds, IAA is also acceptable. + const ExecutionPath path = GetInflateExecutionPath(&dstream); + EXPECT_TRUE(path == IGZIP || path == IAA) + << "Expected IGZIP (fallback) or IAA (hardware success), got " + << static_cast(path); + + EXPECT_EQ(memcmp(uncompressed.data(), input, input_length), 0); + + inflateEnd(&dstream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, InflateDoesNotUseIGZIPWhenFallbackDisabled) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 0); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + ASSERT_EQ(ret, Z_STREAM_END); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + + std::vector uncompressed(input_length); + dstream.next_in = reinterpret_cast(compressed.data()); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = reinterpret_cast(uncompressed.data()); + dstream.avail_out = static_cast(uncompressed.size()); + + inflate(&dstream, Z_FINISH); + + // With fallback disabled, IGZIP must not be selected for an IAA-configured + // inflate stream; it should fall through to zlib. + const ExecutionPath path = GetInflateExecutionPath(&dstream); + EXPECT_NE(path, IGZIP) << "IGZIP must not be used when iaa_fallback_igzip=0"; + + inflateEnd(&dstream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} +#endif // USE_IAA + #endif INSTANTIATE_TEST_SUITE_P( diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 82b55c5..cd010b7 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -203,10 +203,11 @@ struct DeflateSettings { struct InflateSettings { InflateSettings(int _window_bits) - : window_bits(_window_bits), trailer_overconsumption_fixed(0) {} + : window_bits(_window_bits), read_in_correction_applied(0) {} int window_bits; - int trailer_overconsumption_fixed; /* indicates if fix has been applied for - overconsumption issue*/ + int read_in_correction_applied; /* set once per stream when the + read_in_length over-consumption correction + fires; cleared only on inflateReset */ ExecutionPath path = UNDEFINED; struct inflate_state* isal_strm = nullptr; }; @@ -294,6 +295,14 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, Log(LogLevel::LOG_INFO, "deflateSetDictionary Line ", __LINE__, ", strm ", static_cast(strm), ", dictLength ", dictLength, "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); + // Reject mid-stream: if an accelerator is active, the underlying zlib + // stream has not been advanced, so orig_deflateSetDictionary would + // incorrectly accept the call. Per zlib spec, dictionary must be set before + // compression begins. + if (deflate_settings != nullptr && deflate_settings->path != UNDEFINED && + deflate_settings->path != ZLIB) { + return Z_STREAM_ERROR; + } const int ret = orig_deflateSetDictionary(strm, dictionary, dictLength); if (ret == Z_OK) { SetDeflatePath(deflate_settings, strm, ZLIB, @@ -320,18 +329,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { deflate_settings->window_bits, ", total_in ", strm->total_in, ", total_out ", strm->total_out, ", adler ", strm->adler, "\n"); -#ifdef USE_IGZIP - if (configs[USE_IGZIP_COMPRESS] && deflate_settings->path == UNDEFINED && - IGZIPShouldFallbackDeflate(false, flush, strm->avail_in)) { - SetDeflatePath(deflate_settings, strm, ZLIB, "igzip fallback condition"); - } - if (configs[USE_IGZIP_COMPRESS] && - IGZIPShouldFallbackDeflate(deflate_settings->path == IGZIP, flush, - strm->avail_in)) { - SetDeflatePath(deflate_settings, strm, ZLIB, "igzip fallback condition"); - } -#endif - int ret = 1; bool iaa_available = false; bool qat_available = false; @@ -355,9 +352,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { #ifdef USE_IGZIP igzip_stream_active = (deflate_settings->path == IGZIP && deflate_settings->isal_strm != nullptr); - igzip_available = - configs[USE_IGZIP_COMPRESS] && - SupportedOptionsIGZIPCompress(flush, output_len, igzip_stream_active); + igzip_available = configs[USE_IGZIP_COMPRESS]; #endif // If both accelerators are enabled, send configured ratio of requests to @@ -419,11 +414,37 @@ int ZEXPORT deflate(z_streamp strm, int flush) { "selected IGZIP accelerator"); in_call = false; - // INCREMENT_STAT(DEFLATE_IGZIP_COUNT); - // INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); + INCREMENT_STAT(DEFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); #endif } +#ifdef USE_IGZIP + // IAA→IGZIP fallback: if IAA failed and IGZIP is available, retry with + // IGZIP before falling through to software zlib. + if (path_selected == IAA && ret != 0 && configs[IAA_FALLBACK_IGZIP] && + igzip_available) { + // IAA may have modified input_len/output_len on failure — restore them. + input_len = strm->avail_in; + output_len = strm->avail_out; + if (deflate_settings->isal_strm == nullptr) { + deflate_settings->method = 0; + deflate_settings->isal_strm = InitCompressIGZIP( + deflate_settings->level, deflate_settings->window_bits); + } + in_call = true; + ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, + &input_len, strm->next_out, &output_len, + &strm->total_in, &strm->total_out); + SetDeflatePath(deflate_settings, strm, IGZIP, + "IAA failed, IGZIP fallback"); + in_call = false; + path_selected = IGZIP; // use IGZIP return-code semantics below + INCREMENT_STAT(DEFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); + } +#endif // USE_IGZIP iaa fallback + if (ret == 0) { strm->next_in += input_len; strm->avail_in -= input_len; @@ -508,9 +529,8 @@ int ZEXPORT deflateReset(z_streamp strm) { #ifdef USE_IGZIP if (deflate_settings->isal_strm != nullptr) { - isal_deflate_reset(deflate_settings->isal_strm); - deflate_settings->isal_strm->end_of_stream = 0; - deflate_settings->isal_strm->flush = NO_FLUSH; + ResetCompressIGZIP(deflate_settings->isal_strm, + deflate_settings->window_bits); } #endif } @@ -580,32 +600,24 @@ int ZEXPORT inflate(z_streamp strm, int flush) { #ifdef USE_IGZIP const bool igzip_supported_options = - !in_call && configs[USE_IGZIP_UNCOMPRESS] && - SupportedOptionsIGZIPUncompress(inflate_settings->window_bits, - strm->avail_in, strm->avail_out, - igzip_stream_active); + !in_call && configs[USE_IGZIP_UNCOMPRESS]; // Keep stateful IGZIP stream handling on the same engine. // For avail_in==0, let IGZIP process any buffered bits in its internal // state before reporting Z_BUF_ERROR. if (!in_call && igzip_stream_active && strm->avail_in == 0) { - if (!igzip_supported_options) { - SetInflatePath(inflate_settings, strm, ZLIB, - "IGZIP unsupported options for no-input stream"); - } else { - in_call = true; - IGZIPNoInputAction action = IGZIPHandleActiveStreamNoInput( - strm, inflate_settings->isal_strm, inflate_settings->window_bits, - &inflate_settings->trailer_overconsumption_fixed, &ret); - in_call = false; + in_call = true; + IGZIPNoInputAction action = IGZIPHandleActiveStreamNoInput( + strm, inflate_settings->isal_strm, inflate_settings->window_bits, + &inflate_settings->read_in_correction_applied, &ret); + in_call = false; - if (action == IGZIP_NO_INPUT_FALLBACK_ZLIB) { - SetInflatePath(inflate_settings, strm, ZLIB, - "igzip raw input_done ambiguity fallback"); - // Continue through normal zlib fallback path. - } else if (action == IGZIP_NO_INPUT_RETURN) { - return ret; - } + if (action == IGZIP_NO_INPUT_FALLBACK_ZLIB) { + SetInflatePath(inflate_settings, strm, ZLIB, + "igzip raw input_done ambiguity fallback"); + // Continue through normal zlib fallback path. + } else if (action == IGZIP_NO_INPUT_RETURN) { + return ret; } } #endif @@ -622,7 +634,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { } if (!in_call && strm->avail_in > 0 && inflate_settings->path != ZLIB) { +#ifdef USE_IGZIP const uInt pre_avail_in = strm->avail_in; +#endif uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; @@ -700,7 +714,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { const IGZIPInflatePathAction path_action = IGZIPRunInflateAndSelectPathAction( strm, &inflate_settings->isal_strm, inflate_settings->window_bits, - &inflate_settings->trailer_overconsumption_fixed, &input_len, + &inflate_settings->read_in_correction_applied, &input_len, &output_len, &ret, &end_of_stream, pre_avail_in); in_call = false; @@ -725,11 +739,54 @@ int ZEXPORT inflate(z_streamp strm, int flush) { SetInflatePath(inflate_settings, strm, IGZIP, "selected IGZIP accelerator"); } - // INCREMENT_STAT(INFLATE_IGZIP_COUNT); - // INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); + INCREMENT_STAT(INFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); #endif } +#ifdef USE_IGZIP + // IAA→IGZIP fallback: if IAA failed and IGZIP is available, retry with + // IGZIP before falling through to software zlib. + if (path_selected == IAA && ret != 0 && configs[IAA_FALLBACK_IGZIP] && + igzip_available) { + // IAA may have modified input_len/output_len on failure — restore them. + input_len = strm->avail_in; + output_len = strm->avail_out; + end_of_stream = true; + in_call = true; + const IGZIPInflatePathAction path_action = + IGZIPRunInflateAndSelectPathAction( + strm, &inflate_settings->isal_strm, inflate_settings->window_bits, + &inflate_settings->read_in_correction_applied, &input_len, + &output_len, &ret, &end_of_stream, pre_avail_in); + in_call = false; + + if (inflate_settings->isal_strm == nullptr) { + return Z_DATA_ERROR; + } + + if (path_action == IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT) { + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), + " source=igzip (iaa fallback)", " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, "\n"); + SetInflatePath(inflate_settings, strm, ZLIB, + "IAA->IGZIP fallback: Z_NEED_DICT"); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR) { + SetInflatePath(inflate_settings, strm, ZLIB, + "IAA->IGZIP fallback: raw trailer"); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY) { + SetInflatePath(inflate_settings, strm, ZLIB, + "IAA->IGZIP fallback: raw boundary"); + } else if (path_action == IGZIP_INFLATE_PATH_SET_IGZIP && + inflate_settings->path != ZLIB) { + SetInflatePath(inflate_settings, strm, IGZIP, + "IAA failed, IGZIP fallback succeeded"); + } + INCREMENT_STAT(INFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); + } +#endif // USE_IGZIP iaa fallback + if (ret == 0) { strm->next_in += input_len; strm->avail_in -= input_len; @@ -803,8 +860,6 @@ int ZEXPORT inflateReset(z_streamp strm) { Log(LogLevel::LOG_INFO, "inflateReset Line ", __LINE__, ", strm ", static_cast(strm), "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); - const bool was_igzip_path = - (inflate_settings != nullptr && inflate_settings->path == IGZIP); int ret = orig_inflateReset(strm); if (inflate_settings != nullptr) { SetInflatePath(inflate_settings, strm, UNDEFINED, "inflateReset"); @@ -812,11 +867,8 @@ int ZEXPORT inflateReset(z_streamp strm) { if (inflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP ResetUncompressIGZIP(inflate_settings->isal_strm, - &inflate_settings->trailer_overconsumption_fixed); + &inflate_settings->read_in_correction_applied); #endif - if (was_igzip_path) { - inflate_settings->trailer_overconsumption_fixed = 1; - } } return ret; @@ -844,8 +896,7 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(15, input_len); #endif #ifdef USE_IGZIP - igzip_available = configs[USE_IGZIP_COMPRESS] && - SupportedOptionsIGZIPCompress(Z_FINISH, output_len, false); + igzip_available = configs[USE_IGZIP_COMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -943,9 +994,7 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(15, input_len); #endif #ifdef USE_IGZIP - igzip_available = - configs[USE_IGZIP_UNCOMPRESS] && - SupportedOptionsIGZIPUncompress(15, input_len, output_len, false); + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -981,12 +1030,12 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, if (isal_strm == nullptr) { ret = 1; } else { - int tofixed = 0; + int read_in_correction_applied = 0; unsigned long total_in = 0; unsigned long total_out = 0; ret = UncompressIGZIP(isal_strm, const_cast(source), &input_len, - dest, &output_len, 15, &tofixed, &total_in, - &total_out, &end_of_stream); + dest, &output_len, 15, &read_in_correction_applied, + &total_in, &total_out, &end_of_stream); EndUncompressIGZIP(isal_strm); if (ret == 0 && !end_of_stream) { ret = 1; @@ -1245,9 +1294,7 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(31, *input_length); #endif #ifdef USE_IGZIP - igzip_available = - configs[USE_IGZIP_COMPRESS] && - SupportedOptionsIGZIPCompress(Z_FINISH, *output_length, false); + igzip_available = configs[USE_IGZIP_COMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -1321,9 +1368,7 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(31, *input_length); #endif #ifdef USE_IGZIP - igzip_available = - configs[USE_IGZIP_UNCOMPRESS] && - SupportedOptionsIGZIPUncompress(31, *input_length, *output_length, false); + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -1358,12 +1403,12 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, if (isal_strm == nullptr) { ret = 1; } else { - int tofixed = 0; + int read_in_correction_applied = 0; unsigned long total_in = 0; unsigned long total_out = 0; - ret = - UncompressIGZIP(isal_strm, input, input_length, output, output_length, - 31, &tofixed, &total_in, &total_out, end_of_stream); + ret = UncompressIGZIP(isal_strm, input, input_length, output, + output_length, 31, &read_in_correction_applied, + &total_in, &total_out, end_of_stream); EndUncompressIGZIP(isal_strm); } gz->path = IGZIP;