diff --git a/.gitignore b/.gitignore index fc1dadd..aedc64f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ /build/ -/tests/build \ No newline at end of file +/tests/build +cmake.txt +/resources/ + diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b36c5e..445c160 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ if(ENABLE_STATISTICS) add_compile_definitions(ENABLE_STATISTICS) endif() -add_library(${PROJECT_NAME} SHARED config/config_reader.cpp config/config.cpp zlib_accel.cpp iaa.cpp qat.cpp utils.cpp statistics.cpp) +add_library(${PROJECT_NAME} SHARED config/config_reader.cpp config/config.cpp zlib_accel.cpp iaa.cpp qat.cpp utils.cpp statistics.cpp igzip.cpp) add_custom_target(format find .. -iname '*.h' -o -iname '*.cpp' | xargs clang-format -style=Google -i diff --git a/README.md b/README.md index b36a23b..102d44d 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,10 @@ make CMake supports the following options: - USE_QAT (ON/OFF): include QAT acceleration - USE_IAA (ON/OFF): include IAA acceleration +- USE_IGZIP (ON/OFF): include IGZIP acceleration (requires ISA-L) - QPL_PATH: path to QPL for IAA acceleration (if not in a standard directory) - QATZIP_PATH: path to QATzip for QAT acceleration (if not in a standard directory) +- ISAL_PATH: path to ISA-L for IGZIP acceleration (if not in a standard directory) - DEBUG_LOG (ON/OFF): enable logging - ENABLE_STATISTICS (ON/OFF): enable statistics - COVERAGE (ON/OFF): enable test coverage (more details in a later section) @@ -84,6 +86,9 @@ Requirements for IAA - [accel-config](https://github.com/intel/idxd-config) - [Query Processing Library](https://github.com/intel/qpl) +Requirements for IGZIP +- [ISA-L (Intel Intelligent Storage Acceleration Library)](https://github.com/intel/isa-l) + A setup with both QAT and IAA enabled has been tested on an AWS m7i.metal-24xl instance (Ubuntu 22.04, kernel 6.8.0). Refer to the links above for instructions on how to install the dependencies. @@ -178,20 +183,22 @@ use_zlib_uncompress - Enable zlib for decompression - Setting to 1 is recommended, to allow fall back to zlib in case accelerators cannot be used or experience an error. +iaa_fallback_igzip +- Values: 0,1. Default: 0 +- If 1, and an IAA compression or decompression operation fails, the request is retried using IGZIP (if enabled) before falling back to software zlib. Useful on machines where IAA hardware is intermittently unavailable. + iaa_compress_percentage - Values: 0-100. Default: 50 - If both IAA and QAT are enabled, percentage of compression calls to offload to IAA. iaa_prepend_empty_block - Values: 0,1. Default: 0 -- Prepend an empty stored block to the compressed data to "mark" that the data was compressed by IAA. -- IAA has a 4kB history window limit and it is not able to decompress blocks that use a longer history window (up to 32kB per deflate standard). -- During decompression, this marker indicates that the data was compressed by IAA and is therefore guarateed decompressible by IAA. +- **Deprecated.** This option is retained for backward compatibility and will be removed in a future release. Setting it to 1 has no effect on decompression. +- Background: the original design prepended a 5-byte empty stored-block marker to IAA-compressed output so the decompressor could identify IAA-produced data (which uses a 4kB history window). This approach was abandoned because QPL hardware always consumes all `available_in` bytes regardless of where the stream boundary falls, making marker-based detection unreliable when the caller does not supply the exact compressed size. IAA decompression eligibility is now determined by a 512-byte minimum input length threshold: callers such as Java's `ZipInputStream` feed chunks of ≤512 bytes when the compressed size is unknown, while Lucene stored-field reads always supply the exact size (>512 bytes). iaa_uncompress_percentage - Values: 0-100. Default: 50 - If both IAA and QAT are enabled, percentage of decompression calls to offload to IAA. -- If iaa_prepend_empty_block = 1, this percentage is only applied to data with the empty block marker. qat_periodical_polling = 0 - Values: 0,1. Default: 0 diff --git a/common.cmake b/common.cmake index 7c60c0f..a422cb5 100644 --- a/common.cmake +++ b/common.cmake @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 option(USE_IAA "Use IAA (requires QPL)" OFF) +option(USE_IGZIP "Use IGZIP (requires isa-l)" OFF) option(USE_QAT "Use QAT (requires QATzip)" OFF) option(DEBUG_LOG "for logging" ON) option(COVERAGE "for coverage" OFF) @@ -14,6 +15,10 @@ if(USE_IAA) add_compile_definitions(USE_IAA) endif() +if(USE_IGZIP) + add_compile_definitions(USE_IGZIP) +endif() + if(USE_QAT) add_compile_definitions(USE_QAT) endif() @@ -89,6 +94,21 @@ if(USE_IAA) endif() endif() +if(USE_IGZIP) + if(NOT DEFINED ISAL_PATH) + find_package(isal REQUIRED) + if(isal_FOUND) + message(STATUS "Found ISA-L: ${isal_DIR}") + link_libraries(isal) + endif() + else() + message(STATUS "Using ISAL_PATH: ${ISAL_PATH}") + include_directories(${ISAL_PATH}/include) + link_directories(PUBLIC ${ISAL_PATH}/.libs) + link_libraries(isal) + endif() +endif() + if(USE_QAT) if(DEFINED QATZIP_PATH) message(STATUS "Using QATZIP_PATH: ${QATZIP_PATH}") diff --git a/config/config.cpp b/config/config.cpp index 29f93cf..3fa6001 100644 --- a/config/config.cpp +++ b/config/config.cpp @@ -21,6 +21,8 @@ uint32_t configs[CONFIG_MAX] = { 0, /*use_iaa_uncompress*/ 1, /*use_zlib_compress*/ 1, /*use_zlib_uncompress*/ + 0, /*use_igzip_compress*/ + 0, /*use_igzip_uncompress*/ 50, /*iaa_compress_percentage*/ 50, /*iaa_uncompress_percentage*/ 0, /*iaa_prepend_empty_block*/ @@ -28,6 +30,7 @@ uint32_t configs[CONFIG_MAX] = { 1, /*qat_compression_level*/ 0, /*qat_compression_allow_chunking*/ 0, /*ignore_zlib_dictionary*/ + 0, /*iaa_fallback_igzip*/ 2, /*log_level*/ 1000 /*log_stats_samples*/ }; @@ -45,6 +48,8 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { "use_iaa_uncompress", "use_zlib_compress", "use_zlib_uncompress", + "use_igzip_compress", + "use_igzip_uncompress", "iaa_compress_percentage", "iaa_uncompress_percentage", "iaa_prepend_empty_block", @@ -52,6 +57,7 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { "qat_compression_level", "qat_compression_allow_chunking", "ignore_zlib_dictionary", + "iaa_fallback_igzip", "log_level", "log_stats_samples" }; @@ -78,6 +84,8 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { trySetConfig(USE_IAA_UNCOMPRESS, 1, 0); trySetConfig(USE_ZLIB_COMPRESS, 1, 0); trySetConfig(USE_ZLIB_UNCOMPRESS, 1, 0); + trySetConfig(USE_IGZIP_COMPRESS, 1, 0); + trySetConfig(USE_IGZIP_UNCOMPRESS, 1, 0); trySetConfig(IAA_COMPRESS_PERCENTAGE, 100, 0); trySetConfig(IAA_UNCOMPRESS_PERCENTAGE, 100, 0); trySetConfig(IAA_PREPEND_EMPTY_BLOCK, 1, 0); @@ -85,6 +93,7 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { trySetConfig(QAT_COMPRESSION_LEVEL, 9, 1); trySetConfig(QAT_COMPRESSION_ALLOW_CHUNKING, 1, 0); trySetConfig(IGNORE_ZLIB_DICTIONARY, 1, 0); + trySetConfig(IAA_FALLBACK_IGZIP, 1, 0); trySetConfig(LOG_LEVEL, 2, 0); trySetConfig(LOG_STATS_SAMPLES, UINT32_MAX, 0); diff --git a/config/config.h b/config/config.h index acc8055..fff483c 100644 --- a/config/config.h +++ b/config/config.h @@ -16,13 +16,16 @@ enum ConfigOption { USE_IAA_UNCOMPRESS, USE_ZLIB_COMPRESS, USE_ZLIB_UNCOMPRESS, + USE_IGZIP_COMPRESS, + USE_IGZIP_UNCOMPRESS, IAA_COMPRESS_PERCENTAGE, IAA_UNCOMPRESS_PERCENTAGE, - IAA_PREPEND_EMPTY_BLOCK, + IAA_PREPEND_EMPTY_BLOCK, // DEPRECATED — see README for details QAT_PERIODICAL_POLLING, QAT_COMPRESSION_LEVEL, QAT_COMPRESSION_ALLOW_CHUNKING, IGNORE_ZLIB_DICTIONARY, + IAA_FALLBACK_IGZIP, LOG_LEVEL, LOG_STATS_SAMPLES, CONFIG_MAX diff --git a/config/default_config b/config/default_config index 6ae0cc8..be76f5f 100644 --- a/config/default_config +++ b/config/default_config @@ -4,6 +4,8 @@ use_iaa_compress = 0 use_iaa_uncompress = 0 use_zlib_compress = 1 use_zlib_uncompress = 1 +use_igzip_compress = 0 +use_igzip_uncompress = 0 iaa_compress_percentage = 50 iaa_uncompress_percentage = 50 iaa_prepend_empty_block = 0 diff --git a/iaa.cpp b/iaa.cpp index 2bbcd27..5a22f87 100644 --- a/iaa.cpp +++ b/iaa.cpp @@ -99,9 +99,19 @@ int CompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, output_shift += GZIP_EXT_XHDR_SIZE; } - // If prepending an empty block, leave space for it to be added - // For zlib format, we don't need an empty block as a marker, as the zlib - // header includes info about the window size + // DEPRECATED: iaa_prepend_empty_block is no longer used by the decompressor. + // The original design used a 5-byte empty stored-block marker written at the + // start of IAA-compressed output so that the decompressor could detect and + // trust IAA-compressed data (which uses a 4kB history window). This approach + // was abandoned because QPL hardware always consumes all available_in bytes + // regardless of where the BFINAL=1 token falls (overconsumption bug), so a + // caller-supplied exact boundary is required instead. IsIAADecompressible now + // uses a 512-byte minimum input length threshold to gate IAA decompression: + // Java ZipInputStream feeds <=512-byte chunks (csize unknown), triggering + // overconsumption; Lucene stored-field reads always provide the exact + // compressed size (>512 bytes), where consuming all input is correct. + // The config option is retained for backward compatibility and will be + // removed in a future release. bool prepend_empty_block = false; CompressedFormat format = GetCompressedFormat(window_bits); if (format != CompressedFormat::ZLIB && @@ -255,44 +265,32 @@ bool SupportedOptionsIAA(int window_bits, uint32_t input_length, return false; } -bool PrependedEmptyBlockPresent(uint8_t* input, uint32_t input_length, - CompressedFormat format) { - uint32_t header_length = GetHeaderLength(format); - if (header_length + PREPENDED_BLOCK_LENGTH > input_length) { - return false; - } - - if (input[header_length] == 0 && input[header_length + 1] == 0 && - input[header_length + 2] == 0 && input[header_length + 3] == 0xFF && - input[header_length + 4] == 0xFF) { - Log(LogLevel::LOG_INFO, "PrependedEmptyBlockPresent() Line ", __LINE__, - " Empty block detected\n"); - return true; - } - - return false; -} - bool IsIAADecompressible(uint8_t* input, uint32_t input_length, int window_bits) { CompressedFormat format = GetCompressedFormat(window_bits); if (format == CompressedFormat::ZLIB) { int window = GetWindowSizeFromZlibHeader(input, input_length); - Log(LogLevel::LOG_INFO, "IsIAADecompressible() Line ", __LINE__, " window ", - window, "\n"); return window <= 12; - } else { - // if no empty block markers selected, we cannot tell for sure it's - // IAA-decompression, but we assume it is. - if (configs[IAA_PREPEND_EMPTY_BLOCK] == 0) { - return true; - } else if (configs[IAA_PREPEND_EMPTY_BLOCK] == 1 && - PrependedEmptyBlockPresent(input, input_length, format)) { - return true; - } else { - return false; - } } + // For raw deflate and gzip formats, QPL always reports total_in == + // available_in regardless of where BFINAL=1 falls in the stream. This is + // safe only when the caller provides avail_in == actual_compressed_size + // (e.g. Lucene stored-field reads, where the exact compressed size is known + // from the .fdt file format). + // + // Callers that do not know the compressed size a priori — notably Java's + // ZipInputStream, which uses a fixed 512-byte internal buffer and feeds + // chunks of that size to inflate() — will have avail_in > actual_csize. + // QPL consuming all 512 bytes then reporting total_in=512 when actual_csize + // was 2 triggers ZipException at the Java level. + // + // Guard: only attempt IAA when input_length > 512. ZipInputStream always + // feeds chunks of at most 512 bytes, so any call above that threshold is + // guaranteed not to be a ZipInputStream-chunked read. Lucene stored-field + // entries are typically much larger than 512 bytes; the few that are smaller + // fall back to IGZIP which is also correct. + static constexpr uint32_t kZipInputStreamBufferSize = 512; + return input_length > kZipInputStreamBufferSize; } #endif // USE_IAA diff --git a/igzip.cpp b/igzip.cpp new file mode 100644 index 0000000..01f1a47 --- /dev/null +++ b/igzip.cpp @@ -0,0 +1,603 @@ +// Copyright (C) 2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#ifdef USE_IGZIP + +#include "igzip.h" + +#include +#include + +#include "crc.h" +#include "logging.h" + +static uint16_t ClampHistBits(int bits) { + if (bits < 0) { + return 0; + } + if (bits > ISAL_DEF_MAX_HIST_BITS) { + return ISAL_DEF_MAX_HIST_BITS; + } + return (uint16_t)bits; +} + +static void ConfigureDeflateWindow(struct isal_zstream *isal_strm, + int windowBits) { + if (windowBits < 0) { + isal_strm->gzip_flag = IGZIP_DEFLATE; + isal_strm->hist_bits = ClampHistBits(-windowBits); + return; + } + + if (windowBits >= 24 && windowBits <= 31) { + isal_strm->gzip_flag = IGZIP_GZIP; + isal_strm->hist_bits = ClampHistBits(windowBits - 16); + return; + } + + isal_strm->gzip_flag = IGZIP_ZLIB; + isal_strm->hist_bits = ClampHistBits(windowBits); +} + +static void ConfigureInflateWindow(struct inflate_state *isal_strm_inflate, + int windowBits) { + if (windowBits < 0) { + isal_strm_inflate->crc_flag = IGZIP_DEFLATE; + isal_strm_inflate->hist_bits = ClampHistBits(-windowBits); + return; + } + + if ((windowBits >= 24 && windowBits <= 31) || + (windowBits >= 40 && windowBits <= 47)) { + isal_strm_inflate->crc_flag = IGZIP_GZIP; + isal_strm_inflate->hist_bits = + ClampHistBits(windowBits > 31 ? windowBits - 32 : windowBits - 16); + return; + } + + isal_strm_inflate->crc_flag = IGZIP_ZLIB; + isal_strm_inflate->hist_bits = ClampHistBits(windowBits); +} + +bool IsIGZIPDeflateFinished(const struct isal_zstream *stream) { + if (stream == nullptr) { + return false; + } + const enum isal_zstate_state state = stream->internal_state.state; + // ZSTATE_TMP_END is a temporary state and may require reentry to + // flush remaining output; only ZSTATE_END is terminal. + return state == ZSTATE_END; +} + +struct isal_zstream *InitCompressIGZIP(int level, int windowBits) { + Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, + " initializing deflate with level ", level, ", windowBits ", windowBits, + "\n"); + + struct isal_zstream *isal_strm = + (struct isal_zstream *)malloc(sizeof(struct isal_zstream)); + if (!isal_strm) { + Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, + " memory allocation for isal_zstream failed\n"); + return nullptr; + } + + /* Setup ISA-L compression context */ + isal_deflate_init(isal_strm); + + isal_strm->end_of_stream = 0; + isal_strm->flush = NO_FLUSH; + + // Map Zlib levels to ISA-L levels + if (level >= 1 && level <= 2) { + isal_strm->level = 1; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL1_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL1_DEFAULT; + } else if ((level >= 3 && level <= 6) || level == -1) { + isal_strm->level = 2; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL2_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL2_DEFAULT; + } else if (level >= 7 && level <= 9) { + isal_strm->level = 3; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL3_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL3_DEFAULT; + } else { + Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, + " invalid compression level\n"); + free(isal_strm); + return nullptr; + } + + if (!isal_strm->level_buf) { + free(isal_strm); + Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() Line ", __LINE__, + " memory allocation for level_buf failed\n"); + return nullptr; + } + + ConfigureDeflateWindow(isal_strm, windowBits); + + return isal_strm; +} + +int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, + uint32_t *input_length, uint8_t *output, + uint32_t *output_length, unsigned long *total_in, + unsigned long *total_out) { + int ret; + + (void)total_in; + (void)total_out; + if (!isal_strm) { + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " deflate isal_strm is NULL\n"); + return -1; + } + + // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ + isal_strm->next_out = output; + const uint32_t original_avail_out = *output_length; + isal_strm->avail_out = original_avail_out; + isal_strm->next_in = input; + const uint32_t original_avail_in = *input_length; + isal_strm->avail_in = original_avail_in; + isal_strm->total_out = *total_out; + isal_strm->total_in = *total_in; + + // stream->flush mapping + switch (flush) { + case Z_NO_FLUSH: + isal_strm->flush = NO_FLUSH; + break; + case Z_SYNC_FLUSH: + case Z_PARTIAL_FLUSH: + case Z_BLOCK: + isal_strm->flush = SYNC_FLUSH; + break; + case Z_FULL_FLUSH: + isal_strm->flush = FULL_FLUSH; + break; + case Z_FINISH: + isal_strm->flush = FULL_FLUSH; + isal_strm->end_of_stream = 1; + break; + default: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid flush value\n"); + return -1; + } + + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, " gzip_flag ", + isal_strm->gzip_flag, ", hist_bits ", isal_strm->hist_bits, ", flush ", + isal_strm->flush, ", level ", isal_strm->level, ", avail_in ", + isal_strm->avail_in, ", avail_out ", (uint32_t)isal_strm->avail_out, + ", total_out ", (uint32_t)isal_strm->total_out, ", total_in ", + (uint32_t)isal_strm->total_in, "\n"); + + // ISA-L always emits sync bytes on SYNC_FLUSH regardless of pending data. + // When the stream is already byte-aligned (ZSTATE_NEW_HDR) and there is no + // new input, no real progress can be made — return 0 progress so the caller + // reports Z_BUF_ERROR, matching zlib's semantics for empty flush calls. + // ZSTATE_NEW_HDR is the idle/byte-aligned state in ISA-L's internal deflate + // state machine; validated against ISA-L v2.32.0 (commit c196241). + if (isal_strm->avail_in == 0 && isal_strm->flush == SYNC_FLUSH && + isal_strm->end_of_stream == 0 && + isal_strm->internal_state.state == ZSTATE_NEW_HDR) { + *output_length = 0; + *input_length = 0; + return 0; + } + + int comp = isal_deflate(isal_strm); + + *output_length = original_avail_out - isal_strm->avail_out; + *input_length = original_avail_in - isal_strm->avail_in; + input = isal_strm->next_in; + output = isal_strm->next_out; + + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, + " after isal_deflate: avail_in ", isal_strm->avail_in, ", avail_out ", + (uint32_t)isal_strm->avail_out, ", bytes_consumed ", *input_length, + ", bytes_produced ", *output_length, "\n"); + + ret = (comp == COMP_OK) ? 0 : 1; + + if (ret == Z_OK) { + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, + " deflate finished successfully Z_OK\n"); + } else if (ret == Z_STREAM_END) { + Log(LogLevel::LOG_INFO, "CompressIGZIP() Line ", __LINE__, + " deflate finished successfully Z_STREAM_END\n"); + } else { + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " deflate finished with error code ", ret, "\n"); + switch (comp) { + case INVALID_FLUSH: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid flush\n"); + break; + case INVALID_PARAM: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid parameter\n"); + break; + case STATELESS_OVERFLOW: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " stateless overflow\n"); + break; + case ISAL_INVALID_OPERATION: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid operation\n"); + break; + case ISAL_INVALID_STATE: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid state\n"); + break; + case ISAL_INVALID_LEVEL: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid level\n"); + break; + case ISAL_INVALID_LEVEL_BUF: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__, + " invalid level buffer\n"); + break; + } + } + + return ret; +} + +int EndCompressIGZIP(struct isal_zstream *isal_strm) { + if (!isal_strm) { + Log(LogLevel::LOG_ERROR, "EndCompressIGZIP() Line ", __LINE__, + " isal_stream is NULL\n"); + return -1; + } + + // Free allocated memory for level_buf and isal_strm + if (isal_strm->level_buf) { + free(isal_strm->level_buf); + } + free(isal_strm); + + Log(LogLevel::LOG_INFO, "EndCompressIGZIP() Line ", __LINE__, + " deflate end\n"); + return Z_OK; +} + +int deflateSetDictionary(z_streamp strm, unsigned char *dict_data, + unsigned int dict_len) { + if (!strm || !strm->state || !dict_data || dict_len == 0) + return Z_STREAM_ERROR; + + deflate_state *s = (deflate_state *)strm->state; + + if (!s || !s->isal_strm) return Z_STREAM_ERROR; + + return isal_deflate_set_dict(s->isal_strm, dict_data, dict_len); +} + +unsigned long crc32(unsigned long crc, const unsigned char *buf, + unsigned int len) { + return crc32_gzip_refl(crc, buf, len); +} + +unsigned long adler32(unsigned long adler, const unsigned char *buf, + unsigned int len) { + return isal_adler32(adler, buf, len); +} + +struct inflate_state *InitUncompressIGZIP(int windowBits) { + struct inflate_state *isal_strm_inflate = + (struct inflate_state *)malloc(sizeof(struct inflate_state)); + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, "InitUncompressIGZIP() Line ", __LINE__, + " memory allocation for inflate_state failed\n"); + return nullptr; + } + + Log(LogLevel::LOG_INFO, "InitUncompressIGZIP() Line ", __LINE__, + " initializing inflate with windowBits ", windowBits, "\n"); + + /* Setup ISA-L decompression context */ + isal_inflate_init(isal_strm_inflate); + + isal_strm_inflate->avail_in = 0; + isal_strm_inflate->next_in = NULL; + // strm->total_out = 0; + // strm->total_in = 0; + + // s->read_in_correction_applied = 0; + + ConfigureInflateWindow(isal_strm_inflate, windowBits); + + return isal_strm_inflate; +} + +IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( + z_streamp strm, struct inflate_state *isal_strm_inflate, int window_bits, + int *read_in_correction_applied, int *ret) { + if (strm == nullptr || isal_strm_inflate == nullptr || ret == nullptr || + read_in_correction_applied == nullptr || strm->avail_in != 0) { + return IGZIP_NO_INPUT_NOT_HANDLED; + } + + uint32_t input_len = 0; + uint32_t output_len = strm->avail_out; + bool end_of_stream = true; + + *ret = UncompressIGZIP(isal_strm_inflate, strm->next_in, &input_len, + strm->next_out, &output_len, window_bits, + read_in_correction_applied, &strm->total_in, + &strm->total_out, &end_of_stream); + + if (*ret == Z_DATA_ERROR) { + Log(LogLevel::LOG_INFO, "IGZIPHandleActiveStreamNoInput() Line ", __LINE__, + " requested zlib fallback for raw INPUT_DONE ambiguity\n"); + return IGZIP_NO_INPUT_FALLBACK_ZLIB; + } + + if (*ret == 0) { + strm->next_out += output_len; + strm->avail_out -= output_len; + strm->total_out += output_len; + if (output_len > 0) { + *ret = end_of_stream ? Z_STREAM_END : Z_OK; + } else { + *ret = Z_BUF_ERROR; + } + return IGZIP_NO_INPUT_RETURN; + } + + *ret = Z_BUF_ERROR; + return IGZIP_NO_INPUT_RETURN; +} + +IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( + z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits, + int *read_in_correction_applied, uint32_t *input_length, + uint32_t *output_length, int *ret, bool *end_of_stream, + uint32_t pre_avail_in) { + if (strm == nullptr || isal_strm_inflate == nullptr || + input_length == nullptr || output_length == nullptr || ret == nullptr || + end_of_stream == nullptr || read_in_correction_applied == nullptr) { + if (ret != nullptr) { + *ret = Z_DATA_ERROR; + } + return IGZIP_INFLATE_PATH_NONE; + } + + if (*isal_strm_inflate == nullptr) { + *isal_strm_inflate = InitUncompressIGZIP(window_bits); + if (*isal_strm_inflate == nullptr) { + Log(LogLevel::LOG_ERROR, "IGZIPRunInflateAndSelectPathAction() Line ", + __LINE__, " failed to initialize igzip inflate stream\n"); + *ret = Z_DATA_ERROR; + return IGZIP_INFLATE_PATH_NONE; + } + } + + *ret = UncompressIGZIP(*isal_strm_inflate, strm->next_in, input_length, + strm->next_out, output_length, window_bits, + read_in_correction_applied, &strm->total_in, + &strm->total_out, end_of_stream); + + const uint32_t remaining_after_igzip = + (pre_avail_in >= *input_length) ? (pre_avail_in - *input_length) : 0; + + if (*ret == 0 && window_bits < 0 && *end_of_stream && + remaining_after_igzip > 0 && *read_in_correction_applied == 0 && + strm->total_in == 0 && strm->total_out == 0) { + Log(LogLevel::LOG_ERROR, + "IGZIPRunInflateAndSelectPathAction() raw boundary guard FIRED strm=", + static_cast(strm), " bytes_in=", *input_length, + " bytes_out=", *output_length, " pre_avail_in=", pre_avail_in, + " remaining_in=", remaining_after_igzip, "\n"); + *ret = 1; + *end_of_stream = false; + return IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY; + } + + if (*ret == Z_NEED_DICT) { + return IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT; + } + if (*ret == Z_DATA_ERROR) { + return IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR; + } + if (*ret == 0) { + return IGZIP_INFLATE_PATH_SET_IGZIP; + } + + return IGZIP_INFLATE_PATH_NONE; +} + +int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, + uint32_t *input_length, uint8_t *output, + uint32_t *output_length, int window_bits, + int *read_in_correction_applied, unsigned long *total_in, + unsigned long *total_out, bool *end_of_stream) { + (void)total_in; + + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " isal_strm_inflate is NULL\n"); + return Z_STREAM_ERROR; + } + // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ + isal_strm_inflate->next_out = output; + const uint32_t original_avail_out = *output_length; + isal_strm_inflate->avail_out = original_avail_out; + const uint32_t original_avail_in = *input_length; + isal_strm_inflate->avail_in = original_avail_in; + isal_strm_inflate->next_in = input; + isal_strm_inflate->total_out = *total_out; + + const int decomp = isal_inflate(isal_strm_inflate); + + uint32_t consumed_before_adjust = 0; + if (isal_strm_inflate->avail_in <= original_avail_in) { + consumed_before_adjust = original_avail_in - isal_strm_inflate->avail_in; + } else { + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " invalid avail_in ", isal_strm_inflate->avail_in, + " greater than original_avail_in ", original_avail_in, + ", clamping consumed bytes to 0\n"); + consumed_before_adjust = 0; + } + + uint32_t rewind_adjust_bytes = 0; + + // WORKAROUND: ISA-L raw-deflate over-consumption fix. + // ISAL pre-loads input in 8-byte word chunks into a 64-bit shift register + // (read_in). After BLOCK_FINISH, read_in_length >> 3 is the exact byte + // count over-consumed, covering all avail_in scenarios: [0], [1,7], [8], + // and >8 (multi-frame), where prior heuristics were blind or inaccurate. + if (window_bits < 0 && + (decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) && + isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) { + const uint32_t read_in_correction = + (isal_strm_inflate->read_in_length > 0) + ? static_cast(isal_strm_inflate->read_in_length >> 3) + : 0u; + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " raw_finish avail_in ", isal_strm_inflate->avail_in, + " read_in_length_bits ", isal_strm_inflate->read_in_length, + " read_in_correction_bytes ", read_in_correction, "\n"); + if (read_in_correction > 0) { + rewind_adjust_bytes = (read_in_correction <= consumed_before_adjust) + ? read_in_correction + : consumed_before_adjust; + *read_in_correction_applied = 1; + } + } + + // WORKAROUND: BLOCK_INPUT_DONE — output-buffer-limited with ambiguous + // trailer bytes. read_in_length does not apply here (not yet at + // BLOCK_FINISH); request caller fallback to zlib. BLOCK_FINISH is fully + // handled above. + if (window_bits < 0 && decomp == ISAL_DECOMP_OK && + *read_in_correction_applied == 0 && + isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE && + isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) { + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " raw INPUT_DONE ambiguity detected: over_consumed ", + 8u - isal_strm_inflate->avail_in, ", requesting zlib fallback\n"); + return Z_DATA_ERROR; + } + + *output_length = original_avail_out - isal_strm_inflate->avail_out; + *input_length = consumed_before_adjust - rewind_adjust_bytes; + input = isal_strm_inflate->next_in; + output = isal_strm_inflate->next_out; + + if (end_of_stream != nullptr) { + *end_of_stream = (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH); + } + + int ret = 1; + if (decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) { + ret = 0; + } else if (decomp == ISAL_NEED_DICT) { + ret = Z_NEED_DICT; + } + + if (ret == Z_OK) { + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " inflate finished successfully Z_OK\n"); + } else if (ret == Z_STREAM_END) { + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " inflate finished with Z_STREAM_END\n"); + } else { + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " inflate finished with error code ", ret, "\n"); + switch (decomp) { + case ISAL_INVALID_BLOCK: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - Invalid block\n"); + break; + case ISAL_INVALID_SYMBOL: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - Invalid symbol\n"); + break; + case ISAL_INVALID_LOOKBACK: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - Invalid lookback\n"); + break; + case ISAL_END_INPUT: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - End of input reached unexpectedly\n"); + break; + case ISAL_UNSUPPORTED_METHOD: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - Unsupported method\n"); + break; + case ISAL_NEED_DICT: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error - Need dictionary\n"); + break; + default: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() Line ", __LINE__, + " ISA-L error code ", decomp, "\n"); + break; + } + } + + return ret; +} + +int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate) { + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, "EndUncompressIGZIP() Line ", __LINE__, + " z_streamp is NULL\n"); + return Z_STREAM_ERROR; + } + + free(isal_strm_inflate); + + Log(LogLevel::LOG_INFO, "EndUncompressIGZIP() Line ", __LINE__, + " inflate end\n"); + return Z_OK; +} + +int inflateSetDictionary(z_streamp strm, unsigned char *dict_data, + unsigned int dict_len) { + if (!strm || !strm->state || !dict_data || dict_len == 0) + return Z_STREAM_ERROR; + + const inflate_state2 *s = (inflate_state2 *)strm->state; + + if (!s || !s->isal_strm_inflate) return Z_STREAM_ERROR; + + return isal_inflate_set_dict(s->isal_strm_inflate, dict_data, dict_len); +} + +void ResetCompressIGZIP(struct isal_zstream *isal_strm, int windowBits) { + // isal_deflate_reset preserves gzip_flag, hist_bits, level, and level_buf. + // gzip_flag must be restored: after the first chunk ISA-L changes it from + // IGZIP_ZLIB (3) to IGZIP_ZLIB_NO_HDR (4) to suppress the header on + // continuation calls. Without this reset, the next stream reused via + // deflateReset would produce headerless output, causing decompressors + // (e.g. Java Inflater with nowrap=false) to reject every subsequent chunk. + isal_deflate_reset(isal_strm); + isal_strm->end_of_stream = 0; + isal_strm->flush = NO_FLUSH; + ConfigureDeflateWindow(isal_strm, windowBits); +} + +int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, + int *read_in_correction_applied) { + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, "ResetUncompressIGZIP() Line ", __LINE__, + " isal_strm_inflate is NULL\n"); + return Z_STREAM_ERROR; + } + + // Reset ISA-L inflate state. isal_inflate_reset clears the internal + // read_in / read_in_length buffer, so any over-consumption correction + // applied during the previous stream session no longer applies. + // Clear the flag so the new session fires the correction fresh if needed. + isal_inflate_reset(isal_strm_inflate); + *read_in_correction_applied = 0; + + return Z_OK; +} +#endif diff --git a/igzip.h b/igzip.h new file mode 100644 index 0000000..a52ef75 --- /dev/null +++ b/igzip.h @@ -0,0 +1,70 @@ +// Copyright (C) 2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#ifdef USE_IGZIP +#include +#include + +typedef struct internal_state2 { + z_streamp strm; + int level; + int w_bits; + struct inflate_state *isal_strm_inflate; + int read_in_correction_applied; /* Set once per stream session when the + read_in_length over-consumption correction + fires; cleared only on inflateReset */ +} inflate_state2; + +typedef struct internal_state { + z_streamp strm; + int level; + int w_bits; + struct isal_zstream *isal_strm; +} deflate_state; + +struct isal_zstream *InitCompressIGZIP(int level, int windowBits); +int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, + uint32_t *input_length, uint8_t *output, + uint32_t *output_length, unsigned long *total_in, + unsigned long *total_out); +bool IsIGZIPDeflateFinished(const struct isal_zstream *stream); +enum IGZIPNoInputAction { + IGZIP_NO_INPUT_NOT_HANDLED, + IGZIP_NO_INPUT_RETURN, + IGZIP_NO_INPUT_FALLBACK_ZLIB, +}; + +enum IGZIPInflatePathAction { + IGZIP_INFLATE_PATH_NONE, + IGZIP_INFLATE_PATH_SET_IGZIP, + IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT, + IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR, + IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY, +}; + +IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( + z_streamp strm, struct inflate_state *isal_strm_inflate, int window_bits, + int *read_in_correction_applied, int *ret); + +IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( + z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits, + int *read_in_correction_applied, uint32_t *input_length, + uint32_t *output_length, int *ret, bool *end_of_stream, + uint32_t pre_avail_in); + +int EndCompressIGZIP(struct isal_zstream *isal_strm); +void ResetCompressIGZIP(struct isal_zstream *isal_strm, int windowBits); + +struct inflate_state *InitUncompressIGZIP(int windowBits); +int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, + uint32_t *input_length, uint8_t *output, + uint32_t *output_length, int window_bits, + int *read_in_correction_applied, unsigned long *total_in, + unsigned long *total_out, bool *end_of_stream); +int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate); +int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, + int *read_in_correction_applied); +// #define Z_DEFAULT_COMPRESSION 6 +#endif diff --git a/statistics.cpp b/statistics.cpp index 9dabeb6..0610678 100644 --- a/statistics.cpp +++ b/statistics.cpp @@ -18,9 +18,10 @@ using namespace config; const std::array stat_names{ {"deflate_count", "deflate_error_count", "deflate_qat_count", "deflate_qat_error_count", "deflate_iaa_count", "deflate_iaa_error_count", - "deflate_zlib_count", "inflate_count", "inflate_error_count", - "inflate_qat_count", "inflate_qat_error_count", "inflate_iaa_count", - "inflate_iaa_error_count", "inflate_zlib_count"}}; + "deflate_igzip_count", "deflate_igzip_error_count", "deflate_zlib_count", + "inflate_count", "inflate_error_count", "inflate_qat_count", + "inflate_qat_error_count", "inflate_iaa_count", "inflate_iaa_error_count", + "inflate_igzip_count", "inflate_igzip_error_count", "inflate_zlib_count"}}; thread_local std::array stats{}; diff --git a/statistics.h b/statistics.h index 8cee404..b3654ce 100644 --- a/statistics.h +++ b/statistics.h @@ -17,6 +17,8 @@ enum class Statistic : size_t { DEFLATE_QAT_ERROR_COUNT, DEFLATE_IAA_COUNT, DEFLATE_IAA_ERROR_COUNT, + DEFLATE_IGZIP_COUNT, + DEFLATE_IGZIP_ERROR_COUNT, DEFLATE_ZLIB_COUNT, INFLATE_COUNT, INFLATE_ERROR_COUNT, @@ -24,6 +26,8 @@ enum class Statistic : size_t { INFLATE_QAT_ERROR_COUNT, INFLATE_IAA_COUNT, INFLATE_IAA_ERROR_COUNT, + INFLATE_IGZIP_COUNT, + INFLATE_IGZIP_ERROR_COUNT, INFLATE_ZLIB_COUNT, STATS_COUNT }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b0c4564..0ac0465 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -24,7 +24,7 @@ add_executable(zlib_accel_test ) add_custom_target(run - COMMAND ./zlib_accel_test + COMMAND ./zlib_accel_test # --gtest_filter=ConfigLoaderTest.LoadValidConfig DEPENDS zlib_accel_test ) diff --git a/tests/zlib_accel_test.cpp b/tests/zlib_accel_test.cpp index c8ec0ce..b526d0c 100644 --- a/tests/zlib_accel_test.cpp +++ b/tests/zlib_accel_test.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,10 @@ using namespace config; +#ifdef USE_IGZIP +#include "../igzip.h" +#endif + enum BlockCompressibilityType { compressible_block, incompressible_block, @@ -183,6 +188,41 @@ int ZlibUncompressUtility2(const char* input, size_t input_length, return st; } +int ZlibCompressWithLevel(const char* input, size_t input_length, + std::string* output, int level, int window_bits, + int flush, size_t* output_upper_bound, + ExecutionPath* execution_path) { + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + + int st = deflateInit2(&stream, level, Z_DEFLATED, window_bits, 8, + Z_DEFAULT_STRATEGY); + if (st != Z_OK) { + deflateEnd(&stream); + return st; + } + + stream.next_in = (Bytef*)input; + stream.avail_in = static_cast(input_length); + + *output_upper_bound = + deflateBound(&stream, static_cast(input_length)); + output->resize(*output_upper_bound); + stream.avail_out = static_cast(*output_upper_bound); + stream.next_out = reinterpret_cast(&(*output)[0]); + + st = deflate(&stream, flush); + *execution_path = GetDeflateExecutionPath(&stream); + if (st != Z_STREAM_END) { + deflateEnd(&stream); + return st; + } + output->resize(stream.total_out); + + deflateEnd(&stream); + return st; +} + int ZlibCompressGzipFile(const char* input, size_t input_length) { const char* filename = "file.gz"; remove(filename); @@ -240,16 +280,25 @@ void SetCompressPath(ExecutionPath path, bool zlib_fallback, switch (path) { case ZLIB: SetConfig(USE_IAA_COMPRESS, 0); + SetConfig(USE_IGZIP_COMPRESS, 0); SetConfig(USE_QAT_COMPRESS, 0); SetConfig(USE_ZLIB_COMPRESS, 1); break; case QAT: SetConfig(USE_IAA_COMPRESS, 0); + SetConfig(USE_IGZIP_COMPRESS, 0); SetConfig(USE_QAT_COMPRESS, 1); SetConfig(USE_ZLIB_COMPRESS, zlib_fallback ? 1 : 0); break; case IAA: SetConfig(USE_IAA_COMPRESS, 1); + SetConfig(USE_IGZIP_COMPRESS, 0); + SetConfig(USE_QAT_COMPRESS, 0); + SetConfig(USE_ZLIB_COMPRESS, zlib_fallback ? 1 : 0); + break; + case IGZIP: + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(USE_IAA_COMPRESS, 0); SetConfig(USE_QAT_COMPRESS, 0); SetConfig(USE_ZLIB_COMPRESS, zlib_fallback ? 1 : 0); break; @@ -265,16 +314,25 @@ void SetUncompressPath(ExecutionPath path, bool zlib_fallback, switch (path) { case ZLIB: SetConfig(USE_IAA_UNCOMPRESS, 0); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); SetConfig(USE_QAT_UNCOMPRESS, 0); SetConfig(USE_ZLIB_UNCOMPRESS, 1); break; case QAT: SetConfig(USE_IAA_UNCOMPRESS, 0); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); SetConfig(USE_QAT_UNCOMPRESS, 1); SetConfig(USE_ZLIB_UNCOMPRESS, zlib_fallback ? 1 : 0); break; case IAA: SetConfig(USE_IAA_UNCOMPRESS, 1); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); + SetConfig(USE_QAT_UNCOMPRESS, 0); + SetConfig(USE_ZLIB_UNCOMPRESS, zlib_fallback ? 1 : 0); + break; + case IGZIP: + SetConfig(USE_IAA_UNCOMPRESS, 0); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); SetConfig(USE_QAT_UNCOMPRESS, 0); SetConfig(USE_ZLIB_UNCOMPRESS, zlib_fallback ? 1 : 0); break; @@ -331,6 +389,8 @@ struct TestParam { return "QAT"; case IAA: return "IAA"; + case IGZIP: + return "IGZIP"; } return ""; } @@ -590,6 +650,14 @@ TEST_P(ZlibTest, CompressDecompress) { test_param.iaa_prepend_empty_block, test_param.qat_compression_allow_chunking); + // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses + // ISA-L level 3 (stricter match selection), which makes long-history + // limitations consistently observable. + const int compression_level = (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; + size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; char* input = GenerateBlock(input_length, block_type); @@ -598,9 +666,10 @@ TEST_P(ZlibTest, CompressDecompress) { std::string compressed; size_t output_upper_bound; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompress( - input, input_length, &compressed, test_param.window_bits_compress, - test_param.flush_compress, &output_upper_bound, &execution_path); + int ret = ZlibCompressWithLevel( + input, input_length, &compressed, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound, &execution_path); VerifyStatIncremented(Statistic::DEFLATE_COUNT); bool compress_fallback_expected = @@ -621,6 +690,8 @@ TEST_P(ZlibTest, CompressDecompress) { VerifyStatIncremented(Statistic::DEFLATE_QAT_COUNT); } else if (test_param.execution_path_compress == IAA) { VerifyStatIncremented(Statistic::DEFLATE_IAA_COUNT); + } else if (test_param.execution_path_compress == IGZIP) { + VerifyStatIncremented(Statistic::DEFLATE_IGZIP_COUNT); } else if (test_param.execution_path_compress == ZLIB) { VerifyStatIncremented(Statistic::DEFLATE_ZLIB_COUNT); } @@ -674,6 +745,9 @@ TEST_P(ZlibTest, CompressDecompress) { VerifyStatIncremented(Statistic::INFLATE_QAT_COUNT); } else if (test_param.execution_path_uncompress == IAA) { VerifyStatIncremented(Statistic::INFLATE_IAA_COUNT); + } else if (test_param.execution_path_uncompress == IGZIP) { + VerifyStatIncrementedUpTo(Statistic::INFLATE_IGZIP_COUNT, + test_param.input_chunks_uncompress); } else if (test_param.execution_path_uncompress == ZLIB) { VerifyStatIncrementedUpTo(Statistic::INFLATE_ZLIB_COUNT, test_param.input_chunks_uncompress); @@ -729,6 +803,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -740,6 +818,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(-15, 15, 31), @@ -827,6 +909,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -838,6 +924,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(15), @@ -923,6 +1013,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -934,6 +1028,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(15), @@ -960,6 +1058,14 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { test_param.iaa_prepend_empty_block, test_param.qat_compression_allow_chunking); + // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses + // ISA-L level 3 (stricter match selection), which makes long-history + // limitations consistently observable. + const int compression_level = (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; + size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; char* input = GenerateBlock(input_length, block_type); @@ -968,9 +1074,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { std::string compressed; size_t output_upper_bound; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompress( - input, input_length, &compressed, test_param.window_bits_compress, - test_param.flush_compress, &output_upper_bound, &execution_path); + int ret = ZlibCompressWithLevel( + input, input_length, &compressed, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound, &execution_path); bool error_expected = ZlibCompressExpectError(test_param, input_length, output_upper_bound); @@ -998,8 +1105,9 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { window_bits_uncompress, test_param.flush_uncompress, test_param.input_chunks_uncompress, &execution_path); - // Only zlib decompression won't return an error + // zlib and igzip decompression may return partial progress instead of error if (test_param.execution_path_uncompress == ZLIB || + test_param.execution_path_uncompress == IGZIP || test_param.zlib_fallback_uncompress) { ASSERT_EQ(ret, Z_OK); ASSERT_TRUE(uncompressed_length < input_length); @@ -1026,6 +1134,14 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { test_param.iaa_prepend_empty_block, test_param.qat_compression_allow_chunking); + // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses + // ISA-L level 3 (stricter match selection), which makes long-history + // limitations consistently observable. + const int compression_level = (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; + size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; char* input = GenerateBlock(input_length, block_type); @@ -1036,9 +1152,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { size_t input_length1 = input_length / 2; size_t output_upper_bound1; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompress( - input, input_length1, &compressed1, test_param.window_bits_compress, - test_param.flush_compress, &output_upper_bound1, &execution_path); + int ret = ZlibCompressWithLevel( + input, input_length1, &compressed1, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound1, &execution_path); bool error_expected = ZlibCompressExpectError(test_param, input_length1, output_upper_bound1); @@ -1054,9 +1171,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { size_t input_length2 = input_length - input_length / 2; size_t output_upper_bound2; execution_path = UNDEFINED; - ret = ZlibCompress(input + input_length1, input_length2, &compressed2, - test_param.window_bits_compress, test_param.flush_compress, - &output_upper_bound2, &execution_path); + ret = ZlibCompressWithLevel( + input + input_length1, input_length2, &compressed2, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound2, &execution_path); error_expected = ZlibCompressExpectError(test_param, input_length2, output_upper_bound2); @@ -1101,8 +1219,9 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { ASSERT_EQ(uncompressed_length, input_length1); ASSERT_TRUE(memcmp(uncompressed, input, uncompressed_length) == 0); - // IAA does not handle concatenated streams - if (test_param.execution_path_uncompress != IAA) { + // IAA/IGZIP may consume bytes beyond first-stream boundary + if (test_param.execution_path_uncompress != IAA && + test_param.execution_path_uncompress != IGZIP) { ASSERT_EQ(input_consumed, compressed1.length()); } } @@ -1111,6 +1230,1331 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { DestroyBlock(input); } +#ifdef USE_IGZIP +TEST(IGZIPInflateRegressionTest, EmptyInputContinuationKeepsIGZIPPath) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 1 << 20; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output_chunk(8192); + stream.next_in = reinterpret_cast(compressed.data()); + stream.avail_in = static_cast(compressed.size()); + + int iter = 0; + int last_ret = Z_OK; + while (iter++ < 2048) { + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_NE(last_ret, Z_DATA_ERROR); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + if (last_ret == Z_STREAM_END) { + break; + } + + if (stream.avail_in == 0 && last_ret == Z_OK) { + break; + } + } + + ASSERT_NE(last_ret, Z_STREAM_END); + ASSERT_EQ(stream.avail_in, 0u); + + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + stream.next_in = nullptr; + stream.avail_in = 0; + + int continuation_ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_TRUE(continuation_ret == Z_BUF_ERROR || continuation_ret == Z_OK || + continuation_ret == Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, RawContinuationMustNotIncreaseAvailIn) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 16384; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output_chunk(8192); + stream.next_in = reinterpret_cast(compressed.data()); + stream.avail_in = static_cast(compressed.size()); + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + ASSERT_TRUE(ret == Z_OK || ret == Z_STREAM_END); + + while (ret == Z_OK && stream.avail_in > 0) { + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + ASSERT_TRUE(ret == Z_OK || ret == Z_STREAM_END); + } + + ASSERT_EQ(stream.avail_in, 0u); + + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + stream.next_in = nullptr; + stream.avail_in = 0; + ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_TRUE(ret == Z_BUF_ERROR || ret == Z_OK || ret == Z_STREAM_END); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + uint8_t one_byte = 0; + stream.next_in = &one_byte; + stream.avail_in = 1; + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_NE(ret, Z_DATA_ERROR); + EXPECT_LE(stream.avail_in, 1u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, RawTrailingByteMustNotIncreaseAvailIn) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 4096; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + std::string compressed_with_trailing = compressed; + compressed_with_trailing.push_back('\0'); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector uncompressed(input_length * 2); + stream.next_in = reinterpret_cast(compressed_with_trailing.data()); + stream.avail_in = static_cast(compressed_with_trailing.size()); + stream.next_out = reinterpret_cast(uncompressed.data()); + stream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_NE(ret, Z_DATA_ERROR); + EXPECT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + // For valid raw-deflate stream with one extra byte, inflate may leave that + // byte unconsumed, but avail_in must never increase. + EXPECT_LE(stream.avail_in, 1u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, + RawOneByteContinuationMustNotIncreaseAvailInAcrossSizes) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + std::vector input_sizes = {1, 2, 3, 7, 8, 15, 16, + 31, 32, 63, 64, 127, 128, 255, + 256, 511, 512, 1023, 1024, 2047, 2048, + 4095, 4096, 8191, 8192, 16384, 32768}; + + for (size_t input_length : input_sizes) { + std::vector input(input_length); + for (size_t i = 0; i < input_length; ++i) { + input[i] = static_cast((i * 131u + 17u) & 0xFFu); + } + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input.data(), input_length, &compressed, -15, + Z_FINISH, &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output_chunk(1024); + stream.next_in = reinterpret_cast(compressed.data()); + stream.avail_in = static_cast(compressed.size()); + + int last_ret = Z_OK; + for (int iter = 0; iter < 4096; ++iter) { + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + ASSERT_NE(last_ret, Z_DATA_ERROR); + + if (stream.avail_in == 0 || last_ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(stream.avail_in, 0u); + + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + stream.next_in = nullptr; + stream.avail_in = 0; + int empty_ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_TRUE(empty_ret == Z_BUF_ERROR || empty_ret == Z_OK || + empty_ret == Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + uint8_t trailing_byte = 0; + stream.next_in = &trailing_byte; + stream.avail_in = 1; + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + + int one_byte_ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_NE(one_byte_ret, Z_DATA_ERROR); + EXPECT_LE(stream.avail_in, 1u) << "input_length=" << input_length; + + inflateEnd(&stream); + } +} + +TEST(IGZIPInflateRegressionTest, + TinyRawEntryMustNotOverconsumePastStreamBoundary) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + // Empty raw-deflate entries are often tiny (commonly 2 bytes). The + // decompressor must stop exactly at stream end and leave trailing bytes for + // the caller. + const std::string empty_payload; + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = + ZlibCompress(empty_payload.data(), empty_payload.size(), &compressed, -15, + Z_FINISH, &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + ASSERT_GT(compressed.size(), 0u); + + const size_t trailing_len = 510; + std::string input = compressed; + input.append(trailing_len, static_cast(0xA5)); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output(512); + stream.next_in = reinterpret_cast(input.data()); + stream.avail_in = static_cast(input.size()); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + const ExecutionPath observed_path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(observed_path == IGZIP || observed_path == ZLIB); + ASSERT_NE(ret, Z_DATA_ERROR); + + // No output is expected for empty payload. Most importantly, all trailing + // bytes must remain unconsumed for the caller. + EXPECT_EQ(output.size() - stream.avail_out, 0u); + EXPECT_EQ(stream.avail_in, trailing_len) + << "compressed_size=" << compressed.size(); + + inflateEnd(&stream); +} + +TEST(IGZIPInflateRegressionTest, RawStreamEndMustPreserveEightTrailingBytes) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 32768; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + std::string with_trailing = compressed; + with_trailing.append("ABCDEFGH", 8); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output(input_length * 2); + stream.next_in = reinterpret_cast(with_trailing.data()); + stream.avail_in = static_cast(with_trailing.size()); + + int last_ret = Z_OK; + for (int iter = 0; iter < 64; ++iter) { + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + const ExecutionPath path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(last_ret, Z_DATA_ERROR); + if (last_ret == Z_STREAM_END) { + break; + } + if (stream.avail_out > 0 && stream.avail_in == 0) { + break; + } + } + + EXPECT_EQ(last_ret, Z_STREAM_END); + EXPECT_EQ(stream.avail_in, 8u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, RawSplitInputDefersCorrectionUntilStreamEnd) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 65536; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + ASSERT_GT(compressed.size(), 2u); + + std::string with_trailing = compressed; + with_trailing.append("ABCDEFGH", 8); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output(input_length * 2); + const size_t split_offset = compressed.size() - 1; + + stream.next_in = reinterpret_cast(with_trailing.data()); + stream.avail_in = static_cast(split_offset); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + { + const ExecutionPath path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + } + ASSERT_NE(ret, Z_DATA_ERROR); + + stream.next_in = + reinterpret_cast(with_trailing.data() + split_offset); + stream.avail_in = + static_cast(with_trailing.size() - split_offset); + + int last_ret = ret; + for (int iter = 0; iter < 64; ++iter) { + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + const ExecutionPath path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(last_ret, Z_DATA_ERROR); + if (last_ret == Z_STREAM_END) { + break; + } + if (stream.avail_out > 0 && stream.avail_in == 0) { + break; + } + } + + EXPECT_EQ(last_ret, Z_STREAM_END); + EXPECT_EQ(stream.avail_in, 8u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPDeflateRegressionTest, + RepeatedFinishWithEmptyInputMustReturnStreamEnd) { + SetCompressPath(IGZIP, true, false, false); + SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } + + const char* input = "igzip-finish-regression-payload"; + const size_t input_length = strlen(input); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(4096); + stream.next_in = reinterpret_cast(const_cast(input)); + stream.avail_in = static_cast(input_length); + + int ret = Z_OK; + for (int iter = 0; iter < 32; ++iter) { + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + ret = deflate(&stream, Z_FINISH); + const ExecutionPath path = GetDeflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + + for (int iter = 0; iter < 64; ++iter) { + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + int finish_ret = deflate(&stream, Z_FINISH); + EXPECT_EQ(finish_ret, Z_STREAM_END) << "iter=" << iter; + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_TRUE(path == IGZIP || path == ZLIB) << "iter=" << iter; + } + + deflateEnd(&stream); +} + +TEST(IGZIPDeflateRegressionTest, ResetMustNotStallSyncFlushOnSameStream) { + SetCompressPath(IGZIP, true, false, false); + SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(4096); + std::vector input(512, 'A'); + + for (int cycle = 0; cycle < 20; ++cycle) { + stream.next_in = reinterpret_cast(input.data()); + stream.avail_in = static_cast(input.size()); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + int ret = deflate(&stream, Z_NO_FLUSH); + ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR) << "cycle=" << cycle; + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "cycle=" << cycle; + + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = deflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(ret, Z_OK) << "cycle=" << cycle; + ASSERT_LT(stream.avail_out, output.size()) << "cycle=" << cycle; + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "cycle=" << cycle; + + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END) << "cycle=" << cycle; + + ASSERT_EQ(deflateReset(&stream), Z_OK) << "cycle=" << cycle; + } + + deflateEnd(&stream); +} + +TEST(IGZIPDeflateRegressionTest, + RepeatedEmptySyncFlushMustEventuallyReportNoProgress) { + SetCompressPath(IGZIP, true, false, false); + SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(4096); + std::vector input(1024, 'B'); + + stream.next_in = reinterpret_cast(input.data()); + stream.avail_in = static_cast(input.size()); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + int ret = deflate(&stream, Z_NO_FLUSH); + ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP); + + bool observed_buf_error = false; + for (int iter = 0; iter < 128; ++iter) { + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = deflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "iter=" << iter; + ASSERT_NE(ret, Z_DATA_ERROR) << "iter=" << iter; + + if (ret == Z_BUF_ERROR) { + observed_buf_error = true; + break; + } + } + + EXPECT_TRUE(observed_buf_error) + << "Repeated empty Z_SYNC_FLUSH calls must eventually stop producing " + << "new bytes and return Z_BUF_ERROR"; + + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + ret = deflate(&stream, Z_FINISH); + EXPECT_EQ(ret, Z_STREAM_END); + + deflateEnd(&stream); +} + +// Regression test for: deflateReset on a reused IGZIP stream must restore the +// zlib header (gzip_flag = IGZIP_ZLIB) so that each independent chunk is +// self-contained and decompressible by a fresh zlib inflater. Without the +// fix, isal_deflate_reset preserved gzip_flag = IGZIP_ZLIB_NO_HDR (4) and the +// second and subsequent chunks were emitted without a zlib header, causing +// Java's Inflater (nowrap=false) to report "incorrect header check". +TEST(IGZIPDeflateRegressionTest, + ResetMustRestoreZlibHeaderForSubsequentChunks) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + z_stream cstream; + memset(&cstream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, + /*windowBits=*/15, /*memLevel=*/8, Z_DEFAULT_STRATEGY), + Z_OK); + + const int kChunks = 5; + const int kChunkSize = 16384; + const int kCompBound = deflateBound(&cstream, kChunkSize); + + for (int chunk = 0; chunk < kChunks; ++chunk) { + // Each chunk is independent data compressed with a fresh IGZIP stream + // (via deflateReset). + std::vector input(kChunkSize, + static_cast('A' + chunk % 26)); + std::vector compressed(kCompBound); + + cstream.next_in = input.data(); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + ASSERT_EQ(deflate(&cstream, Z_FINISH), Z_STREAM_END) << "chunk=" << chunk; + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP) << "chunk=" << chunk; + + const size_t compressed_size = compressed.size() - cstream.avail_out; + + // Decompress with a fresh zlib inflater — requires a valid zlib header. + std::vector decompressed(kChunkSize); + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit(&dstream), Z_OK) << "chunk=" << chunk; + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = decompressed.data(); + dstream.avail_out = static_cast(decompressed.size()); + ASSERT_EQ(inflate(&dstream, Z_FINISH), Z_STREAM_END) + << "chunk=" << chunk + << ": decompression failed (missing zlib header after deflateReset?)"; + ASSERT_EQ(decompressed, input) << "chunk=" << chunk; + inflateEnd(&dstream); + + ASSERT_EQ(deflateReset(&cstream), Z_OK) << "chunk=" << chunk; + } + + deflateEnd(&cstream); +} + +TEST(IGZIPDeflateRegressionTest, DictionaryStreamMustStayOnZlibAcrossReset) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + const char dict[] = "0123456789abcdef"; + ASSERT_EQ(deflateSetDictionary(&stream, reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), + Z_OK); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + + std::vector out1(4096); + const char* msg1 = "dictionary-stream-first-message"; + stream.next_in = reinterpret_cast(const_cast(msg1)); + stream.avail_in = static_cast(strlen(msg1)); + int ret = Z_OK; + for (int iter = 0; iter < 16; ++iter) { + stream.next_out = reinterpret_cast(out1.data()); + stream.avail_out = static_cast(out1.size()); + ret = deflate(&stream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + const size_t out1_size = out1.size() - stream.avail_out; + + ASSERT_EQ(deflateReset(&stream), Z_OK); + + // Re-set dictionary after reset (standard zlib API usage). + ASSERT_EQ(deflateSetDictionary(&stream, reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), + Z_OK); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + + std::vector out2(4096); + const char* msg2 = "dictionary-stream-second-message"; + stream.next_in = reinterpret_cast(const_cast(msg2)); + stream.avail_in = static_cast(strlen(msg2)); + ret = Z_OK; + for (int iter = 0; iter < 16; ++iter) { + stream.next_out = reinterpret_cast(out2.data()); + stream.avail_out = static_cast(out2.size()); + ret = deflate(&stream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + const size_t out2_size = out2.size() - stream.avail_out; + + z_stream verify; + memset(&verify, 0, sizeof(verify)); + ASSERT_EQ(inflateInit2(&verify, 15), Z_OK); + + std::vector verify_out(1024); + verify.next_in = reinterpret_cast(out2.data()); + verify.avail_in = static_cast(out2_size); + verify.next_out = reinterpret_cast(verify_out.data()); + verify.avail_out = static_cast(verify_out.size()); + + ret = inflate(&verify, Z_NO_FLUSH); + EXPECT_EQ(ret, Z_NEED_DICT); + + ASSERT_EQ(inflateSetDictionary(&verify, reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), + Z_OK); + ret = inflate(&verify, Z_FINISH); + EXPECT_EQ(ret, Z_STREAM_END); + + inflateEnd(&verify); + + EXPECT_GT(out1_size, 0u); + EXPECT_GT(out2_size, 0u); + + deflateEnd(&stream); +} + +TEST(IGZIPDeflateRegressionTest, + ParallelDictionaryResetStreamsRemainRoundTripSafe) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + const std::string dictionary = + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const int thread_count = 4; + const int iterations_per_thread = 120; + + std::atomic failed{false}; + std::string failure_reason; + std::mutex failure_mutex; + + auto set_failure = [&](const std::string& message) { + bool expected = false; + if (failed.compare_exchange_strong(expected, true)) { + std::lock_guard lock(failure_mutex); + failure_reason = message; + } + }; + + auto worker = [&](int worker_id) { + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + int ret = deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { + set_failure("deflateInit2 failed"); + return; + } + + for (int i = 0; i < iterations_per_thread && !failed.load(); ++i) { + if (deflateReset(&cstream) != Z_OK) { + set_failure("deflateReset failed"); + break; + } + + ret = deflateSetDictionary( + &cstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())); + if (ret != Z_OK) { + set_failure("deflateSetDictionary failed"); + break; + } + + if (GetDeflateExecutionPath(&cstream) != ZLIB) { + set_failure("dictionary stream did not stay on zlib"); + break; + } + + std::string input; + input.reserve(64 * 1024); + for (int j = 0; j < 1024; ++j) { + input += "tid="; + input += std::to_string(worker_id); + input += " iter="; + input += std::to_string(i); + input += " row="; + input += std::to_string(j); + input += " payload="; + input += dictionary; + input += "\n"; + } + + std::vector compressed(compressBound(input.size())); + cstream.next_in = + reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + int def_ret = Z_OK; + for (int guard = 0; guard < 64; ++guard) { + def_ret = deflate(&cstream, Z_FINISH); + if (def_ret == Z_STREAM_END) { + break; + } + if (def_ret != Z_OK && def_ret != Z_BUF_ERROR) { + break; + } + } + if (def_ret != Z_STREAM_END) { + set_failure("deflate did not reach Z_STREAM_END"); + break; + } + + const size_t compressed_size = compressed.size() - cstream.avail_out; + if (compressed_size == 0) { + set_failure("compressed payload was empty"); + break; + } + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ret = inflateInit2(&dstream, 15); + if (ret != Z_OK) { + set_failure("inflateInit2 failed"); + break; + } + + std::vector output(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + ret = inflate(&dstream, Z_NO_FLUSH); + if (ret == Z_NEED_DICT) { + ret = inflateSetDictionary( + &dstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())); + if (ret == Z_OK) { + ret = inflate(&dstream, Z_FINISH); + } + } + + const size_t produced = output.size() - dstream.avail_out; + const bool output_matches = + (produced == input.size()) && + (memcmp(output.data(), input.data(), input.size()) == 0); + + inflateEnd(&dstream); + + if (ret != Z_STREAM_END) { + set_failure("inflate did not reach Z_STREAM_END"); + break; + } + if (!output_matches) { + set_failure("round-trip data mismatch"); + break; + } + } + + deflateEnd(&cstream); + }; + + std::vector workers; + workers.reserve(thread_count); + for (int worker_id = 0; worker_id < thread_count; ++worker_id) { + workers.emplace_back(worker, worker_id); + } + for (auto& worker_thread : workers) { + worker_thread.join(); + } + + ASSERT_FALSE(failed.load()) << failure_reason; +} + +TEST(IGZIPDeflateRegressionTest, + IGZIPDictionaryOutputMustBeZlibDictionaryCompatible) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + const std::string dictionary = + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + std::string input; + for (int i = 0; i < 512; ++i) { + input += "doc:"; + input += dictionary; + input += "|"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + ASSERT_EQ(deflateSetDictionary( + &cstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + + std::vector compressed(compressBound(input.size())); + cstream.next_in = reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + int ret = Z_OK; + for (int iter = 0; iter < 32; ++iter) { + ret = deflate(&cstream, Z_FINISH); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + const size_t compressed_size = compressed.size() - cstream.avail_out; + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector uncompressed(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = uncompressed.data(); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_NO_FLUSH); + if (ret == Z_NEED_DICT) { + ASSERT_EQ(inflateSetDictionary( + &dstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ret = inflate(&dstream, Z_FINISH); + } + + EXPECT_EQ(ret, Z_STREAM_END); + const size_t produced = uncompressed.size() - dstream.avail_out; + ASSERT_EQ(produced, input.size()); + EXPECT_EQ(memcmp(uncompressed.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +TEST(IGZIPDeflateRegressionTest, + FinishWithTinyOutputBufferMustNotTruncateStream) { + SetCompressPath(IGZIP, true, false, false); + SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } + + std::string input; + input.reserve(256 * 1024); + for (int i = 0; i < 4096; ++i) { + input += "{\"k\":"; + input += std::to_string(i); + input += ",\"msg\":\"abcdefghijklmnopqrstuvwxyz\"}"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + cstream.next_in = reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + + std::vector compressed; + compressed.reserve(input.size()); + + int ret = Z_OK; + for (int iter = 0; iter < 20000; ++iter) { + unsigned char out_chunk[64]; + cstream.next_out = out_chunk; + cstream.avail_out = sizeof(out_chunk); + + ret = deflate(&cstream, Z_FINISH); + const ExecutionPath path = GetDeflateExecutionPath(&cstream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(ret, Z_DATA_ERROR) << "iter=" << iter; + + const size_t produced = sizeof(out_chunk) - cstream.avail_out; + compressed.insert(compressed.end(), out_chunk, out_chunk + produced); + + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector output(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + for (int iter = 0; iter < 1024; ++iter) { + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + if (ret == Z_BUF_ERROR && dstream.avail_in == 0) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + const size_t out_size = output.size() - dstream.avail_out; + ASSERT_EQ(out_size, input.size()); + EXPECT_EQ(memcmp(output.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +TEST(IGZIPDeflateRegressionTest, SyncFlushWithInputMustStayOnIGZIPPath) { + // Originally this test asserted the path was ZLIB (IGZIPShouldFallbackDeflate + // redirected SYNC_FLUSH to zlib). That function is removed; IGZIP now handles + // SYNC_FLUSH natively and the stream must stay on IGZIP throughout. + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + std::string input; + input.reserve(64 * 1024); + for (int i = 0; i < 1024; ++i) { + input += "record-"; + input += std::to_string(i); + input += "-abcdefghijklmnopqrstuvwxyz"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector compressed; + compressed.reserve(input.size()); + + cstream.next_in = reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + + unsigned char sync_chunk[256]; + cstream.next_out = sync_chunk; + cstream.avail_out = sizeof(sync_chunk); + + int ret = deflate(&cstream, Z_SYNC_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP); + const size_t sync_produced = sizeof(sync_chunk) - cstream.avail_out; + compressed.insert(compressed.end(), sync_chunk, sync_chunk + sync_produced); + + for (int iter = 0; iter < 8192; ++iter) { + unsigned char out_chunk[256]; + cstream.next_out = out_chunk; + cstream.avail_out = sizeof(out_chunk); + cstream.next_in = nullptr; + cstream.avail_in = 0; + + ret = deflate(&cstream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP); + + const size_t produced = sizeof(out_chunk) - cstream.avail_out; + compressed.insert(compressed.end(), out_chunk, out_chunk + produced); + + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector output(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + for (int iter = 0; iter < 1024; ++iter) { + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + const size_t out_size = output.size() - dstream.avail_out; + ASSERT_EQ(out_size, input.size()); + EXPECT_EQ(memcmp(output.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +TEST(IGZIPInflateRegressionTest, + NeedDictFromIGZIPMustFallbackToZlibOnFirstInflateCall) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + const std::string dictionary = + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + std::string input; + for (int i = 0; i < 512; ++i) { + input += "doc:"; + input += dictionary; + input += "|"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + ASSERT_EQ(deflateSetDictionary( + &cstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + + std::vector compressed(compressBound(input.size())); + cstream.next_in = reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + int ret = Z_OK; + for (int iter = 0; iter < 64; ++iter) { + ret = deflate(&cstream, Z_FINISH); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + const size_t compressed_size = compressed.size() - cstream.avail_out; + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector uncompressed(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = uncompressed.data(); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&dstream), ZLIB); + ASSERT_EQ(ret, Z_NEED_DICT); + + ASSERT_EQ(inflateSetDictionary( + &dstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ret = inflate(&dstream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + const size_t out_size = uncompressed.size() - dstream.avail_out; + ASSERT_EQ(out_size, input.size()); + EXPECT_EQ(memcmp(uncompressed.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +#ifdef USE_IAA +// IAA->IGZIP fallback tests. +// On machines without IAA hardware, CompressIAA/UncompressIAA return non-zero, +// which naturally triggers the fallback path. These tests verify: +// - When iaa_fallback_igzip=1: the stream lands on IGZIP after IAA fails. +// - When iaa_fallback_igzip=0: the stream falls through to zlib, not IGZIP. + +TEST(IAAFallbackIGZIPTest, DeflateUsesIGZIPWhenIAAFailsAndFallbackEnabled) { + SetCompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false, + /*qat_compression_allow_chunking=*/false); + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 1); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(deflateBound(&stream, input_length)); + stream.next_in = reinterpret_cast(input); + stream.avail_in = static_cast(input_length); + stream.next_out = output.data(); + stream.avail_out = static_cast(output.size()); + + int ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // If IAA hardware is absent, the fallback must have routed to IGZIP. + // If IAA hardware is present and succeeds, IAA path is also acceptable. + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_TRUE(path == IGZIP || path == IAA) + << "Expected IGZIP (fallback) or IAA (hardware success), got " + << static_cast(path); + + deflateEnd(&stream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, DeflateDoesNotUseIGZIPWhenFallbackDisabled) { + SetCompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false, + /*qat_compression_allow_chunking=*/false); + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 0); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(deflateBound(&stream, input_length)); + stream.next_in = reinterpret_cast(input); + stream.avail_in = static_cast(input_length); + stream.next_out = output.data(); + stream.avail_out = static_cast(output.size()); + + deflate(&stream, Z_FINISH); + + // With fallback disabled, IGZIP must not be selected for an IAA-configured + // stream; it should fall through to zlib. + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_NE(path, IGZIP) << "IGZIP must not be used when iaa_fallback_igzip=0"; + + deflateEnd(&stream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, InflateUsesIGZIPWhenIAAFailsAndFallbackEnabled) { + // Compress with IGZIP so the output is IGZIP-compatible (4kB window). + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 1); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + ASSERT_EQ(ret, Z_STREAM_END); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + + std::vector uncompressed(input_length); + dstream.next_in = reinterpret_cast(compressed.data()); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = reinterpret_cast(uncompressed.data()); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // If IAA hardware is absent, fallback must route to IGZIP. + // If IAA hardware is present and succeeds, IAA is also acceptable. + const ExecutionPath path = GetInflateExecutionPath(&dstream); + EXPECT_TRUE(path == IGZIP || path == IAA) + << "Expected IGZIP (fallback) or IAA (hardware success), got " + << static_cast(path); + + EXPECT_EQ(memcmp(uncompressed.data(), input, input_length), 0); + + inflateEnd(&dstream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, InflateDoesNotUseIGZIPWhenFallbackDisabled) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 0); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + ASSERT_EQ(ret, Z_STREAM_END); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + + std::vector uncompressed(input_length); + dstream.next_in = reinterpret_cast(compressed.data()); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = reinterpret_cast(uncompressed.data()); + dstream.avail_out = static_cast(uncompressed.size()); + + inflate(&dstream, Z_FINISH); + + // With fallback disabled, IGZIP must not be selected for an IAA-configured + // inflate stream; it should fall through to zlib. + const ExecutionPath path = GetInflateExecutionPath(&dstream); + EXPECT_NE(path, IGZIP) << "IGZIP must not be used when iaa_fallback_igzip=0"; + + inflateEnd(&dstream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} +#endif // USE_IAA + +#endif + INSTANTIATE_TEST_SUITE_P( CompressDecompress, ZlibPartialAndMultiStreamTest, testing::Combine( @@ -1122,6 +2566,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -1133,6 +2581,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(-15, 15, 31), @@ -1203,6 +2655,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -1214,6 +2670,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(31), @@ -1226,6 +2686,120 @@ INSTANTIATE_TEST_SUITE_P( class ConfigLoaderTest : public ::testing::Test {}; +#if defined(USE_IGZIP) || defined(USE_QAT) || defined(USE_IAA) +class DictionaryMidstreamFallbackRegressionTest : public ::testing::Test {}; + +static void RunMidstreamSetDictionaryRegression(ExecutionPath accel_path) { + SetCompressPath(accel_path, true, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + std::string first_chunk; + std::string second_chunk; + for (int i = 0; i < 256; ++i) { + first_chunk += "first:"; + first_chunk += std::to_string(i); + first_chunk += ":abcdefghijklmnopqrstuvwxyz|"; + + second_chunk += "second:"; + second_chunk += std::to_string(i); + second_chunk += ":0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ|"; + } + const std::string expected = first_chunk + second_chunk; + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector compressed(compressBound(expected.size()) + 512); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + cstream.next_in = + reinterpret_cast(const_cast(first_chunk.data())); + cstream.avail_in = static_cast(first_chunk.size()); + int ret = deflate(&cstream, Z_NO_FLUSH); + ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR); + const ExecutionPath observed_before_dict = GetDeflateExecutionPath(&cstream); + EXPECT_TRUE(observed_before_dict == accel_path || + observed_before_dict == ZLIB); + + const unsigned char dict[] = "midstream-dictionary"; + const int dict_ret = + deflateSetDictionary(&cstream, dict, static_cast(sizeof(dict) - 1)); + + // zlib semantics require dictionary to be set before any deflate output. + // Accepting this call midstream can desynchronize accelerator and zlib state. + EXPECT_EQ(dict_ret, Z_STREAM_ERROR); + // Rejected dictionary update must not mutate active stream path. + EXPECT_EQ(GetDeflateExecutionPath(&cstream), observed_before_dict); + + cstream.next_in = + reinterpret_cast(const_cast(second_chunk.data())); + cstream.avail_in = static_cast(second_chunk.size()); + for (int guard = 0; guard < 128; ++guard) { + ret = deflate(&cstream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + + const size_t compressed_size = compressed.size() - cstream.avail_out; + ASSERT_GT(compressed_size, 0u); + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector uncompressed(expected.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = uncompressed.data(); + dstream.avail_out = static_cast(uncompressed.size()); + + for (int guard = 0; guard < 128; ++guard) { + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + const size_t produced = uncompressed.size() - dstream.avail_out; + ASSERT_EQ(produced, expected.size()); + EXPECT_EQ(memcmp(uncompressed.data(), expected.data(), expected.size()), 0); + + inflateEnd(&dstream); +} + +#ifdef USE_IGZIP +TEST_F(DictionaryMidstreamFallbackRegressionTest, + IGZIPRejectsMidstreamSetDictionary) { + RunMidstreamSetDictionaryRegression(IGZIP); +} +#endif + +#ifdef USE_QAT +TEST_F(DictionaryMidstreamFallbackRegressionTest, + QATRejectsMidstreamSetDictionary) { + RunMidstreamSetDictionaryRegression(QAT); +} +#endif + +#ifdef USE_IAA +TEST_F(DictionaryMidstreamFallbackRegressionTest, + IAARejectsMidstreamSetDictionary) { + RunMidstreamSetDictionaryRegression(IAA); +} +#endif +#endif + void CreateAndWriteTempConfigFile(const char* file_path) { std::ofstream temp_file(file_path); temp_file << "use_qat_compress=5000\n"; @@ -1272,6 +2846,8 @@ TEST_F(ConfigLoaderTest, LoadValidConfig) { EXPECT_EQ(GetConfig(USE_QAT_UNCOMPRESS), 1); EXPECT_EQ(GetConfig(USE_IAA_COMPRESS), 0); EXPECT_EQ(GetConfig(USE_IAA_UNCOMPRESS), 0); + EXPECT_EQ(GetConfig(USE_IGZIP_COMPRESS), 0); + EXPECT_EQ(GetConfig(USE_IGZIP_UNCOMPRESS), 0); EXPECT_EQ(GetConfig(USE_ZLIB_COMPRESS), 1); EXPECT_EQ(GetConfig(USE_ZLIB_UNCOMPRESS), 1); EXPECT_EQ(GetConfig(LOG_LEVEL), 2); diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 26188e7..cd010b7 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -20,6 +21,9 @@ #ifdef USE_IAA #include "iaa.h" #endif +#ifdef USE_IGZIP +#include "igzip.h" +#endif #ifdef USE_QAT #include "qat.h" #endif @@ -177,6 +181,8 @@ static void cleanup_zlib_accel(void) { // Avoid recursive call (e.g., if QATzip falls back to zlib internally) static thread_local bool in_call = false; +constexpr uint8_t ZLIB_FDICT_MASK = 0x20; + struct DeflateSettings { DeflateSettings(int _level, int _method, int _window_bits, int _mem_level, int _strategy) @@ -192,12 +198,18 @@ struct DeflateSettings { int mem_level; int strategy; ExecutionPath path = UNDEFINED; + struct isal_zstream* isal_strm = nullptr; }; struct InflateSettings { - InflateSettings(int _window_bits) : window_bits(_window_bits) {} + InflateSettings(int _window_bits) + : window_bits(_window_bits), read_in_correction_applied(0) {} int window_bits; + int read_in_correction_applied; /* set once per stream when the + read_in_length over-consumption correction + fires; cleared only on inflateReset */ ExecutionPath path = UNDEFINED; + struct inflate_state* isal_strm = nullptr; }; class DeflateStreamSettings { @@ -234,6 +246,26 @@ class InflateStreamSettings { }; InflateStreamSettings inflate_stream_settings; +static void SetDeflatePath(DeflateSettings* settings, z_streamp strm, + ExecutionPath new_path, const char* reason) { + if (settings == nullptr || settings->path == new_path) { + return; + } + (void)strm; + (void)reason; + settings->path = new_path; +} + +static void SetInflatePath(InflateSettings* settings, z_streamp strm, + ExecutionPath new_path, const char* reason) { + if (settings == nullptr || settings->path == new_path) { + return; + } + (void)strm; + (void)reason; + settings->path = new_path; +} + int ZEXPORT deflateInit_(z_streamp strm, int level, const char* version, int stream_size) { Log(LogLevel::LOG_INFO, "deflateInit_ Line ", __LINE__, ", strm ", @@ -263,8 +295,20 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, Log(LogLevel::LOG_INFO, "deflateSetDictionary Line ", __LINE__, ", strm ", static_cast(strm), ", dictLength ", dictLength, "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); - deflate_settings->path = ZLIB; - return orig_deflateSetDictionary(strm, dictionary, dictLength); + // Reject mid-stream: if an accelerator is active, the underlying zlib + // stream has not been advanced, so orig_deflateSetDictionary would + // incorrectly accept the call. Per zlib spec, dictionary must be set before + // compression begins. + if (deflate_settings != nullptr && deflate_settings->path != UNDEFINED && + deflate_settings->path != ZLIB) { + return Z_STREAM_ERROR; + } + const int ret = orig_deflateSetDictionary(strm, dictionary, dictLength); + if (ret == Z_OK) { + SetDeflatePath(deflate_settings, strm, ZLIB, + "deflateSetDictionary called"); + } + return ret; } Log(LogLevel::LOG_INFO, "deflateSetDictionary Line ", __LINE__, " ignored because ignore_zlib_dictionary is set to ", @@ -280,30 +324,43 @@ int ZEXPORT deflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", - static_cast(deflate_settings->path), "\n"); + static_cast(deflate_settings->path), ", path_name ", + static_cast(deflate_settings->path), ", window_bits ", + deflate_settings->window_bits, ", total_in ", strm->total_in, + ", total_out ", strm->total_out, ", adler ", strm->adler, "\n"); int ret = 1; bool iaa_available = false; bool qat_available = false; - if (!in_call && flush == Z_FINISH && deflate_settings->path != ZLIB) { + bool igzip_available = false; + if (!in_call && deflate_settings->path != ZLIB) { uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; + bool igzip_stream_active = false; #ifdef USE_IAA - iaa_available = configs[USE_IAA_COMPRESS] && + iaa_available = (flush == Z_FINISH) && configs[USE_IAA_COMPRESS] && SupportedOptionsIAA(deflate_settings->window_bits, input_len, output_len); #endif #ifdef USE_QAT qat_available = - configs[USE_QAT_COMPRESS] && output_len >= QAT_DEST_BUFFER_MIN_SIZE && + (flush == Z_FINISH) && configs[USE_QAT_COMPRESS] && + output_len >= QAT_DEST_BUFFER_MIN_SIZE && SupportedOptionsQAT(deflate_settings->window_bits, input_len); #endif +#ifdef USE_IGZIP + igzip_stream_active = (deflate_settings->path == IGZIP && + deflate_settings->isal_strm != nullptr); + igzip_available = configs[USE_IGZIP_COMPRESS]; +#endif // If both accelerators are enabled, send configured ratio of requests to // one or the other ExecutionPath path_selected = ZLIB; - if (iaa_available && qat_available) { + if (igzip_stream_active) { + path_selected = IGZIP; + } else if (iaa_available && qat_available) { if (static_cast(std::rand() % 100) < configs[IAA_COMPRESS_PERCENTAGE]) { path_selected = IAA; @@ -314,6 +371,8 @@ int ZEXPORT deflate(z_streamp strm, int flush) { path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -325,7 +384,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { ret = CompressIAA(strm->next_in, &input_len, strm->next_out, &output_len, qpl_path_hardware, deflate_settings->window_bits, max_compressed_size); - deflate_settings->path = IAA; + SetDeflatePath(deflate_settings, strm, IAA, "selected IAA accelerator"); in_call = false; INCREMENT_STAT(DEFLATE_IAA_COUNT); INCREMENT_STAT_COND(ret != 0, DEFLATE_IAA_ERROR_COUNT); @@ -335,12 +394,56 @@ int ZEXPORT deflate(z_streamp strm, int flush) { in_call = true; ret = CompressQAT(strm->next_in, &input_len, strm->next_out, &output_len, deflate_settings->window_bits); - deflate_settings->path = QAT; + SetDeflatePath(deflate_settings, strm, QAT, "selected QAT accelerator"); in_call = false; INCREMENT_STAT(DEFLATE_QAT_COUNT); INCREMENT_STAT_COND(ret != 0, DEFLATE_QAT_ERROR_COUNT); #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + if (deflate_settings->isal_strm == nullptr) { + deflate_settings->method = 0; + deflate_settings->isal_strm = InitCompressIGZIP( + deflate_settings->level, deflate_settings->window_bits); + } + in_call = true; + ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, + &input_len, strm->next_out, &output_len, + &strm->total_in, &strm->total_out); + SetDeflatePath(deflate_settings, strm, IGZIP, + "selected IGZIP accelerator"); + in_call = false; + + INCREMENT_STAT(DEFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); +#endif + } + +#ifdef USE_IGZIP + // IAA→IGZIP fallback: if IAA failed and IGZIP is available, retry with + // IGZIP before falling through to software zlib. + if (path_selected == IAA && ret != 0 && configs[IAA_FALLBACK_IGZIP] && + igzip_available) { + // IAA may have modified input_len/output_len on failure — restore them. + input_len = strm->avail_in; + output_len = strm->avail_out; + if (deflate_settings->isal_strm == nullptr) { + deflate_settings->method = 0; + deflate_settings->isal_strm = InitCompressIGZIP( + deflate_settings->level, deflate_settings->window_bits); + } + in_call = true; + ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, + &input_len, strm->next_out, &output_len, + &strm->total_in, &strm->total_out); + SetDeflatePath(deflate_settings, strm, IGZIP, + "IAA failed, IGZIP fallback"); + in_call = false; + path_selected = IGZIP; // use IGZIP return-code semantics below + INCREMENT_STAT(DEFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); } +#endif // USE_IGZIP iaa fallback if (ret == 0) { strm->next_in += input_len; @@ -349,25 +452,45 @@ int ZEXPORT deflate(z_streamp strm, int flush) { strm->next_out += output_len; strm->avail_out -= output_len; strm->total_out += output_len; - if (strm->avail_in == 0) { - ret = Z_STREAM_END; + if (path_selected == IGZIP) { + const bool no_progress = (input_len == 0 && output_len == 0); + bool finish_done = false; +#ifdef USE_IGZIP + finish_done = (flush == Z_FINISH) && + IsIGZIPDeflateFinished(deflate_settings->isal_strm); +#endif + + if (finish_done) { + ret = Z_STREAM_END; + } else if (!no_progress) { + ret = Z_OK; + } else { + ret = Z_BUF_ERROR; + } } else { - ret = Z_BUF_ERROR; + if (strm->avail_in == 0) { + ret = Z_STREAM_END; + } else { + ret = Z_BUF_ERROR; + } } Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", accelerator return code ", ret, - ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, - ", path ", static_cast(deflate_settings->path), "\n"); + ", bytes_in ", input_len, ", bytes_out ", output_len, ", avail_in ", + strm->avail_in, ", avail_out ", strm->avail_out, ", path ", + static_cast(deflate_settings->path), ", path_name ", + static_cast(deflate_settings->path), "\n"); return ret; } } - if (in_call || configs[USE_ZLIB_COMPRESS]) { + if (in_call || configs[USE_ZLIB_COMPRESS] || deflate_settings->path == ZLIB) { ret = orig_deflate(strm, flush); INCREMENT_STAT(DEFLATE_ZLIB_COUNT); if (!in_call) { - deflate_settings->path = ZLIB; + SetDeflatePath(deflate_settings, strm, ZLIB, + "zlib fallback or configured zlib"); } } else { ret = Z_DATA_ERROR; @@ -376,6 +499,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", zlib return code ", ret, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", + static_cast(deflate_settings->path), ", path_name ", static_cast(deflate_settings->path), "\n"); INCREMENT_STAT_COND(ret < 0, DEFLATE_ERROR_COUNT); @@ -385,6 +509,12 @@ int ZEXPORT deflate(z_streamp strm, int flush) { int ZEXPORT deflateEnd(z_streamp strm) { Log(LogLevel::LOG_INFO, "deflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); + DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); + if (deflate_settings->isal_strm != nullptr) { +#ifdef USE_IGZIP + EndCompressIGZIP(deflate_settings->isal_strm); +#endif + } deflate_stream_settings.Unset(strm); return orig_deflateEnd(strm); } @@ -393,18 +523,25 @@ int ZEXPORT deflateReset(z_streamp strm) { Log(LogLevel::LOG_INFO, "deflateReset Line ", __LINE__, ", strm ", static_cast(strm), "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); + int ret = orig_deflateReset(strm); if (deflate_settings != nullptr) { - deflate_settings->path = UNDEFINED; + SetDeflatePath(deflate_settings, strm, UNDEFINED, "deflateReset"); + +#ifdef USE_IGZIP + if (deflate_settings->isal_strm != nullptr) { + ResetCompressIGZIP(deflate_settings->isal_strm, + deflate_settings->window_bits); + } +#endif } - return orig_deflateReset(strm); + return ret; } int ZEXPORT inflateInit_(z_streamp strm, const char* version, int stream_size) { inflate_stream_settings.Set(strm, 15); Log(LogLevel::LOG_INFO, "inflateInit_ Line ", __LINE__, ", strm ", static_cast(strm), "\n"); - return orig_inflateInit_(strm, version, stream_size); } @@ -413,7 +550,6 @@ int ZEXPORT inflateInit2_(z_streamp strm, int window_bits, const char* version, inflate_stream_settings.Set(strm, window_bits); Log(LogLevel::LOG_INFO, "inflateInit2_ Line ", __LINE__, ", strm ", static_cast(strm), ", window_bits ", window_bits, "\n"); - return orig_inflateInit2_(strm, window_bits, version, stream_size); } @@ -421,10 +557,14 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, uInt dictLength) { if (!configs[IGNORE_ZLIB_DICTIONARY]) { Log(LogLevel::LOG_INFO, "inflateSetDictionary Line ", __LINE__, ", strm ", - static_cast(strm), "dictLength ", dictLength, "\n"); + static_cast(strm), ", dictLength ", dictLength, "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); - inflate_settings->path = ZLIB; - return orig_inflateSetDictionary(strm, dictionary, dictLength); + const int ret = orig_inflateSetDictionary(strm, dictionary, dictLength); + if (ret == Z_OK) { + SetInflatePath(inflate_settings, strm, ZLIB, + "inflateSetDictionary called"); + } + return ret; } Log(LogLevel::LOG_INFO, "inflateSetDictionary Line ", __LINE__, " ignored because ignore_zlib_dictionary is set to ", @@ -434,13 +574,18 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, int ZEXPORT inflate(z_streamp strm, int flush) { InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); + INCREMENT_STAT(INFLATE_COUNT); PrintStats(); Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", - static_cast(inflate_settings->path), "\n"); + static_cast(inflate_settings->path), ", path_name ", + static_cast(inflate_settings->path), ", window_bits ", + inflate_settings->window_bits, ", total_in ", strm->total_in, + ", total_out ", strm->total_out, ", adler ", strm->adler, "\n"); + PrintDeflateBlockHeader(LogLevel::LOG_INFO, strm->next_in, strm->avail_in, inflate_settings->window_bits); @@ -448,7 +593,50 @@ int ZEXPORT inflate(z_streamp strm, int flush) { bool end_of_stream = true; bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; + + const bool igzip_stream_active = (inflate_settings->path == IGZIP && + inflate_settings->isal_strm != nullptr); + +#ifdef USE_IGZIP + const bool igzip_supported_options = + !in_call && configs[USE_IGZIP_UNCOMPRESS]; + + // Keep stateful IGZIP stream handling on the same engine. + // For avail_in==0, let IGZIP process any buffered bits in its internal + // state before reporting Z_BUF_ERROR. + if (!in_call && igzip_stream_active && strm->avail_in == 0) { + in_call = true; + IGZIPNoInputAction action = IGZIPHandleActiveStreamNoInput( + strm, inflate_settings->isal_strm, inflate_settings->window_bits, + &inflate_settings->read_in_correction_applied, &ret); + in_call = false; + + if (action == IGZIP_NO_INPUT_FALLBACK_ZLIB) { + SetInflatePath(inflate_settings, strm, ZLIB, + "igzip raw input_done ambiguity fallback"); + // Continue through normal zlib fallback path. + } else if (action == IGZIP_NO_INPUT_RETURN) { + return ret; + } + } +#endif + + // Early detection: if this is a zlib-format stream with the FDICT bit set + // in the header, pin to ZLIB immediately so dictionary streams never reach + // any accelerator (QAT/IAA/IGZIP don't support preset dictionaries). + if (!in_call && inflate_settings->path == UNDEFINED && + inflate_settings->window_bits >= 8 && + inflate_settings->window_bits <= 15 && strm->avail_in >= 2 && + (strm->next_in[1] & ZLIB_FDICT_MASK)) { + SetInflatePath(inflate_settings, strm, ZLIB, + "FDICT bit detected in zlib header"); + } + if (!in_call && strm->avail_in > 0 && inflate_settings->path != ZLIB) { +#ifdef USE_IGZIP + const uInt pre_avail_in = strm->avail_in; +#endif uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; @@ -465,11 +653,16 @@ int ZEXPORT inflate(z_streamp strm, int flush) { configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(inflate_settings->window_bits, input_len); #endif +#ifdef USE_IGZIP + igzip_available = igzip_supported_options; +#endif // If both accelerators are enabled, send configured ratio of requests to // one or the other ExecutionPath path_selected = ZLIB; - if (iaa_available && qat_available) { + if (igzip_stream_active) { + path_selected = IGZIP; + } else if (iaa_available && qat_available) { if (static_cast(std::rand()) % 100 < configs[IAA_UNCOMPRESS_PERCENTAGE]) { path_selected = IAA; @@ -480,15 +673,16 @@ int ZEXPORT inflate(z_streamp strm, int flush) { path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } - if (path_selected == IAA) { #ifdef USE_IAA in_call = true; ret = UncompressIAA(strm->next_in, &input_len, strm->next_out, &output_len, qpl_path_hardware, inflate_settings->window_bits, &end_of_stream); - inflate_settings->path = IAA; + SetInflatePath(inflate_settings, strm, IAA, "selected IAA accelerator"); // IAA inflate is stateless in this wrapper. If stream end was not // reached, use zlib for stateful continuation. if (!end_of_stream) { @@ -504,7 +698,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ret = UncompressQAT(strm->next_in, &input_len, strm->next_out, &output_len, inflate_settings->window_bits, &end_of_stream); - inflate_settings->path = QAT; + SetInflatePath(inflate_settings, strm, QAT, "selected QAT accelerator"); // QATzip does not support stateful decompression // Fall back to zlib if end-of-stream not reached in one call if (!end_of_stream) { @@ -514,8 +708,85 @@ int ZEXPORT inflate(z_streamp strm, int flush) { INCREMENT_STAT(INFLATE_QAT_COUNT); INCREMENT_STAT_COND(ret != 0, INFLATE_QAT_ERROR_COUNT); #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + const IGZIPInflatePathAction path_action = + IGZIPRunInflateAndSelectPathAction( + strm, &inflate_settings->isal_strm, inflate_settings->window_bits, + &inflate_settings->read_in_correction_applied, &input_len, + &output_len, &ret, &end_of_stream, pre_avail_in); + in_call = false; + + if (inflate_settings->isal_strm == nullptr) { + return Z_DATA_ERROR; + } + + if (path_action == IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT) { + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), + " source=igzip", " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, "\n"); + SetInflatePath(inflate_settings, strm, ZLIB, + "IGZIP returned Z_NEED_DICT"); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR) { + SetInflatePath(inflate_settings, strm, ZLIB, + "IGZIP requested raw trailer fallback"); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY) { + SetInflatePath(inflate_settings, strm, ZLIB, + "raw boundary guard fallback"); + } else if (path_action == IGZIP_INFLATE_PATH_SET_IGZIP && + inflate_settings->path != ZLIB) { + SetInflatePath(inflate_settings, strm, IGZIP, + "selected IGZIP accelerator"); + } + INCREMENT_STAT(INFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); +#endif } +#ifdef USE_IGZIP + // IAA→IGZIP fallback: if IAA failed and IGZIP is available, retry with + // IGZIP before falling through to software zlib. + if (path_selected == IAA && ret != 0 && configs[IAA_FALLBACK_IGZIP] && + igzip_available) { + // IAA may have modified input_len/output_len on failure — restore them. + input_len = strm->avail_in; + output_len = strm->avail_out; + end_of_stream = true; + in_call = true; + const IGZIPInflatePathAction path_action = + IGZIPRunInflateAndSelectPathAction( + strm, &inflate_settings->isal_strm, inflate_settings->window_bits, + &inflate_settings->read_in_correction_applied, &input_len, + &output_len, &ret, &end_of_stream, pre_avail_in); + in_call = false; + + if (inflate_settings->isal_strm == nullptr) { + return Z_DATA_ERROR; + } + + if (path_action == IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT) { + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), + " source=igzip (iaa fallback)", " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, "\n"); + SetInflatePath(inflate_settings, strm, ZLIB, + "IAA->IGZIP fallback: Z_NEED_DICT"); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR) { + SetInflatePath(inflate_settings, strm, ZLIB, + "IAA->IGZIP fallback: raw trailer"); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY) { + SetInflatePath(inflate_settings, strm, ZLIB, + "IAA->IGZIP fallback: raw boundary"); + } else if (path_action == IGZIP_INFLATE_PATH_SET_IGZIP && + inflate_settings->path != ZLIB) { + SetInflatePath(inflate_settings, strm, IGZIP, + "IAA failed, IGZIP fallback succeeded"); + } + INCREMENT_STAT(INFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); + } +#endif // USE_IGZIP iaa fallback + if (ret == 0) { strm->next_in += input_len; strm->avail_in -= input_len; @@ -535,18 +806,27 @@ int ZEXPORT inflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", accelerator return code ", ret, - ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, - ", end_of_stream ", end_of_stream, ", path ", - static_cast(inflate_settings->path), "\n"); + ", bytes_in ", input_len, ", bytes_out ", output_len, ", avail_in ", + strm->avail_in, ", avail_out ", strm->avail_out, ", end_of_stream ", + end_of_stream, ", path ", static_cast(inflate_settings->path), + ", path_name ", static_cast(inflate_settings->path), + ", window_bits ", inflate_settings->window_bits, "\n"); return ret; } } - if (in_call || configs[USE_ZLIB_UNCOMPRESS]) { + if (in_call || configs[USE_ZLIB_UNCOMPRESS] || + inflate_settings->path == ZLIB) { ret = orig_inflate(strm, flush); + if (ret == Z_NEED_DICT) { + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), + " source=zlib", " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, "\n"); + } INCREMENT_STAT(INFLATE_ZLIB_COUNT); if (!in_call) { - inflate_settings->path = ZLIB; + SetInflatePath(inflate_settings, strm, ZLIB, + "zlib fallback or configured zlib"); } } else { ret = Z_DATA_ERROR; @@ -555,7 +835,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", zlib return code ", ret, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", - static_cast(inflate_settings->path), "\n"); + static_cast(inflate_settings->path), ", path_name ", + static_cast(inflate_settings->path), ", window_bits ", + inflate_settings->window_bits, "\n"); INCREMENT_STAT_COND(ret < 0, INFLATE_ERROR_COUNT); return ret; @@ -564,6 +846,12 @@ int ZEXPORT inflate(z_streamp strm, int flush) { int ZEXPORT inflateEnd(z_streamp strm) { Log(LogLevel::LOG_INFO, "inflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); + InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); + if (inflate_settings->isal_strm != nullptr) { +#ifdef USE_IGZIP + EndUncompressIGZIP(inflate_settings->isal_strm); +#endif + } inflate_stream_settings.Unset(strm); return orig_inflateEnd(strm); } @@ -572,11 +860,18 @@ int ZEXPORT inflateReset(z_streamp strm) { Log(LogLevel::LOG_INFO, "inflateReset Line ", __LINE__, ", strm ", static_cast(strm), "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); + int ret = orig_inflateReset(strm); if (inflate_settings != nullptr) { - inflate_settings->path = UNDEFINED; + SetInflatePath(inflate_settings, strm, UNDEFINED, "inflateReset"); + } + if (inflate_settings->isal_strm != nullptr) { +#ifdef USE_IGZIP + ResetUncompressIGZIP(inflate_settings->isal_strm, + &inflate_settings->read_in_correction_applied); +#endif } - return orig_inflateReset(strm); + return ret; } int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, @@ -591,6 +886,7 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA iaa_available = configs[USE_IAA_COMPRESS] && SupportedOptionsIAA(15, input_len, output_len); @@ -599,12 +895,17 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, qat_available = configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(15, input_len); #endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_COMPRESS]; +#endif ExecutionPath path_selected = ZLIB; if (iaa_available) { path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -621,6 +922,24 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, &output_len, 15); in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct isal_zstream* isal_strm = InitCompressIGZIP(level, 15); + if (isal_strm == nullptr) { + ret = 1; + } else { + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = CompressIGZIP(isal_strm, Z_FINISH, const_cast(source), + &input_len, dest, &output_len, &total_in, &total_out); + EndCompressIGZIP(isal_strm); + if (ret == 0 && input_len != sourceLen) { + ret = 1; + } + } + in_call = false; +#endif } if (ret == 0) { @@ -663,6 +982,7 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA iaa_available = configs[USE_IAA_UNCOMPRESS] && @@ -673,12 +993,17 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, qat_available = configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(15, input_len); #endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; +#endif ExecutionPath path_selected = ZLIB; if (iaa_available) { path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -698,6 +1023,26 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, &output_len, 15, &end_of_stream); in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct inflate_state* isal_strm = InitUncompressIGZIP(15); + if (isal_strm == nullptr) { + ret = 1; + } else { + int read_in_correction_applied = 0; + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = UncompressIGZIP(isal_strm, const_cast(source), &input_len, + dest, &output_len, 15, &read_in_correction_applied, + &total_in, &total_out, &end_of_stream); + EndUncompressIGZIP(isal_strm); + if (ret == 0 && !end_of_stream) { + ret = 1; + } + } + in_call = false; +#endif } if (ret == 0) { @@ -938,6 +1283,7 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, int ret = 1; bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA iaa_available = configs[USE_IAA_COMPRESS] && @@ -947,12 +1293,17 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, qat_available = configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(31, *input_length); #endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_COMPRESS]; +#endif ExecutionPath path_selected = ZLIB; if (qat_available) { path_selected = QAT; } else if (iaa_available) { path_selected = IAA; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -970,6 +1321,23 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, gz->path = QAT; in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct isal_zstream* isal_strm = + InitCompressIGZIP(Z_DEFAULT_COMPRESSION, 31); + if (isal_strm == nullptr) { + ret = 1; + } else { + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = CompressIGZIP(isal_strm, Z_FINISH, input, input_length, output, + output_length, &total_in, &total_out); + EndCompressIGZIP(isal_strm); + } + gz->path = IGZIP; + in_call = false; +#endif } return ret; } @@ -988,6 +1356,7 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, int ret = 1; bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA iaa_available = configs[USE_IAA_UNCOMPRESS] && @@ -998,12 +1367,17 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, qat_available = configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(31, *input_length); #endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; +#endif ExecutionPath path_selected = ZLIB; if (qat_available) { path_selected = QAT; } else if (iaa_available) { path_selected = IAA; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -1022,6 +1396,24 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, gz->path = QAT; in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct inflate_state* isal_strm = InitUncompressIGZIP(31); + if (isal_strm == nullptr) { + ret = 1; + } else { + int read_in_correction_applied = 0; + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = UncompressIGZIP(isal_strm, input, input_length, output, + output_length, 31, &read_in_correction_applied, + &total_in, &total_out, end_of_stream); + EndUncompressIGZIP(isal_strm); + } + gz->path = IGZIP; + in_call = false; +#endif } return ret; } @@ -1105,8 +1497,9 @@ int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len) { static_cast(file), ", buf ", buf, ", len ", len, "\n"); unsigned int written_bytes = 0; - bool accelerator_selected = - configs[USE_IAA_COMPRESS] || configs[USE_QAT_COMPRESS]; + bool accelerator_selected = configs[USE_IAA_COMPRESS] || + configs[USE_QAT_COMPRESS] || + configs[USE_IGZIP_COMPRESS]; if (gz->path != ZLIB && accelerator_selected) { gz->AllocateBuffers(); gz->data_buf_size = 256 << 10; @@ -1164,8 +1557,9 @@ int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { int ret = 1; uint32_t read_bytes = 0; - bool accelerator_selected = - configs[USE_IAA_UNCOMPRESS] || configs[USE_QAT_UNCOMPRESS]; + bool accelerator_selected = configs[USE_IAA_UNCOMPRESS] || + configs[USE_QAT_UNCOMPRESS] || + configs[USE_IGZIP_UNCOMPRESS]; if (gz->path != ZLIB && accelerator_selected) { gz->AllocateBuffers(); gz->data_buf_size = 512 << 10; @@ -1348,7 +1742,6 @@ int ZEXPORT gzclose(gzFile file) { } else { ret = orig_gzclose(file); } - Log(LogLevel::LOG_INFO, "gzclose Line ", __LINE__, ", file ", static_cast(file), ", return code ", ret, ", buffered processed ", gz->data_buf_pos, "\n"); diff --git a/zlib_accel.h b/zlib_accel.h index c763ea9..2dd2c78 100644 --- a/zlib_accel.h +++ b/zlib_accel.h @@ -7,7 +7,7 @@ #include // Visible for testing -enum ExecutionPath { UNDEFINED, ZLIB, QAT, IAA }; +enum ExecutionPath { UNDEFINED, ZLIB, QAT, IAA, IGZIP }; ExecutionPath GetDeflateExecutionPath(z_streamp strm); ExecutionPath GetInflateExecutionPath(z_streamp strm);