diff --git a/.gitignore b/.gitignore index fc1dadd..aedc64f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ /build/ -/tests/build \ No newline at end of file +/tests/build +cmake.txt +/resources/ + diff --git a/README.md b/README.md index b36a23b..102d44d 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,10 @@ make CMake supports the following options: - USE_QAT (ON/OFF): include QAT acceleration - USE_IAA (ON/OFF): include IAA acceleration +- USE_IGZIP (ON/OFF): include IGZIP acceleration (requires ISA-L) - QPL_PATH: path to QPL for IAA acceleration (if not in a standard directory) - QATZIP_PATH: path to QATzip for QAT acceleration (if not in a standard directory) +- ISAL_PATH: path to ISA-L for IGZIP acceleration (if not in a standard directory) - DEBUG_LOG (ON/OFF): enable logging - ENABLE_STATISTICS (ON/OFF): enable statistics - COVERAGE (ON/OFF): enable test coverage (more details in a later section) @@ -84,6 +86,9 @@ Requirements for IAA - [accel-config](https://github.com/intel/idxd-config) - [Query Processing Library](https://github.com/intel/qpl) +Requirements for IGZIP +- [ISA-L (Intel Intelligent Storage Acceleration Library)](https://github.com/intel/isa-l) + A setup with both QAT and IAA enabled has been tested on an AWS m7i.metal-24xl instance (Ubuntu 22.04, kernel 6.8.0). Refer to the links above for instructions on how to install the dependencies. @@ -178,20 +183,22 @@ use_zlib_uncompress - Enable zlib for decompression - Setting to 1 is recommended, to allow fall back to zlib in case accelerators cannot be used or experience an error. +iaa_fallback_igzip +- Values: 0,1. Default: 0 +- If 1, and an IAA compression or decompression operation fails, the request is retried using IGZIP (if enabled) before falling back to software zlib. Useful on machines where IAA hardware is intermittently unavailable. + iaa_compress_percentage - Values: 0-100. Default: 50 - If both IAA and QAT are enabled, percentage of compression calls to offload to IAA. iaa_prepend_empty_block - Values: 0,1. Default: 0 -- Prepend an empty stored block to the compressed data to "mark" that the data was compressed by IAA. -- IAA has a 4kB history window limit and it is not able to decompress blocks that use a longer history window (up to 32kB per deflate standard). -- During decompression, this marker indicates that the data was compressed by IAA and is therefore guarateed decompressible by IAA. +- **Deprecated.** This option is retained for backward compatibility and will be removed in a future release. Setting it to 1 has no effect on decompression. +- Background: the original design prepended a 5-byte empty stored-block marker to IAA-compressed output so the decompressor could identify IAA-produced data (which uses a 4kB history window). This approach was abandoned because QPL hardware always consumes all `available_in` bytes regardless of where the stream boundary falls, making marker-based detection unreliable when the caller does not supply the exact compressed size. IAA decompression eligibility is now determined by a 512-byte minimum input length threshold: callers such as Java's `ZipInputStream` feed chunks of ≤512 bytes when the compressed size is unknown, while Lucene stored-field reads always supply the exact size (>512 bytes). iaa_uncompress_percentage - Values: 0-100. Default: 50 - If both IAA and QAT are enabled, percentage of decompression calls to offload to IAA. -- If iaa_prepend_empty_block = 1, this percentage is only applied to data with the empty block marker. qat_periodical_polling = 0 - Values: 0,1. Default: 0 diff --git a/config/config.cpp b/config/config.cpp index 51a30af..3fa6001 100644 --- a/config/config.cpp +++ b/config/config.cpp @@ -30,6 +30,7 @@ uint32_t configs[CONFIG_MAX] = { 1, /*qat_compression_level*/ 0, /*qat_compression_allow_chunking*/ 0, /*ignore_zlib_dictionary*/ + 0, /*iaa_fallback_igzip*/ 2, /*log_level*/ 1000 /*log_stats_samples*/ }; @@ -56,6 +57,7 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { "qat_compression_level", "qat_compression_allow_chunking", "ignore_zlib_dictionary", + "iaa_fallback_igzip", "log_level", "log_stats_samples" }; @@ -91,6 +93,7 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { trySetConfig(QAT_COMPRESSION_LEVEL, 9, 1); trySetConfig(QAT_COMPRESSION_ALLOW_CHUNKING, 1, 0); trySetConfig(IGNORE_ZLIB_DICTIONARY, 1, 0); + trySetConfig(IAA_FALLBACK_IGZIP, 1, 0); trySetConfig(LOG_LEVEL, 2, 0); trySetConfig(LOG_STATS_SAMPLES, UINT32_MAX, 0); diff --git a/config/config.h b/config/config.h index 12d3563..fff483c 100644 --- a/config/config.h +++ b/config/config.h @@ -20,11 +20,12 @@ enum ConfigOption { USE_IGZIP_UNCOMPRESS, IAA_COMPRESS_PERCENTAGE, IAA_UNCOMPRESS_PERCENTAGE, - IAA_PREPEND_EMPTY_BLOCK, + IAA_PREPEND_EMPTY_BLOCK, // DEPRECATED — see README for details QAT_PERIODICAL_POLLING, QAT_COMPRESSION_LEVEL, QAT_COMPRESSION_ALLOW_CHUNKING, IGNORE_ZLIB_DICTIONARY, + IAA_FALLBACK_IGZIP, LOG_LEVEL, LOG_STATS_SAMPLES, CONFIG_MAX diff --git a/iaa.cpp b/iaa.cpp index 2bbcd27..5a22f87 100644 --- a/iaa.cpp +++ b/iaa.cpp @@ -99,9 +99,19 @@ int CompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, output_shift += GZIP_EXT_XHDR_SIZE; } - // If prepending an empty block, leave space for it to be added - // For zlib format, we don't need an empty block as a marker, as the zlib - // header includes info about the window size + // DEPRECATED: iaa_prepend_empty_block is no longer used by the decompressor. + // The original design used a 5-byte empty stored-block marker written at the + // start of IAA-compressed output so that the decompressor could detect and + // trust IAA-compressed data (which uses a 4kB history window). This approach + // was abandoned because QPL hardware always consumes all available_in bytes + // regardless of where the BFINAL=1 token falls (overconsumption bug), so a + // caller-supplied exact boundary is required instead. IsIAADecompressible now + // uses a 512-byte minimum input length threshold to gate IAA decompression: + // Java ZipInputStream feeds <=512-byte chunks (csize unknown), triggering + // overconsumption; Lucene stored-field reads always provide the exact + // compressed size (>512 bytes), where consuming all input is correct. + // The config option is retained for backward compatibility and will be + // removed in a future release. bool prepend_empty_block = false; CompressedFormat format = GetCompressedFormat(window_bits); if (format != CompressedFormat::ZLIB && @@ -255,44 +265,32 @@ bool SupportedOptionsIAA(int window_bits, uint32_t input_length, return false; } -bool PrependedEmptyBlockPresent(uint8_t* input, uint32_t input_length, - CompressedFormat format) { - uint32_t header_length = GetHeaderLength(format); - if (header_length + PREPENDED_BLOCK_LENGTH > input_length) { - return false; - } - - if (input[header_length] == 0 && input[header_length + 1] == 0 && - input[header_length + 2] == 0 && input[header_length + 3] == 0xFF && - input[header_length + 4] == 0xFF) { - Log(LogLevel::LOG_INFO, "PrependedEmptyBlockPresent() Line ", __LINE__, - " Empty block detected\n"); - return true; - } - - return false; -} - bool IsIAADecompressible(uint8_t* input, uint32_t input_length, int window_bits) { CompressedFormat format = GetCompressedFormat(window_bits); if (format == CompressedFormat::ZLIB) { int window = GetWindowSizeFromZlibHeader(input, input_length); - Log(LogLevel::LOG_INFO, "IsIAADecompressible() Line ", __LINE__, " window ", - window, "\n"); return window <= 12; - } else { - // if no empty block markers selected, we cannot tell for sure it's - // IAA-decompression, but we assume it is. - if (configs[IAA_PREPEND_EMPTY_BLOCK] == 0) { - return true; - } else if (configs[IAA_PREPEND_EMPTY_BLOCK] == 1 && - PrependedEmptyBlockPresent(input, input_length, format)) { - return true; - } else { - return false; - } } + // For raw deflate and gzip formats, QPL always reports total_in == + // available_in regardless of where BFINAL=1 falls in the stream. This is + // safe only when the caller provides avail_in == actual_compressed_size + // (e.g. Lucene stored-field reads, where the exact compressed size is known + // from the .fdt file format). + // + // Callers that do not know the compressed size a priori — notably Java's + // ZipInputStream, which uses a fixed 512-byte internal buffer and feeds + // chunks of that size to inflate() — will have avail_in > actual_csize. + // QPL consuming all 512 bytes then reporting total_in=512 when actual_csize + // was 2 triggers ZipException at the Java level. + // + // Guard: only attempt IAA when input_length > 512. ZipInputStream always + // feeds chunks of at most 512 bytes, so any call above that threshold is + // guaranteed not to be a ZipInputStream-chunked read. Lucene stored-field + // entries are typically much larger than 512 bytes; the few that are smaller + // fall back to IGZIP which is also correct. + static constexpr uint32_t kZipInputStreamBufferSize = 512; + return input_length > kZipInputStreamBufferSize; } #endif // USE_IAA diff --git a/igzip.cpp b/igzip.cpp index 954dbd7..01f1a47 100644 --- a/igzip.cpp +++ b/igzip.cpp @@ -21,8 +21,6 @@ static uint16_t ClampHistBits(int bits) { return (uint16_t)bits; } -static constexpr uint32_t kIGZIPMinFinishOutputSize = 256; - static void ConfigureDeflateWindow(struct isal_zstream *isal_strm, int windowBits) { if (windowBits < 0) { @@ -71,72 +69,6 @@ bool IsIGZIPDeflateFinished(const struct isal_zstream *stream) { return state == ZSTATE_END; } -bool SupportedOptionsIGZIPCompress(int flush, uint32_t output_length, - bool stream_on_igzip_path) { - if (flush != Z_FINISH) { - Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPCompress() Line ", __LINE__, - " flush ", flush, " is not Z_FINISH; IGZIP deflate path disabled\n"); - return false; - } - if (!stream_on_igzip_path && output_length < kIGZIPMinFinishOutputSize) { - Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPCompress() Line ", __LINE__, - " output length ", output_length, - " is less than minimum finish buffer ", kIGZIPMinFinishOutputSize, - "\n"); - return false; - } - return true; -} - -bool SupportedOptionsIGZIPUncompress(int window_bits, uint32_t input_length, - uint32_t output_length, - bool stream_on_igzip_path) { - (void)window_bits; - (void)output_length; - - if (!stream_on_igzip_path && input_length == 0) { - Log(LogLevel::LOG_INFO, "SupportedOptionsIGZIPUncompress() Line ", __LINE__, - " fallback reason=no_input_on_new_stream input_length ", input_length, - " output_length ", output_length, "\n"); - return false; - } - - return true; -} - -static bool IsIGZIPSyncFlush(int flush) { - return flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || flush == Z_BLOCK; -} - -bool IGZIPShouldFallbackDeflate(bool stream_on_igzip_path, int flush, - uint32_t avail_in) { - const bool is_streaming_flush = - (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH || - flush == Z_FULL_FLUSH || flush == Z_BLOCK); - - if (!stream_on_igzip_path && is_streaming_flush && avail_in > 0) { - Log(LogLevel::LOG_INFO, "IGZIPShouldFallbackDeflate() Line ", __LINE__, - " fallback reason=streaming_flush_with_input flush ", flush, - " avail_in ", avail_in, "\n"); - return true; - } - - if (!stream_on_igzip_path || avail_in != 0) { - return false; - } - - if (IsIGZIPSyncFlush(flush)) { - Log(LogLevel::LOG_INFO, "IGZIPShouldFallbackDeflate() Line ", __LINE__, - " fallback reason=empty_sync_flush_reentry flush ", flush, " avail_in ", - avail_in, "\n"); - return true; - } - if (flush == Z_FINISH) { - return false; - } - return false; -} - struct isal_zstream *InitCompressIGZIP(int level, int windowBits) { Log(LogLevel::LOG_INFO, "InitCompressIGZIP() Line ", __LINE__, " initializing deflate with level ", level, ", windowBits ", windowBits, @@ -242,6 +174,20 @@ int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, ", total_out ", (uint32_t)isal_strm->total_out, ", total_in ", (uint32_t)isal_strm->total_in, "\n"); + // ISA-L always emits sync bytes on SYNC_FLUSH regardless of pending data. + // When the stream is already byte-aligned (ZSTATE_NEW_HDR) and there is no + // new input, no real progress can be made — return 0 progress so the caller + // reports Z_BUF_ERROR, matching zlib's semantics for empty flush calls. + // ZSTATE_NEW_HDR is the idle/byte-aligned state in ISA-L's internal deflate + // state machine; validated against ISA-L v2.32.0 (commit c196241). + if (isal_strm->avail_in == 0 && isal_strm->flush == SYNC_FLUSH && + isal_strm->end_of_stream == 0 && + isal_strm->internal_state.state == ZSTATE_NEW_HDR) { + *output_length = 0; + *input_length = 0; + return 0; + } + int comp = isal_deflate(isal_strm); *output_length = original_avail_out - isal_strm->avail_out; @@ -360,7 +306,7 @@ struct inflate_state *InitUncompressIGZIP(int windowBits) { // strm->total_out = 0; // strm->total_in = 0; - // s->trailer_overconsumption_fixed = 0; // Initialize the workaround flag + // s->read_in_correction_applied = 0; ConfigureInflateWindow(isal_strm_inflate, windowBits); @@ -369,9 +315,9 @@ struct inflate_state *InitUncompressIGZIP(int windowBits) { IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( z_streamp strm, struct inflate_state *isal_strm_inflate, int window_bits, - int *tofixed, int *ret) { + int *read_in_correction_applied, int *ret) { if (strm == nullptr || isal_strm_inflate == nullptr || ret == nullptr || - tofixed == nullptr || strm->avail_in != 0) { + read_in_correction_applied == nullptr || strm->avail_in != 0) { return IGZIP_NO_INPUT_NOT_HANDLED; } @@ -380,8 +326,9 @@ IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( bool end_of_stream = true; *ret = UncompressIGZIP(isal_strm_inflate, strm->next_in, &input_len, - strm->next_out, &output_len, window_bits, tofixed, - &strm->total_in, &strm->total_out, &end_of_stream); + strm->next_out, &output_len, window_bits, + read_in_correction_applied, &strm->total_in, + &strm->total_out, &end_of_stream); if (*ret == Z_DATA_ERROR) { Log(LogLevel::LOG_INFO, "IGZIPHandleActiveStreamNoInput() Line ", __LINE__, @@ -407,11 +354,12 @@ IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits, - int *tofixed, uint32_t *input_length, uint32_t *output_length, int *ret, - bool *end_of_stream, uint32_t pre_avail_in) { + int *read_in_correction_applied, uint32_t *input_length, + uint32_t *output_length, int *ret, bool *end_of_stream, + uint32_t pre_avail_in) { if (strm == nullptr || isal_strm_inflate == nullptr || input_length == nullptr || output_length == nullptr || ret == nullptr || - end_of_stream == nullptr || tofixed == nullptr) { + end_of_stream == nullptr || read_in_correction_applied == nullptr) { if (ret != nullptr) { *ret = Z_DATA_ERROR; } @@ -429,17 +377,18 @@ IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( } *ret = UncompressIGZIP(*isal_strm_inflate, strm->next_in, input_length, - strm->next_out, output_length, window_bits, tofixed, - &strm->total_in, &strm->total_out, end_of_stream); + strm->next_out, output_length, window_bits, + read_in_correction_applied, &strm->total_in, + &strm->total_out, end_of_stream); const uint32_t remaining_after_igzip = (pre_avail_in >= *input_length) ? (pre_avail_in - *input_length) : 0; if (*ret == 0 && window_bits < 0 && *end_of_stream && - remaining_after_igzip > 0 && strm->total_in == 0 && - strm->total_out == 0) { + remaining_after_igzip > 0 && *read_in_correction_applied == 0 && + strm->total_in == 0 && strm->total_out == 0) { Log(LogLevel::LOG_ERROR, - "IGZIPRunInflateAndSelectPathAction() raw boundary guard strm=", + "IGZIPRunInflateAndSelectPathAction() raw boundary guard FIRED strm=", static_cast(strm), " bytes_in=", *input_length, " bytes_out=", *output_length, " pre_avail_in=", pre_avail_in, " remaining_in=", remaining_after_igzip, "\n"); @@ -463,9 +412,9 @@ IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t *input_length, uint8_t *output, - uint32_t *output_length, int window_bits, int *tofixed, - unsigned long *total_in, unsigned long *total_out, - bool *end_of_stream) { + uint32_t *output_length, int window_bits, + int *read_in_correction_applied, unsigned long *total_in, + unsigned long *total_out, bool *end_of_stream) { (void)total_in; if (!isal_strm_inflate) { @@ -497,33 +446,44 @@ int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t rewind_adjust_bytes = 0; - // WORKAROUND: ISA-L over-consumption fix for raw deflate mode. - // Option 2 behavior: if ambiguity appears at INPUT_DONE, request caller - // fallback to zlib instead of carrying deferred state. - if (window_bits < 0 && decomp == ISAL_DECOMP_OK && *tofixed == 0 && - (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH || - isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE) && - isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) { - const uint32_t expected_trailer_bytes = 8; - const uint32_t over_consumed = - expected_trailer_bytes - isal_strm_inflate->avail_in; - if (over_consumed >= 1 && over_consumed <= 7) { - if (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) { - rewind_adjust_bytes = consumed_before_adjust < over_consumed - ? consumed_before_adjust - : over_consumed; - if (rewind_adjust_bytes > 0) { - *tofixed = 1; - } - } else { - Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, - " raw INPUT_DONE ambiguity detected: over_consumed ", over_consumed, - ", requesting zlib fallback\n"); - return Z_DATA_ERROR; - } + // WORKAROUND: ISA-L raw-deflate over-consumption fix. + // ISAL pre-loads input in 8-byte word chunks into a 64-bit shift register + // (read_in). After BLOCK_FINISH, read_in_length >> 3 is the exact byte + // count over-consumed, covering all avail_in scenarios: [0], [1,7], [8], + // and >8 (multi-frame), where prior heuristics were blind or inaccurate. + if (window_bits < 0 && + (decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) && + isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) { + const uint32_t read_in_correction = + (isal_strm_inflate->read_in_length > 0) + ? static_cast(isal_strm_inflate->read_in_length >> 3) + : 0u; + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " raw_finish avail_in ", isal_strm_inflate->avail_in, + " read_in_length_bits ", isal_strm_inflate->read_in_length, + " read_in_correction_bytes ", read_in_correction, "\n"); + if (read_in_correction > 0) { + rewind_adjust_bytes = (read_in_correction <= consumed_before_adjust) + ? read_in_correction + : consumed_before_adjust; + *read_in_correction_applied = 1; } } + // WORKAROUND: BLOCK_INPUT_DONE — output-buffer-limited with ambiguous + // trailer bytes. read_in_length does not apply here (not yet at + // BLOCK_FINISH); request caller fallback to zlib. BLOCK_FINISH is fully + // handled above. + if (window_bits < 0 && decomp == ISAL_DECOMP_OK && + *read_in_correction_applied == 0 && + isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE && + isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) { + Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__, + " raw INPUT_DONE ambiguity detected: over_consumed ", + 8u - isal_strm_inflate->avail_in, ", requesting zlib fallback\n"); + return Z_DATA_ERROR; + } + *output_length = original_avail_out - isal_strm_inflate->avail_out; *input_length = consumed_before_adjust - rewind_adjust_bytes; input = isal_strm_inflate->next_in; @@ -610,19 +570,33 @@ int inflateSetDictionary(z_streamp strm, unsigned char *dict_data, return isal_inflate_set_dict(s->isal_strm_inflate, dict_data, dict_len); } +void ResetCompressIGZIP(struct isal_zstream *isal_strm, int windowBits) { + // isal_deflate_reset preserves gzip_flag, hist_bits, level, and level_buf. + // gzip_flag must be restored: after the first chunk ISA-L changes it from + // IGZIP_ZLIB (3) to IGZIP_ZLIB_NO_HDR (4) to suppress the header on + // continuation calls. Without this reset, the next stream reused via + // deflateReset would produce headerless output, causing decompressors + // (e.g. Java Inflater with nowrap=false) to reject every subsequent chunk. + isal_deflate_reset(isal_strm); + isal_strm->end_of_stream = 0; + isal_strm->flush = NO_FLUSH; + ConfigureDeflateWindow(isal_strm, windowBits); +} + int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, - int *tofixed) { + int *read_in_correction_applied) { if (!isal_strm_inflate) { Log(LogLevel::LOG_ERROR, "ResetUncompressIGZIP() Line ", __LINE__, " isal_strm_inflate is NULL\n"); return Z_STREAM_ERROR; } - // Reset ISA-L inflate state + // Reset ISA-L inflate state. isal_inflate_reset clears the internal + // read_in / read_in_length buffer, so any over-consumption correction + // applied during the previous stream session no longer applies. + // Clear the flag so the new session fires the correction fresh if needed. isal_inflate_reset(isal_strm_inflate); - - // Reset workaround flag - *tofixed = 0; + *read_in_correction_applied = 0; return Z_OK; } diff --git a/igzip.h b/igzip.h index 0317907..a52ef75 100644 --- a/igzip.h +++ b/igzip.h @@ -12,8 +12,9 @@ typedef struct internal_state2 { int level; int w_bits; struct inflate_state *isal_strm_inflate; - int trailer_overconsumption_fixed; /* Indicates if fix has been applied for - gzip trailer overconsumption issue */ + int read_in_correction_applied; /* Set once per stream session when the + read_in_length over-consumption correction + fires; cleared only on inflateReset */ } inflate_state2; typedef struct internal_state { @@ -29,11 +30,6 @@ int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input, uint32_t *output_length, unsigned long *total_in, unsigned long *total_out); bool IsIGZIPDeflateFinished(const struct isal_zstream *stream); -bool SupportedOptionsIGZIPCompress(int flush, uint32_t output_length, - bool stream_on_igzip_path); -bool SupportedOptionsIGZIPUncompress(int window_bits, uint32_t input_length, - uint32_t output_length, - bool stream_on_igzip_path); enum IGZIPNoInputAction { IGZIP_NO_INPUT_NOT_HANDLED, IGZIP_NO_INPUT_RETURN, @@ -50,24 +46,25 @@ enum IGZIPInflatePathAction { IGZIPNoInputAction IGZIPHandleActiveStreamNoInput( z_streamp strm, struct inflate_state *isal_strm_inflate, int window_bits, - int *tofixed, int *ret); + int *read_in_correction_applied, int *ret); IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits, - int *tofixed, uint32_t *input_length, uint32_t *output_length, int *ret, - bool *end_of_stream, uint32_t pre_avail_in); + int *read_in_correction_applied, uint32_t *input_length, + uint32_t *output_length, int *ret, bool *end_of_stream, + uint32_t pre_avail_in); -bool IGZIPShouldFallbackDeflate(bool stream_on_igzip_path, int flush, - uint32_t avail_in); int EndCompressIGZIP(struct isal_zstream *isal_strm); +void ResetCompressIGZIP(struct isal_zstream *isal_strm, int windowBits); struct inflate_state *InitUncompressIGZIP(int windowBits); int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input, uint32_t *input_length, uint8_t *output, - uint32_t *output_length, int window_bits, int *tofixed, - unsigned long *total_in, unsigned long *total_out, - bool *end_of_stream); + uint32_t *output_length, int window_bits, + int *read_in_correction_applied, unsigned long *total_in, + unsigned long *total_out, bool *end_of_stream); int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate); -int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, int *tofixed); +int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate, + int *read_in_correction_applied); // #define Z_DEFAULT_COMPRESSION 6 #endif diff --git a/statistics.cpp b/statistics.cpp index 9dabeb6..0610678 100644 --- a/statistics.cpp +++ b/statistics.cpp @@ -18,9 +18,10 @@ using namespace config; const std::array stat_names{ {"deflate_count", "deflate_error_count", "deflate_qat_count", "deflate_qat_error_count", "deflate_iaa_count", "deflate_iaa_error_count", - "deflate_zlib_count", "inflate_count", "inflate_error_count", - "inflate_qat_count", "inflate_qat_error_count", "inflate_iaa_count", - "inflate_iaa_error_count", "inflate_zlib_count"}}; + "deflate_igzip_count", "deflate_igzip_error_count", "deflate_zlib_count", + "inflate_count", "inflate_error_count", "inflate_qat_count", + "inflate_qat_error_count", "inflate_iaa_count", "inflate_iaa_error_count", + "inflate_igzip_count", "inflate_igzip_error_count", "inflate_zlib_count"}}; thread_local std::array stats{}; diff --git a/statistics.h b/statistics.h index 8cee404..b3654ce 100644 --- a/statistics.h +++ b/statistics.h @@ -17,6 +17,8 @@ enum class Statistic : size_t { DEFLATE_QAT_ERROR_COUNT, DEFLATE_IAA_COUNT, DEFLATE_IAA_ERROR_COUNT, + DEFLATE_IGZIP_COUNT, + DEFLATE_IGZIP_ERROR_COUNT, DEFLATE_ZLIB_COUNT, INFLATE_COUNT, INFLATE_ERROR_COUNT, @@ -24,6 +26,8 @@ enum class Statistic : size_t { INFLATE_QAT_ERROR_COUNT, INFLATE_IAA_COUNT, INFLATE_IAA_ERROR_COUNT, + INFLATE_IGZIP_COUNT, + INFLATE_IGZIP_ERROR_COUNT, INFLATE_ZLIB_COUNT, STATS_COUNT }; diff --git a/tests/zlib_accel_test.cpp b/tests/zlib_accel_test.cpp index d09bf44..b526d0c 100644 --- a/tests/zlib_accel_test.cpp +++ b/tests/zlib_accel_test.cpp @@ -690,6 +690,8 @@ TEST_P(ZlibTest, CompressDecompress) { VerifyStatIncremented(Statistic::DEFLATE_QAT_COUNT); } else if (test_param.execution_path_compress == IAA) { VerifyStatIncremented(Statistic::DEFLATE_IAA_COUNT); + } else if (test_param.execution_path_compress == IGZIP) { + VerifyStatIncremented(Statistic::DEFLATE_IGZIP_COUNT); } else if (test_param.execution_path_compress == ZLIB) { VerifyStatIncremented(Statistic::DEFLATE_ZLIB_COUNT); } @@ -743,6 +745,9 @@ TEST_P(ZlibTest, CompressDecompress) { VerifyStatIncremented(Statistic::INFLATE_QAT_COUNT); } else if (test_param.execution_path_uncompress == IAA) { VerifyStatIncremented(Statistic::INFLATE_IAA_COUNT); + } else if (test_param.execution_path_uncompress == IGZIP) { + VerifyStatIncrementedUpTo(Statistic::INFLATE_IGZIP_COUNT, + test_param.input_chunks_uncompress); } else if (test_param.execution_path_uncompress == ZLIB) { VerifyStatIncrementedUpTo(Statistic::INFLATE_ZLIB_COUNT, test_param.input_chunks_uncompress); @@ -1697,7 +1702,7 @@ TEST(IGZIPDeflateRegressionTest, ResetMustNotStallSyncFlushOnSameStream) { int ret = deflate(&stream, Z_NO_FLUSH); ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR) << "cycle=" << cycle; - ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "cycle=" << cycle; stream.next_in = nullptr; stream.avail_in = 0; @@ -1707,7 +1712,7 @@ TEST(IGZIPDeflateRegressionTest, ResetMustNotStallSyncFlushOnSameStream) { ret = deflate(&stream, Z_SYNC_FLUSH); ASSERT_EQ(ret, Z_OK) << "cycle=" << cycle; ASSERT_LT(stream.avail_out, output.size()) << "cycle=" << cycle; - ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "cycle=" << cycle; stream.next_in = nullptr; stream.avail_in = 0; @@ -1746,7 +1751,7 @@ TEST(IGZIPDeflateRegressionTest, stream.avail_out = static_cast(output.size()); int ret = deflate(&stream, Z_NO_FLUSH); ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR); - ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP); bool observed_buf_error = false; for (int iter = 0; iter < 128; ++iter) { @@ -1756,7 +1761,7 @@ TEST(IGZIPDeflateRegressionTest, stream.avail_out = static_cast(output.size()); ret = deflate(&stream, Z_SYNC_FLUSH); - ASSERT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "iter=" << iter; ASSERT_NE(ret, Z_DATA_ERROR) << "iter=" << iter; if (ret == Z_BUF_ERROR) { @@ -1779,6 +1784,65 @@ TEST(IGZIPDeflateRegressionTest, deflateEnd(&stream); } +// Regression test for: deflateReset on a reused IGZIP stream must restore the +// zlib header (gzip_flag = IGZIP_ZLIB) so that each independent chunk is +// self-contained and decompressible by a fresh zlib inflater. Without the +// fix, isal_deflate_reset preserved gzip_flag = IGZIP_ZLIB_NO_HDR (4) and the +// second and subsequent chunks were emitted without a zlib header, causing +// Java's Inflater (nowrap=false) to report "incorrect header check". +TEST(IGZIPDeflateRegressionTest, + ResetMustRestoreZlibHeaderForSubsequentChunks) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + z_stream cstream; + memset(&cstream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, + /*windowBits=*/15, /*memLevel=*/8, Z_DEFAULT_STRATEGY), + Z_OK); + + const int kChunks = 5; + const int kChunkSize = 16384; + const int kCompBound = deflateBound(&cstream, kChunkSize); + + for (int chunk = 0; chunk < kChunks; ++chunk) { + // Each chunk is independent data compressed with a fresh IGZIP stream + // (via deflateReset). + std::vector input(kChunkSize, + static_cast('A' + chunk % 26)); + std::vector compressed(kCompBound); + + cstream.next_in = input.data(); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + ASSERT_EQ(deflate(&cstream, Z_FINISH), Z_STREAM_END) << "chunk=" << chunk; + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP) << "chunk=" << chunk; + + const size_t compressed_size = compressed.size() - cstream.avail_out; + + // Decompress with a fresh zlib inflater — requires a valid zlib header. + std::vector decompressed(kChunkSize); + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit(&dstream), Z_OK) << "chunk=" << chunk; + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = decompressed.data(); + dstream.avail_out = static_cast(decompressed.size()); + ASSERT_EQ(inflate(&dstream, Z_FINISH), Z_STREAM_END) + << "chunk=" << chunk + << ": decompression failed (missing zlib header after deflateReset?)"; + ASSERT_EQ(decompressed, input) << "chunk=" << chunk; + inflateEnd(&dstream); + + ASSERT_EQ(deflateReset(&cstream), Z_OK) << "chunk=" << chunk; + } + + deflateEnd(&cstream); +} + TEST(IGZIPDeflateRegressionTest, DictionaryStreamMustStayOnZlibAcrossReset) { SetCompressPath(IGZIP, false, false, false); SetUncompressPath(ZLIB, false, false); @@ -2166,8 +2230,10 @@ TEST(IGZIPDeflateRegressionTest, inflateEnd(&dstream); } -TEST(IGZIPDeflateRegressionTest, - SyncFlushWithInputMustFallbackToZlibForStreamSafety) { +TEST(IGZIPDeflateRegressionTest, SyncFlushWithInputMustStayOnIGZIPPath) { + // Originally this test asserted the path was ZLIB (IGZIPShouldFallbackDeflate + // redirected SYNC_FLUSH to zlib). That function is removed; IGZIP now handles + // SYNC_FLUSH natively and the stream must stay on IGZIP throughout. SetCompressPath(IGZIP, false, false, false); SetUncompressPath(ZLIB, false, false); @@ -2197,7 +2263,7 @@ TEST(IGZIPDeflateRegressionTest, int ret = deflate(&cstream, Z_SYNC_FLUSH); ASSERT_NE(ret, Z_DATA_ERROR); - ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP); const size_t sync_produced = sizeof(sync_chunk) - cstream.avail_out; compressed.insert(compressed.end(), sync_chunk, sync_chunk + sync_produced); @@ -2210,7 +2276,7 @@ TEST(IGZIPDeflateRegressionTest, ret = deflate(&cstream, Z_FINISH); ASSERT_NE(ret, Z_DATA_ERROR); - ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP); const size_t produced = sizeof(out_chunk) - cstream.avail_out; compressed.insert(compressed.end(), out_chunk, out_chunk + produced); @@ -2320,6 +2386,173 @@ TEST(IGZIPInflateRegressionTest, inflateEnd(&dstream); } +#ifdef USE_IAA +// IAA->IGZIP fallback tests. +// On machines without IAA hardware, CompressIAA/UncompressIAA return non-zero, +// which naturally triggers the fallback path. These tests verify: +// - When iaa_fallback_igzip=1: the stream lands on IGZIP after IAA fails. +// - When iaa_fallback_igzip=0: the stream falls through to zlib, not IGZIP. + +TEST(IAAFallbackIGZIPTest, DeflateUsesIGZIPWhenIAAFailsAndFallbackEnabled) { + SetCompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false, + /*qat_compression_allow_chunking=*/false); + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 1); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(deflateBound(&stream, input_length)); + stream.next_in = reinterpret_cast(input); + stream.avail_in = static_cast(input_length); + stream.next_out = output.data(); + stream.avail_out = static_cast(output.size()); + + int ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // If IAA hardware is absent, the fallback must have routed to IGZIP. + // If IAA hardware is present and succeeds, IAA path is also acceptable. + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_TRUE(path == IGZIP || path == IAA) + << "Expected IGZIP (fallback) or IAA (hardware success), got " + << static_cast(path); + + deflateEnd(&stream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, DeflateDoesNotUseIGZIPWhenFallbackDisabled) { + SetCompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false, + /*qat_compression_allow_chunking=*/false); + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 0); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(deflateBound(&stream, input_length)); + stream.next_in = reinterpret_cast(input); + stream.avail_in = static_cast(input_length); + stream.next_out = output.data(); + stream.avail_out = static_cast(output.size()); + + deflate(&stream, Z_FINISH); + + // With fallback disabled, IGZIP must not be selected for an IAA-configured + // stream; it should fall through to zlib. + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_NE(path, IGZIP) << "IGZIP must not be used when iaa_fallback_igzip=0"; + + deflateEnd(&stream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, InflateUsesIGZIPWhenIAAFailsAndFallbackEnabled) { + // Compress with IGZIP so the output is IGZIP-compatible (4kB window). + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 1); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + ASSERT_EQ(ret, Z_STREAM_END); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + + std::vector uncompressed(input_length); + dstream.next_in = reinterpret_cast(compressed.data()); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = reinterpret_cast(uncompressed.data()); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // If IAA hardware is absent, fallback must route to IGZIP. + // If IAA hardware is present and succeeds, IAA is also acceptable. + const ExecutionPath path = GetInflateExecutionPath(&dstream); + EXPECT_TRUE(path == IGZIP || path == IAA) + << "Expected IGZIP (fallback) or IAA (hardware success), got " + << static_cast(path); + + EXPECT_EQ(memcmp(uncompressed.data(), input, input_length), 0); + + inflateEnd(&dstream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, InflateDoesNotUseIGZIPWhenFallbackDisabled) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + SetConfig(IAA_FALLBACK_IGZIP, 0); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + ASSERT_EQ(ret, Z_STREAM_END); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + + std::vector uncompressed(input_length); + dstream.next_in = reinterpret_cast(compressed.data()); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = reinterpret_cast(uncompressed.data()); + dstream.avail_out = static_cast(uncompressed.size()); + + inflate(&dstream, Z_FINISH); + + // With fallback disabled, IGZIP must not be selected for an IAA-configured + // inflate stream; it should fall through to zlib. + const ExecutionPath path = GetInflateExecutionPath(&dstream); + EXPECT_NE(path, IGZIP) << "IGZIP must not be used when iaa_fallback_igzip=0"; + + inflateEnd(&dstream); + SetConfig(IAA_FALLBACK_IGZIP, 0); + DestroyBlock(input); +} +#endif // USE_IAA + #endif INSTANTIATE_TEST_SUITE_P( diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 82b55c5..cd010b7 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -203,10 +203,11 @@ struct DeflateSettings { struct InflateSettings { InflateSettings(int _window_bits) - : window_bits(_window_bits), trailer_overconsumption_fixed(0) {} + : window_bits(_window_bits), read_in_correction_applied(0) {} int window_bits; - int trailer_overconsumption_fixed; /* indicates if fix has been applied for - overconsumption issue*/ + int read_in_correction_applied; /* set once per stream when the + read_in_length over-consumption correction + fires; cleared only on inflateReset */ ExecutionPath path = UNDEFINED; struct inflate_state* isal_strm = nullptr; }; @@ -294,6 +295,14 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, Log(LogLevel::LOG_INFO, "deflateSetDictionary Line ", __LINE__, ", strm ", static_cast(strm), ", dictLength ", dictLength, "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); + // Reject mid-stream: if an accelerator is active, the underlying zlib + // stream has not been advanced, so orig_deflateSetDictionary would + // incorrectly accept the call. Per zlib spec, dictionary must be set before + // compression begins. + if (deflate_settings != nullptr && deflate_settings->path != UNDEFINED && + deflate_settings->path != ZLIB) { + return Z_STREAM_ERROR; + } const int ret = orig_deflateSetDictionary(strm, dictionary, dictLength); if (ret == Z_OK) { SetDeflatePath(deflate_settings, strm, ZLIB, @@ -320,18 +329,6 @@ int ZEXPORT deflate(z_streamp strm, int flush) { deflate_settings->window_bits, ", total_in ", strm->total_in, ", total_out ", strm->total_out, ", adler ", strm->adler, "\n"); -#ifdef USE_IGZIP - if (configs[USE_IGZIP_COMPRESS] && deflate_settings->path == UNDEFINED && - IGZIPShouldFallbackDeflate(false, flush, strm->avail_in)) { - SetDeflatePath(deflate_settings, strm, ZLIB, "igzip fallback condition"); - } - if (configs[USE_IGZIP_COMPRESS] && - IGZIPShouldFallbackDeflate(deflate_settings->path == IGZIP, flush, - strm->avail_in)) { - SetDeflatePath(deflate_settings, strm, ZLIB, "igzip fallback condition"); - } -#endif - int ret = 1; bool iaa_available = false; bool qat_available = false; @@ -355,9 +352,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { #ifdef USE_IGZIP igzip_stream_active = (deflate_settings->path == IGZIP && deflate_settings->isal_strm != nullptr); - igzip_available = - configs[USE_IGZIP_COMPRESS] && - SupportedOptionsIGZIPCompress(flush, output_len, igzip_stream_active); + igzip_available = configs[USE_IGZIP_COMPRESS]; #endif // If both accelerators are enabled, send configured ratio of requests to @@ -419,11 +414,37 @@ int ZEXPORT deflate(z_streamp strm, int flush) { "selected IGZIP accelerator"); in_call = false; - // INCREMENT_STAT(DEFLATE_IGZIP_COUNT); - // INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); + INCREMENT_STAT(DEFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); #endif } +#ifdef USE_IGZIP + // IAA→IGZIP fallback: if IAA failed and IGZIP is available, retry with + // IGZIP before falling through to software zlib. + if (path_selected == IAA && ret != 0 && configs[IAA_FALLBACK_IGZIP] && + igzip_available) { + // IAA may have modified input_len/output_len on failure — restore them. + input_len = strm->avail_in; + output_len = strm->avail_out; + if (deflate_settings->isal_strm == nullptr) { + deflate_settings->method = 0; + deflate_settings->isal_strm = InitCompressIGZIP( + deflate_settings->level, deflate_settings->window_bits); + } + in_call = true; + ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, + &input_len, strm->next_out, &output_len, + &strm->total_in, &strm->total_out); + SetDeflatePath(deflate_settings, strm, IGZIP, + "IAA failed, IGZIP fallback"); + in_call = false; + path_selected = IGZIP; // use IGZIP return-code semantics below + INCREMENT_STAT(DEFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); + } +#endif // USE_IGZIP iaa fallback + if (ret == 0) { strm->next_in += input_len; strm->avail_in -= input_len; @@ -508,9 +529,8 @@ int ZEXPORT deflateReset(z_streamp strm) { #ifdef USE_IGZIP if (deflate_settings->isal_strm != nullptr) { - isal_deflate_reset(deflate_settings->isal_strm); - deflate_settings->isal_strm->end_of_stream = 0; - deflate_settings->isal_strm->flush = NO_FLUSH; + ResetCompressIGZIP(deflate_settings->isal_strm, + deflate_settings->window_bits); } #endif } @@ -580,32 +600,24 @@ int ZEXPORT inflate(z_streamp strm, int flush) { #ifdef USE_IGZIP const bool igzip_supported_options = - !in_call && configs[USE_IGZIP_UNCOMPRESS] && - SupportedOptionsIGZIPUncompress(inflate_settings->window_bits, - strm->avail_in, strm->avail_out, - igzip_stream_active); + !in_call && configs[USE_IGZIP_UNCOMPRESS]; // Keep stateful IGZIP stream handling on the same engine. // For avail_in==0, let IGZIP process any buffered bits in its internal // state before reporting Z_BUF_ERROR. if (!in_call && igzip_stream_active && strm->avail_in == 0) { - if (!igzip_supported_options) { - SetInflatePath(inflate_settings, strm, ZLIB, - "IGZIP unsupported options for no-input stream"); - } else { - in_call = true; - IGZIPNoInputAction action = IGZIPHandleActiveStreamNoInput( - strm, inflate_settings->isal_strm, inflate_settings->window_bits, - &inflate_settings->trailer_overconsumption_fixed, &ret); - in_call = false; + in_call = true; + IGZIPNoInputAction action = IGZIPHandleActiveStreamNoInput( + strm, inflate_settings->isal_strm, inflate_settings->window_bits, + &inflate_settings->read_in_correction_applied, &ret); + in_call = false; - if (action == IGZIP_NO_INPUT_FALLBACK_ZLIB) { - SetInflatePath(inflate_settings, strm, ZLIB, - "igzip raw input_done ambiguity fallback"); - // Continue through normal zlib fallback path. - } else if (action == IGZIP_NO_INPUT_RETURN) { - return ret; - } + if (action == IGZIP_NO_INPUT_FALLBACK_ZLIB) { + SetInflatePath(inflate_settings, strm, ZLIB, + "igzip raw input_done ambiguity fallback"); + // Continue through normal zlib fallback path. + } else if (action == IGZIP_NO_INPUT_RETURN) { + return ret; } } #endif @@ -622,7 +634,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { } if (!in_call && strm->avail_in > 0 && inflate_settings->path != ZLIB) { +#ifdef USE_IGZIP const uInt pre_avail_in = strm->avail_in; +#endif uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; @@ -700,7 +714,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { const IGZIPInflatePathAction path_action = IGZIPRunInflateAndSelectPathAction( strm, &inflate_settings->isal_strm, inflate_settings->window_bits, - &inflate_settings->trailer_overconsumption_fixed, &input_len, + &inflate_settings->read_in_correction_applied, &input_len, &output_len, &ret, &end_of_stream, pre_avail_in); in_call = false; @@ -725,11 +739,54 @@ int ZEXPORT inflate(z_streamp strm, int flush) { SetInflatePath(inflate_settings, strm, IGZIP, "selected IGZIP accelerator"); } - // INCREMENT_STAT(INFLATE_IGZIP_COUNT); - // INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); + INCREMENT_STAT(INFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); #endif } +#ifdef USE_IGZIP + // IAA→IGZIP fallback: if IAA failed and IGZIP is available, retry with + // IGZIP before falling through to software zlib. + if (path_selected == IAA && ret != 0 && configs[IAA_FALLBACK_IGZIP] && + igzip_available) { + // IAA may have modified input_len/output_len on failure — restore them. + input_len = strm->avail_in; + output_len = strm->avail_out; + end_of_stream = true; + in_call = true; + const IGZIPInflatePathAction path_action = + IGZIPRunInflateAndSelectPathAction( + strm, &inflate_settings->isal_strm, inflate_settings->window_bits, + &inflate_settings->read_in_correction_applied, &input_len, + &output_len, &ret, &end_of_stream, pre_avail_in); + in_call = false; + + if (inflate_settings->isal_strm == nullptr) { + return Z_DATA_ERROR; + } + + if (path_action == IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT) { + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), + " source=igzip (iaa fallback)", " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, "\n"); + SetInflatePath(inflate_settings, strm, ZLIB, + "IAA->IGZIP fallback: Z_NEED_DICT"); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR) { + SetInflatePath(inflate_settings, strm, ZLIB, + "IAA->IGZIP fallback: raw trailer"); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY) { + SetInflatePath(inflate_settings, strm, ZLIB, + "IAA->IGZIP fallback: raw boundary"); + } else if (path_action == IGZIP_INFLATE_PATH_SET_IGZIP && + inflate_settings->path != ZLIB) { + SetInflatePath(inflate_settings, strm, IGZIP, + "IAA failed, IGZIP fallback succeeded"); + } + INCREMENT_STAT(INFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); + } +#endif // USE_IGZIP iaa fallback + if (ret == 0) { strm->next_in += input_len; strm->avail_in -= input_len; @@ -803,8 +860,6 @@ int ZEXPORT inflateReset(z_streamp strm) { Log(LogLevel::LOG_INFO, "inflateReset Line ", __LINE__, ", strm ", static_cast(strm), "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); - const bool was_igzip_path = - (inflate_settings != nullptr && inflate_settings->path == IGZIP); int ret = orig_inflateReset(strm); if (inflate_settings != nullptr) { SetInflatePath(inflate_settings, strm, UNDEFINED, "inflateReset"); @@ -812,11 +867,8 @@ int ZEXPORT inflateReset(z_streamp strm) { if (inflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP ResetUncompressIGZIP(inflate_settings->isal_strm, - &inflate_settings->trailer_overconsumption_fixed); + &inflate_settings->read_in_correction_applied); #endif - if (was_igzip_path) { - inflate_settings->trailer_overconsumption_fixed = 1; - } } return ret; @@ -844,8 +896,7 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(15, input_len); #endif #ifdef USE_IGZIP - igzip_available = configs[USE_IGZIP_COMPRESS] && - SupportedOptionsIGZIPCompress(Z_FINISH, output_len, false); + igzip_available = configs[USE_IGZIP_COMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -943,9 +994,7 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(15, input_len); #endif #ifdef USE_IGZIP - igzip_available = - configs[USE_IGZIP_UNCOMPRESS] && - SupportedOptionsIGZIPUncompress(15, input_len, output_len, false); + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -981,12 +1030,12 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, if (isal_strm == nullptr) { ret = 1; } else { - int tofixed = 0; + int read_in_correction_applied = 0; unsigned long total_in = 0; unsigned long total_out = 0; ret = UncompressIGZIP(isal_strm, const_cast(source), &input_len, - dest, &output_len, 15, &tofixed, &total_in, - &total_out, &end_of_stream); + dest, &output_len, 15, &read_in_correction_applied, + &total_in, &total_out, &end_of_stream); EndUncompressIGZIP(isal_strm); if (ret == 0 && !end_of_stream) { ret = 1; @@ -1245,9 +1294,7 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(31, *input_length); #endif #ifdef USE_IGZIP - igzip_available = - configs[USE_IGZIP_COMPRESS] && - SupportedOptionsIGZIPCompress(Z_FINISH, *output_length, false); + igzip_available = configs[USE_IGZIP_COMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -1321,9 +1368,7 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(31, *input_length); #endif #ifdef USE_IGZIP - igzip_available = - configs[USE_IGZIP_UNCOMPRESS] && - SupportedOptionsIGZIPUncompress(31, *input_length, *output_length, false); + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -1358,12 +1403,12 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, if (isal_strm == nullptr) { ret = 1; } else { - int tofixed = 0; + int read_in_correction_applied = 0; unsigned long total_in = 0; unsigned long total_out = 0; - ret = - UncompressIGZIP(isal_strm, input, input_length, output, output_length, - 31, &tofixed, &total_in, &total_out, end_of_stream); + ret = UncompressIGZIP(isal_strm, input, input_length, output, + output_length, 31, &read_in_correction_applied, + &total_in, &total_out, end_of_stream); EndUncompressIGZIP(isal_strm); } gz->path = IGZIP;