From a80102371bdb692aacde4061c6e7b0abc7ef3152 Mon Sep 17 00:00:00 2001 From: Rodrigo Batista de Moraes Date: Tue, 22 Sep 2026 22:57:24 -0300 Subject: [PATCH 1/3] Build fastlz without the alignment sanitizer --- src/meson.build | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/meson.build b/src/meson.build index 1987fa5..97db785 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,5 +1,4 @@ libfst_sources = [ - 'fastlz.c', 'fstapi.c', 'lz4.c', ] @@ -14,10 +13,19 @@ libfst_dependencies = [ thread_dep, ] +# fastlz.c intentionally do unaligned writes, which trigger sanitizers +libfastlz = static_library( + 'fastlz', + 'fastlz.c', + c_args: cc.get_supported_arguments('-fno-sanitize=alignment'), + pic: true, +) + libfst = library( 'fst', sources: libfst_sources, dependencies: libfst_dependencies, + link_whole: libfastlz, version: meson.project_version(), install: true, include_directories: config_inc, @@ -37,4 +45,4 @@ pkg.generate( libraries: libfst, description: 'FST reader and writer library', subdirs: 'libfst', -) \ No newline at end of file +) From ad254f7a6eb98e4658602ee9a1a143e114f9dbd1 Mon Sep 17 00:00:00 2001 From: Rodrigo Batista de Moraes Date: Tue, 22 Sep 2026 22:58:01 -0300 Subject: [PATCH 2/3] Update fastlz to upstream master Update the vendored copy with `fastlz.c` and `fastlz.h` from the master branch of https://github.com/ariya/FastLZ at commit b1342da, from August 2024. The file is labeled as version 0.5.0, but that version was first released in February 2020. The vendored version before this commit was based on one from 2007. The new version bounds-checks the input, so it supersedes some of the checks previously introduced in this vendored copy. The new bounds checks cover more areas than the previous ones, but they still do not cover everything (this is fixed in the follow-up commit). --- src/fastlz.c | 845 ++++++++++++++++++++++++--------------------------- src/fastlz.h | 68 ++--- 2 files changed, 430 insertions(+), 483 deletions(-) diff --git a/src/fastlz.c b/src/fastlz.c index b52a799..f99bb10 100644 --- a/src/fastlz.c +++ b/src/fastlz.c @@ -1,9 +1,6 @@ /* - FastLZ - lightning-fast lossless compression library - - Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) + FastLZ - Byte-aligned LZ77 compression library + Copyright (C) 2005-2020 Ariya Hidayat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,528 +19,490 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - SPDX-License-Identifier: MIT */ #include "fastlz.h" -#if !defined(FASTLZ__COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) - -/* - * Always check for bound when decompressing. - * Generally it is best to leave it defined. - */ -#define FASTLZ_SAFE +#include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wimplicit-fallthrough" /* * Give hints to the compiler for branch prediction optimization. */ -#if defined(__GNUC__) && (__GNUC__ > 2) -#define FASTLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1)) -#define FASTLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0)) +#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 2)) +#define FASTLZ_LIKELY(c) (__builtin_expect(!!(c), 1)) +#define FASTLZ_UNLIKELY(c) (__builtin_expect(!!(c), 0)) #else -#define FASTLZ_EXPECT_CONDITIONAL(c) (c) -#define FASTLZ_UNEXPECT_CONDITIONAL(c) (c) +#define FASTLZ_LIKELY(c) (c) +#define FASTLZ_UNLIKELY(c) (c) #endif /* - * Use inlined functions for supported systems. + * Specialize custom 64-bit implementation for speed improvements. */ -#if defined(__GNUC__) || defined(__DMC__) || defined(__POCC__) || defined(__WATCOMC__) || defined(__SUNPRO_C) -#define FASTLZ_INLINE inline -#elif defined(__BORLANDC__) || defined(_MSC_VER) || defined(__LCC__) -#define FASTLZ_INLINE __inline -#else -#define FASTLZ_INLINE +#if defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__) +#define FLZ_ARCH64 #endif /* - * Prevent accessing more than 8-bit at once, except on x86 architectures. + * Workaround for DJGPP to find uint8_t, uint16_t, etc. */ -#if !defined(FASTLZ_STRICT_ALIGN) -#define FASTLZ_STRICT_ALIGN -#if defined(__i386__) || defined(__386) /* GNU C, Sun Studio */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(__i486__) || defined(__i586__) || defined(__i686__) || defined(__amd64) /* GNU C */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(_M_IX86) /* Intel, MSVC */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(__386) -#undef FASTLZ_STRICT_ALIGN -#elif defined(_X86_) /* MinGW */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(__I86__) /* Digital Mars */ -#undef FASTLZ_STRICT_ALIGN -#endif +#if defined(__MSDOS__) && defined(__GNUC__) +#include #endif -/* prototypes */ -int fastlz_compress(const void* input, int length, void* output); -int fastlz_compress_level(int level, const void* input, int length, void* output); -int fastlz_decompress(const void* input, int length, void* output, int maxout); +#if defined(FASTLZ_USE_MEMMOVE) && (FASTLZ_USE_MEMMOVE == 0) -#define MAX_COPY 32 -#define MAX_LEN 264 /* 256 + 8 */ -#define MAX_DISTANCE 8192 +static void fastlz_memmove(uint8_t* dest, const uint8_t* src, uint32_t count) { + do { + *dest++ = *src++; + } while (--count); +} + +static void fastlz_memcpy(uint8_t* dest, const uint8_t* src, uint32_t count) { + return fastlz_memmove(dest, src, count); +} -#if !defined(FASTLZ_STRICT_ALIGN) -#define FASTLZ_READU16(p) *((const flzuint16*)(p)) #else -#define FASTLZ_READU16(p) ((p)[0] | (p)[1]<<8) + +#include + +static void fastlz_memmove(uint8_t* dest, const uint8_t* src, uint32_t count) { + if ((count > 4) && (dest >= src + count)) { + memmove(dest, src, count); + } else { + switch (count) { + default: + do { + *dest++ = *src++; + } while (--count); + break; + case 3: + *dest++ = *src++; + case 2: + *dest++ = *src++; + case 1: + *dest++ = *src++; + case 0: + break; + } + } +} + +static void fastlz_memcpy(uint8_t* dest, const uint8_t* src, uint32_t count) { memcpy(dest, src, count); } + #endif -#define HASH_LOG 13 -#define HASH_SIZE (1<< HASH_LOG) -#define HASH_MASK (HASH_SIZE-1) -#define HASH_FUNCTION(v,p) { v = FASTLZ_READU16(p); v ^= FASTLZ_READU16(p+1)^(v>>(16-HASH_LOG));v &= HASH_MASK; } - -#undef FASTLZ_LEVEL -#define FASTLZ_LEVEL 1 - -#undef FASTLZ_COMPRESSOR -#undef FASTLZ_DECOMPRESSOR -#define FASTLZ_COMPRESSOR fastlz1_compress -#define FASTLZ_DECOMPRESSOR fastlz1_decompress -static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output); -static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout); -#include "fastlz.c" - -#undef FASTLZ_LEVEL -#define FASTLZ_LEVEL 2 - -#undef MAX_DISTANCE -#define MAX_DISTANCE 8191 -#define MAX_FARDISTANCE (65535+MAX_DISTANCE-1) - -#undef FASTLZ_COMPRESSOR -#undef FASTLZ_DECOMPRESSOR -#define FASTLZ_COMPRESSOR fastlz2_compress -#define FASTLZ_DECOMPRESSOR fastlz2_decompress -static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output); -static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout); -#include "fastlz.c" - -int fastlz_compress(const void* input, int length, void* output) -{ - /* for short block, choose fastlz1 */ - if(length < 65536) - return fastlz1_compress(input, length, output); +#if defined(FLZ_ARCH64) - /* else... */ - return fastlz2_compress(input, length, output); +static uint32_t flz_readu32(const void* ptr) { return *(const uint32_t*)ptr; } + +static uint32_t flz_cmp(const uint8_t* p, const uint8_t* q, const uint8_t* r) { + const uint8_t* start = p; + + if (flz_readu32(p) == flz_readu32(q)) { + p += 4; + q += 4; + } + while (q < r) + if (*p++ != *q++) break; + return p - start; } -int fastlz_decompress(const void* input, int length, void* output, int maxout) -{ - /* magic identifier for compression level */ - int level = ((*(const flzuint8*)input) >> 5) + 1; +#endif /* FLZ_ARCH64 */ - if(level == 1) - return fastlz1_decompress(input, length, output, maxout); - if(level == 2) - return fastlz2_decompress(input, length, output, maxout); +#if !defined(FLZ_ARCH64) - /* unknown level, trigger error */ - return 0; +static uint32_t flz_readu32(const void* ptr) { + const uint8_t* p = (const uint8_t*)ptr; + return (p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0]; } -int fastlz_compress_level(int level, const void* input, int length, void* output) -{ - if(level == 1) - return fastlz1_compress(input, length, output); - if(level == 2) - return fastlz2_compress(input, length, output); +static uint32_t flz_cmp(const uint8_t* p, const uint8_t* q, const uint8_t* r) { + const uint8_t* start = p; + while (q < r) + if (*p++ != *q++) break; + return p - start; +} - return 0; +#endif /* !FLZ_ARCH64 */ + +#define MAX_COPY 32 +#define MAX_LEN 264 /* 256 + 8 */ +#define MAX_L1_DISTANCE 8192 +#define MAX_L2_DISTANCE 8191 +#define MAX_FARDISTANCE (65535 + MAX_L2_DISTANCE - 1) + +#define HASH_LOG 13 +#define HASH_SIZE (1 << HASH_LOG) +#define HASH_MASK (HASH_SIZE - 1) + +static uint16_t flz_hash(uint32_t v) { + uint32_t h = (v * 2654435769LL) >> (32 - HASH_LOG); + return h & HASH_MASK; } -#else /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */ - -static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output) -{ - const flzuint8* ip = (const flzuint8*) input; - const flzuint8* ip_bound = ip + length - 2; - const flzuint8* ip_limit = ip + length - 12; - flzuint8* op = (flzuint8*) output; - - const flzuint8* htab[HASH_SIZE]; - const flzuint8** hslot; - flzuint32 hval; - - flzuint32 copy; - - /* sanity check */ - if(FASTLZ_UNEXPECT_CONDITIONAL(length < 4)) - { - if(length) - { - /* create literal copy only */ - *op++ = length-1; - ip_bound++; - while(ip <= ip_bound) - *op++ = *ip++; - return length+1; +/* special case of memcpy: at most MAX_COPY bytes */ +static void flz_smallcopy(uint8_t* dest, const uint8_t* src, uint32_t count) { +#if defined(FLZ_ARCH64) + if (count >= 4) { + const uint32_t* p = (const uint32_t*)src; + uint32_t* q = (uint32_t*)dest; + while (count > 4) { + *q++ = *p++; + count -= 4; + dest += 4; + src += 4; } - else - return 0; } +#endif + fastlz_memcpy(dest, src, count); +} - /* initializes hash table */ - for (hslot = htab; hslot < htab + HASH_SIZE; hslot++) - *hslot = ip; +/* special case of memcpy: exactly MAX_COPY bytes */ +static void flz_maxcopy(void* dest, const void* src) { +#if defined(FLZ_ARCH64) + const uint32_t* p = (const uint32_t*)src; + uint32_t* q = (uint32_t*)dest; + *q++ = *p++; + *q++ = *p++; + *q++ = *p++; + *q++ = *p++; + *q++ = *p++; + *q++ = *p++; + *q++ = *p++; + *q++ = *p++; +#else + fastlz_memcpy(dest, src, MAX_COPY); +#endif +} - /* we start with literal copy */ - copy = 2; - *op++ = MAX_COPY-1; - *op++ = *ip++; - *op++ = *ip++; +static uint8_t* flz_literals(uint32_t runs, const uint8_t* src, uint8_t* dest) { + while (runs >= MAX_COPY) { + *dest++ = MAX_COPY - 1; + flz_maxcopy(dest, src); + src += MAX_COPY; + dest += MAX_COPY; + runs -= MAX_COPY; + } + if (runs > 0) { + *dest++ = runs - 1; + flz_smallcopy(dest, src, runs); + dest += runs; + } + return dest; +} - /* main loop */ - while(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit)) - { - const flzuint8* ref; - flzuint32 distance; - - /* minimum match length */ - flzuint32 len = 3; - - /* comparison starting-point */ - const flzuint8* anchor = ip; - - /* check for a run */ -#if FASTLZ_LEVEL==2 - if(ip[0] == ip[-1] && FASTLZ_READU16(ip-1)==FASTLZ_READU16(ip+1)) - { - distance = 1; - /* ip += 3; */ /* scan-build, never used */ - ref = anchor - 1 + 3; - goto match; +static uint8_t* flz1_match(uint32_t len, uint32_t distance, uint8_t* op) { + --distance; + if (FASTLZ_UNLIKELY(len > MAX_LEN - 2)) + while (len > MAX_LEN - 2) { + *op++ = (7 << 5) + (distance >> 8); + *op++ = MAX_LEN - 2 - 7 - 2; + *op++ = (distance & 255); + len -= MAX_LEN - 2; } -#endif + if (len < 7) { + *op++ = (len << 5) + (distance >> 8); + *op++ = (distance & 255); + } else { + *op++ = (7 << 5) + (distance >> 8); + *op++ = len - 7; + *op++ = (distance & 255); + } + return op; +} - /* find potential match */ - HASH_FUNCTION(hval,ip); - hslot = htab + hval; - ref = htab[hval]; +#define FASTLZ_BOUND_CHECK(cond) \ + if (FASTLZ_UNLIKELY(!(cond))) return 0; - /* calculate distance to the match */ - distance = anchor - ref; +static int fastlz1_compress(const void* input, int length, void* output) { + const uint8_t* ip = (const uint8_t*)input; + const uint8_t* ip_start = ip; + const uint8_t* ip_bound = ip + length - 4; /* because readU32 */ + const uint8_t* ip_limit = ip + length - 12 - 1; + uint8_t* op = (uint8_t*)output; - /* update hash table */ - *hslot = anchor; + uint32_t htab[HASH_SIZE]; + uint32_t seq, hash; - /* is this a match? check the first 3 bytes */ - if(distance==0 || -#if FASTLZ_LEVEL==1 - (distance >= MAX_DISTANCE) || -#else - (distance >= MAX_FARDISTANCE) || -#endif - *ref++ != *ip++ || *ref++!=*ip++ || *ref++!=*ip++) - goto literal; + /* initializes hash table */ + for (hash = 0; hash < HASH_SIZE; ++hash) htab[hash] = 0; -#if FASTLZ_LEVEL==2 - /* far, needs at least 5-byte match */ - if(distance >= MAX_DISTANCE) - { - if(*ip++ != *ref++ || *ip++!= *ref++) - goto literal; - len += 2; + /* we start with literal copy */ + const uint8_t* anchor = ip; + ip += 2; + + /* main loop */ + while (FASTLZ_LIKELY(ip < ip_limit)) { + const uint8_t* ref; + uint32_t distance, cmp; + + /* find potential match */ + do { + seq = flz_readu32(ip) & 0xffffff; + hash = flz_hash(seq); + ref = ip_start + htab[hash]; + htab[hash] = ip - ip_start; + distance = ip - ref; + cmp = FASTLZ_LIKELY(distance < MAX_L1_DISTANCE) ? flz_readu32(ref) & 0xffffff : 0x1000000; + if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; + ++ip; + } while (seq != cmp); + + if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; + --ip; + + if (FASTLZ_LIKELY(ip > anchor)) { + op = flz_literals(ip - anchor, anchor, op); } - match: -#endif + uint32_t len = flz_cmp(ref + 3, ip + 3, ip_bound); + op = flz1_match(len, distance, op); - /* last matched byte */ - ip = anchor + len; + /* update the hash at match boundary */ + ip += len; + seq = flz_readu32(ip); + hash = flz_hash(seq & 0xffffff); + htab[hash] = ip++ - ip_start; + seq >>= 8; + hash = flz_hash(seq); + htab[hash] = ip++ - ip_start; + + anchor = ip; + } - /* distance is biased */ - distance--; + uint32_t copy = (uint8_t*)input + length - anchor; + op = flz_literals(copy, anchor, op); - if(!distance) - { - /* zero distance means a run */ - flzuint8 x = ip[-1]; - while(ip < ip_bound) - if(*ref++ != x) break; else ip++; - } - else - for(;;) - { - /* safe because the outer check against ip limit */ - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - while(ip < ip_bound) - if(*ref++ != *ip++) break; - break; - } + return op - (uint8_t*)output; +} - /* if we have copied something, adjust the copy count */ - if(copy) - /* copy is biased, '0' means 1 byte copy */ - *(op-copy-1) = copy-1; - else - /* back, to overwrite the copy count */ - op--; - - /* reset literal counter */ - copy = 0; - - /* length is biased, '1' means a match of 3 bytes */ - ip -= 3; - len = ip - anchor; - - /* encode the match */ -#if FASTLZ_LEVEL==2 - if(distance < MAX_DISTANCE) - { - if(len < 7) - { - *op++ = (len << 5) + (distance >> 8); - *op++ = (distance & 255); - } - else - { - *op++ = (7 << 5) + (distance >> 8); - for(len-=7; len >= 255; len-= 255) - *op++ = 255; - *op++ = len; - *op++ = (distance & 255); - } - } - else - { - /* far away, but not yet in the another galaxy... */ - if(len < 7) - { - distance -= MAX_DISTANCE; - *op++ = (len << 5) + 31; - *op++ = 255; - *op++ = distance >> 8; - *op++ = distance & 255; - } - else - { - distance -= MAX_DISTANCE; - *op++ = (7 << 5) + 31; - for(len-=7; len >= 255; len-= 255) - *op++ = 255; - *op++ = len; - *op++ = 255; - *op++ = distance >> 8; - *op++ = distance & 255; +static int fastlz1_decompress(const void* input, int length, void* output, int maxout) { + const uint8_t* ip = (const uint8_t*)input; + const uint8_t* ip_limit = ip + length; + const uint8_t* ip_bound = ip_limit - 2; + uint8_t* op = (uint8_t*)output; + uint8_t* op_limit = op + maxout; + uint32_t ctrl = (*ip++) & 31; + + while (1) { + if (ctrl >= 32) { + uint32_t len = (ctrl >> 5) - 1; + uint32_t ofs = (ctrl & 31) << 8; + const uint8_t* ref = op - ofs - 1; + if (len == 7 - 1) { + FASTLZ_BOUND_CHECK(ip <= ip_bound); + len += *ip++; } + ref -= *ip++; + len += 3; + FASTLZ_BOUND_CHECK(op + len <= op_limit); + FASTLZ_BOUND_CHECK(ref >= (uint8_t*)output); + fastlz_memmove(op, ref, len); + op += len; + } else { + ctrl++; + FASTLZ_BOUND_CHECK(op + ctrl <= op_limit); + FASTLZ_BOUND_CHECK(ip + ctrl <= ip_limit); + fastlz_memcpy(op, ip, ctrl); + ip += ctrl; + op += ctrl; } -#else - if(FASTLZ_UNEXPECT_CONDITIONAL(len > MAX_LEN-2)) - while(len > MAX_LEN-2) - { - *op++ = (7 << 5) + (distance >> 8); - *op++ = MAX_LEN - 2 - 7 -2; - *op++ = (distance & 255); - len -= MAX_LEN-2; - } + if (FASTLZ_UNLIKELY(ip > ip_bound)) break; + ctrl = *ip++; + } - if(len < 7) - { + return op - (uint8_t*)output; +} + +static uint8_t* flz2_match(uint32_t len, uint32_t distance, uint8_t* op) { + --distance; + if (distance < MAX_L2_DISTANCE) { + if (len < 7) { *op++ = (len << 5) + (distance >> 8); *op++ = (distance & 255); - } - else - { + } else { *op++ = (7 << 5) + (distance >> 8); - *op++ = len - 7; + for (len -= 7; len >= 255; len -= 255) *op++ = 255; + *op++ = len; *op++ = (distance & 255); } -#endif + } else { + /* far away, but not yet in the another galaxy... */ + if (len < 7) { + distance -= MAX_L2_DISTANCE; + *op++ = (len << 5) + 31; + *op++ = 255; + *op++ = distance >> 8; + *op++ = distance & 255; + } else { + distance -= MAX_L2_DISTANCE; + *op++ = (7 << 5) + 31; + for (len -= 7; len >= 255; len -= 255) *op++ = 255; + *op++ = len; + *op++ = 255; + *op++ = distance >> 8; + *op++ = distance & 255; + } + } + return op; +} - /* update the hash at match boundary */ - HASH_FUNCTION(hval,ip); - htab[hval] = ip++; - HASH_FUNCTION(hval,ip); - htab[hval] = ip++; - - /* assuming literal copy */ - *op++ = MAX_COPY-1; - - continue; - - literal: - *op++ = *anchor++; - ip = anchor; - copy++; - if(FASTLZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY)) - { - copy = 0; - *op++ = MAX_COPY-1; +static int fastlz2_compress(const void* input, int length, void* output) { + const uint8_t* ip = (const uint8_t*)input; + const uint8_t* ip_start = ip; + const uint8_t* ip_bound = ip + length - 4; /* because readU32 */ + const uint8_t* ip_limit = ip + length - 12 - 1; + uint8_t* op = (uint8_t*)output; + + uint32_t htab[HASH_SIZE]; + uint32_t seq, hash; + + /* initializes hash table */ + for (hash = 0; hash < HASH_SIZE; ++hash) htab[hash] = 0; + + /* we start with literal copy */ + const uint8_t* anchor = ip; + ip += 2; + + /* main loop */ + while (FASTLZ_LIKELY(ip < ip_limit)) { + const uint8_t* ref; + uint32_t distance, cmp; + + /* find potential match */ + do { + seq = flz_readu32(ip) & 0xffffff; + hash = flz_hash(seq); + ref = ip_start + htab[hash]; + htab[hash] = ip - ip_start; + distance = ip - ref; + cmp = FASTLZ_LIKELY(distance < MAX_FARDISTANCE) ? flz_readu32(ref) & 0xffffff : 0x1000000; + if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; + ++ip; + } while (seq != cmp); + + if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; + + --ip; + + /* far, needs at least 5-byte match */ + if (distance >= MAX_L2_DISTANCE) { + if (ref[3] != ip[3] || ref[4] != ip[4]) { + ++ip; + continue; } - } + } - /* left-over as literal copy */ - ip_bound++; - while(ip <= ip_bound) - { - *op++ = *ip++; - copy++; - if(copy == MAX_COPY) - { - copy = 0; - *op++ = MAX_COPY-1; + if (FASTLZ_LIKELY(ip > anchor)) { + op = flz_literals(ip - anchor, anchor, op); } + + uint32_t len = flz_cmp(ref + 3, ip + 3, ip_bound); + op = flz2_match(len, distance, op); + + /* update the hash at match boundary */ + ip += len; + seq = flz_readu32(ip); + hash = flz_hash(seq & 0xffffff); + htab[hash] = ip++ - ip_start; + seq >>= 8; + hash = flz_hash(seq); + htab[hash] = ip++ - ip_start; + + anchor = ip; } - /* if we have copied something, adjust the copy length */ - if(copy) - *(op-copy-1) = copy-1; - else - op--; + uint32_t copy = (uint8_t*)input + length - anchor; + op = flz_literals(copy, anchor, op); -#if FASTLZ_LEVEL==2 /* marker for fastlz2 */ - *(flzuint8*)output |= (1 << 5); -#endif + *(uint8_t*)output |= (1 << 5); - return op - (flzuint8*)output; + return op - (uint8_t*)output; } -static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout) -{ - const flzuint8* ip = (const flzuint8*) input; - const flzuint8* ip_limit = ip + length; - flzuint8* op = (flzuint8*) output; - flzuint8* op_limit = op + maxout; - flzuint32 ctrl = (*ip++) & 31; - int loop = 1; - - do - { - const flzuint8* ref = op; - flzuint32 len = ctrl >> 5; - flzuint32 ofs = (ctrl & 31) << 8; - - if(ctrl >= 32) - { -#if FASTLZ_LEVEL==2 - flzuint8 code; -#endif - len--; - ref -= ofs; - if (len == 7-1) -#if FASTLZ_LEVEL==1 - len += *ip++; - ref -= *ip++; -#else - do - { +static int fastlz2_decompress(const void* input, int length, void* output, int maxout) { + const uint8_t* ip = (const uint8_t*)input; + const uint8_t* ip_limit = ip + length; + const uint8_t* ip_bound = ip_limit - 2; + uint8_t* op = (uint8_t*)output; + uint8_t* op_limit = op + maxout; + uint32_t ctrl = (*ip++) & 31; + + while (1) { + if (ctrl >= 32) { + uint32_t len = (ctrl >> 5) - 1; + uint32_t ofs = (ctrl & 31) << 8; + const uint8_t* ref = op - ofs - 1; + + uint8_t code; + if (len == 7 - 1) do { + FASTLZ_BOUND_CHECK(ip <= ip_bound); code = *ip++; len += code; - } while (code==255); + } while (code == 255); code = *ip++; ref -= code; + len += 3; /* match from 16-bit distance */ - if(FASTLZ_UNEXPECT_CONDITIONAL(code==255)) - if(FASTLZ_EXPECT_CONDITIONAL(ofs==(31 << 8))) - { - ofs = (*ip++) << 8; - ofs += *ip++; - ref = op - ofs - MAX_DISTANCE; - } -#endif + if (FASTLZ_UNLIKELY(code == 255)) + if (FASTLZ_LIKELY(ofs == (31 << 8))) { + FASTLZ_BOUND_CHECK(ip < ip_bound); + ofs = (*ip++) << 8; + ofs += *ip++; + ref = op - ofs - MAX_L2_DISTANCE - 1; + } -#ifdef FASTLZ_SAFE - if (FASTLZ_UNEXPECT_CONDITIONAL(op + len + 3 > op_limit)) - return 0; + FASTLZ_BOUND_CHECK(op + len <= op_limit); + FASTLZ_BOUND_CHECK(ref >= (uint8_t*)output); + fastlz_memmove(op, ref, len); + op += len; + } else { + ctrl++; + FASTLZ_BOUND_CHECK(op + ctrl <= op_limit); + FASTLZ_BOUND_CHECK(ip + ctrl <= ip_limit); + fastlz_memcpy(op, ip, ctrl); + ip += ctrl; + op += ctrl; + } - if (FASTLZ_UNEXPECT_CONDITIONAL(ref-1 < (flzuint8 *)output)) - return 0; -#endif + if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; + ctrl = *ip++; + } - if(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit)) - ctrl = *ip++; - else - loop = 0; - - if(ref == op) - { - /* optimize copy for a run */ - flzuint8 b = ref[-1]; - *op++ = b; - *op++ = b; - *op++ = b; - for(; len; --len) - *op++ = b; - } - else - { -#if !defined(FASTLZ_STRICT_ALIGN) - const flzuint16* p; - flzuint16* q; -#endif - /* copy from reference */ - ref--; - *op++ = *ref++; - *op++ = *ref++; - *op++ = *ref++; - -#if !defined(FASTLZ_STRICT_ALIGN) - /* copy a byte, so that now it's word aligned */ - if(len & 1) - { - *op++ = *ref++; - len--; - } + return op - (uint8_t*)output; +} - /* copy 16-bit at once */ - q = (flzuint16*) op; - op += len; - p = (const flzuint16*) ref; - for(len>>=1; len > 4; len-=4) - { - *q++ = *p++; - *q++ = *p++; - *q++ = *p++; - *q++ = *p++; - } - for(; len; --len) - *q++ = *p++; -#else - for(; len; --len) - *op++ = *ref++; -#endif - } - } - else - { - ctrl++; -#ifdef FASTLZ_SAFE - if (FASTLZ_UNEXPECT_CONDITIONAL(op + ctrl > op_limit)) - return 0; - if (FASTLZ_UNEXPECT_CONDITIONAL(ip + ctrl > ip_limit)) - return 0; -#endif +int fastlz_compress(const void* input, int length, void* output) { + /* for short block, choose fastlz1 */ + if (length < 65536) return fastlz1_compress(input, length, output); - *op++ = *ip++; - for(--ctrl; ctrl; ctrl--) - *op++ = *ip++; + /* else... */ + return fastlz2_compress(input, length, output); +} - loop = FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit); - if(loop) - ctrl = *ip++; - } - } - while(FASTLZ_EXPECT_CONDITIONAL(loop)); +int fastlz_decompress(const void* input, int length, void* output, int maxout) { + /* magic identifier for compression level */ + int level = ((*(const uint8_t*)input) >> 5) + 1; + + if (level == 1) return fastlz1_decompress(input, length, output, maxout); + if (level == 2) return fastlz2_decompress(input, length, output, maxout); + + /* unknown level, trigger error */ + return 0; +} - return op - (flzuint8*)output; +int fastlz_compress_level(int level, const void* input, int length, void* output) { + if (level == 1) return fastlz1_compress(input, length, output); + if (level == 2) return fastlz2_compress(input, length, output); + + return 0; } -#endif /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */ +#pragma GCC diagnostic pop diff --git a/src/fastlz.h b/src/fastlz.h index 1ce44a3..9172d74 100644 --- a/src/fastlz.h +++ b/src/fastlz.h @@ -1,9 +1,6 @@ /* - FastLZ - lightning-fast lossless compression library - - Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) + FastLZ - Byte-aligned LZ77 compression library + Copyright (C) 2005-2020 Ariya Hidayat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,29 +19,20 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - SPDX-License-Identifier: MIT */ #ifndef FASTLZ_H #define FASTLZ_H -#include - -#define flzuint8 uint8_t -#define flzuint16 uint16_t -#define flzuint32 uint32_t - - -#define FASTLZ_VERSION 0x000100 +#define FASTLZ_VERSION 0x000500 -#define FASTLZ_VERSION_MAJOR 0 -#define FASTLZ_VERSION_MINOR 0 -#define FASTLZ_VERSION_REVISION 0 +#define FASTLZ_VERSION_MAJOR 0 +#define FASTLZ_VERSION_MINOR 5 +#define FASTLZ_VERSION_REVISION 0 -#define FASTLZ_VERSION_STRING "0.1.0" +#define FASTLZ_VERSION_STRING "0.5.0" -#if defined (__cplusplus) +#if defined(__cplusplus) extern "C" { #endif @@ -60,9 +48,17 @@ extern "C" { length (input buffer size). The input buffer and the output buffer can not overlap. + + Compression level can be specified in parameter level. At the moment, + only level 1 and level 2 are supported. + Level 1 is the fastest compression and generally useful for short data. + Level 2 is slightly slower but it gives better compression ratio. + + Note that the compressed data, regardless of the level, can always be + decompressed using the function fastlz_decompress below. */ -int fastlz_compress(const void* input, int length, void* output); +int fastlz_compress_level(int level, const void* input, int length, void* output); /** Decompress a block of compressed data and returns the size of the @@ -74,35 +70,27 @@ int fastlz_compress(const void* input, int length, void* output); Decompression is memory safe and guaranteed not to write the output buffer more than what is specified in maxout. + + Note that the decompression will always work, regardless of the + compression level specified in fastlz_compress_level above (when + producing the compressed block). */ int fastlz_decompress(const void* input, int length, void* output, int maxout); /** - Compress a block of data in the input buffer and returns the size of - compressed block. The size of input buffer is specified by length. The - minimum input buffer size is 16. - - The output buffer must be at least 5% larger than the input buffer - and can not be smaller than 66 bytes. - - If the input is not compressible, the return value might be larger than - length (input buffer size). - - The input buffer and the output buffer can not overlap. + DEPRECATED. - Compression level can be specified in parameter level. At the moment, - only level 1 and level 2 are supported. - Level 1 is the fastest compression and generally useful for short data. - Level 2 is slightly slower but it gives better compression ratio. + This is similar to fastlz_compress_level above, but with the level + automatically chosen. - Note that the compressed data, regardless of the level, can always be - decompressed using the function fastlz_decompress above. + This function is deprecated and it will be completely removed in some future + version. */ -int fastlz_compress_level(int level, const void* input, int length, void* output); +int fastlz_compress(const void* input, int length, void* output); -#if defined (__cplusplus) +#if defined(__cplusplus) } #endif From a7c4cb0e3ba62b7904d5b617cb957cae5fc60687 Mon Sep 17 00:00:00 2001 From: Obi Ebuka David Date: Mon, 7 Sep 2026 13:52:49 -0500 Subject: [PATCH 3/3] Bounds-check the two unchecked input reads in the decompressors (CWE-125 OOB read) Pulled from https://github.com/ariya/FastLZ/pull/12, commit d29d1abeb1d503db15586788249011ea8a753b70. --- src/fastlz.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fastlz.c b/src/fastlz.c index f99bb10..8272cf2 100644 --- a/src/fastlz.c +++ b/src/fastlz.c @@ -297,6 +297,7 @@ static int fastlz1_decompress(const void* input, int length, void* output, int m FASTLZ_BOUND_CHECK(ip <= ip_bound); len += *ip++; } + FASTLZ_BOUND_CHECK(ip <= ip_bound); ref -= *ip++; len += 3; FASTLZ_BOUND_CHECK(op + len <= op_limit); @@ -446,6 +447,7 @@ static int fastlz2_decompress(const void* input, int length, void* output, int m code = *ip++; len += code; } while (code == 255); + FASTLZ_BOUND_CHECK(ip <= ip_bound); code = *ip++; ref -= code; len += 3; @@ -488,6 +490,7 @@ int fastlz_compress(const void* input, int length, void* output) { } int fastlz_decompress(const void* input, int length, void* output, int maxout) { + if (length < 1) return 0; /* magic identifier for compression level */ int level = ((*(const uint8_t*)input) >> 5) + 1;