Skip to content

Commit a6fd5fd

Browse files
aglab2aglab2
committed
Introduce savestate compression via LZ4-like algo
1 parent ef02d4f commit a6fd5fd

4 files changed

Lines changed: 313 additions & 4 deletions

File tree

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ OBJ_DIR = obj
33
TOOL_DIR = Hacktice
44

55
# Alter these 4 variables according to your need
6-
CPP_FILES = main savestate text_manager input_viewer level_reset cfg level_conv strings string_conv wallkick_frame distance levitate speed timer checkpoint action interaction music death death_floor version soft_reset custom_text colors debug_box
6+
CPP_FILES = main savestate text_manager input_viewer level_reset cfg level_conv strings string_conv wallkick_frame distance levitate speed timer checkpoint action interaction music death death_floor version soft_reset custom_text colors debug_box compress
77

88
# If _start will have more elements, adjust this variable
99
PAYLOAD_HEADER_SIZE = 64
@@ -28,7 +28,7 @@ PAYLOAD_DATA = $(TOOL_DIR)/payload_data
2828
CC = clang
2929
AR = llvm-ar
3030
LD = ld.lld
31-
CFLAGS = -DBINARY -flto -Wall -Wdouble-promotion -Oz -mfix4300 -march=mips2 --target=mips-img-elf -fomit-frame-pointer -G0 -I $(INCLUDE_PATH) -I $(INCLUDE_PATH)/libc -mno-check-zero-division -fno-exceptions -fno-builtin -fno-rtti -fno-common -mno-abicalls -DTARGET_N64 -mfpxx
31+
CFLAGS = -DBINARY -flto -Wall -Wdouble-promotion -Os -mfix4300 -march=mips2 --target=mips-img-elf -fomit-frame-pointer -G0 -I $(INCLUDE_PATH) -I $(INCLUDE_PATH)/libc -mno-check-zero-division -fno-exceptions -fno-builtin -fno-rtti -fno-common -mno-abicalls -DTARGET_N64 -mfpxx
3232

3333
all: $(OBJ_DIR) $(ROM) $(PAYLOAD_HEADER) $(PAYLOAD_DATA)
3434

‎src/compress.c‎

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
// This is a very baby version of LZ4 compression algorithm
2+
3+
#include "compress.h"
4+
5+
#include "bool.h"
6+
#include "libc/string.h"
7+
8+
#define PACKED __attribute__((packed))
9+
#define __GET_UNALIGNED_T(type, ptr) \
10+
({ \
11+
const struct { \
12+
type x; \
13+
} PACKED *__pptr = (typeof(__pptr)) (ptr); \
14+
__pptr->x; \
15+
})
16+
17+
#define __PUT_UNALIGNED_T(type, val, ptr) \
18+
do { \
19+
struct { \
20+
type x; \
21+
} PACKED *__pptr = (typeof(__pptr)) (ptr); \
22+
__pptr->x = (val); \
23+
} while (0)
24+
25+
#define GET_UNALIGNED4(ptr) __GET_UNALIGNED_T(uint32_t, (ptr))
26+
#define PUT_UNALIGNED4(val, ptr) __PUT_UNALIGNED_T(uint32_t, (val), (ptr))
27+
28+
#define MINMATCH 3
29+
30+
typedef struct
31+
{
32+
uint32_t match3;
33+
uint32_t match4;
34+
} MinMatch;
35+
36+
static MinMatch readMinMatch(const uint8_t* buf)
37+
{
38+
const uint32_t mask = 0xffffff00;
39+
const uint32_t match4 = GET_UNALIGNED4(buf);
40+
const uint32_t match3 = match4 & mask;
41+
return (MinMatch) { match3, match4 };
42+
}
43+
44+
static uint32_t readMinMatch3(const uint8_t* buf)
45+
{
46+
const uint32_t mask = 0xffffff00;
47+
const uint32_t match4 = GET_UNALIGNED4(buf);
48+
return match4 & mask;
49+
}
50+
51+
#define HASH_CHAINS_BITS 12
52+
#define HASH_CHAINS_SIZE (1 << HASH_CHAINS_BITS)
53+
54+
typedef struct
55+
{
56+
uint32_t hash3;
57+
uint32_t hash4;
58+
} MinMatcHash;
59+
60+
static MinMatcHash hashMinMatch(MinMatch match)
61+
{
62+
return (MinMatcHash) { ((match.match3 * 506832829U) << 8) >> (32 - HASH_CHAINS_BITS), (match.match4 * 2654435761U) >> (32 - HASH_CHAINS_BITS)};
63+
}
64+
65+
static void wildCopy4(uint8_t* dst, const uint8_t* src, const uint8_t* dstEnd)
66+
{
67+
do
68+
{
69+
PUT_UNALIGNED4(GET_UNALIGNED4(src), dst);
70+
src += 4;
71+
dst += 4;
72+
} while (dst < dstEnd);
73+
}
74+
75+
#define WILD_COPY_MOVE(dst, src, amount) do{ uint8_t* start = dst; dst += (amount); wildCopy4(start, src, dst); }while(0)
76+
77+
static void encodeLargeInt(uint8_t** _out, int leftToEncode)
78+
{
79+
#define out (*_out)
80+
bool keepEncoding = true;
81+
while (keepEncoding)
82+
{
83+
keepEncoding = leftToEncode >= 255;
84+
*(out++) = (uint8_t)(keepEncoding ? 255 : leftToEncode);
85+
leftToEncode -= 255;
86+
}
87+
#undef out
88+
}
89+
90+
typedef struct
91+
{
92+
int matchLen;
93+
const uint8_t* inMatch;
94+
} MinMatchPtr;
95+
96+
static MinMatchPtr tryMinMatch(uint32_t potentialMinMatchOffset, MinMatch minMatch, const uint8_t* inCursor, const uint8_t* inLimit, int inCursorOffset)
97+
{
98+
const uint8_t* potentialMinMatch = inCursor - inCursorOffset + potentialMinMatchOffset;
99+
100+
// need to do some fun wraparound stuff
101+
if (potentialMinMatch > inCursor - 4)
102+
potentialMinMatch -= 256;
103+
104+
if (minMatch.match3 == readMinMatch3(potentialMinMatch))
105+
{
106+
// find the longest match we have with the potential match
107+
int matchLen = MINMATCH;
108+
while (1)
109+
{
110+
if (inCursor + matchLen >= inLimit)
111+
break;
112+
if (inCursor[matchLen] != potentialMinMatch[matchLen])
113+
break;
114+
115+
matchLen++;
116+
}
117+
118+
return (MinMatchPtr) { matchLen, potentialMinMatch };
119+
}
120+
else
121+
{
122+
return (MinMatchPtr) { 0, NULL };
123+
}
124+
}
125+
126+
void mlz4_compress(const uint8_t* in, const uint32_t inSize, uint8_t* out, uint32_t* outSize)
127+
{
128+
uint8_t* compressedData = out;
129+
130+
uint8_t* outCursor = compressedData;
131+
#ifndef TARGET_N64
132+
uint8_t hashChains3[HASH_CHAINS_SIZE] = {};
133+
uint8_t hashChains4[HASH_CHAINS_SIZE] = {};
134+
#else
135+
// placed in gDecompressionHeap
136+
uint8_t* hashChains3 = (uint8_t*) 0x801c1000;
137+
uint8_t* hashChains4 = (uint8_t*) 0x801c1000 + HASH_CHAINS_SIZE;
138+
#endif
139+
140+
// should be safe to do
141+
*(uint32_t*)outCursor = inSize;
142+
outCursor += 4;
143+
144+
// uncompressible data, just write it out as literals
145+
if (inSize <= 8)
146+
{
147+
outCursor[0] = ((uint8_t)inSize) << 4;
148+
outCursor++;
149+
WILD_COPY_MOVE(outCursor, in, inSize);
150+
*outSize = (size_t) (outCursor - compressedData);
151+
return;
152+
}
153+
154+
const uint8_t* inEnd = in + inSize;
155+
const uint8_t* inCursor = in + 4;
156+
const uint8_t* inLimit = inEnd - 4;
157+
const uint8_t* inLiteralStart = in;
158+
159+
// start the loop
160+
while (1)
161+
{
162+
// push match we took last time
163+
{
164+
MinMatch prevMinMatch = readMinMatch(inCursor - 4);
165+
MinMatcHash hashValue = hashMinMatch(prevMinMatch);
166+
uint8_t offset = (uint8_t)(inCursor - 4 - in);
167+
hashChains3[hashValue.hash3] = offset;
168+
hashChains4[hashValue.hash4] = offset;
169+
}
170+
171+
if (inCursor >= inLimit)
172+
{
173+
// printf("Final literals: %lld\n", inEnd - inLiteralStart);
174+
const int literalsCount = inEnd - inLiteralStart;
175+
if (literalsCount >= 15)
176+
{
177+
*(outCursor++) = 0xf0;
178+
encodeLargeInt(&outCursor, literalsCount - 15);
179+
}
180+
else
181+
{
182+
*(outCursor++) = ((uint8_t) literalsCount) << 4;
183+
}
184+
185+
WILD_COPY_MOVE(outCursor, inLiteralStart, literalsCount);
186+
*outSize = (uint32_t)(outCursor - compressedData);
187+
return;
188+
}
189+
190+
MinMatch minMatch = readMinMatch(inCursor);
191+
MinMatcHash hashValue = hashMinMatch(minMatch);
192+
uint8_t inCursorOffset = (uint8_t) (inCursor - in);
193+
194+
uint8_t potentialMinMatchOffset3 = hashChains3[hashValue.hash3];
195+
uint8_t potentialMinMatchOffset4 = hashChains4[hashValue.hash4];
196+
197+
MinMatchPtr match3 = tryMinMatch(potentialMinMatchOffset3, minMatch, inCursor, inLimit, inCursorOffset);
198+
MinMatchPtr match4 = tryMinMatch(potentialMinMatchOffset4, minMatch, inCursor, inLimit, inCursorOffset);
199+
200+
MinMatchPtr match = match3.matchLen > match4.matchLen ? match3 : match4;
201+
if (match.matchLen)
202+
{
203+
// write the match...
204+
uint8_t* pnibble = outCursor++;
205+
const int literalsCount = (int) (inCursor - inLiteralStart);
206+
if (literalsCount >= 15)
207+
{
208+
*pnibble = 0xf0;
209+
encodeLargeInt(&outCursor, literalsCount - 15);
210+
}
211+
else
212+
{
213+
*pnibble = ((uint8_t)literalsCount) << 4;
214+
}
215+
216+
WILD_COPY_MOVE(outCursor, inLiteralStart, literalsCount);
217+
*(outCursor++) = (uint8_t)(inCursor - match.inMatch - 4);
218+
219+
const int matchesCount = match.matchLen - MINMATCH;
220+
if (matchesCount >= 15)
221+
{
222+
*pnibble |= 0x0f;
223+
encodeLargeInt(&outCursor, matchesCount - 15);
224+
}
225+
else
226+
{
227+
*pnibble |= (uint8_t)matchesCount;
228+
}
229+
230+
// printf("Literals: %lld, Offset: %lld, Match: %d\n", inCursor - inLiteralStart, inCursor - match.inMatch, match.matchLen);
231+
inCursor += match.matchLen;
232+
inLiteralStart = inCursor;
233+
}
234+
else
235+
{
236+
inCursor++;
237+
}
238+
}
239+
}
240+
241+
void mlz4_decompress(const uint8_t* in, uint8_t* out)
242+
{
243+
const uint8_t* inCursor = in;
244+
uint32_t originalSize = *(uint32_t*)inCursor;
245+
inCursor += 4;
246+
247+
uint8_t* outCursor = out;
248+
uint8_t* outEnd = out + originalSize;
249+
uint8_t* outSafetyMargin = outEnd - 4;
250+
while (1)
251+
{
252+
uint8_t nibble = *(inCursor++);
253+
int literalsCount = nibble >> 4;
254+
if (literalsCount == 15)
255+
{
256+
while (1)
257+
{
258+
uint8_t nextNibble = *(inCursor++);
259+
literalsCount += nextNibble;
260+
if (nextNibble != 255)
261+
break;
262+
}
263+
}
264+
265+
if (literalsCount)
266+
{
267+
uint8_t* literalsEnd = outCursor + literalsCount;
268+
if (literalsEnd >= outSafetyMargin)
269+
{
270+
// last 4 literals must be in safety margin
271+
memcpy(outCursor, inCursor, literalsCount);
272+
outCursor += literalsCount;
273+
break;
274+
}
275+
else
276+
{
277+
wildCopy4(outCursor, inCursor, literalsEnd);
278+
outCursor = literalsEnd;
279+
inCursor += literalsCount;
280+
}
281+
}
282+
283+
int offset = 4 + *(inCursor++);
284+
int matchesCount = nibble & 0x0f;
285+
if (matchesCount == 15)
286+
{
287+
while (1)
288+
{
289+
uint8_t nextNibble = *(inCursor++);
290+
matchesCount += nextNibble;
291+
if (nextNibble != 255)
292+
break;
293+
}
294+
}
295+
matchesCount += MINMATCH;
296+
297+
const uint8_t* matchStart = outCursor - offset;
298+
WILD_COPY_MOVE(outCursor, matchStart, matchesCount);
299+
}
300+
}

‎src/compress.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#include "types.h"
2+
3+
#define uint8_t u8
4+
#define uint32_t u32
5+
6+
void mlz4_compress(const uint8_t* in, const uint32_t inSize, uint8_t* out, uint32_t* outSize);
7+
void mlz4_decompress(const uint8_t* in, uint8_t* out);

‎src/savestate.c‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "savestate.h"
22

33
#include "binary.h"
4+
#include "compress.h"
45
#include "cfg.h"
56
#include "text_manager.h"
67

@@ -47,15 +48,16 @@ void SaveState_onNormal()
4748
Hacktice_gState->area = gCurrAreaIndex;
4849
Hacktice_gState->level = gCurrLevelNum;
4950
Hacktice_gState->size = sizeof(State);
50-
memcpy(Hacktice_gState->memory, _hackticeStateDataStart, _hackticeStateDataEnd - _hackticeStateDataStart);
51+
uint32_t stateSize = 0;
52+
mlz4_compress(_hackticeStateDataStart, _hackticeStateDataEnd - _hackticeStateDataStart, Hacktice_gState->memory, &stateSize);
5153
}
5254
else
5355
{
5456
if (Config_action() == Config_ButtonAction_LOAD_STATE)
5557
{
5658
if (Hacktice_gState->area == gCurrAreaIndex && Hacktice_gState->level == gCurrLevelNum)
5759
{
58-
memcpy(_hackticeStateDataStart, Hacktice_gState->memory, _hackticeStateDataEnd - _hackticeStateDataStart);
60+
mlz4_decompress(Hacktice_gState->memory, _hackticeStateDataStart);
5961
resetCamera();
6062
}
6163
}

0 commit comments

Comments
 (0)