Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
158 commits
Select commit Hold shift + click to select a range
e42124f
feat(59-01): create vector module skeleton and AlignedBuffer type
TinDang97 Mar 29, 2026
75e2488
feat(59-01): scalar distance kernels and DistanceTable OnceLock dispatch
TinDang97 Mar 29, 2026
18bf96b
docs(59-01): update .planning submodule for 59-01 completion
TinDang97 Mar 29, 2026
e9a299b
feat(59-02): AVX2+FMA and AVX-512 distance kernels
TinDang97 Mar 29, 2026
6c0131c
feat(59-02): NEON kernels and DistanceTable SIMD dispatch wiring
TinDang97 Mar 29, 2026
2685417
docs(59-02): update .planning submodule for 59-02 completion
TinDang97 Mar 29, 2026
9116dc1
bench(59-03): add Criterion benchmarks for distance kernels
TinDang97 Mar 29, 2026
25715d1
test(59-03): add exhaustive SIMD-vs-scalar correctness tests
TinDang97 Mar 29, 2026
a278596
docs(59-03): update .planning submodule ref
TinDang97 Mar 29, 2026
52e0e23
fix(59-03): replace deprecated criterion::black_box with std::hint::b…
TinDang97 Mar 29, 2026
d2f9e2e
fix(59): close verification gaps — add core newtypes, SAFETY comments
TinDang97 Mar 29, 2026
c9edb54
docs(phase-59): complete SIMD vector distance engine
TinDang97 Mar 29, 2026
1eb4309
feat(60-01): FWHT scalar+AVX2 and Lloyd-Max 4-bit codebook
TinDang97 Mar 29, 2026
4ef1837
feat(60-01): TurboQuant MSE encoder/decoder with nibble packing
TinDang97 Mar 29, 2026
121214f
docs(60-01): complete TurboQuant 4-bit encoder plan
TinDang97 Mar 29, 2026
9ff685f
feat(60-02): CollectionMetadata with XXHash64 checksum and TQ ADC sca…
TinDang97 Mar 29, 2026
2688c24
feat(60-02): wire TQ ADC into DistanceTable and init FWHT dispatch
TinDang97 Mar 29, 2026
0e9a2e5
docs(60-02): update .planning submodule ref
TinDang97 Mar 29, 2026
97b8d6b
docs(phase-60): complete TurboQuant 4-bit encoder
TinDang97 Mar 29, 2026
318ee6f
feat(61-01): HnswGraph struct with BFS reorder and dual prefetch
TinDang97 Mar 29, 2026
f64ab31
feat(61-01): HnswBuilder with insert, neighbor pruning, and BFS reorder
TinDang97 Mar 29, 2026
6e50d0c
docs(61-01): update .planning submodule for 61-01 completion
TinDang97 Mar 29, 2026
4718a1c
fix(60): remove unused imports in collection.rs and tq_adc.rs
TinDang97 Mar 29, 2026
7a5221a
test(61-02): add BitVec and SearchScratch with TDD tests
TinDang97 Mar 29, 2026
619ff25
feat(61-02): HNSW beam search with dual prefetch and TQ-ADC distance
TinDang97 Mar 29, 2026
18d505a
docs(61-02): update .planning submodule for 61-02 completion
TinDang97 Mar 29, 2026
bd9fb1c
fix(61): remove unused variables in HNSW graph and build tests
TinDang97 Mar 29, 2026
a45a171
docs(phase-61): complete HNSW core
TinDang97 Mar 29, 2026
7667b0c
feat(62-01): segment architecture with MutableSegment, ImmutableSegme…
TinDang97 Mar 30, 2026
016b39c
docs(62-01): update .planning submodule for segment architecture plan
TinDang97 Mar 30, 2026
8c48869
feat(62-02): compaction pipeline with HNSW+TQ encoding and recall ver…
TinDang97 Mar 30, 2026
8d42bb7
docs(62-02): update .planning submodule for compaction pipeline compl…
TinDang97 Mar 30, 2026
ecddba2
docs(phase-62): complete segment architecture
TinDang97 Mar 30, 2026
54a44f3
feat(63-01): add VectorStore, VectorIndex, IndexMeta types
TinDang97 Mar 30, 2026
5ed8b15
feat(63-01): add FT.CREATE/FT.DROPINDEX/FT.INFO commands and Shard in…
TinDang97 Mar 30, 2026
307e2c4
docs(63-01): update .planning submodule ref
TinDang97 Mar 30, 2026
76de2f8
feat(63-02): add FT.SEARCH handler, SQ quantization, ShardMessage::Ve…
TinDang97 Mar 30, 2026
9db69e4
feat(63-02): SPSC handler FT.* interception + HSET auto-indexing hook
TinDang97 Mar 30, 2026
15a6412
docs(63-02): update .planning submodule ref
TinDang97 Mar 30, 2026
27faf90
feat(63-03): add merge_search_results and scatter_vector_search coord…
TinDang97 Mar 30, 2026
77fbbd8
test(63-03): end-to-end integration tests for FT.CREATE + HSET + FT.S…
TinDang97 Mar 30, 2026
76717d4
docs(63-03): update .planning submodule for 63-03 completion
TinDang97 Mar 30, 2026
9b88118
fix(vector): add DEL/HDEL/UNLINK auto-delete hook for vector indexes
TinDang97 Mar 30, 2026
c38adf1
feat(vector): route FT.* commands from connection handlers to VectorS…
TinDang97 Mar 30, 2026
a682ecb
docs(phase-63): complete FT.* Redis commands + shard integration
TinDang97 Mar 30, 2026
f342973
feat(64-01): add FilterExpr AST and PayloadIndex with Roaring bitmaps
TinDang97 Mar 30, 2026
472fc2a
feat(64-01): add cost-based filter strategy selection
TinDang97 Mar 30, 2026
16ffbab
docs(64-01): update .planning submodule ref
TinDang97 Mar 30, 2026
d422f5c
test(64-02): add failing tests for filtered search functions
TinDang97 Mar 30, 2026
0b8822e
feat(64-02): filtered search functions with ACORN 2-hop expansion
TinDang97 Mar 30, 2026
e3ce43d
test(64-02): add failing tests for FILTER clause parsing and PayloadI…
TinDang97 Mar 30, 2026
9c96625
feat(64-02): FILTER clause parsing + PayloadIndex wiring in FT.SEARCH
TinDang97 Mar 30, 2026
ba5e0ee
docs(64-02): update .planning submodule ref
TinDang97 Mar 30, 2026
84e0d50
fix(64): populate PayloadIndex during HSET auto-indexing
TinDang97 Mar 30, 2026
881396d
docs(phase-64): complete filtered search engine
TinDang97 Mar 30, 2026
b09b784
feat(65-01): TransactionManager with begin/commit/abort/acquire_write
TinDang97 Mar 30, 2026
45798ca
feat(65-01): MVCC visibility check for snapshot isolation entry filte…
TinDang97 Mar 30, 2026
2855544
test(65-02): add failing MVCC search + VectorStore txn_manager tests
TinDang97 Mar 30, 2026
b5b2db8
feat(65-02): MVCC-aware mutable segment search + TransactionManager o…
TinDang97 Mar 30, 2026
38952ba
test(65-02): add failing SegmentHolder search_mvcc + dirty set merge …
TinDang97 Mar 30, 2026
5014bea
feat(65-02): thread MVCC through SegmentHolder search + dirty set mer…
TinDang97 Mar 30, 2026
e70ef84
docs(65-02): update .planning submodule ref
TinDang97 Mar 30, 2026
5849d9a
docs(phase-65): complete MVCC transaction protocol
TinDang97 Mar 30, 2026
18570d2
feat(66-01): VectorWalRecord enum with CRC32 framing and HnswGraph se…
TinDang97 Mar 30, 2026
d2eee8e
feat(66-01): immutable segment disk I/O with checksum verification an…
TinDang97 Mar 30, 2026
94691e1
docs(66-01): update planning submodule for plan completion
TinDang97 Mar 30, 2026
bcb4913
feat(66-02): WAL writer extension + vector crash recovery engine
TinDang97 Mar 30, 2026
dc7148b
feat(66-02): shard restore integration + VectorStore pending segments
TinDang97 Mar 30, 2026
3a1f4aa
docs(66-02): update planning submodule for plan completion
TinDang97 Mar 30, 2026
95177fa
docs(phase-66): complete vector persistence
TinDang97 Mar 30, 2026
f9e8918
feat(67-01): IVF data structures + FAISS-interleaved PostingList + LU…
TinDang97 Mar 30, 2026
99cde81
feat(67-01): VPSHUFB FastScan AVX2 kernel + scalar fallback + dispatch
TinDang97 Mar 30, 2026
f7a413d
docs(67-01): complete IVF segment + VPSHUFB FastScan plan
TinDang97 Mar 30, 2026
01bff13
feat(67-02): IVF search, k-means clustering, and build_ivf_segment
TinDang97 Mar 30, 2026
2756eb2
feat(67-02): SegmentHolder fan-out integration for IVF segments
TinDang97 Mar 30, 2026
4ab3aad
docs(67-02): update .planning submodule ref
TinDang97 Mar 30, 2026
4ba0b7c
docs(phase-67): complete IVF + FastScan
TinDang97 Mar 30, 2026
3331334
feat(68-01): add gpu-cuda feature flag and build.rs CUDA detection
TinDang97 Mar 30, 2026
e24516a
feat(68-01): add GPU module tree with types, context, CAGRA build, an…
TinDang97 Mar 30, 2026
b7b7cfa
docs(68-01): update .planning submodule ref
TinDang97 Mar 30, 2026
a50404b
feat(68-02): GPU-aware compaction pipeline with CPU fallback
TinDang97 Mar 30, 2026
87eafc3
docs(phase-68): complete GPU pipeline
TinDang97 Mar 30, 2026
9753bec
test(69-01): add vector engine stress tests (10K interleaved ops + co…
TinDang97 Mar 30, 2026
75b2170
feat(69-02): Criterion benchmarks for HNSW build/search and FWHT tran…
TinDang97 Mar 30, 2026
40f1a63
test(69-01): add vector engine edge case and FT.* command hardening t…
TinDang97 Mar 30, 2026
ffb5232
test(69-02): memory audit validating per-vector overhead budget
TinDang97 Mar 30, 2026
7bdc4fb
docs(phase-69): complete vector engine hardening
TinDang97 Mar 30, 2026
e2acb0b
chore: update Cargo.lock for vector engine dependencies (arc-swap, ro…
TinDang97 Mar 30, 2026
c749c20
docs: add phase 70 — production benchmark, profiling, monitoring, tuning
TinDang97 Mar 30, 2026
4440ee1
feat(70-01): add vector metrics module with atomic counters
TinDang97 Mar 30, 2026
cb34b72
feat(70-02): create vector engine benchmark script
TinDang97 Mar 30, 2026
5a70ab5
feat(70-02): add flamegraph profiling script and 768d Criterion bench…
TinDang97 Mar 30, 2026
54f80ff
feat(70-01): wire vector metrics into FT commands and INFO output
TinDang97 Mar 30, 2026
bbbbd0f
docs(70-02): update .planning submodule for benchmark infrastructure …
TinDang97 Mar 30, 2026
17b7b8b
docs(70-01): update .planning submodule for vector metrics plan
TinDang97 Mar 30, 2026
b02e585
perf(70-03): optimize HNSW search hot path for >10% QPS improvement
TinDang97 Mar 30, 2026
204e9a2
docs(phase-70): complete production benchmark and tuning
TinDang97 Mar 30, 2026
a400f6b
bench: add comprehensive vector engine production benchmark script
TinDang97 Mar 30, 2026
d2a1c48
bench: add competitive benchmark vs Redis 8.6.1 and Qdrant
TinDang97 Mar 30, 2026
4614773
fix(vector): critical recall bugs — BFS reorder + dimension-adaptive …
TinDang97 Mar 30, 2026
c781e74
fix(vector): root cause HNSW recall — BitVec bug + add f32 search path
TinDang97 Mar 30, 2026
55908a7
docs: add phase 71 — fix TurboQuant integration
TinDang97 Mar 30, 2026
d36d937
feat(71-01): wire ImmutableSegment search to f32 HNSW traversal
TinDang97 Mar 30, 2026
9204475
test(71-03): add QJL transform module with sign-bit random projection
TinDang97 Mar 30, 2026
70f8989
feat(71-03): implement TurboQuant_prod encoder and inner-product scorer
TinDang97 Mar 30, 2026
098e18b
docs(71-01): update .planning submodule for HNSW recall fix
TinDang97 Mar 30, 2026
e361d6d
docs(71-03): update .planning submodule for TurboQuant IP mode comple…
TinDang97 Mar 30, 2026
255db2b
feat(71-02): add TQ-ADC brute-force scan for exhaustive nearest neigh…
TinDang97 Mar 30, 2026
716bbb1
docs(71-02): update .planning submodule for TQ-ADC brute-force scan
TinDang97 Mar 30, 2026
568546b
fix(71-04): fix all 9 broken tests from plans 01-03, add f32 recall b…
TinDang97 Mar 30, 2026
713b637
docs(71-04): update .planning submodule for integration validation
TinDang97 Mar 30, 2026
dedce36
docs(phase-71): complete TurboQuant integration fix
TinDang97 Mar 30, 2026
85570ca
docs: add phase 72 — multi-bit TQ, NEON tuning, benchmark
TinDang97 Mar 30, 2026
cbf6024
feat(72-01): multi-bit codebooks (1/2/3-bit) and variable bit packing
TinDang97 Mar 30, 2026
4ad5474
feat(72-02): extend QuantizationConfig with TurboQuant1/2/3 variants
TinDang97 Mar 30, 2026
c71a9cd
feat(72-01): multi-bit ADC distance computation (1/2/3-bit)
TinDang97 Mar 30, 2026
25f719f
feat(72-03): fix int8 NEON dispatch, add NEON FWHT butterfly
TinDang97 Mar 30, 2026
3b4cc95
docs(72-01): complete multi-bit TurboQuant codebooks and ADC plan
TinDang97 Mar 30, 2026
b8d93b7
docs(72-02): update .planning submodule for config/persistence plan
TinDang97 Mar 30, 2026
462fd68
feat(72-03): convert SmallVec upper layers to CSR format
TinDang97 Mar 30, 2026
00beb69
docs(72-03): update .planning submodule for ARM NEON + CSR plan
TinDang97 Mar 30, 2026
06f9178
feat(72-04): server-mode benchmark scripts and competitive results
TinDang97 Mar 30, 2026
287add5
docs(72-04): complete competitive benchmark plan
TinDang97 Mar 30, 2026
7987f3e
docs(phase-72): complete multi-bit TQ + benchmark
TinDang97 Mar 30, 2026
3b9a1d2
fix(vector): prevent SPSC self-send crash in FT.* command routing
TinDang97 Mar 30, 2026
c7b8070
fix: prevent panics in DashTable get_mut-after-insert and SPSC self-send
TinDang97 Mar 30, 2026
5fa2e97
feat(vector): enable FT.* commands on --shards 1 via shared VectorStore
TinDang97 Mar 30, 2026
c648a3f
bench: fair TCP benchmark infrastructure (Moon insert works, search c…
TinDang97 Mar 30, 2026
01aec25
fix(vector): complete server-mode vector search — 3 bugs fixed
TinDang97 Mar 30, 2026
f45cb46
docs: update .planning submodule (debug session)
TinDang97 Mar 30, 2026
2b23a81
fix(vector): TurboQuant codebook fix, memory optimization, 2-stage HN…
TinDang97 Mar 31, 2026
f5bcbf3
feat(vector): TQ-at-insert architecture — 5.5x memory reduction
TinDang97 Mar 31, 2026
2a27ea3
fix(vector): restore f32 reranking — recall 0.18 → 0.99
TinDang97 Mar 31, 2026
4b26e7f
fix(vector): mutable uses f32 L2 brute-force, not TQ-ADC
TinDang97 Mar 31, 2026
2bad801
feat(vector): TurboQuant_prod scoring — unbiased L2 without f32
TinDang97 Mar 31, 2026
36ee475
feat(vector): M=4 multi-projection QJL — recall 83% → 90%
TinDang97 Mar 31, 2026
4a056eb
perf(vector): SRHT replaces dense Gaussian QJL — 25x faster insert
TinDang97 Mar 31, 2026
5ea8c22
fix(vector): restore dense Gaussian QJL — recall 88% → 94.5%
TinDang97 Mar 31, 2026
f07ada6
feat(vector): sub-centroid refinement, multi-shard FT.*, fast insert
TinDang97 Mar 31, 2026
08a78c1
perf(vector): auto-compact mutable → HNSW on first search
TinDang97 Mar 31, 2026
10f16de
perf(vector): f32 HNSW build + LUT ADC + adaptive ef — recall 87→93%,…
TinDang97 Mar 31, 2026
56914cc
perf(vector): sub-centroid LUT in HNSW beam — eliminate rerank, +23% QPS
TinDang97 Mar 31, 2026
ecb3d4c
feat(vector): configurable EF_RUNTIME and COMPACT_THRESHOLD per index
TinDang97 Mar 31, 2026
82de46e
docs: vector engine report + user guide
TinDang97 Mar 31, 2026
4abd7f7
feat(vector): Light/Exact build modes — Light default, 5× less memory…
TinDang97 Mar 31, 2026
6f91362
docs: vector engine report + user guide
TinDang97 Mar 31, 2026
03512ed
fix: resolve merge conflicts with main (pub/sub branch)
TinDang97 Mar 31, 2026
5c8df60
fix: CI failures + address 23 code review findings
TinDang97 Mar 31, 2026
5afe4f7
fix: address 10 critical code review findings
TinDang97 Mar 31, 2026
3383bf5
fix: remove #[inline(always)] from target_feature functions (unstable…
TinDang97 Mar 31, 2026
95e5edf
fix: add # Safety docs to AVX2/AVX-512 unsafe fns, relax IVF recall t…
TinDang97 Apr 1, 2026
5721d6a
chore: clean up dead code, unused files, and test warnings
TinDang97 Apr 1, 2026
ceeb39d
Merge branch 'main' into feat/vector-engine
TinDang97 Apr 1, 2026
7dee89f
fix: address 15 Qodo review findings — 6 bugs, 9 rule violations
TinDang97 Apr 1, 2026
c74e379
fix: address inline review findings — scripts, init safety, codebook API
TinDang97 Apr 1, 2026
777be2b
fix: clarify codebook accessor docs, add debug_assert for non-4-bit u…
TinDang97 Apr 1, 2026
439a986
fix: remove remaining panic!/unwrap in library code, eliminate format…
TinDang97 Apr 1, 2026
6054c64
fix: fail fast on invalid codebook construction instead of swallowing…
TinDang97 Apr 1, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions scripts/test-commands.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ while [[ $# -gt 0 ]]; do
echo " pubsub - Pub/Sub commands (SUBSCRIBE, PUBLISH, etc.)"
echo " transaction - Transaction commands (MULTI, EXEC, DISCARD)"
echo " scripting - Lua scripting (EVAL, EVALSHA)"
echo " vector - Vector search commands (FT.CREATE, FT.SEARCH, FT.INFO, FT.DROPINDEX)"
echo " persistence - Persistence commands (BGSAVE, BGREWRITEAOF, etc.)"
echo " blocking - Blocking commands (BLPOP, BRPOP, BZPOPMIN, etc.)"
echo " benchmark - redis-benchmark throughput for all benchmarkable commands"
Expand Down Expand Up @@ -665,6 +666,42 @@ fi
# PERSISTENCE COMMANDS
# ===========================================================================

# ===========================================================================
# VECTOR SEARCH COMMANDS (moon-only — Redis uses different syntax)
# ===========================================================================

if should_run "vector"; then
echo ""
echo "=== VECTOR SEARCH COMMANDS ==="
mcli FLUSHALL >/dev/null 2>&1

# FT.CREATE — create a vector index
assert_moon "FT.CREATE basic" "OK" FT.CREATE myidx ON HASH PREFIX 1 doc: SCHEMA embedding VECTOR FLAT 6 DIM 4 DISTANCE_METRIC L2 TYPE FLOAT32

# FT.INFO — index metadata
TOTAL=$((TOTAL + 1)); FT_INFO=$(mcli FT.INFO myidx 2>&1)
if echo "$FT_INFO" | grep -q "myidx"; then PASS=$((PASS + 1)); echo " PASS: FT.INFO returns index name"; else FAIL=$((FAIL + 1)); echo " FAIL: FT.INFO returns index name"; fi

# Insert vectors via HSET (auto-indexed) — use python3 to avoid null byte stripping in bash
python3 -c "import struct,sys; sys.stdout.buffer.write(struct.pack('<4f',1.0,0.0,0.0,0.0))" | redis-cli -x -p "$PORT_RUST" HSET doc:1 embedding >/dev/null 2>&1
python3 -c "import struct,sys; sys.stdout.buffer.write(struct.pack('<4f',0.0,1.0,0.0,0.0))" | redis-cli -x -p "$PORT_RUST" HSET doc:2 embedding >/dev/null 2>&1

# FT.SEARCH — verify command doesn't error (redis-cli can't pass binary args directly)
TOTAL=$((TOTAL + 1)); FT_SEARCH=$(mcli FT.SEARCH myidx "*" 2>&1)
if ! echo "$FT_SEARCH" | grep -qi "err"; then PASS=$((PASS + 1)); echo " PASS: FT.SEARCH does not error"; else FAIL=$((FAIL + 1)); echo " FAIL: FT.SEARCH returned error"; fi

# FT.DROPINDEX — remove index
assert_moon "FT.DROPINDEX" "OK" FT.DROPINDEX myidx

# FT.INFO after drop should error
TOTAL=$((TOTAL + 1)); FT_INFO_AFTER=$(mcli FT.INFO myidx 2>&1)
if echo "$FT_INFO_AFTER" | grep -qi "err\|not found"; then PASS=$((PASS + 1)); echo " PASS: FT.INFO after drop errors"; else FAIL=$((FAIL + 1)); echo " FAIL: FT.INFO after drop errors"; fi
fi

# ===========================================================================
# PERSISTENCE COMMANDS
# ===========================================================================

if should_run "persistence"; then
echo ""
echo "=== PERSISTENCE COMMANDS ==="
Expand Down
25 changes: 25 additions & 0 deletions scripts/test-consistency.sh
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,31 @@ assert_both "GET with 500-char key" GET "$LONGKEY"
# ===========================================================================

echo ""
# ===========================================================================
# Vector Search (moon-only — FT.* not available in Redis)
# ===========================================================================
log "=== Vector Search (moon-only) ==="

# Create index on moon only
FT_CREATE=$(redis-cli -p "$PORT_RUST" FT.CREATE vecidx ON HASH PREFIX 1 vec: SCHEMA embedding VECTOR FLAT 6 DIM 4 DISTANCE_METRIC L2 TYPE FLOAT32 2>&1)
assert_eq "FT.CREATE" "OK" "$FT_CREATE"

# Insert vectors — use python3 to avoid null byte stripping in bash
python3 -c "import struct,sys; sys.stdout.buffer.write(struct.pack('<4f',1.0,0.0,0.0,0.0))" | redis-cli -x -p "$PORT_RUST" HSET vec:1 embedding >/dev/null 2>&1
python3 -c "import struct,sys; sys.stdout.buffer.write(struct.pack('<4f',0.0,1.0,0.0,0.0))" | redis-cli -x -p "$PORT_RUST" HSET vec:2 embedding >/dev/null 2>&1

# FT.INFO should show index
FT_INFO=$(redis-cli -p "$PORT_RUST" FT.INFO vecidx 2>&1)
if echo "$FT_INFO" | grep -q "vecidx"; then
PASS=$((PASS + 1))
else
FAIL=$((FAIL + 1)); echo " FAIL: FT.INFO should show vecidx"
fi

# FT.DROPINDEX
FT_DROP=$(redis-cli -p "$PORT_RUST" FT.DROPINDEX vecidx 2>&1)
assert_eq "FT.DROPINDEX" "OK" "$FT_DROP"

echo "============================================"
echo " Data Consistency Test Results"
echo "============================================"
Expand Down
Loading
Loading