diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5174261..860f57f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,7 +16,7 @@ if this touches public Rust, Python, C ABI, Go, Manifest, persisted-format, examples/docs, feature, or MSRV surfaces - [ ] `CHANGELOG.md` updated under `Unreleased` if user-facing -- [ ] `cargo deny check` passes (licenses / advisories / bans / sources) +- [ ] `cargo deny --workspace --all-features --locked check` passes (licenses / advisories / bans / sources) - [ ] If `ordvec-python/` changed: `cargo clippy -p ordvec-python --all-targets -- -D warnings`, then `maturin develop` + `pytest ordvec-python/tests` pass ## Notes diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index e9daa2d..8c54e25 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -10,8 +10,10 @@ name: audit # Read-only token, no `run:` steps (no injection surface). Because this runs # UNATTENDED on a cron schedule, the third-party actions are SHA-pinned (not the # mutable @v4/@v2 tags ci.yml uses on human-triggered push/PR) so a compromised -# tag update cannot auto-execute on a scheduled runner. Scoped to -# `check advisories`. +# tag update cannot auto-execute on a scheduled runner. Scoped to `check +# advisories`, with separate locked roots for the main workspace and the +# standalone fuzz crate so neither committed lockfile can fall outside the +# advisory graph. on: schedule: @@ -36,6 +38,16 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 + - name: Audit root workspace advisories + uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 with: - command: check advisories + command: check + arguments: --workspace --all-features --locked + command-arguments: advisories + - name: Audit standalone fuzz lockfile advisories + uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 + with: + command: check + manifest-path: fuzz/Cargo.toml + arguments: --all-features --locked + command-arguments: advisories diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a0658e..a121ea6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -350,9 +350,18 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 + - name: Check root workspace policy + uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 with: command: check + arguments: --workspace --all-features --locked + - name: Audit standalone fuzz lockfile advisories + uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 + with: + command: check + manifest-path: fuzz/Cargo.toml + arguments: --all-features --locked + command-arguments: advisories # ---------------------------------------------------------------------- # AVX-512 coverage via Intel SDE. GitHub-hosted runners lack AVX-512, so we diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4052b3d..9edc611 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -161,6 +161,7 @@ jobs: needs: guard if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest + timeout-minutes: 35 permissions: contents: read actions: read @@ -174,26 +175,105 @@ jobs: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} SHA: ${{ github.sha }} + POLL_INTERVAL_SECONDS: 30 + POLL_TIMEOUT_SECONDS: 1800 run: | set -euo pipefail - MAIN_SHA="$(gh api "repos/${REPO}/commits/main" --jq '.sha')" - echo "current main sha: ${MAIN_SHA}" - if [ "$SHA" != "$MAIN_SHA" ]; then - echo "::error::release tag points at ${SHA}, but current main is ${MAIN_SHA}. Move the tag to the current protected main HEAD and re-run." - exit 1 - fi - # Require a SUCCESSFUL push run for this SHA *on main* for each workflow. - # Filtering on branch as well as head_sha stops a green run for the same - # commit on an unrelated branch from satisfying the gate. - for wf in ci.yml python.yml fuzz.yml codeql.yml actionlint.yml zizmor.yml; do - ok="$(gh api \ - "repos/${REPO}/actions/workflows/${wf}/runs?head_sha=${SHA}&branch=main&event=push&status=success&per_page=20" \ - --jq '[.workflow_runs[] | select(.head_branch == "main" and .event == "push" and .conclusion == "success")] | length')" - echo "successful push ${wf} runs for ${SHA} on main: ${ok}" - if [ "${ok}" -lt 1 ]; then - echo "::error::no successful push ${wf} run for ${SHA} on main. Push to main, let CI pass, then re-tag." + deadline=$((SECONDS + POLL_TIMEOUT_SECONDS)) + while :; do + MAIN_SHA="$(gh api "repos/${REPO}/commits/main" --jq '.sha')" + echo "current main sha: ${MAIN_SHA}" + if [ "$SHA" != "$MAIN_SHA" ]; then + echo "::error::release tag points at ${SHA}, but current main is ${MAIN_SHA}. Move the tag to the current protected main HEAD and re-run." + exit 1 + fi + + all_green=true + for wf in ci.yml python.yml fuzz.yml codeql.yml actionlint.yml zizmor.yml; do + # Query every status, then inspect the latest exact-SHA push run. + # A success-only filter cannot distinguish normal run-visibility + # delay from terminal failure and was the source of issue #272. + api_error="$(mktemp)" + if runs_json="$(gh api \ + "repos/${REPO}/actions/workflows/${wf}/runs?head_sha=${SHA}&branch=main&event=push&per_page=100" \ + 2>"$api_error")"; then + : + else + api_exit_status=$? + api_message="$(<"$api_error")" + api_status="$(sed -nE 's/.*HTTP ([0-9]{3}).*/\1/p' "$api_error" | tail -n 1)" + retryable_api_error=false + case "$api_status" in + 408|429|5??) + retryable_api_error=true + ;; + "") + if [ "$api_exit_status" -ne 4 ] && grep -Eiq \ + 'timeout|timed out|connection (reset|refused|closed)|could not resolve|temporary failure in name resolution|network is unreachable|TLS handshake|unexpected EOF|error connecting to' \ + "$api_error"; then + retryable_api_error=true + fi + ;; + 403) + if grep -Eiq 'rate.?limit|secondary rate|abuse detection' "$api_error"; then + retryable_api_error=true + fi + ;; + esac + rm -f "$api_error" + if "$retryable_api_error"; then + echo "waiting for ${wf}: transient API query failure${api_status:+ (HTTP ${api_status})}: ${api_message}" + all_green=false + continue + fi + echo "::error::${wf}: API query failed permanently${api_status:+ (HTTP ${api_status})}: ${api_message}" + exit 1 + fi + rm -f "$api_error" + + latest="$(jq -r --arg sha "$SHA" ' + [.workflow_runs[] + | select(.head_sha == $sha and .head_branch == "main" and .event == "push")] + | sort_by([.run_number, .run_attempt, .id]) + | last + | if . == null then "" + else [.status, (.conclusion // ""), (.html_url // "")] | join("|") + end + ' <<<"$runs_json")" + if [ -z "$latest" ]; then + echo "waiting for ${wf}: no exact-SHA main push run is visible yet" + all_green=false + continue + fi + + IFS='|' read -r status conclusion run_url <<<"$latest" + if [ "$status" != "completed" ]; then + echo "waiting for ${wf}: latest run is ${status} (${run_url})" + all_green=false + elif [ "$conclusion" = "success" ]; then + echo "green ${wf}: ${run_url}" + else + echo "::error::${wf} completed with ${conclusion:-no conclusion} for ${SHA}: ${run_url}" + exit 1 + fi + done + + if "$all_green"; then + FINAL_MAIN_SHA="$(gh api "repos/${REPO}/commits/main" --jq '.sha')" + echo "final current main sha: ${FINAL_MAIN_SHA}" + if [ "$SHA" != "$FINAL_MAIN_SHA" ]; then + echo "::error::main advanced from release tag ${SHA} to ${FINAL_MAIN_SHA} while CI was being checked. Tag only the new protected main HEAD after its own CI is green." + exit 1 + fi + echo "all release-gated workflows are green for ${SHA} on main" + break + fi + if (( SECONDS >= deadline )); then + echo "::error::timed out after ${POLL_TIMEOUT_SECONDS}s waiting for exact-SHA main push CI. No artifacts or draft release were created." exit 1 fi + echo "CI is not settled; polling again in ${POLL_INTERVAL_SECONDS}s" + sleep "$POLL_INTERVAL_SECONDS" done release-avx512: @@ -264,7 +344,7 @@ jobs: notes: name: release notes (git-cliff) + draft Release - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest permissions: @@ -302,7 +382,7 @@ jobs: build-crate: name: build .crate + SBOM - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest steps: @@ -393,7 +473,7 @@ jobs: build-wheels: name: wheel ${{ matrix.platform.target }} (${{ matrix.platform.runner }}) - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ${{ matrix.platform.runner }} strategy: @@ -536,7 +616,7 @@ jobs: build-sdist: name: build sdist + SBOM - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest steps: @@ -609,7 +689,7 @@ jobs: build-manifest-wheels: name: manifest wheel ${{ matrix.platform.target }} (${{ matrix.platform.runner }}) - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ${{ matrix.platform.runner }} strategy: @@ -674,7 +754,7 @@ jobs: build-manifest-sdist: name: build manifest sdist + SBOM - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 15580ed..2bc72fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,14 @@ _No unreleased changes._ ### Changed +- **First-run and release surfaces are aligned for v0.6.0.** The root and PyPI + READMEs now lead with a tiny deterministic retrieval that prints a checked + result, show persistence/reopen, and route users to the Rust, Python, CLI, + and manifest packages. Both manifest packages include an end-to-end + create-and-verify path, and every `ordvec-manifest` command and option now + has CLI help text. Release invariants pin the current package version, + manifest schema, feature defaults, public metadata links, and first-run + examples so the next release cannot silently drift across surfaces. - **BREAKING (`ordvec-manifest`): deterministic manifest schema v2.** The manifest schema version is now `ordvec.index_manifest.v2`. `manifest_id` and `created_at` are removed from `IndexManifest`, creation omits the @@ -154,6 +162,19 @@ _No unreleased changes._ 0.40's build dependency used a library feature unavailable on Rust 1.89; the all-features manifest suite now runs in the permanent MSRV lane. +### Security + +- The release CI gate now waits, with a 30-minute fail-closed deadline, for + the latest exact-HEAD push runs instead of treating normal Actions + visibility/in-progress delay as an immediate failure. It rejects terminal + non-successes and a moving `main`, and every draft-release or artifact-build + job is directly blocked behind the gate. +- The advisory policy now roots `cargo-deny` at every workspace member and + enables every feature, closing the blind spot that omitted dev-only + benchmark dependencies. `anyhow` is updated from 1.0.102 to 1.0.103 to + clear RUSTSEC-2026-0190; the documented dev-only `bincode` 1.x + unmaintained advisory remains explicitly triaged. + ## 0.5.0 - 2026-06-19 ### Security diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7c7eab..a2340e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ cargo test --no-default-features cargo +1.89.0 build # MSRV cargo build --locked RUSTFLAGS="-D warnings" cargo build -cargo deny check # licenses / advisories / bans / sources +cargo deny --workspace --all-features --locked check # full locked workspace graph ``` SIMD dispatches at runtime — AVX-512 / AVX2 on x86_64, NEON on aarch64, diff --git a/Cargo.lock b/Cargo.lock index dfaa738..b36e6f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,9 +95,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "autocfg" diff --git a/README.md b/README.md index 0442cd9..453cfbd 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,84 @@ Training-free ordinal & sign quantization for vector retrieval. that quantizes the **ordinal (rank) and sign structure** of an embedding — no codebook, no learned rotation, no graph to build. +## Choose your surface + +| You want to... | Use | Install | +|---|---|---| +| Build retrieval in Rust | [`ordvec`](https://crates.io/crates/ordvec) | `cargo add ordvec@0.6` | +| Build retrieval in Python | [`ordvec`](https://pypi.org/project/ordvec/) | `python -m pip install ordvec` | +| Create or verify index manifests from Rust or a CLI | [`ordvec-manifest`](https://crates.io/crates/ordvec-manifest) | `cargo install ordvec-manifest` | +| Verify manifests from Python | [`ordvec-manifest`](https://pypi.org/project/ordvec-manifest/) | `python -m pip install ordvec-manifest` | + +## Quickstart: see a result in 30 seconds + +Python: + +```bash +python -m pip install --upgrade ordvec +``` + +```python +import numpy as np +from ordvec import RankQuant + +documents = np.array([ + [8, 7, 6, 5, 4, 3, 2, 1], + [1, 2, 3, 4, 5, 6, 7, 8], + [8, 1, 7, 2, 6, 3, 5, 4], +], dtype=np.float32) +query = np.array([[8, 7, 6, 5, 4, 3, 2, 1]], dtype=np.float32) + +index = RankQuant(dim=8, bits=1) +index.add(documents) +scores, ids = index.search_asymmetric(query, k=1) +print(f"top document: {ids[0, 0]} (score {scores[0, 0]:.3f})") +``` + +```text +top document: 0 (score 0.396) +``` + +Rust: + +```bash +cargo add ordvec@0.6 +``` + +Or add the dependency directly: + +```toml +[dependencies] +ordvec = "0.6" +``` + +```rust +use ordvec::RankQuant; + +let documents = [ + 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, + 8.0, 1.0, 7.0, 2.0, 6.0, 3.0, 5.0, 4.0, +]; +let query = [8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0]; + +let mut index = RankQuant::new(8, 1); +index.add(&documents); +let results = index.search_asymmetric(&query, 1); +assert_eq!(results.indices_for_query(0)[0], 0); +``` + +The runnable Rust version is [`examples/quickstart.rs`](examples/quickstart.rs). + +Persist and reopen the same index without keeping the original float corpus: + +```python +index.write("quickstart.ovrq") +reopened = RankQuant.load("quickstart.ovrq") +_, reopened_ids = reopened.search_asymmetric(query, k=1) +assert reopened_ids[0, 0] == 0 +``` + ## What is ordinal retrieval? Ordinal retrieval is a retrieval family where the index operates on order/sign @@ -78,7 +156,7 @@ and the gap widens over the committed subsampling sweep: layers are the right comparison target; this README does not claim public million-scale HNSW crossover or GPU bandwidth numbers until the underlying run artifacts are committed. -- **Reproducible on your machine, one command:** +- **Reproducible on your machine:** ```sh make bench-beir-setup # Python deps + CUDA llama-cpp-python (GGUF Q8 encoder) @@ -124,7 +202,7 @@ structure of each vector on its own: known before you see any data (256 B at dim = 1024, 2-bit), with `bits ∈ {1, 2, 4}` the size/recall knob. (`b = 8` is an opt-in evidence/refinement width — asymmetric scoring at any dim, symmetric only - when `dim % 256 == 0` — not a broad retrieval mode. In v0.5.0 it is + when `dim % 256 == 0` — not a broad retrieval mode. In v0.6.0 it is Rust-only, in-memory, not accepted by the Python `RankQuant` constructor, and not persistable to `.ovrq`; each prepared asymmetric query owns a `dim * 256` `f32` LUT, about 64 MiB at the maximum dimension.) @@ -148,7 +226,7 @@ large-scale serving rather than competing with one. - **`RankQuant`** — ranks bucketed into `1 << bits` equal-width bins, `bits` bits per coordinate (`dim * bits / 8` bytes/doc). Both a symmetric (Spearman) and asymmetric (float-query LUT) scorer. `bits ∈ - {1, 2, 4}` are the cross-language persisted retrieval widths in v0.5.0; + {1, 2, 4}` are the cross-language persisted retrieval widths in v0.6.0; `b = 8` is Rust-only and in-memory for evidence/refinement. - **`Bitmap`** — a top-bucket bitmap per document (one bit per coordinate); scoring is `popcount(Q AND D)`, a coarsened rank overlap. @@ -162,8 +240,8 @@ Two further paths, for callers who need them: scalar dispatch) for absolute-minimum stage-1 scan latency, at 2× the RankQuant b=2 footprint (`dim/2` bytes/doc) and 8-bit LUT scoring noise. It persists to `.ovfs` (magic `OVFS`) through direct - `RankQuantFastscan::{write,load}` calls. In v0.5.0, `.ovfs` is not yet part - of the `probe_index_metadata()` / `ordvec-manifest` v1 contract; bind it with + `RankQuantFastscan::{write,load}` calls. In v0.6.0, `.ovfs` is not yet part + of the `probe_index_metadata()` / `ordvec-manifest` v2 contract; bind it with an application-owned digest or attestation if it crosses a trust boundary. Reach for it only when scan latency at b=2 is the binding constraint; the headline retrieval surface is still `RankQuant` / `Bitmap` / two-stage. @@ -208,36 +286,6 @@ empirical contract to measure. Details in [`docs/RANK_MODES.md`](docs/RANK_MODES.md). -## Quickstart - -```toml -[dependencies] -ordvec = "0.6" - -# Or, to track unreleased `main`, use a git dependency instead: -# ordvec = { git = "https://github.com/Project-Navi/ordvec" } -``` - -```rust -use ordvec::RankQuant; - -let dim = 1024; -let n_docs = 10_000; -let mut index = RankQuant::new(dim, 2); // 2 bits/coord → 256 bytes/doc - -// `add` takes a flat, row-major buffer of `dim * n_docs` f32s. -// Replace this with your real embeddings. -let doc_embeddings: Vec = vec![0.0; dim * n_docs]; -index.add(&doc_embeddings); - -// Asymmetric scan: full-precision queries vs bucketed docs (recommended). -let query_embeddings: Vec = vec![0.0; dim * 4]; // 4 queries, row-major -let results = index.search_asymmetric(&query_embeddings, 10); - -let top_ids = results.indices_for_query(0); // top-10 doc ids for query 0 -let top_scores = results.scores_for_query(0); -``` - For the two-stage compressed-scan path (`Bitmap` / `SignBitmap` candidate generation → `RankQuant` rerank) and the full mode comparison, see [`docs/RANK_MODES.md`](docs/RANK_MODES.md). @@ -342,9 +390,6 @@ candidate slices passed to `Search` until the call returns. [`theorem-map`](https://github.com/Project-Navi/ordvec-formalization/blob/main/docs/theorem-map.md), and [`reviewer brief`](https://github.com/Project-Navi/ordvec-formalization/blob/main/docs/reviewer-brief.md). - **API docs:** , -- **Paper (OrdVec / RankQuant):** _link TBD — see - [Research collaboration](#research-collaboration)._ - ## Benchmarks ### BEIR retrieval (public datasets, reproducible) @@ -517,8 +562,8 @@ The probe/manifest-covered on-disk formats (`.ovr` / `.ovrq` / `.ovbm` / **no built-in checksum, MAC, or signature — by design.** The loaders validate *structure* (magic, version, bounds, exact-length payload) but not *origin*: a structurally valid file can still be untrusted. `RankQuantFastscan` also writes -and loads `.ovfs` directly, but in v0.5.0 that format is not yet covered by -`probe_index_metadata()` or `ordvec-manifest` v1. If an index file crosses a +and loads `.ovfs` directly, but in v0.6.0 that format is not yet covered by +`probe_index_metadata()` or `ordvec-manifest` v2. If an index file crosses a trust boundary (network transfer, shared storage), verify it before loading. `ordvec-manifest` binds supported index files to a JSON manifest by SHA-256, header metadata, row identity, named auxiliary sidecars, and attestation shape diff --git a/RELEASING.md b/RELEASING.md index 9af5e7c..fbed381 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -24,8 +24,12 @@ The unified `release.yml`: workflow but skip every job below the gate; - runs a **`require-ci-green`** gate confirming the tag points at current `main` HEAD and that per-commit CI is green on `main` for that SHA — `ci.yml`, - `python.yml`, `fuzz.yml`, `codeql.yml`, `actionlint.yml`, `zizmor.yml` (a - *successful* run for that exact SHA on `main`); + `python.yml`, `fuzz.yml`, `codeql.yml`, `actionlint.yml`, `zizmor.yml`. The + gate polls the latest exact-SHA push run every 30 seconds for at most 30 + minutes: missing, queued, and in-progress runs wait; a terminal non-success + fails immediately; and a moved `main` HEAD fails immediately. Every draft- + release and build-artifact job depends directly on this gate, so an unsettled + or failed gate creates no release artifacts and no draft GitHub Release; - publishes via **OIDC trusted publishing** (no long-lived crates.io / PyPI tokens in the repo) for both Rust crates and both Python distributions; - canonicalizes each Python dist before attestation and release upload: for a @@ -167,8 +171,11 @@ the OIDC exchange (no risk of a bad publish; just a failed run). `ordvec-manifest-python/Cargo.toml`, `ordvec-manifest-python/pyproject.toml`, `ordvec-manifest-python/python/ordvec_manifest/__init__.py`, and - `ordvec-ffi/Cargo.toml`) and update `CHANGELOG.md` with migration notes for - every intentional compatibility break. Commit on `main`. + `ordvec-ffi/Cargo.toml`). Also bump every internal path-dependency version: + `ordvec-manifest` → `ordvec`, both Python binding aliases, `ordvec-ffi` → + `ordvec`, and `benchmarks/beir-bench` → `ordvec`. Update `CHANGELOG.md` + with migration notes for every intentional compatibility break. Commit on + `main`. - Run `python tests/release_publish_invariants.py` after the bump; it checks lockstep versions, MSRV/docs drift, registry metadata parity, Python classifier/URL parity, docs.rs feature policy, package contents, and @@ -181,8 +188,11 @@ the OIDC exchange (no risk of a bad publish; just a failed run). crates.io releases instead of a pre-release git revision. 4. Confirm CI is **green for current `main` HEAD**. `require-ci-green` checks `main` HEAD's SHA — which needs a **completed, successful** (not - `cancelled`, not in-progress) run of `ci.yml`, `python.yml`, `fuzz.yml`, - `codeql.yml`, `actionlint.yml`, and `zizmor.yml`. + `cancelled`) latest run of `ci.yml`, `python.yml`, `fuzz.yml`, `codeql.yml`, + `actionlint.yml`, and `zizmor.yml`. A just-pushed tag may reach the release + workflow before Actions exposes or completes those runs, so the gate waits + up to 30 minutes and polls every 30 seconds. It fails immediately on a + terminal non-success or if `main` advances, and fails closed on timeout. - Routine `ci.yml` / `coverage.yml` runs may warn and skip SDE-dependent steps when Intel's downloadmirror challenges GitHub-hosted runners. That keeps external mirror outages from holding `main` red, but it does **not** diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index a096f29..aad3d55 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -1,6 +1,6 @@ # Threat Model — `ordvec` -> **Status:** v0.6.0 (pre-1.0), 2026-06-15. This is the maintained threat model +> **Status:** v0.6.0 (pre-1.0), 2026-07-19. This is the maintained threat model > for the `ordvec` Rust crate, C ABI, Go wrapper, PyO3/maturin Python bindings, > and the `ordvec-manifest` sidecar verifier. It is reviewed when the > attack surface changes (new persistence formats, new `unsafe` kernels, new @@ -55,10 +55,10 @@ merged, (2) enforceable by tests or CI, (3) local to the library boundary, and (4) unlikely to add operational burden downstream. Heavyweight controls (mandatory index signing, long-running fuzz farms, service-level admission control) are documented as **deployment guidance** unless the project has -maintainer capacity to own them. Release publication requires a non-triggering -approver through protected GitHub Environments; the residual release -supply-chain risk is approver account compromise / collusion, not a -single-owner project structure (see THREAT-SUPPLY-001). +maintainer capacity to own them. Release policy requires a non-triggering +approver through protected GitHub Environments, but that policy is only in +force when the pre-tag environment-settings audit passes. The live settings +currently fail that audit and block v0.6.0 (see THREAT-SUPPLY-001). --- @@ -306,7 +306,7 @@ applications must validate paths before calling"). ## 5. Supply-chain threats (THREAT-SUPPLY) -### 5.1 Existing controls (verified) +### 5.1 Workflow controls (verified) and environment audit status **Workflow code (all workflows):** third-party actions pinned by commit SHA (the one mandated exception is the SLSA reusable workflow, which the SLSA @@ -327,6 +327,11 @@ attestation fails the release closed. Each Rust publish job proves pre- and post-publish crates.io byte identity against the attested `.crate`; PyPI additionally gets **PEP 740** attestations via Trusted Publishing. +The environment protections are mutable GitHub configuration outside the +workflow. `tests/release_environment_settings.sh` audits them before a tag is +created. As of 2026-07-19 that audit fails; the exact drift and release gate are +recorded in THREAT-SUPPLY-001 below. + **Static / supply-chain analysis:** **CodeQL** scans Rust, Python, and Actions (no-build databases); **OpenSSF Scorecard** publishes SARIF to code scanning and the score badge; **zizmor** audits workflow hardening (pinned); a @@ -337,23 +342,25 @@ scoped to the wheel. ### 5.2 Risks -**THREAT-SUPPLY-001 (mitigated; residual = release-approver account -compromise / collusion): Release configuration and ownership.** The release -**environments** (`pypi`, `crates-io`) list `Fieldnote-Echo` and `toadkicker` as -required reviewers, enable **prevent self-review**, enforce a **30-minute wait -timer**, and restrict deployment to the **release-tag pattern -`v[0-9]*.[0-9]*.[0-9]*`** (the tag-triggered workflow runs on -`refs/tags/...`, not `refs/heads/main`, so a branch-only allowlist would -deadlock publishing — see RELEASING.md). The `require-ci-green` gate -independently verifies the tag SHA has a successful push-event CI run on `main`, -and `main` itself is branch-protected (PR review, no force-push) — so a release -cannot be cut from an unmerged or attacker branch, and no publish runs without -an explicit human approval by a listed release approver who did not trigger the -deployment. The remaining residual is compromise or misuse of an eligible -approver account, or collusion between release participants. *Mitigations:* -strong 2FA / passkeys on both approver accounts, a small reviewed approver list, -and the 30-minute deployment window for the non-triggering approver to inspect -or cancel a bad release. See [`RELEASING.md`](RELEASING.md). +**THREAT-SUPPLY-001 (open release blocker as of 2026-07-19): Release +configuration and ownership.** The intended release-environment policy for +both `pypi` and `crates-io` is: `Fieldnote-Echo` and `toadkicker` are eligible +reviewers, self-review is prevented, a 30-minute wait timer is enforced, and +deployment is restricted to the release-tag pattern +`v[0-9]*.[0-9]*.[0-9]*`. The live environments currently retain the correct +tag pattern but list only `Fieldnote-Echo`, permit self-review, and have no wait +timer. Consequently `tests/release_environment_settings.sh` fails and **no +v0.6.0 tag or publish should proceed**. Restore `toadkicker`, prevent +self-review, and the 30-minute wait on both environments, then rerun the audit +before tagging. The `require-ci-green` gate independently verifies the tag SHA +has a successful push-event CI run on `main`, and `main` itself is +branch-protected (PR review, no force-push), but those controls do not replace +the independent publication approval gate. Once the environment audit passes, +the remaining residual is compromise or misuse of an eligible approver account, +or collusion between release participants. *Mitigations:* strong 2FA / passkeys +on both approver accounts, a small reviewed approver list, and the 30-minute +deployment window for the non-triggering approver to inspect or cancel a bad +release. See [`RELEASING.md`](RELEASING.md). **THREAT-SUPPLY-002 (mitigated): Release immutability and tag integrity.** Published artifacts are **immutable by registry design** — crates.io is @@ -507,7 +514,7 @@ blast radius of a compromised dependency separately. | THREAT-FFI-003 | FFI | Binding | Accidental telemetry through ABI stats | Low | Low | **Mitigated** — caller-owned stats, no logging | | THREAT-FFI-004 | FFI | Binding | Concurrent input mutation during released-GIL call | Medium | Medium | **P2** — documented contract | | THREAT-FFI-005 | FFI | Binding | Unsanitized path forwarding | Medium | Medium | **P2** — documented contract | -| THREAT-SUPPLY-001 | Supply chain | Config | Release config / dual-approver gate | Low | Critical | **Mitigated** (two approvers, self-review blocked, 30-minute wait timer, `require-ci-green` main-SHA gate); residual = approver compromise / collusion | +| THREAT-SUPPLY-001 | Supply chain | Config | Release config / dual-approver gate | Medium | Critical | **Release blocked** until both environment settings pass the pre-tag audit; tag policy and `require-ci-green` remain in place | | THREAT-SUPPLY-002 | Supply chain | Config | Release immutability / tag integrity | Low | High | **Mitigated** — registries immutable; GitHub immutable releases on + `main` protected | | THREAT-SUPPLY-003 | Supply chain | Config | Typosquatting adjacent names | Medium | Medium | P3 | | THREAT-QUERY-001 | Resource | Deployment | Batch / `k` exhaustion in serving | Medium | Medium | **P2** — deployment docs | diff --git a/benchmarks/beir-bench/Cargo.toml b/benchmarks/beir-bench/Cargo.toml index 5122bf5..04349b6 100644 --- a/benchmarks/beir-bench/Cargo.toml +++ b/benchmarks/beir-bench/Cargo.toml @@ -13,7 +13,7 @@ name = "beir-bench" path = "src/main.rs" [dependencies] -ordvec = { path = "../.." } +ordvec = { version = "0.6.0", path = "../.." } # Pure-Rust HNSW (Malkov–Yashunin); no system/C++ deps. The faithful portable # stand-in for the C++ hnswlib (no maintained Rust binding to that exists). hnsw_rs = "0.3" diff --git a/deny.toml b/deny.toml index 17acba9..5e20903 100644 --- a/deny.toml +++ b/deny.toml @@ -1,6 +1,6 @@ # cargo-deny configuration for ordvec. # -# Run locally with: cargo deny check +# Run locally with: cargo deny --workspace --all-features --locked check # Schema validated against cargo-deny 0.19.x. # # ordvec ships no system/numerical dependencies (no BLAS, faer, ndarray, @@ -36,6 +36,8 @@ ignore = ["RUSTSEC-2025-0141"] # * BSD-2-Clause — zerocopy / zerocopy-derive are # `BSD-2-Clause OR Apache-2.0 OR MIT`; the OR resolves via Apache/MIT, but # allowing BSD-2-Clause keeps the choice explicit and audit-clear. +# * Zlib — foldhash, transitively used by the workspace's +# dev-only HNSW benchmark and the optional manifest SQLite implementation. # * Apache-2.0 WITH LLVM-exception — wasi's expression includes it; only # pulled on wasm targets, harmless to allow and keeps cross-target clean. allow = [ @@ -44,6 +46,7 @@ allow = [ "Apache-2.0 WITH LLVM-exception", "Unicode-3.0", "BSD-2-Clause", + "Zlib", ] # License text must match an SPDX template at >= this confidence to count. confidence-threshold = 0.8 diff --git a/docs/INDEX_PROVENANCE.md b/docs/INDEX_PROVENANCE.md index 03fa45e..4e92113 100644 --- a/docs/INDEX_PROVENANCE.md +++ b/docs/INDEX_PROVENANCE.md @@ -4,8 +4,8 @@ `.ovrq` / `.ovbm` / `.ovsb` files and reloads them through `Rank::load`, `RankQuant::load`, `Bitmap::load`, and `SignBitmap::load`. `RankQuantFastscan` also writes and loads `.ovfs` through its direct API, but in -v0.5.0 `.ovfs` is not covered by `probe_index_metadata()` or -`ordvec-manifest` v1. This note states exactly **what the loaders guarantee and +v0.6.0 `.ovfs` is not covered by `probe_index_metadata()` or +`ordvec-manifest` v2. This note states exactly **what the loaders guarantee and what they do not**, so you can decide whether an index file needs out-of-band verification before you load it. For the byte layout and versioning of the persisted formats themselves, see [`PERSISTED_FORMAT.md`](PERSISTED_FORMAT.md). @@ -105,12 +105,12 @@ The manifest verifier checks: least one subject SHA-256 matching the artifact when attestations are supplied. -The v1 verifier intentionally does not create or verify `.ovfs` FastScan +The v2 verifier intentionally does not create or verify `.ovfs` FastScan artifacts yet. If a `RankQuantFastscan` artifact crosses a trust boundary in -v0.5.0, bind the bytes with a caller-owned checksum, artifact-store control, or +v0.6.0, bind the bytes with a caller-owned checksum, artifact-store control, or attestation and load it directly only after that policy check succeeds. The direct `.ovfs` loader still rejects invalid nibbles, non-canonical block-tail -padding, and rows that violate b=2 constant composition; manifest v1 simply +padding, and rows that violate b=2 constant composition; manifest v2 simply does not bind or probe those bytes yet. Auxiliary artifacts are for application-owned sidecars such as metadata, diff --git a/docs/PERSISTED_FORMAT.md b/docs/PERSISTED_FORMAT.md index 3f19677..556b6b7 100644 --- a/docs/PERSISTED_FORMAT.md +++ b/docs/PERSISTED_FORMAT.md @@ -65,7 +65,7 @@ cache in their own manifests: `n_top`; - `file_size_bytes`: total observed file size. -In v0.5.0, `probe_index_metadata(path)` rejects `OVFS` with an unsupported +In v0.6.0, `probe_index_metadata(path)` rejects `OVFS` with an unsupported metadata-probe error rather than returning a partial descriptor. Load `.ovfs` only through `RankQuantFastscan::load` unless and until the FastScan metadata contract is promoted in a later minor release; the direct loader rejects @@ -227,7 +227,7 @@ payload invariants: - `SignBitmap::load`: no additional row invariant exists. `RankQuantFastscan::load` has its own direct loader path for `.ovfs`; it is not -covered by this probe-versus-load contract in v0.5.0. +covered by this probe-versus-load contract in v0.6.0. Loader success is the primitive binary-safety boundary. It is not a provenance or deployment-policy decision. diff --git a/docs/artifact-platform-matrix.md b/docs/artifact-platform-matrix.md index 8218093..d88a20b 100644 --- a/docs/artifact-platform-matrix.md +++ b/docs/artifact-platform-matrix.md @@ -13,7 +13,7 @@ environment matching a platform family is supported. | Surface | Published where | Platform/build contract | Release verification | | --- | --- | --- | --- | | `ordvec` Rust crate | crates.io package `ordvec`; GitHub Release `.crate` asset | Rust 1.89 MSRV; default features empty; pure Rust, no BLAS/LAPACK/system numeric dependency | `cargo package --locked`; GitHub/Sigstore/SLSA provenance; pre-publish and post-publish byte identity against crates.io | -| `ordvec-manifest` Rust crate | crates.io package `ordvec-manifest`; GitHub Release `.crate` asset | Rust 1.89 MSRV; default features empty; optional `cli`, `sqlite`, and `sqlite-bundled` features | Built after matching `ordvec` exists; GitHub/Sigstore/SLSA provenance; byte identity against crates.io | +| `ordvec-manifest` Rust crate | crates.io package `ordvec-manifest`; GitHub Release `.crate` asset | Rust 1.89 MSRV; default feature `cli`; optional `sqlite` and `sqlite-bundled` features; library-only consumers can disable defaults | Built after matching `ordvec` exists; GitHub/Sigstore/SLSA provenance; byte identity against crates.io | | Python `ordvec` | PyPI package `ordvec`; GitHub Release wheels and sdist | CPython 3.10+ abi3; `numpy>=2.2`; wheels for Linux x86_64 and Linux aarch64 are manylinux/glibc wheels; no musllinux/Alpine wheel is shipped yet; macOS aarch64 and Windows x64 wheels are also published; native extension modules are embedded in the wheel and do not load a separate `ordvec_ffi` library | Canonical wheel/sdist selection; linux/aarch64 native smoke; PyPI hash verification; PEP 740 attestation on fresh upload | | Python `ordvec-manifest` | PyPI package `ordvec-manifest`; GitHub Release wheels and sdist | CPython 3.10+ abi3; Linux wheels are manylinux/glibc for x86_64 and aarch64; no musllinux/Alpine wheel is shipped yet; macOS aarch64 and Windows x64 wheels are also published; native extension modules are embedded in the wheel | Canonical wheel/sdist selection; linux/aarch64 native smoke; PyPI hash verification; PEP 740 attestation on fresh upload | | Node/WASM | Not shipped; no npm package is published yet | Placeholder for issue #138; no JavaScript, TypeScript, or wasm package support is promised by this release | No release verification until a future packaging lane adds build jobs | diff --git a/docs/compatibility-policy.md b/docs/compatibility-policy.md index a471b51..a2e4491 100644 --- a/docs/compatibility-policy.md +++ b/docs/compatibility-policy.md @@ -63,9 +63,9 @@ The `experimental` feature is a default-off research surface. Today it exposes `RankQuantFastscan` is a stable, public (but specialized) type, covered by the normal pre-1.0 compatibility policy above. Its direct `.ovfs` -`RankQuantFastscan::{write,load}` path is supported, but in v0.5.0 `.ovfs` is +`RankQuantFastscan::{write,load}` path is supported, but in v0.6.0 `.ovfs` is not yet part of the primitive persisted-format, `probe_index_metadata()`, or -`ordvec-manifest` v1 contract. Feature-gated `#[doc(hidden)]` exports such as +`ordvec-manifest` v2 contract. Feature-gated `#[doc(hidden)]` exports such as `search_asymmetric_byte_lut` are reachable for internal benchmarks and parity tests only when explicitly enabled, and are not part of the stable default API. @@ -114,7 +114,7 @@ a new ABI version or clear migration notes. The Go wrapper follows the C ABI. Source-breaking Go API changes require the same compatibility classification in release notes. -The `ordvec-manifest` CLI, library API, and v1 JSON schema are treated as +The `ordvec-manifest` CLI, library API, and v2 JSON schema are treated as stable release surfaces. Patch releases should not introduce breaking changes to the CLI arguments, emitted error codes, library report shapes, or JSON schema structure. Minor releases may introduce schema, CLI, or library updates @@ -136,7 +136,7 @@ loaders. Writers no longer emit those magics. `RankQuantFastscan` writes and loads `.ovfs` / `OVFS` directly, but that specialized format is not included in the primitive probe/manifest contract for -v0.5.0. Promoting `.ovfs` into `probe_index_metadata()` and `ordvec-manifest` +v0.6.0. Promoting `.ovfs` into `probe_index_metadata()` and `ordvec-manifest` requires an explicit future compatibility review. Patch releases should keep valid files from the same minor series loadable. diff --git a/examples/quickstart.rs b/examples/quickstart.rs new file mode 100644 index 0000000..79a926b --- /dev/null +++ b/examples/quickstart.rs @@ -0,0 +1,20 @@ +use ordvec::RankQuant; + +fn main() { + // Three tiny embeddings with distinct coordinate orderings. + let documents = [ + 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, // document 0 + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, // document 1 + 8.0, 1.0, 7.0, 2.0, 6.0, 3.0, 5.0, 4.0, // document 2 + ]; + let query = [8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0]; + + let mut index = RankQuant::new(8, 1); + index.add(&documents); + let results = index.search_asymmetric(&query, 1); + + let top_document = results.indices_for_query(0)[0]; + let top_score = results.scores_for_query(0)[0]; + assert_eq!(top_document, 0); + println!("top document: {top_document} (score {top_score:.3})"); +} diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index ecb390e..f67f073 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arbitrary" diff --git a/ordvec-ffi/Cargo.toml b/ordvec-ffi/Cargo.toml index 177a92b..c5bd08d 100644 --- a/ordvec-ffi/Cargo.toml +++ b/ordvec-ffi/Cargo.toml @@ -11,4 +11,4 @@ name = "ordvec_ffi" crate-type = ["rlib", "cdylib", "staticlib"] [dependencies] -ordvec = { path = ".." } +ordvec = { version = "0.6.0", path = ".." } diff --git a/ordvec-manifest-python/Cargo.toml b/ordvec-manifest-python/Cargo.toml index 490ef70..946ae8b 100644 --- a/ordvec-manifest-python/Cargo.toml +++ b/ordvec-manifest-python/Cargo.toml @@ -14,7 +14,7 @@ name = "_ordvec_manifest" crate-type = ["cdylib"] [dependencies] -ordvec_manifest_core = { package = "ordvec-manifest", path = "../ordvec-manifest", default-features = false } +ordvec_manifest_core = { package = "ordvec-manifest", version = "0.6.0", path = "../ordvec-manifest", default-features = false } pyo3 = { version = "0.29.0", features = ["extension-module", "abi3-py310"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/ordvec-manifest-python/README.md b/ordvec-manifest-python/README.md index 9fa3b11..674e8cc 100644 --- a/ordvec-manifest-python/README.md +++ b/ordvec-manifest-python/README.md @@ -2,24 +2,41 @@ Python bindings for the `ordvec-manifest` verifier. -Install from PyPI: +## First verified index ```bash -python -m pip install ordvec-manifest +python -m pip install --upgrade ordvec ordvec-manifest ``` -Import as `ordvec_manifest`. The package exposes the Rust manifest verifier as -dict-returning Python functions: - ```python +import numpy as np +from ordvec import RankQuant import ordvec_manifest -report = ordvec_manifest.verify_manifest("index.manifest.json") -if not report["ok"]: - raise RuntimeError(report["errors"]) +documents = np.array([ + [8, 7, 6, 5, 4, 3, 2, 1], + [1, 2, 3, 4, 5, 6, 7, 8], +], dtype=np.float32) +index = RankQuant(dim=8, bits=1) +index.add(documents) +index.write("quickstart.ovrq") + +ordvec_manifest.create_manifest( + "quickstart.ovrq", + "quickstart.manifest.json", + "quickstart-embedding-v1", + row_id_is_identity=True, +) +report = ordvec_manifest.verify_manifest("quickstart.manifest.json") +print(f"verified: {report['ok']}") +``` + +```text +verified: True ``` -Create manifests with caller-owned sidecars by passing dictionaries with +The package exposes the Rust manifest verifier as dict-returning Python +functions. To bind existing caller-owned sidecars, pass dictionaries with `name`, `path`, and optional `required`: ```python diff --git a/ordvec-manifest-python/pyproject.toml b/ordvec-manifest-python/pyproject.toml index 1e2f099..19f9f14 100644 --- a/ordvec-manifest-python/pyproject.toml +++ b/ordvec-manifest-python/pyproject.toml @@ -42,6 +42,8 @@ classifiers = [ Homepage = "https://github.com/Project-Navi/ordvec" Repository = "https://github.com/Project-Navi/ordvec" Issues = "https://github.com/Project-Navi/ordvec/issues" +Documentation = "https://github.com/Project-Navi/ordvec/blob/v0.6.0/ordvec-manifest-python/README.md" +Changelog = "https://github.com/Project-Navi/ordvec/blob/main/CHANGELOG.md" [tool.maturin] manifest-path = "Cargo.toml" diff --git a/ordvec-manifest/README.md b/ordvec-manifest/README.md index 05ed039..332f90a 100644 --- a/ordvec-manifest/README.md +++ b/ordvec-manifest/README.md @@ -16,6 +16,44 @@ From a workspace checkout, use the CLI with `cargo run -p ordvec-manifest --`. Library-only consumers that do not need the CLI can depend on the crate with `default-features = false`. +## First verified index + +The shortest end-to-end check uses the Python core package to write a tiny real +ordvec index, then creates and verifies its manifest with the CLI: + +```sh +python -m pip install --upgrade ordvec +cargo install ordvec-manifest + +python - <<'PY' +import numpy as np +from ordvec import RankQuant + +documents = np.array([ + [8, 7, 6, 5, 4, 3, 2, 1], + [1, 2, 3, 4, 5, 6, 7, 8], +], dtype=np.float32) +index = RankQuant(dim=8, bits=1) +index.add(documents) +index.write("quickstart.ovrq") +PY + +ordvec-manifest create \ + --index quickstart.ovrq \ + --row-id-is-identity \ + --embedding-model quickstart-embedding-v1 \ + --out quickstart.manifest.json +ordvec-manifest verify --manifest quickstart.manifest.json +``` + +```text +quickstart.manifest.json +verified +``` + +For an existing bundle with a caller-owned ID sidecar, bind both files before +loading either one: + ```sh ordvec-manifest create \ --index path/to/index.ovrq \ diff --git a/ordvec-manifest/src/lib.rs b/ordvec-manifest/src/lib.rs index 2144667..6bdbb6a 100644 --- a/ordvec-manifest/src/lib.rs +++ b/ordvec-manifest/src/lib.rs @@ -13,6 +13,25 @@ //! verified snapshot of the canonical artifact path and related load metadata. //! The `ordvec-manifest` binary exposes the same bounded verification surfaces //! for command-line use. +//! +//! ```no_run +//! use ordvec_manifest::{verify_for_load, VerifyOptions}; +//! +//! # fn main() -> Result<(), Box> { +//! let plan = verify_for_load("quickstart.manifest.json", VerifyOptions::default())?; +//! println!( +//! "verified {} rows at {}", +//! plan.metadata().vector_count, +//! plan.artifact_path().display() +//! ); +//! // Load the index from plan.artifact_path() immediately, while the verified +//! // bytes remain under the caller's control. +//! # Ok(()) +//! # } +//! ``` +//! +//! See the crate README for a copy-and-run index creation and CLI verification +//! path. use chrono::{DateTime, SecondsFormat, Utc}; use ordvec::{ diff --git a/ordvec-manifest/src/main.rs b/ordvec-manifest/src/main.rs index f4188e4..8b73fd7 100644 --- a/ordvec-manifest/src/main.rs +++ b/ordvec-manifest/src/main.rs @@ -14,66 +14,93 @@ const EXIT_USAGE_OR_CONFIG: i32 = 2; #[derive(Parser)] #[command(name = "ordvec-manifest")] -#[command(about = "Verify ordvec index manifests", version)] +#[command(about = "Create and verify ordvec index manifests", version)] +#[command(after_help = "Run `ordvec-manifest --help` for command options.")] struct Cli { + /// Manifest operation to run. #[command(subcommand)] command: Commands, } #[derive(Subcommand)] enum Commands { + /// Compute a file's SHA-256 digest and byte length. Hash { + /// File to hash. path: PathBuf, + /// Emit a machine-readable JSON object. #[arg(long)] json: bool, }, + /// Print a manifest summary without verifying its artifacts. Inspect { + /// Manifest JSON to inspect. manifest: PathBuf, #[command(flatten)] limits: LimitArgs, + /// Emit the parsed manifest as JSON. #[arg(long)] json: bool, }, + /// Verify a manifest and every declared artifact. Verify { + /// Manifest JSON to verify. #[arg(long)] manifest: PathBuf, + /// Override the primary index path declared by the manifest. #[arg(long)] index: Option, + /// Permit absolute artifact paths (disabled by default). #[arg(long)] allow_absolute_paths: bool, + /// Permit relative paths that escape the manifest directory. #[arg(long)] allow_path_escape: bool, + /// Permit duplicate db_id values in a JSONL row map. #[arg(long)] allow_duplicate_db_ids: bool, #[command(flatten)] limits: LimitArgs, + /// Emit the verification report as JSON. #[arg(long)] json: bool, }, + /// Create a deterministic manifest for an existing ordvec index. Create { + /// Existing ordvec index to bind. #[arg(long)] index: PathBuf, + /// JSONL row-identity map to bind. #[arg(long)] row_map: Option, + /// Declare row IDs as the zero-based index row numbers. #[arg(long)] row_id_is_identity: bool, + /// Bind a required caller-owned sidecar as NAME=PATH (repeatable). #[arg(long = "aux", value_name = "NAME=PATH", value_parser = parse_auxiliary_artifact_arg)] auxiliary_artifacts: Vec, + /// Bind an optional caller-owned sidecar as NAME=PATH (repeatable). #[arg(long = "optional-aux", value_name = "NAME=PATH", value_parser = parse_auxiliary_artifact_arg)] optional_auxiliary_artifacts: Vec, + /// Identifier of the embedding model that produced the index. #[arg(long)] embedding_model: String, + /// Destination manifest JSON path. #[arg(long)] out: PathBuf, + /// Permit absolute artifact paths in the emitted manifest. #[arg(long)] allow_absolute_paths: bool, + /// Permit emitted relative paths to escape the manifest directory. #[arg(long)] allow_path_escape: bool, #[command(flatten)] limits: LimitArgs, }, #[cfg(feature = "sqlite")] + /// Verify or activate manifests with a SQLite report cache. Sqlite { + /// SQLite-backed operation to run. #[command(subcommand)] command: SqliteCommands, }, @@ -104,7 +131,7 @@ fn parse_auxiliary_artifact_arg(value: &str) -> Result panic!("expected verify command"), } } + + #[test] + fn help_describes_commands_and_safety_relevant_options() { + let mut root = Cli::command(); + let root_help = root.render_long_help().to_string(); + for expected in [ + "Compute a file's SHA-256 digest", + "Print a manifest summary", + "Verify a manifest and every declared artifact", + "Create a deterministic manifest", + ] { + assert!( + root_help.contains(expected), + "missing help text: {expected}" + ); + } + + let mut verify = Cli::command() + .find_subcommand("verify") + .expect("verify subcommand") + .clone(); + let verify_help = verify.render_long_help().to_string(); + for expected in [ + "Manifest JSON to verify", + "Permit absolute artifact paths", + "Permit relative paths that escape", + "Emit the verification report as JSON", + ] { + assert!( + verify_help.contains(expected), + "missing verify help text: {expected}" + ); + } + } } #[cfg(feature = "sqlite")] #[derive(Subcommand)] enum SqliteCommands { + /// Verify a manifest, optionally reusing a valid cached report. Verify { + /// SQLite cache database path. #[arg(long)] db: PathBuf, + /// Manifest JSON to verify. #[arg(long)] manifest: PathBuf, + /// Reuse a matching cached report when available. #[arg(long)] use_cache: bool, + /// Override the primary index path declared by the manifest. #[arg(long)] index: Option, + /// Permit absolute artifact paths (disabled by default). #[arg(long)] allow_absolute_paths: bool, + /// Permit relative paths that escape the manifest directory. #[arg(long)] allow_path_escape: bool, + /// Permit duplicate db_id values in a JSONL row map. #[arg(long)] allow_duplicate_db_ids: bool, #[command(flatten)] limits: LimitArgs, + /// Emit the verification report as JSON. #[arg(long)] json: bool, }, + /// Verify and mark a manifest active in the SQLite cache. Activate { + /// SQLite cache database path. #[arg(long)] db: PathBuf, + /// Manifest JSON to verify and activate. #[arg(long)] manifest: PathBuf, + /// Activate even when verification reports errors. #[arg(long)] force: bool, + /// Override the primary index path declared by the manifest. #[arg(long)] index: Option, + /// Permit absolute artifact paths (disabled by default). #[arg(long)] allow_absolute_paths: bool, + /// Permit relative paths that escape the manifest directory. #[arg(long)] allow_path_escape: bool, + /// Permit duplicate db_id values in a JSONL row map. #[arg(long)] allow_duplicate_db_ids: bool, #[command(flatten)] limits: LimitArgs, + /// Emit the verification report as JSON. #[arg(long)] json: bool, }, @@ -198,26 +277,37 @@ enum SqliteCommands { #[derive(Args, Clone, Debug, Default)] struct LimitArgs { + /// Maximum manifest JSON bytes to read. #[arg(long)] max_manifest_bytes: Option, + /// Maximum bytes in one JSONL row-map line. #[arg(long)] max_row_map_line_bytes: Option, + /// Maximum JSONL row-map rows to inspect. #[arg(long)] max_row_map_rows: Option, + /// Maximum bytes retained while checking duplicate db_id values. #[arg(long)] max_row_map_tracked_id_bytes: Option, + /// Maximum number of declared auxiliary artifacts. #[arg(long)] max_auxiliary_artifacts: Option, + /// Maximum bytes permitted for each auxiliary artifact. #[arg(long)] max_auxiliary_artifact_bytes: Option, + /// Maximum bytes permitted for the primary index artifact. #[arg(long)] max_index_artifact_bytes: Option, + /// Maximum bytes permitted for a calibration profile. #[arg(long)] max_calibration_profile_bytes: Option, + /// Maximum bytes permitted for an encoder-distortion profile. #[arg(long)] max_encoder_distortion_profile_bytes: Option, + /// Maximum detail issues retained in a verification report. #[arg(long)] max_report_issues: Option, + /// Maximum bytes accepted for a cached SQLite report. #[arg(long)] max_cached_report_bytes: Option, } diff --git a/ordvec-python/Cargo.toml b/ordvec-python/Cargo.toml index fb0a3cd..297f30f 100644 --- a/ordvec-python/Cargo.toml +++ b/ordvec-python/Cargo.toml @@ -20,6 +20,6 @@ bench-utils = ["ordvec_core/bench-utils"] [dependencies] # Alias the core crate as `ordvec_core` so binding code is unambiguous and never # mixes `ordvec::` with the Python-facing `ordvec` package name. -ordvec_core = { package = "ordvec", path = ".." } +ordvec_core = { package = "ordvec", version = "0.6.0", path = ".." } pyo3 = { version = "0.29.0", features = ["extension-module", "abi3-py310"] } numpy = "0.29.0" diff --git a/ordvec-python/README.md b/ordvec-python/README.md index 8375d28..f3d414d 100644 --- a/ordvec-python/README.md +++ b/ordvec-python/README.md @@ -5,14 +5,40 @@ training-free **ordinal & sign** vector-quantization library for compressed nearest-neighbour retrieval over high-dimensional embeddings. Pure-Rust core, zero system dependencies; SIMD-accelerated at runtime (AVX-512 / AVX2 / scalar). +## Quickstart + +```bash +python -m pip install --upgrade ordvec +``` + ```python import numpy as np -import ordvec +from ordvec import RankQuant + +documents = np.array([ + [8, 7, 6, 5, 4, 3, 2, 1], + [1, 2, 3, 4, 5, 6, 7, 8], + [8, 1, 7, 2, 6, 3, 5, 4], +], dtype=np.float32) +query = np.array([[8, 7, 6, 5, 4, 3, 2, 1]], dtype=np.float32) + +index = RankQuant(dim=8, bits=1) +index.add(documents) +scores, ids = index.search_asymmetric(query, k=1) +print(f"top document: {ids[0, 0]} (score {scores[0, 0]:.3f})") +``` + +```text +top document: 0 (score 0.396) +``` + +Persist and reopen without the original float corpus: -q = ordvec.RankQuant(1024, 2) # 1024-dim, 2 bits/coord -q.add(np.random.randn(10_000, 1024).astype(np.float32)) -# asymmetric: full-precision float queries vs bucketed docs (recommended) -scores, ids = q.search_asymmetric(np.random.randn(8, 1024).astype(np.float32), k=10) +```python +index.write("quickstart.ovrq") +reopened = RankQuant.load("quickstart.ovrq") +_, reopened_ids = reopened.search_asymmetric(query, k=1) +assert reopened_ids[0, 0] == 0 ``` ## Classes @@ -25,17 +51,28 @@ scores, ids = q.search_asymmetric(np.random.randn(8, 1024).astype(np.float32), k | `SignBitmap` | Sign bitmap for sign-cosine candidate generation; separate from the constant-weight bitmap theorem. | The Rust crate's `b = 8` RankQuant evidence/refinement width is not exposed -through the v0.5 Python `RankQuant` constructor and cannot be persisted to +through the v0.6.0 Python `RankQuant` constructor and cannot be persisted to `.ovrq`; use `bits` 1, 2, or 4 from Python. ## Two-stage retrieval (subset rerank) A `Bitmap` / `SignBitmap` probe yields a candidate shortlist that -`RankQuant.search_asymmetric_subset(query, candidates, k)` reranks exactly: +`RankQuant.search_asymmetric_subset(query, candidates, k)` reranks exactly. +Continuing from the quickstart's `documents` and `query`, tile the tiny rows to +the bitmap kernel's 64-coordinate minimum: ```python -cands = bm.top_m_candidates(query, m=256) # uint32 shortlist -scores, ids = rq.search_asymmetric_subset(query, cands, k=10) +from ordvec import Bitmap + +documents64 = np.tile(documents, (1, 8)) +query64 = np.tile(query, (1, 8)) +reranker = RankQuant(dim=64, bits=1) +reranker.add(documents64) +probe = Bitmap(dim=64, n_top=16) +probe.add(documents64) +candidates = probe.top_m_candidates(query64[0], m=2) +scores, ids = reranker.search_asymmetric_subset(query64[0], candidates, k=1) +assert ids[0] == 0 ``` Both returned arrays have length **`min(k, len(candidates))`**, not `k`. When @@ -56,11 +93,7 @@ calibrates that threshold by the hypergeometric upper tail. This is not a deployment guarantee for every encoder or corpus. Real-corpus recall, monotonicity, and null fit remain empirical diagnostics. -## Installation - -```bash -pip install ordvec -``` +## Installation details Wheels target CPython 3.10+ (abi3) and require `numpy>=2.2`. Building from source needs a Rust toolchain (MSRV 1.89) and @@ -74,7 +107,7 @@ buffers first, so ordinary Python in-place NumPy mutation from another thread cannot race the detached Rust scan. Large calls may temporarily require an additional input-sized buffer. The cross-language ownership and lifetime contract is maintained in -[`docs/bindings-safety.md`](https://github.com/Project-Navi/ordvec/blob/v0.5.0/docs/bindings-safety.md) +[`docs/bindings-safety.md`](https://github.com/Project-Navi/ordvec/blob/v0.6.0/docs/bindings-safety.md) for this release line. ## Type stubs diff --git a/ordvec-python/pyproject.toml b/ordvec-python/pyproject.toml index 79b26f7..240545e 100644 --- a/ordvec-python/pyproject.toml +++ b/ordvec-python/pyproject.toml @@ -49,6 +49,8 @@ dependencies = [ Homepage = "https://github.com/Project-Navi/ordvec" Repository = "https://github.com/Project-Navi/ordvec" Issues = "https://github.com/Project-Navi/ordvec/issues" +Documentation = "https://github.com/Project-Navi/ordvec/blob/v0.6.0/ordvec-python/README.md" +Changelog = "https://github.com/Project-Navi/ordvec/blob/main/CHANGELOG.md" Formalization = "https://github.com/Project-Navi/ordvec-formalization" [tool.maturin] diff --git a/ordvec-python/tests/test_rank_quant.py b/ordvec-python/tests/test_rank_quant.py index a6e1e2f..ca4fce3 100644 --- a/ordvec-python/tests/test_rank_quant.py +++ b/ordvec-python/tests/test_rank_quant.py @@ -16,7 +16,7 @@ import numpy as np import pytest -from ordvec import RankQuant, rankquant_eval_search +from ordvec import Bitmap, RankQuant, rankquant_eval_search def unit_vectors(n: int, dim: int, seed: int = 0) -> np.ndarray: @@ -117,6 +117,41 @@ def test_search_asymmetric_shape(bits): assert indices.shape == (3, 10) +def test_readme_quickstart_has_visible_deterministic_result(tmp_path): + documents = np.array( + [ + [8, 7, 6, 5, 4, 3, 2, 1], + [1, 2, 3, 4, 5, 6, 7, 8], + [8, 1, 7, 2, 6, 3, 5, 4], + ], + dtype=np.float32, + ) + query = np.array([[8, 7, 6, 5, 4, 3, 2, 1]], dtype=np.float32) + + index = RankQuant(dim=8, bits=1) + index.add(documents) + scores, ids = index.search_asymmetric(query, k=1) + + assert int(ids[0, 0]) == 0 + assert round(float(scores[0, 0]), 3) == 0.396 + + path = str(tmp_path / "quickstart.ovrq") + index.write(path) + reopened = RankQuant.load(path) + _, reopened_ids = reopened.search_asymmetric(query, k=1) + assert int(reopened_ids[0, 0]) == 0 + + documents64 = np.tile(documents, (1, 8)) + query64 = np.tile(query, (1, 8)) + reranker = RankQuant(dim=64, bits=1) + reranker.add(documents64) + probe = Bitmap(dim=64, n_top=16) + probe.add(documents64) + candidates = probe.top_m_candidates(query64[0], m=2) + _, subset_ids = reranker.search_asymmetric_subset(query64[0], candidates, k=1) + assert int(subset_ids[0]) == 0 + + def test_search_asymmetric_read_concurrent_results_match_baseline(): corpus = np.ascontiguousarray(unit_vectors(64, 128, seed=101)) queries = np.ascontiguousarray(unit_vectors(5, 128, seed=202)) diff --git a/src/lib.rs b/src/lib.rs index 754b6a4..d5d494b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -53,16 +53,20 @@ //! in-model theorem, not a claim that real encoders automatically satisfy the //! quotient, symmetry, or null assumptions. //! -//! ```no_run -//! use ordvec::{Rank, RankQuant}; +//! ``` +//! use ordvec::RankQuant; //! -//! let mut idx = RankQuant::new(1024, 2); -//! let docs: Vec = vec![0.0; 1024 * 10_000]; -//! idx.add(&docs); +//! let documents = [ +//! 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, +//! 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, +//! 8.0, 1.0, 7.0, 2.0, 6.0, 3.0, 5.0, 4.0, +//! ]; +//! let query = [8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0]; //! -//! let queries: Vec = vec![0.0; 1024 * 4]; -//! let res = idx.search_asymmetric(&queries, 10); -//! assert_eq!(res.k, 10); +//! let mut index = RankQuant::new(8, 1); +//! index.add(&documents); +//! let results = index.search_asymmetric(&query, 1); +//! assert_eq!(results.indices_for_query(0)[0], 0); //! ``` // Every unsafe operation in the crate must sit inside an explicit `unsafe {}` diff --git a/tests/release_publish_invariants.py b/tests/release_publish_invariants.py index e4d39a8..e2f266f 100644 --- a/tests/release_publish_invariants.py +++ b/tests/release_publish_invariants.py @@ -23,6 +23,7 @@ WORKFLOW_PATH = os.environ.get("RELEASE_WORKFLOW_PATH", ".github/workflows/release.yml") CI_WORKFLOW_PATH = os.environ.get("CI_WORKFLOW_PATH", ".github/workflows/ci.yml") +AUDIT_WORKFLOW_PATH = os.environ.get("AUDIT_WORKFLOW_PATH", ".github/workflows/audit.yml") PYTHON_WORKFLOW_PATH = os.environ.get("PYTHON_WORKFLOW_PATH", ".github/workflows/python.yml") STRICT_STABLE_TAG_PATTERN = r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$" CHANGELOG_RELEASE_HEADING_RE = re.compile( @@ -464,30 +465,120 @@ def check_release_version_sync() -> None: if version != core_version: fail(f"{label} is {version}, expected lockstep version {core_version}") - manifest = load_toml("ordvec-manifest/Cargo.toml") - dependencies = mapping(manifest.get("dependencies"), "ordvec-manifest/Cargo.toml: dependencies") - ordvec_dep = mapping( - dependencies.get("ordvec"), "ordvec-manifest/Cargo.toml: dependencies.ordvec" + internal_requirements = ( + ("ordvec-manifest/Cargo.toml", "ordvec"), + ("ordvec-python/Cargo.toml", "ordvec_core"), + ("ordvec-manifest-python/Cargo.toml", "ordvec_manifest_core"), + ("ordvec-ffi/Cargo.toml", "ordvec"), + ("benchmarks/beir-bench/Cargo.toml", "ordvec"), ) - dep_version = ordvec_dep.get("version") - if dep_version != core_version: - fail( - "ordvec-manifest/Cargo.toml: dependencies.ordvec.version " - f"is {dep_version!r}, expected {core_version!r}" + for path, dependency_name in internal_requirements: + manifest = load_toml(path) + dependencies = mapping(manifest.get("dependencies"), f"{path}: dependencies") + dependency = mapping( + dependencies.get(dependency_name), + f"{path}: dependencies.{dependency_name}", ) + dep_version = dependency.get("version") + if dep_version != core_version: + fail( + f"{path}: dependencies.{dependency_name}.version is {dep_version!r}, " + f"expected lockstep version {core_version!r}" + ) changelog = read_text("CHANGELOG.md") - if not re.search(rf"^## \[?{re.escape(core_version)}\]? - \d{{4}}-\d{{2}}-\d{{2}}$", changelog, re.MULTILINE): + release_heading = re.search( + rf"^## \[?{re.escape(core_version)}\]? - (\d{{4}}-\d{{2}}-\d{{2}})$", + changelog, + re.MULTILINE, + ) + if release_heading is None: fail(f"CHANGELOG.md must contain a dated section for {core_version}") check_unreleased_section_empty_for_dated_version(changelog, core_version) threat_model = read_text("THREAT_MODEL.md") - if not re.search( - rf"^\>\s+\*\*Status:\*\*\s+v{re.escape(core_version)}\s+\(pre-1\.0\),", - threat_model, + release_date = release_heading.group(1) + expected_threat_status = ( + f"> **Status:** v{core_version} (pre-1.0), {release_date}." + ) + if expected_threat_status not in threat_model: + fail( + "THREAT_MODEL.md status must match the current changelog release " + f"and date: {expected_threat_status!r}" + ) + + semver_capture = r"v((?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))" + release_contract_patterns = { + "README.md": ( + rf"\bIn {semver_capture} it is\s+Rust-only", + rf"cross-language persisted retrieval widths in {semver_capture}", + rf"`RankQuantFastscan::\{{write,load\}}` calls\. In {semver_capture}", + rf"loads `\.ovfs` directly, but in {semver_capture}", + ), + "ordvec-python/README.md": ( + rf"through the {semver_capture} Python `RankQuant` constructor", + rf"/blob/{semver_capture}/docs/bindings-safety\.md", + ), + "docs/INDEX_PROVENANCE.md": ( + rf"^{semver_capture} `\.ovfs` is not covered", + rf"crosses a trust boundary in\s+{semver_capture}, bind the bytes", + ), + "docs/PERSISTED_FORMAT.md": ( + rf"In {semver_capture}, `probe_index_metadata\(path\)` rejects", + rf"probe-versus-load contract in {semver_capture}\.", + ), + "docs/compatibility-policy.md": ( + rf"supported, but in {semver_capture} `\.ovfs`", + rf"^{semver_capture}\. Promoting `\.ovfs`", + ), + } + for path, patterns in release_contract_patterns.items(): + text = read_text(path) + for pattern in patterns: + match = re.search(pattern, text, re.MULTILINE) + if match is None: + fail(f"{path}: current-release contract marker is missing: {pattern!r}") + if match.group(1) != core_version: + fail( + f"{path}: current-release contract says v{match.group(1)}, " + f"expected v{core_version}" + ) + + schema_source = read_text("ordvec-manifest/src/lib.rs") + schema_match = re.search( + r'^pub const SCHEMA_VERSION: &str = "ordvec\.index_manifest\.v([1-9][0-9]*)";$', + schema_source, re.MULTILINE, - ): - fail(f"THREAT_MODEL.md status must mention v{core_version}") + ) + if schema_match is None: + fail("ordvec-manifest/src/lib.rs: could not read the manifest schema major") + schema_major = schema_match.group(1) + schema_contract_patterns = { + "README.md": ( + r"`probe_index_metadata\(\)` / `ordvec-manifest` v([1-9][0-9]*) contract", + r"`probe_index_metadata\(\)` or `ordvec-manifest` v([1-9][0-9]*)\.", + ), + "docs/INDEX_PROVENANCE.md": ( + r"`ordvec-manifest` v([1-9][0-9]*)\. This note", + r"The v([1-9][0-9]*) verifier intentionally does not create or verify `\.ovfs`", + r"manifest v([1-9][0-9]*) simply\s+does not bind or probe those bytes", + ), + "docs/compatibility-policy.md": ( + r"`ordvec-manifest` v([1-9][0-9]*) contract\. Feature-gated", + r"and v([1-9][0-9]*) JSON schema are treated as\s+stable release surfaces", + ), + } + for path, patterns in schema_contract_patterns.items(): + text = read_text(path) + for pattern in patterns: + match = re.search(pattern, text, re.MULTILINE) + if match is None: + fail(f"{path}: current-schema contract marker is missing: {pattern!r}") + if match.group(1) != schema_major: + fail( + f"{path}: current-schema contract says v{match.group(1)}, " + f"expected v{schema_major}" + ) fuzz_lock = read_text("fuzz/Cargo.lock") if not re.search( @@ -660,6 +751,29 @@ def check_manifest_cli_defaults() -> None: if "cargo install ordvec-manifest" not in readme: fail("ordvec-manifest/README.md: must document default cargo install") + matrix = read_text("docs/artifact-platform-matrix.md") + matrix_row = re.search( + r"^\| `ordvec-manifest` Rust crate \|.*$", matrix, re.MULTILINE + ) + if matrix_row is None: + fail("docs/artifact-platform-matrix.md: ordvec-manifest Rust crate row is missing") + row_text = matrix_row.group(0) + if re.search(r"default features? (?:are )?empty", row_text, re.IGNORECASE): + fail( + "docs/artifact-platform-matrix.md: ordvec-manifest defaults must not be " + "documented as empty" + ) + for required_fragment in ( + "default feature `cli`", + "optional `sqlite` and `sqlite-bundled`", + "disable defaults", + ): + if required_fragment not in row_text: + fail( + "docs/artifact-platform-matrix.md: ordvec-manifest row must mention " + f"{required_fragment!r}" + ) + def check_publication_model() -> None: expected_publish = { @@ -679,6 +793,7 @@ def check_publication_model() -> None: def check_python_package_metadata() -> None: + release_tag = f"v{package_version('Cargo.toml')}" pyproject = load_toml("ordvec-python/pyproject.toml") project = mapping(pyproject.get("project"), "ordvec-python/pyproject.toml: project") if project.get("name") != "ordvec": @@ -718,6 +833,11 @@ def check_python_package_metadata() -> None: "Homepage": "https://github.com/Project-Navi/ordvec", "Repository": "https://github.com/Project-Navi/ordvec", "Issues": "https://github.com/Project-Navi/ordvec/issues", + "Documentation": ( + f"https://github.com/Project-Navi/ordvec/blob/{release_tag}/" + "ordvec-python/README.md" + ), + "Changelog": "https://github.com/Project-Navi/ordvec/blob/main/CHANGELOG.md", "Formalization": "https://github.com/Project-Navi/ordvec-formalization", }.items(): if urls.get(key) != expected: @@ -779,6 +899,11 @@ def check_python_package_metadata() -> None: "Homepage": "https://github.com/Project-Navi/ordvec", "Repository": "https://github.com/Project-Navi/ordvec", "Issues": "https://github.com/Project-Navi/ordvec/issues", + "Documentation": ( + f"https://github.com/Project-Navi/ordvec/blob/{release_tag}/" + "ordvec-manifest-python/README.md" + ), + "Changelog": "https://github.com/Project-Navi/ordvec/blob/main/CHANGELOG.md", }.items(): if manifest_urls.get(key) != expected: fail( @@ -827,6 +952,69 @@ def check_python_package_metadata() -> None: fail(".github/dependabot.yml must keep the Python NumPy floor comment at >=2.2") +def check_first_run_docs() -> None: + readme = read_text("README.md") + quickstart_pos = readme.find("## Quickstart: see a result in 30 seconds") + benchmark_pos = readme.find("## Benchmark at a glance") + if quickstart_pos < 0 or benchmark_pos < 0 or quickstart_pos > benchmark_pos: + fail("README.md: the runnable quickstart must appear before benchmark detail") + for required in ( + "## Choose your surface", + "top document: 0 (score 0.396)", + 'index.write("quickstart.ovrq")', + 'RankQuant.load("quickstart.ovrq")', + "examples/quickstart.rs", + ): + if required not in readme: + fail(f"README.md: first-run path must include {required!r}") + if "link TBD" in readme: + fail("README.md: public release links must not contain a TBD placeholder") + + crate_docs = read_text("src/lib.rs") + for path, text in (("README.md", readme), ("src/lib.rs", crate_docs)): + example_pos = text.find("RankQuant::new") + if example_pos < 0: + fail(f"{path}: expected RankQuant::new in the first-run example") + if "10_000" in text[example_pos : example_pos + 800]: + fail(f"{path}: first example must not allocate a 10,000-row placeholder corpus") + + python_readme = read_text("ordvec-python/README.md") + install_pos = python_readme.find("python -m pip install --upgrade ordvec") + import_pos = python_readme.find("from ordvec import RankQuant") + if install_pos < 0 or import_pos < 0 or install_pos > import_pos: + fail("ordvec-python/README.md: installation must precede the first import") + for required in ( + "top document: 0 (score 0.396)", + 'index.write("quickstart.ovrq")', + "Continuing from the quickstart", + ): + if required not in python_readme: + fail(f"ordvec-python/README.md: first-run path must include {required!r}") + if "np.random" in python_readme: + fail("ordvec-python/README.md: first-run examples must be deterministic") + + manifest_readme = read_text("ordvec-manifest/README.md") + for required in ( + "## First verified index", + 'index.write("quickstart.ovrq")', + "quickstart.manifest.json\nverified", + ): + if required not in manifest_readme: + fail(f"ordvec-manifest/README.md: first-run path must include {required!r}") + + manifest_python_readme = read_text("ordvec-manifest-python/README.md") + for required in ( + "python -m pip install --upgrade ordvec ordvec-manifest", + 'index.write("quickstart.ovrq")', + "verified: True", + ): + if required not in manifest_python_readme: + fail( + "ordvec-manifest-python/README.md: first-run path must include " + f"{required!r}" + ) + + def check_release_docs_include_manifest_pypi_lane() -> None: releasing = read_text("RELEASING.md") normalized_releasing = " ".join(releasing.split()) @@ -950,6 +1138,7 @@ def check_package_contents() -> None: "docs/compatibility-policy.md", "docs/determinism.md", "examples/bench_rank.rs", + "examples/quickstart.rs", "src/lib.rs", "tests/index/main.rs", "tests/persistence_compat.rs", @@ -1306,10 +1495,14 @@ def check_release_security_gates(workflow: dict[str, Any], path: str) -> None: jobs = mapping(workflow.get("jobs"), f"{path}: jobs") require_job = mapping(jobs.get("require-ci-green"), f"{path}: jobs.require-ci-green") + timeout_minutes = require_job.get("timeout-minutes") + if not isinstance(timeout_minutes, int) or not 30 < timeout_minutes <= 60: + fail(f"{path}: require-ci-green must have a bounded 31-60 minute job timeout") steps = sequence(require_job.get("steps"), f"{path}: jobs.require-ci-green.steps") gated_workflows = ("ci.yml", "python.yml", "fuzz.yml", "codeql.yml", "actionlint.yml", "zizmor.yml") found_loop: tuple[str, ...] | None = None found_gate_run: str | None = None + found_gate_env: dict[str, Any] | None = None for index, raw_step in enumerate(steps): step = mapping(raw_step, f"{path}: jobs.require-ci-green.steps[{index}]") run = step.get("run") @@ -1319,6 +1512,9 @@ def check_release_security_gates(workflow: dict[str, Any], path: str) -> None: if match: found_loop = tuple(shlex.split(match.group(1))) found_gate_run = run + found_gate_env = mapping( + step.get("env"), f"{path}: jobs.require-ci-green.steps[{index}].env" + ) break if found_loop is None: fail(f"{path}: require-ci-green must loop over the release-gated workflow filenames") @@ -1328,12 +1524,56 @@ def check_release_security_gates(workflow: dict[str, Any], path: str) -> None: ) if found_gate_run is None or "event=push" not in found_gate_run or '.event == "push"' not in found_gate_run: fail(f"{path}: require-ci-green must require successful push workflow runs") + if "status=success" in found_gate_run: + fail(f"{path}: require-ci-green must query all run states before classifying them") if ( found_gate_run is None or "repos/${REPO}/commits/main" not in found_gate_run or "MAIN_SHA" not in found_gate_run ): fail(f"{path}: require-ci-green must verify the release tag points at current main") + if found_gate_env is None: + fail(f"{path}: require-ci-green poll environment was not found") + if found_gate_env.get("POLL_INTERVAL_SECONDS") != 30: + fail(f"{path}: require-ci-green must poll at the reviewed 30-second interval") + if found_gate_env.get("POLL_TIMEOUT_SECONDS") != 1800: + fail(f"{path}: require-ci-green must fail closed after the reviewed 30-minute deadline") + required_poll_fragments = ( + "while :; do", + ".head_sha == $sha", + 'sort_by([.run_number, .run_attempt, .id])', + "| last", + 'status\" != \"completed', + 'conclusion\" = \"success', + "SECONDS >= deadline", + 'sleep \"$POLL_INTERVAL_SECONDS\"', + 'api_status=\"$(sed -nE', + "api_exit_status=$?", + "408|429|5??)", + "connection (reset|refused|closed)", + "rate.?limit|secondary rate|abuse detection", + "retryable_api_error=true", + "API query failed permanently", + ) + for fragment in required_poll_fragments: + if fragment not in found_gate_run: + fail(f"{path}: require-ci-green is missing bounded-poll behavior {fragment!r}") + loop_index = found_gate_run.find("while :; do") + main_check_index = found_gate_run.find('MAIN_SHA="$(gh api', loop_index) + workflow_loop_index = found_gate_run.find("for wf in ", loop_index) + if main_check_index < loop_index or main_check_index > workflow_loop_index: + fail(f"{path}: require-ci-green must re-check current main before every workflow poll") + success_index = found_gate_run.find('if "$all_green"; then', workflow_loop_index) + final_main_index = found_gate_run.find('FINAL_MAIN_SHA="$(gh api', success_index) + success_break_index = found_gate_run.find("break", success_index) + if ( + success_index < workflow_loop_index + or final_main_index < success_index + or success_break_index < final_main_index + or '$SHA" != "$FINAL_MAIN_SHA' not in found_gate_run[final_main_index:success_break_index] + ): + fail(f"{path}: require-ci-green must re-check current main immediately before success") + allowed_id_token_jobs = { "attest", @@ -1377,6 +1617,74 @@ def check_release_security_gates(workflow: dict[str, Any], path: str) -> None: fail(f"{path}: jobs.{job_name} must use environment {environment!r}; got {actual!r}") +def check_cargo_deny_workspace_scope( + workflow: dict[str, Any], path: str, job_name: str, *, advisories_only: bool +) -> None: + jobs = mapping(workflow.get("jobs"), f"{path}: jobs") + job = mapping(jobs.get(job_name), f"{path}: jobs.{job_name}") + steps = sequence(job.get("steps"), f"{path}: jobs.{job_name}.steps") + deny_steps = [ + mapping(raw_step, f"{path}: jobs.{job_name}.steps[{index}]") + for index, raw_step in enumerate(steps) + if action_name(mapping(raw_step, f"{path}: jobs.{job_name}.steps[{index}]")) + == "embarkstudios/cargo-deny-action" + ] + if len(deny_steps) != 2: + fail( + f"{path}: jobs.{job_name} must use exactly two cargo-deny actions " + "(root workspace plus standalone fuzz lockfile)" + ) + + scoped_command_arguments: dict[str, str] = {} + for index, deny_step in enumerate(deny_steps): + inputs = mapping( + deny_step.get("with"), f"{path}: jobs.{job_name}.cargo-deny[{index}].with" + ) + if inputs.get("command") != "check": + fail(f"{path}: jobs.{job_name} cargo-deny command must be check") + argument_words = shlex.split(str(inputs.get("arguments", ""))) + if "--all-features" not in argument_words: + fail(f"{path}: jobs.{job_name} cargo-deny must activate all features") + if "--locked" not in argument_words: + fail(f"{path}: jobs.{job_name} cargo-deny must audit a committed lockfile") + + if "--manifest-path" in argument_words: + fail( + f"{path}: jobs.{job_name} cargo-deny must use the action's " + "manifest-path input instead of duplicating the CLI option" + ) + manifest_path = norm_path(inputs.get("manifest-path")) + has_fuzz_manifest = manifest_path == "fuzz/Cargo.toml" + if manifest_path and not has_fuzz_manifest: + fail( + f"{path}: jobs.{job_name} cargo-deny has unexpected manifest-path " + f"{manifest_path!r}" + ) + if "--workspace" in argument_words and not has_fuzz_manifest: + scope = "workspace" + elif has_fuzz_manifest and "--workspace" not in argument_words: + scope = "fuzz" + else: + fail( + f"{path}: jobs.{job_name} cargo-deny action must target either " + "--workspace or --manifest-path fuzz/Cargo.toml" + ) + if scope in scoped_command_arguments: + fail(f"{path}: jobs.{job_name} has duplicate cargo-deny scope {scope!r}") + scoped_command_arguments[scope] = str(inputs.get("command-arguments", "")).strip() + + if set(scoped_command_arguments) != {"workspace", "fuzz"}: + fail(f"{path}: jobs.{job_name} must audit both workspace and fuzz lockfile scopes") + expected_workspace_command = "advisories" if advisories_only else "" + if scoped_command_arguments["workspace"] != expected_workspace_command: + fail( + f"{path}: jobs.{job_name} workspace cargo-deny command arguments must be " + f"{expected_workspace_command!r}" + ) + if scoped_command_arguments["fuzz"] != "advisories": + fail(f"{path}: jobs.{job_name} fuzz cargo-deny must be scoped to advisories") + + def check_aarch64_smoke_selector(workflow: dict[str, Any], path: str) -> None: jobs = mapping(workflow.get("jobs"), f"{path}: jobs") job = mapping(jobs.get("smoke-linux-aarch64-wheel"), f"{path}: jobs.smoke-linux-aarch64-wheel") @@ -2057,6 +2365,7 @@ def check_sde_cache_invariants() -> None: def main() -> None: workflow = load_workflow(WORKFLOW_PATH) ci_workflow = load_workflow(CI_WORKFLOW_PATH) + audit_workflow = load_workflow(AUDIT_WORKFLOW_PATH) check_release_version_sync() check_release_compatibility_sync() check_python_binding_safety_docs_sync() @@ -2064,10 +2373,17 @@ def main() -> None: check_manifest_cli_defaults() check_publication_model() check_python_package_metadata() + check_first_run_docs() check_release_docs_include_manifest_pypi_lane() check_strict_release_tag_patterns(workflow, WORKFLOW_PATH) check_package_contents() check_ci_package_guards(ci_workflow, CI_WORKFLOW_PATH) + check_cargo_deny_workspace_scope( + ci_workflow, CI_WORKFLOW_PATH, "deny", advisories_only=False + ) + check_cargo_deny_workspace_scope( + audit_workflow, AUDIT_WORKFLOW_PATH, "advisories", advisories_only=True + ) check_hash_requirement_temp_paths( [WORKFLOW_PATH, PYTHON_WORKFLOW_PATH, CI_WORKFLOW_PATH, COVERAGE_WORKFLOW_PATH] ) diff --git a/tests/release_signed_release_invariants.sh b/tests/release_signed_release_invariants.sh index 2994fa7..be9c475 100755 --- a/tests/release_signed_release_invariants.sh +++ b/tests/release_signed_release_invariants.sh @@ -112,6 +112,15 @@ job_downloads_artifact_to_path() { ' } +# No draft Release, build artifact, wheel, or sdist may be created until the +# exact-head main CI gate has settled green. Keeping these dependencies direct +# makes the fail-closed front door visible and prevents a future DAG refactor +# from accidentally starting release work before the gate. +for job in notes build-crate build-wheels build-sdist build-manifest-wheels build-manifest-sdist; do + job_needs "$job" require-ci-green \ + || fail "$job must \`needs: require-ci-green\` before creating release artifacts" +done + # ---------------------------------------------------------------------- # (1) release-assets-draft needs attest + provenance + require-ci-green + notes # + fail-closed release AVX-512 proof + exact linux/aarch64 wheel smoke