diff --git a/.github/agents/unsafe-reviewer.agent.md b/.github/agents/unsafe-reviewer.agent.md new file mode 100644 index 000000000..86c990b2f --- /dev/null +++ b/.github/agents/unsafe-reviewer.agent.md @@ -0,0 +1,22 @@ +--- +name: unsafe-reviewer +description: Reviews unsafe Rust code for soundness and proper SAFETY documentation +tools: ['read', 'search'] +--- + +# Unsafe Code Reviewer + +You are a specialist in reviewing `unsafe` Rust code for soundness. + +## Focus Areas + +- Every `unsafe` block must have a `// SAFETY:` comment explaining the invariant +- Check that safety invariants actually hold (not just documented) +- Verify SIMD intrinsics match the target architecture feature gates +- Look for undefined behavior: uninitialized memory, alignment violations, data races + +## Constraints + +- Review only — do not modify code +- Flag issues with specific line references and suggested fixes + diff --git a/.github/hooks/security.json b/.github/hooks/security.json new file mode 100644 index 000000000..9f075b750 --- /dev/null +++ b/.github/hooks/security.json @@ -0,0 +1,10 @@ +{ + "version": 1, + "hooks": { + "preToolUse": [{ + "type": "command", + "bash": "./scripts/security-check.sh", + "powershell": "pwsh -File scripts/security-check.ps1" + }] + } +} diff --git a/.github/instructions/ci-workflows.instructions.md b/.github/instructions/ci-workflows.instructions.md new file mode 100644 index 000000000..b3919af7c --- /dev/null +++ b/.github/instructions/ci-workflows.instructions.md @@ -0,0 +1,9 @@ +--- +applyTo: '.github/workflows/*.yml' +--- +# CI Workflow Conventions +**When to read:** Modifying GitHub Actions workflow files. + +- Do not modify CI workflows unless explicitly asked +- The `RUST_CONFIG` env var in ci.yml sets `-Dwarnings` — do not remove or weaken this +- Test jobs run on both Linux and Windows — ensure changes are cross-platform diff --git a/.github/instructions/unsafe-code.instructions.md b/.github/instructions/unsafe-code.instructions.md new file mode 100644 index 000000000..444ab9788 --- /dev/null +++ b/.github/instructions/unsafe-code.instructions.md @@ -0,0 +1,9 @@ +--- +applyTo: '**/*.rs' +--- +# Unsafe Code Conventions +**When to read:** Writing or modifying Rust source files that use `unsafe`. + +- Document every `unsafe` block with a `// SAFETY:` comment explaining the invariant +- Prefer safe abstractions over raw unsafe code +- When touching architecture-specific intrinsics (SIMD), validate across platforms per diskann-wide/README.md diff --git a/.github/prompts/generate-tests.prompt.md b/.github/prompts/generate-tests.prompt.md new file mode 100644 index 000000000..96e2329f7 --- /dev/null +++ b/.github/prompts/generate-tests.prompt.md @@ -0,0 +1,14 @@ +--- +mode: 'agent' +description: 'Generate tests for the current file following DiskANN conventions' +--- + +Generate tests for the code in #{file:${file}}. + +Follow these DiskANN testing conventions: +- Do not add tests for derived traits (Clone, Debug, PartialEq) +- Do not add tests for enums unless they have explicit functionality +- Look for existing test infrastructure before creating new patterns +- Use `VirtualStorageProvider::new_overlay()` for storage in tests (never `new_physical()`) +- Use functions from `random.rs` instead of `rand::thread_rng` +- Check if the crate has a `test` module with shared helpers diff --git a/.github/skills/ci-failure-triage/SKILL.md b/.github/skills/ci-failure-triage/SKILL.md new file mode 100644 index 000000000..11cb73edc --- /dev/null +++ b/.github/skills/ci-failure-triage/SKILL.md @@ -0,0 +1,41 @@ +--- +name: ci-failure-triage +description: 'Diagnose CI failures. Use when "CI failed", "build broken", "test failure in CI", or "triage CI".' +--- + +# CI Failure Triage + +Systematic approach to diagnosing failures in the DiskANN CI pipeline. + +## When to Use + +- A CI workflow run has failed +- Tests pass locally but fail in CI +- Platform-specific failures (Linux vs Windows) + +## Process + +### Step 1: Identify Failure Type + +Check which CI job failed: +- `format` — run `cargo fmt --all --check` +- `clippy` / `clippy-features` — run `cargo clippy --workspace --all-targets --config 'build.rustflags=["-Dwarnings"]'` +- `test` — run `cargo test` +- `sde-*` — architecture-specific issues (AVX-512, baseline x86-64) + +### Step 2: Reproduce Locally + + +- For format/clippy/test: run `./scripts/verify` +- For feature-gated failures: check `DISKANN_FEATURES` env var in ci.yml +- For SDE failures: see diskann-wide/README.md for cross-platform validation + +### Step 3: Fix and Verify + +- Apply fix, then run `./scripts/verify` +- If SIMD-related, validate per diskann-wide/README.md + +## Constraints + +- Do not disable or weaken CI checks to fix failures +- Do not skip platform-specific test runs diff --git a/.github/skills/new-crate/SKILL.md b/.github/skills/new-crate/SKILL.md new file mode 100644 index 000000000..e1fb2b554 --- /dev/null +++ b/.github/skills/new-crate/SKILL.md @@ -0,0 +1,47 @@ +--- +name: new-crate +description: 'Scaffold a new workspace crate. Use when "add a crate", "create a new crate", "new package", or "scaffold crate".' +--- + +# New Crate Scaffolding + +Scaffold a new Cargo workspace crate following the tiered architecture. + +## When to Use + +- Adding a new crate to the workspace +- Creating a new module that should be a separate crate + +## Process + +### Step 1: Determine Tier + + +- Tier 1 (Foundation): SIMD, vector primitives, platform utils +- Tier 2 (Core Libraries): Linear algebra, utilities, quantization +- Tier 3 (Algorithm & Storage): Core algorithm, providers, disk indexing +- Tier 4 (Infrastructure & Tools): Benchmarks, CLI tools + +### Step 2: Create Crate + +1. Run `cargo new --lib` in the repo root +2. Add to `[workspace.members]` in root `Cargo.toml` +3. Add to `default-members` if it's Tier 1 or Tier 2 +4. Set `version`, `edition`, `license` from `workspace.package` +5. Add `[lints] workspace = true` to inherit workspace lints + +### Step 3: Configure Dependencies + +- Only depend on crates from equal or lower tiers +- Add workspace dependency entry in root `Cargo.toml` if other crates will depend on it +- Follow dependency rules documented in AGENTS.md + +## Constraints + +- Never add Tier 3 dependencies to benchmark-runner, benchmark-core, or benchmark-simd +- Use `workspace = true` for shared dependencies + +## Validation + +- `cargo check -p ` +- `cargo clippy -p --all-targets --config 'build.rustflags=["-Dwarnings"]'` diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 000000000..71a2e4be5 --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,13 @@ +name: "Copilot Setup Steps" +on: workflow_dispatch +jobs: + copilot-setup-steps: + runs-on: ubuntu-latest + environment: copilot + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@v5 + - name: Setup environment + run: ./scripts/setup diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..ea05caf7a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,104 @@ +# AGENTS.md + +## Verification + +- Run: `cargo fmt --all --check && cargo clippy --workspace --all-targets --config 'build.rustflags=["-Dwarnings"]' && cargo test` +- Or: `./scripts/verify` (bash) / `./scripts/verify.ps1` (PowerShell) +- If it fails, fix the root cause and re-run before committing. + +## Environment + +- Rust edition 2021, toolchain version in `rust-toolchain.toml` +- Cargo workspace with 18 crates across 4 tiers (see `Cargo.toml`) +- CI uses `cargo-nextest` — see `.cargo/nextest.toml` for timeouts/retries +- `rustfmt` and `clippy` not installed by default — run `./scripts/setup` first + +### Crate Organization + +**Tier 1: Foundation** +- `diskann-wide/` - Low-level SIMD, bit manipulation, type width abstractions +- `diskann-vector/` - Vector primitives and operations +- `diskann-platform/` - Platform-specific utilities + +**Tier 2: Core Libraries** +- `diskann-linalg/` - Linear algebra operations +- `diskann-utils/` - Shared utilities (Reborrow, MatrixView traits) +- `diskann-quantization/` - Vector quantization (PQ, SQ) + +**Tier 3: Algorithm & Storage** +- `diskann/` - Core ANN graph algorithm and in-memory indexing (CENTRAL crate) +- `diskann-providers/` - Storage abstraction layer +- `diskann-disk/` - Disk-based indexing with io_uring support +- `diskann-label-filter/` - Inverted index for filtered search + +**Tier 4: Infrastructure & Tools** +- `diskann-benchmark-runner/` - Test runner infrastructure +- `diskann-benchmark-core/` - Benchmark framework +- `diskann-benchmark-simd/` - SIMD-specific benchmarks +- `diskann-benchmark/` - Benchmark definitions and runners +- `diskann-tools/` - CLI utilities (autotuner, etc.) + +## Crate Dependency Rules + +- Tier 1 and Tier 2 crates may be added as dependencies of any internal crate +- `diskann` may be added as a dependency of any equal or higher tier internal crate except those below +- Do not add Tier 3 crates as dependencies of these Tier 4 crates: + - `diskann-benchmark-runner` + - `diskann-benchmark-core` (`diskann` is allowed) + - `diskann-benchmark-simd` + +## Error Handling + +Three regimes — use the one matching the crate's tier: + +- **Low-level crates**: Bespoke, precise, non-allocating error types. Use `thiserror`. Chain with `std::error::Error::source`. Do not use `ANNError`. +- **Mid-level (diskann algorithms)**: Use `diskann::ANNError` with its context machinery. Use `#[track_caller]` when converting to ANNError. Use only for unrecoverable errors. Consider `diskann::error::ToRanked` for traits with associated error types. +- **High-level (tooling)**: Use `anyhow::Error`. +- **Never** use a single crate-level error enum — it provides no per-function failure docs, generates oversized structs, and bloats code. + +## Guardrails + +- Do not use `rand::thread_rng` — use functions from `random.rs` +- Do not use rayon global thread pool methods — use `*_in_pool` wrappers from `rayon_utils.rs` (see `clippy.toml`) +- Do not use `VirtualStorageProvider::new_physical()` in tests — use `new_overlay()` +- Do not use `vfs::PhysicalFS::new` in tests — use `VirtualStorageProvider::new_overlay()` +- When touching architecture-specific intrinsics, run cross-platform validation per `diskann-wide/README.md` + +## Testing + +- Baseline caching system for regression detection — see `diskann/src/test/cache.rs` and `diskann/src/test/cmp.rs` +- Do not add tests for derived traits (Clone, Debug, PartialEq) +- Do not add tests for enums unless they have explicit functionality +- Look for existing setup/execution infrastructure before creating new patterns +- Code coverage of changes is required for PRs + +## AVX-512, Aarch64, and multi-platform + +When touching architecture-specific intrinsics, run cross-platform validation per `diskann-wide/README.md`: + +- Testing AVX-512 code on non-AVX-512 capable x86-64 machines. +- Testing Aarch64 code on x86-64 machines. +- Testing code compiled for and running on the `x86-64` CPU (no AVX/AVX2) does not execute unsupported instructions. + +## Constraints + +- Keep diffs minimal and scoped to the request +- Update or add tests for any behavior change +- Do not modify CI, dependency versions, or security settings unless asked +- Never print, log, or commit secrets + +## Definition of Done + +- `./scripts/verify` passes (fmt + clippy + tests) +- No new lint warnings introduced +- Changes are scoped to the request — no drive-by refactors + +## Where to find more + +- Path-specific rules: `.github/instructions/` +- Multi-step workflows: `.github/skills/*/SKILL.md` + +## Best Practices for Writing an Effective AGENTS.md + +- Keep AGENTS.md concise — ideally within 30–80 lines. +- Do not include structural details such as directory listings or README‑style content, as research shows these can degrade agent performance. \ No newline at end of file diff --git a/agents.md b/agents.md deleted file mode 100644 index 0feeca54d..000000000 --- a/agents.md +++ /dev/null @@ -1,233 +0,0 @@ -# DiskANN Repository - Agent Onboarding Guide - -**Last Updated**: 2026-02-11 (based on v0.45.0, Rust 1.92) - -This guide helps coding agents understand how to work efficiently with the DiskANN repository. - ---- - -## Table of Contents - -1. [Repository Overview](#repository-overview) -2. [Repository Structure](#repository-structure) -3. [Testing](#testing) -4. [Code Quality & Linting](#code-quality--linting) - ---- - -## Repository Overview - -**DiskANN** is a Rust implementation of scalable approximate nearest neighbor (ANN) search algorithms. The project is a rewrite from C++ to Rust. - -- **Language**: Rust (Edition 2021), toolchain version in [`rust-toolchain.toml`](rust-toolchain.toml) -- **License**: MIT (see [`LICENSE.txt`](LICENSE.txt)) -- **Version**: See [`Cargo.toml`](Cargo.toml) -- **Architecture**: Cargo workspace with 15+ crates -- **Legacy Code**: Older C++ code is on the `cpp_main` branch (not maintained) - -### Key Resources -- **Contributing**: See [`CONTRIBUTING.md`](CONTRIBUTING.md) (requires CLA) -- **Code of Conduct**: See [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md) - ---- - -## Repository Structure - -The repository uses a Cargo workspace with crates organized into functional tiers. See [`Cargo.toml`](Cargo.toml) for: -- Workspace members and their dependencies -- Shared dependency versions -- Build profiles (release, ci) -- Workspace-level lints - -### Crate Organization - -**Tier 1: Foundation** -- `diskann-wide/` - Low-level SIMD, bit manipulation, type width abstractions -- `diskann-vector/` - Vector primitives and operations -- `diskann-platform/` - Platform-specific utilities - -**Tier 2: Core Libraries** -- `diskann-linalg/` - Linear algebra operations -- `diskann-utils/` - Shared utilities (Reborrow, MatrixView traits) -- `diskann-quantization/` - Vector quantization (PQ, SQ) - -**Tier 3: Algorithm & Storage** -- `diskann/` - Core ANN graph algorithm and in-memory indexing (CENTRAL crate) -- `diskann-providers/` - Storage abstraction layer -- `diskann-disk/` - Disk-based indexing with io_uring support -- `diskann-label-filter/` - Inverted index for filtered search - -**Tier 4: Infrastructure & Tools** -- `diskann-benchmark-runner/` - Test runner infrastructure -- `diskann-benchmark-core/` - Benchmark framework -- `diskann-benchmark-simd/` - SIMD-specific benchmarks -- `diskann-benchmark/` - Benchmark definitions and runners -- `diskann-tools/` - CLI utilities (autotuner, etc.) - ---- - -## Dependencies - -### Internal - -- Tier 1 and Tier 2 crates may be added as dependencies of any internal crate -- `diskann` may be added as a dependency of any equal or higher tier internal crate except those below -- Do not add Tier 3 crates as dependencies of these Tier 4 crates: - - `diskann-benchmark-runner` - - `diskann-benchmark-core` (`diskann` is allowed) - - `diskann-benchmark-simd` - ---- - -## Testing - -### Test Execution - -```bash -# Run all tests -cargo test - -# Run tests for specific crate -cargo test -p diskann - -# Run specific test -cargo test -p diskann -- --exact test_name - -# Run with CI profile (faster) -cargo test --profile ci - -# Run doc tests -cargo test --doc -``` - -**Note**: CI uses `cargo-nextest` for running tests. See [`.cargo/nextest.toml`](.cargo/nextest.toml) for test configuration (timeouts, retries, etc.). - -### Test Baseline Caching System - -DiskANN uses a baseline caching system for regression detection. See [`diskann/README.md`](diskann/README.md) for a high-level overview of how the baseline system works. For implementation and API details, refer directly to: -- [`diskann/src/test/cache.rs`](diskann/src/test/cache.rs) — core baseline caching APIs -- [`diskann/src/test/cmp.rs`](diskann/src/test/cmp.rs) — `VerboseEq` and related helpers for better test error messages - -### AVX-512, Aarch64, and multi-platform - -When touching architecture-specific intrinsics, run cross-platform validation per `diskann-wide/README.md`: - -- Testing AVX-512 code on non-AVX-512 capable x86-64 machines. -- Testing Aarch64 code on x86-64 machines. -- Testing code compiled for and running on the `x86-64` CPU (no AVX/AVX2) does not execute unsupported instructions. - ---- - -## Code Quality & Linting - -### Error Handling - -There are three regimes of error handling and the strategy to use depends on the regime. - -#### Low-Level - -Low-level crates should use bespoke, precise, non-allocating error types. Use `thiserror` for boilerplate. Chain with `std::error::Error::source`. - -`diskann::ANNError` is not a suitable low-level error type. - -#### Mid-Level (diskann algorithms) - -Use `diskann::ANNError` and its context machinery. This type: - -- Has a small size and `Drop` implementation, so is efficient in function ABIs. -- Records stack trace of its first creation under `RUST_BACKTRACE=1`. -- Precisely records line numbers of creation. -- Has a context layering machinery to add additional information as an error is passed up the stack. - -When converting to `ANNError`, use `#[track_caller]` for better source reporting. - -Traits with associated error types should consider constraining with `diskann::error::ToRanked` instead of `Into` if non-critical errors should be supported. -`diskann::ANNError` should be used only for unrecoverable errors. - -#### High Level (tooling) - -At this level `anyhow::Error` is an appropriate type to use. - -#### Do Not - -Do not use a single crate-level error enum. Problems: - -- Provides no documentation on how an individual function could fail -- Encourages **worse** error messages than bespoke types -- Generates large structs that blow up the stack -- Branch-heavy `Drop` implementations which bloat code - -### Formatting - -**Note**: `rustfmt` is not installed by default. Run `rustup component add rustfmt` if needed. - -```bash -# Check formatting (matches CI) -cargo fmt --all --check - -# Apply formatting to all crates -cargo fmt --all -``` - -See [`rustfmt.toml`](rustfmt.toml) for formatting configuration. - -### Clippy (Linting) - -**Note**: `clippy` is not installed by default. Run `rustup component add clippy` if needed. - -```bash -# Basic clippy check -cargo clippy --workspace --all-targets - -# CI-style check (warnings as errors) -cargo clippy --workspace --all-targets --config 'build.rustflags=["-Dwarnings"]' - -# Check with no default features (for specific crates) -cargo clippy -p diskann --no-default-features -``` - -See [`clippy.toml`](clippy.toml) for linting rules, including: -- Disallowed methods (rayon global thread pool, rand::thread_rng, etc.) -- Required documentation for unsafe blocks - -### Code Coverage - -Code coverage of changes is required for PRs. See [`.codecov.yml`](.codecov.yml) for coverage policy and thresholds. - -### CI Pipeline - -CI workflow is defined in [`.github/workflows/ci.yml`](.github/workflows/ci.yml). Key jobs include: -- Format and clippy checks -- Tests on multiple platforms (Linux, Windows) -- Code coverage -- Architecture compatibility (SDE) - -### Test Patterns - -**DO**: -- Look for existing setup/execution infrastructure -- Factor out common patterns - -**DON'T**: -- Add tests for derived traits (Clone, Debug, PartialEq) -- Add tests for enums unless they have explicit functionality - ---- - -## Pre-commit Checklist - -Before committing changes, always run: - -```bash -# Format all code -cargo fmt --all - -# Run clippy with warnings as errors -cargo clippy --workspace --all-targets -- -D warnings -``` - ---- - -**End of Agent Onboarding Guide** - -*This guide should be updated when major changes occur to the repository structure or development workflows.* diff --git a/scripts/security-check.ps1 b/scripts/security-check.ps1 new file mode 100644 index 000000000..81ed75f17 --- /dev/null +++ b/scripts/security-check.ps1 @@ -0,0 +1,10 @@ +$ErrorActionPreference = 'Stop' +# Block destructive commands — customize this blocklist for your repo +$blockedPatterns = @('rm -rf /', 'DROP DATABASE', 'format C:', 'mkfs') +$input = $args -join ' ' +foreach ($pattern in $blockedPatterns) { + if ($input -match [regex]::Escape($pattern)) { + Write-Error "Blocked: destructive pattern detected" + exit 1 + } +} diff --git a/scripts/security-check.sh b/scripts/security-check.sh new file mode 100644 index 000000000..7c622489b --- /dev/null +++ b/scripts/security-check.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +# Block destructive commands — customize this blocklist for your repo +BLOCKED_PATTERNS=("rm -rf /" "DROP DATABASE" "format C:" "mkfs") +for pattern in "${BLOCKED_PATTERNS[@]}"; do + if echo "$*" | grep -qi "$pattern"; then + echo "❌ Blocked: destructive pattern detected" >&2 + exit 1 + fi +done diff --git a/scripts/setup b/scripts/setup new file mode 100644 index 000000000..9cca94b19 --- /dev/null +++ b/scripts/setup @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +# Setup: one-time environment bootstrap +rustup component add rustfmt clippy diff --git a/scripts/setup.ps1 b/scripts/setup.ps1 new file mode 100644 index 000000000..076a91440 --- /dev/null +++ b/scripts/setup.ps1 @@ -0,0 +1,3 @@ +$ErrorActionPreference = 'Stop' +# Setup: one-time environment bootstrap +rustup component add rustfmt clippy diff --git a/scripts/verify b/scripts/verify new file mode 100644 index 000000000..7a413c437 --- /dev/null +++ b/scripts/verify @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +# Verify: repeatable health check (assumes rustfmt + clippy installed) +cargo fmt --all --check +cargo clippy --workspace --all-targets --config 'build.rustflags=["-Dwarnings"]' +cargo test diff --git a/scripts/verify.ps1 b/scripts/verify.ps1 new file mode 100644 index 000000000..84c4ba936 --- /dev/null +++ b/scripts/verify.ps1 @@ -0,0 +1,8 @@ +$ErrorActionPreference = 'Stop' +# Verify: repeatable health check (assumes rustfmt + clippy installed) +cargo fmt --all --check +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +cargo clippy --workspace --all-targets --config 'build.rustflags=["-Dwarnings"]' +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +cargo test +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }