diff --git a/.github/workflows/production-streaming-load.yml b/.github/workflows/production-streaming-load.yml index e26fb16..9e3d97d 100644 --- a/.github/workflows/production-streaming-load.yml +++ b/.github/workflows/production-streaming-load.yml @@ -93,6 +93,21 @@ on: required: false default: "" type: string + pgvector_dsns: + description: "Optional comma/newline-separated pgvector shard DSNs. Falls back to secret WAVEMIND_PGVECTOR_DSNS." + required: false + default: "" + type: string + provision_pgvector_shards: + description: "Provision isolated pgvector services on the ephemeral runner when external DSNs are unavailable." + required: true + default: false + type: boolean + pgvector_shard_count: + description: "Number of ephemeral pgvector services to provision. Strict 10M evidence requires four." + required: true + default: "4" + type: string faiss_ivfpq_path: description: "Optional persisted FAISS IVF-PQ path. Defaults under ./state." required: false @@ -117,6 +132,7 @@ permissions: jobs: production-streaming-load: runs-on: ${{ inputs.runner_label }} + timeout-minutes: 340 env: ENGINE: ${{ inputs.engine }} SIZE: ${{ inputs.size }} @@ -141,11 +157,15 @@ jobs: INPUT_QDRANT_URL: ${{ inputs.qdrant_url }} INPUT_QDRANT_URLS: ${{ inputs.qdrant_urls }} INPUT_PGVECTOR_DSN: ${{ inputs.pgvector_dsn }} + INPUT_PGVECTOR_DSNS: ${{ inputs.pgvector_dsns }} + PROVISION_PGVECTOR_SHARDS: ${{ inputs.provision_pgvector_shards }} + PGVECTOR_SHARD_COUNT: ${{ inputs.pgvector_shard_count }} INPUT_FAISS_IVFPQ_PATH: ${{ inputs.faiss_ivfpq_path }} RUNNER_STORAGE_ROOT: ${{ inputs.runner_storage_root }} SECRET_QDRANT_URL: ${{ secrets.WAVEMIND_QDRANT_URL }} SECRET_QDRANT_URLS: ${{ secrets.WAVEMIND_QDRANT_URLS }} SECRET_PGVECTOR_DSN: ${{ secrets.WAVEMIND_PGVECTOR_DSN }} + SECRET_PGVECTOR_DSNS: ${{ secrets.WAVEMIND_PGVECTOR_DSNS }} SECRET_QDRANT_API_KEY: ${{ secrets.WAVEMIND_QDRANT_API_KEY }} SECRET_QDRANT_API_KEYS: ${{ secrets.WAVEMIND_QDRANT_API_KEYS }} @@ -161,6 +181,61 @@ jobs: python -m pip install --upgrade pip python -m pip install -e ".[dev,bench,indexes,postgres]" + - name: Provision isolated pgvector services + if: ${{ inputs.engine == 'pgvector-service' && inputs.provision_pgvector_shards }} + run: | + case "$PGVECTOR_SHARD_COUNT" in + ''|*[!0-9]*) echo "pgvector_shard_count must be an integer" >&2; exit 1 ;; + esac + if [ "$PGVECTOR_SHARD_COUNT" -lt 2 ] || [ "$PGVECTOR_SHARD_COUNT" -gt 8 ]; then + echo "pgvector_shard_count must be between 2 and 8" >&2 + exit 1 + fi + + sudo rm -rf /usr/local/lib/android /opt/ghc /usr/share/dotnet /opt/hostedtoolcache/CodeQL + docker network create wavemind-pgvector-ci + : > pgvector-managed-dsns.txt + for shard in $(seq 0 $((PGVECTOR_SHARD_COUNT - 1))); do + name="wavemind-pgvector-${shard}" + port=$((55432 + shard)) + docker run --detach \ + --name "$name" \ + --network wavemind-pgvector-ci \ + --shm-size 1g \ + --publish "127.0.0.1:${port}:5432" \ + --env POSTGRES_PASSWORD=wavemind-ci \ + --env POSTGRES_DB=wavemind \ + "pgvector/pgvector:pg16" \ + postgres \ + -c shared_buffers=512MB \ + -c maintenance_work_mem=768MB \ + -c effective_cache_size=2GB \ + -c max_parallel_maintenance_workers=1 \ + -c max_wal_size=4GB \ + -c min_wal_size=1GB \ + -c checkpoint_timeout=30min \ + -c checkpoint_completion_target=0.9 \ + -c wal_compression=on + printf 'postgresql://postgres:wavemind-ci@127.0.0.1:%s/wavemind\n' "$port" \ + >> pgvector-managed-dsns.txt + done + + for shard in $(seq 0 $((PGVECTOR_SHARD_COUNT - 1))); do + name="wavemind-pgvector-${shard}" + for attempt in $(seq 1 90); do + if docker exec "$name" pg_isready --username postgres --dbname wavemind; then + break + fi + if [ "$attempt" -eq 90 ]; then + docker logs "$name" + exit 1 + fi + sleep 2 + done + done + df -h + free -h + - name: Restore previous checkpoint artifact if: ${{ inputs.resume_run_id != '' }} env: @@ -225,10 +300,38 @@ jobs: if api_keys: env["WAVEMIND_QDRANT_API_KEYS"] = api_keys elif engine == "pgvector-service": + shard_values = pick("INPUT_PGVECTOR_DSNS", "SECRET_PGVECTOR_DSNS") value = pick("INPUT_PGVECTOR_DSN", "SECRET_PGVECTOR_DSN") - if not value: - raise SystemExit("pgvector-service requires pgvector_dsn or secret WAVEMIND_PGVECTOR_DSN") - env["WAVEMIND_PGVECTOR_DSN"] = value + managed_shards = os.environ.get("PROVISION_PGVECTOR_SHARDS", "").lower() == "true" + if managed_shards: + shard_values = Path("pgvector-managed-dsns.txt").read_text( + encoding="utf-8" + ).strip() + env["WAVEMIND_PGVECTOR_EVIDENCE_TOPOLOGY"] = ( + "github-hosted-isolated-service-processes" + ) + if shard_values: + env["WAVEMIND_PGVECTOR_DSNS"] = shard_values + env.pop("WAVEMIND_PGVECTOR_DSN", None) + elif value: + env["WAVEMIND_PGVECTOR_DSN"] = value + else: + raise SystemExit( + "pgvector-service requires pgvector_dsns/WAVEMIND_PGVECTOR_DSNS " + "or pgvector_dsn/WAVEMIND_PGVECTOR_DSN" + ) + env.setdefault("WAVEMIND_PGVECTOR_STORAGE_TYPE", "halfvec") + env.setdefault("WAVEMIND_PGVECTOR_INSERT_MODE", "copy") + env.setdefault("WAVEMIND_PGVECTOR_INDEX_TYPE", "hnsw") + env.setdefault("WAVEMIND_PGVECTOR_HNSW_M", "16") + env.setdefault("WAVEMIND_PGVECTOR_HNSW_EF_CONSTRUCTION", "256") + env.setdefault("WAVEMIND_PGVECTOR_EF_SEARCH", "800") + env.setdefault("WAVEMIND_PGVECTOR_QUERY_ROUTING", "namespace") + env.setdefault("WAVEMIND_PGVECTOR_PREWARM_INDEX", "1") + if managed_shards: + env["WAVEMIND_PGVECTOR_INDEX_BUILD_WORKERS"] = os.environ[ + "PGVECTOR_SHARD_COUNT" + ] elif engine == "faiss-ivfpq-persisted": value = os.environ.get("INPUT_FAISS_IVFPQ_PATH", "").strip() env["WAVEMIND_FAISS_IVFPQ_PATH"] = value or str(runner_storage_root / f"faiss-ivfpq-{size}.faiss") @@ -311,6 +414,36 @@ jobs: raise SystemExit(f"production benchmark skipped: {row.get('reason', 'unknown reason')}") if int(row.get("vectors", 0) or 0) != expected_vectors: raise SystemExit("production benchmark did not ingest the requested vector count") + if ( + os.environ["ENGINE"] == "pgvector-service" + and os.environ.get("PROVISION_PGVECTOR_SHARDS", "").lower() == "true" + ): + expected_shards = int(os.environ["PGVECTOR_SHARD_COUNT"]) + if int(row.get("shard_count", 0) or 0) != expected_shards: + raise SystemExit( + f"expected {expected_shards} pgvector shards; " + f"found {row.get('shard_count')}" + ) + expected_per_shard = expected_vectors // expected_shards + if row.get("shard_row_counts") != [expected_per_shard] * expected_shards: + raise SystemExit( + "pgvector shard row counts do not prove an exact balanced layout: " + f"{row.get('shard_row_counts')}" + ) + if row.get("shard_misplaced_rows") != [0] * expected_shards: + raise SystemExit( + f"pgvector shards contain misplaced rows: {row.get('shard_misplaced_rows')}" + ) + if row.get("query_routing") != "namespace": + raise SystemExit("managed pgvector evidence must use namespace routing") + if int(row.get("index_build_workers", 0) or 0) != expected_shards: + raise SystemExit( + "managed pgvector evidence must build independent shard indexes in parallel" + ) + if row.get("evidence_topology") != "github-hosted-isolated-service-processes": + raise SystemExit( + "managed pgvector evidence is missing its isolated-service topology attestation" + ) print( f"validated {expected_engine}: vectors={row['vectors']} " f"recall={row.get('recall_at_k')} p99_ms={row.get('p99_latency_ms')}" @@ -420,6 +553,18 @@ jobs: echo "::warning::Evidence branch $branch was published, but repository settings prevented github-actions from opening the PR." fi + - name: Capture pgvector service diagnostics + if: ${{ always() && inputs.engine == 'pgvector-service' && inputs.provision_pgvector_shards }} + run: | + mkdir -p state/pgvector-service-diagnostics + docker stats --no-stream > state/pgvector-service-diagnostics/docker-stats.txt || true + df -h > state/pgvector-service-diagnostics/disk.txt || true + free -h > state/pgvector-service-diagnostics/memory.txt || true + for shard in $(seq 0 $((PGVECTOR_SHARD_COUNT - 1))); do + docker logs "wavemind-pgvector-${shard}" \ + > "state/pgvector-service-diagnostics/shard-${shard}.log" 2>&1 || true + done + - uses: actions/upload-artifact@v7 if: always() with: diff --git a/README.md b/README.md index f9b542d..2dc2258 100644 --- a/README.md +++ b/README.md @@ -523,7 +523,7 @@ results: |---|---|---:|---|---| | `qdrant-10m` | `qdrant-service` | 10000000 | `WAVEMIND_QDRANT_URL` | `benchmarks/production_streaming_load_qdrant_10m_results.json` | | `qdrant-sharded-10m` | `qdrant-sharded-service` | 10000000 | `WAVEMIND_QDRANT_URLS` | `benchmarks/production_streaming_load_qdrant_sharded_10m_results.json` | -| `pgvector-10m` | `pgvector-service` | 10000000 | `WAVEMIND_PGVECTOR_DSN` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | +| `pgvector-10m` | `pgvector-service` | 10000000 | `WAVEMIND_PGVECTOR_DSNS` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | | `faiss-ivfpq-50m` | `faiss-ivfpq-persisted` | 50000000 | `WAVEMIND_FAISS_IVFPQ_PATH` | `benchmarks/production_streaming_load_ivfpq_50m_results.json` | | `qdrant-sharded-100m` | `qdrant-sharded-service` | 100000000 | `WAVEMIND_QDRANT_URLS` | `benchmarks/production_streaming_load_qdrant_sharded_100m_results.json` | @@ -2031,6 +2031,43 @@ next actions into one publishable status contract. Large-N service plans include resumable `--checkpoint-path` commands so interrupted 10M/50M/100M ingest runs can continue from completed batches instead of restarting from zero. +The pgvector large-N runner uses bounded `COPY` batches and supports +`WAVEMIND_PGVECTOR_STORAGE_TYPE=halfvec` for a smaller PostgreSQL and HNSW +footprint. A completed checkpoint is accepted only when the remote collection +still contains the exact expected row count and the requested HNSW index is +present; otherwise the evidence run fails instead of silently benchmarking a +partial corpus. Use a dedicated table when switching between `vector` and +`halfvec` because PostgreSQL column types are intentionally validated. +Set `WAVEMIND_PGVECTOR_PREWARM_INDEX=1` for steady-state service evidence. The +runner loads the selected candidate-index relation through PostgreSQL +`pg_prewarm` and records the +number of cached index blocks, so warm latency is explicit rather than inferred. +Large pgvector profiles can select +`WAVEMIND_PGVECTOR_INDEX_TYPE=hnsw|hnsw-binary|ivfflat`. +For IVFFlat, `WAVEMIND_PGVECTOR_IVFFLAT_LISTS` controls partition count and +`WAVEMIND_PGVECTOR_IVFFLAT_PROBES` controls the recall/latency tradeoff. +The strict 10M profile uses four modulo-sharded PostgreSQL services with +`halfvec` HNSW (`m=16`, `ef_construction=256`, `ef_search=800`). This profile was +selected from measured binary-HNSW, IVFFlat, and full-HNSW sweeps: the larger +construction depth reached the recall target in dedicated-shard sweeps, while +namespace routing avoids broadcasting ordinary scoped queries across every +service. The full four-service 10M artifact remains a release gate and is not +claimed from those component measurements. Binary HNSW remains available +for storage-constrained experiments through `WAVEMIND_PGVECTOR_INDEX_TYPE`, but +it is not the strict-evidence default. +For horizontal service sharding, set `WAVEMIND_PGVECTOR_DSNS` to two or more +comma-, semicolon-, or newline-separated PostgreSQL DSNs. The runner assigns +`memory_id` values by modulo, validates exact per-shard counts and placement, +builds one index per service, and fanout-merges shard-local results by original +vector distance. Checkpoints are committed only after every shard accepts the +batch, so interrupted multi-service ingest resumes without silently losing a +shard. +`WAVEMIND_PGVECTOR_QUERY_ROUTING=namespace` is the strict production default: +the benchmark label supplies the same namespace ownership information that a +real WaveMind request carries, and the query is sent only to its owning shard. +Set it to `fanout` only for explicit cross-namespace search. Results record the +routing mode and do not present namespace-scoped latency as global-corpus +fanout latency. Manual strict-evidence runners include `.github/workflows/production-streaming-load.yml`, `.github/workflows/external-http-cluster-load.yml`, `.github/workflows/external-http-active-active.yml`, and @@ -2138,7 +2175,7 @@ public claim boundaries stable: | Claim area | Current public status | Source of truth | Not proven yet | |---|---|---|---| | Production readiness | WaveMind core readiness is gated by checked-in artifacts before release. | `benchmarks/production_readiness_results.json`, `benchmarks/PRODUCTION_READINESS.md` | Missing external competitor credentials should not be treated as WaveMind core failure, but they still limit competitor claims. | -| Strict production evidence | The gate now passes `4/8` requirements: persisted 50M FAISS reaches recall@10 `0.9705` and p99 `73.11 ms`; single-service 10M Qdrant reaches `0.975` and `43.27 ms`; four-service 10M sharded Qdrant reaches `0.9925` and `71.28 ms`; the non-loopback Kubernetes cluster passes success/failover `1.00`, query p99 `84.80 ms`, batch p99 `148.82 ms`, and physical-worker attestation `10/10`. | `benchmarks/production_evidence_results.json`, `benchmarks/PRODUCTION_EVIDENCE.md`, `benchmarks/production_evidence_gate.py`, `wavemind production-evidence --strict` | Four requirements remain: remote active-active, managed serverless telemetry, 10M pgvector, and the 100M sharded service run. | +| Strict production evidence | The gate now passes `4/8` requirements: persisted 50M FAISS reaches recall@10 `0.9705` and p99 `73.11 ms`; single-service 10M Qdrant reaches `0.975` and `43.27 ms`; four-service 10M sharded Qdrant reaches `0.9925` and `71.28 ms`; the non-loopback Kubernetes cluster passes success/failover `1.00`, query p99 `79.44 ms`, batch p99 `186.78 ms`, and physical-worker attestation `10/10`. | `benchmarks/production_evidence_results.json`, `benchmarks/PRODUCTION_EVIDENCE.md`, `benchmarks/production_evidence_gate.py`, `wavemind production-evidence --strict` | Four requirements remain: remote active-active, managed serverless telemetry, 10M pgvector, and the 100M sharded service run. | | Production evidence preflight | Remote endpoint/env/path prerequisites are checked before launching expensive strict-evidence jobs. | `benchmarks/production_evidence_preflight_results.json`, `benchmarks/PRODUCTION_EVIDENCE_PREFLIGHT.md`, `wavemind production-evidence-preflight --write-artifacts` | A ready preflight is not a passing evidence result; it only proves the environment is ready to run the remote/large-N jobs. | | Production evidence env contract | Secret-safe operator map from every strict-evidence env var to the workflows, claims, artifacts, GitHub Actions secrets, input bindings, and `.env.example` placeholders it unlocks. | `benchmarks/production_evidence_env_contract.json`, `benchmarks/PRODUCTION_EVIDENCE_ENV.md`, `deploy/cluster/production-evidence.env.example`, `wavemind production-evidence-env --write-artifacts` | It does not unlock production claims; it prevents ambiguous or unsafe production evidence launches and keeps secrets out of checked-in artifacts. | | Production evidence dispatch | Secret-safe workflow dispatch contract for every unfinished strict-evidence job, including safe `commit_results=false` launch commands, publish commands, required env/secrets, and artifact promotion commands. | `benchmarks/production_evidence_dispatch_results.json`, `benchmarks/PRODUCTION_EVIDENCE_DISPATCH.md`, `wavemind production-evidence-dispatch --write-artifacts` | A dispatch plan only launches or reviews evidence runs; it does not unlock production claims until downloaded artifacts pass ingest and strict validation. | @@ -2148,7 +2185,7 @@ public claim boundaries stable: | Agent impact leaderboard | Behavioral benchmark evidence is aggregated across agent coherence, dynamic-memory policy, long-memory retrieval, and LongMemEval answer quality. | `benchmarks/agent_impact_results.json`, `benchmarks/AGENT_IMPACT.md`, `benchmarks/agent_impact_leaderboard.py` | It proves lift on the listed checked-in scenarios only; it does not claim general agent success outside those tasks. | | Memory OS intelligence | Adaptive-worker evidence is aggregated across scale readiness, agent coherence, staging canary, and admission artifacts. It tracks hot-query prewarm, transition-learned predictive prefetch, priority learning, adaptive forgetting, concept consolidation, Redis coordination, canary status, and production-admission boundaries. | `benchmarks/memory_os_intelligence_results.json`, `benchmarks/MEMORY_OS_INTELLIGENCE.md`, `benchmarks/memory_os_intelligence_report.py` | It proves Memory OS behavior on checked-in fixtures; unattended production automation remains locked until real shared Redis, distributed lock, runtime env, and large-scale evidence pass. | | Cluster autoscale | Cluster/operator evidence is pulled into a dedicated public report. It tracks deterministic shard placement, node/zone loss availability, autoscale planning, rebalance checkpoints, Kubernetes operator reconciliation, quorum safety, HTTP sharding, active-active convergence, CRDT field state, and the 100M capacity envelope. | `benchmarks/cluster_autoscale_results.json`, `benchmarks/CLUSTER_AUTOSCALE.md`, `benchmarks/cluster_autoscale_report.py` | It is a deterministic capacity and operator evidence report, not a real 100M vector-query latency benchmark or managed Kubernetes production SLO. | -| Kubernetes physical worker failure | Four WaveMind pod-DNS endpoints across three kind worker zones retain `1.00` recall while one worker container is physically paused, recover without pod replacement, then pass the mixed cluster load at query p99 `84.80 ms` and batch p99 `148.82 ms`. | `benchmarks/kubernetes_cluster_network_smoke_results.json`, `benchmarks/http_cluster_load_results.json`, `benchmarks/kubernetes_cluster_network_smoke.py`, [workflow run 29070578441](https://github.com/CaspianG/wavemind/actions/runs/29070578441) | This unlocks the non-loopback Kubernetes service-node SLO. It does not claim managed Kubernetes, independent remote regions, or 10M-100M distributed scale. | +| Kubernetes physical worker failure | Four WaveMind pod-DNS endpoints across three kind worker zones retain `1.00` recall while one worker container is physically paused, recover without pod replacement, then pass the mixed cluster load at query p99 `79.44 ms` and batch p99 `186.78 ms`. | `benchmarks/kubernetes_cluster_network_smoke_results.json`, `benchmarks/http_cluster_load_results.json`, `benchmarks/kubernetes_cluster_network_smoke.py`, [workflow run 29165761261](https://github.com/CaspianG/wavemind/actions/runs/29165761261) | This unlocks the non-loopback Kubernetes service-node SLO. It does not claim managed Kubernetes, independent remote regions, or 10M-100M distributed scale. | | Kubernetes active-active region failure | Three PVC-backed replicated regions in three worker zones converge `48` initial writes, continue `32` writes plus a delete while region B is physically unavailable, then recover at `1.00` convergence and delete suppression with an idempotent final sync. | `benchmarks/kubernetes_active_active_region_smoke_results.json`, `benchmarks/kubernetes_active_active_region_smoke.py`, `wavemind active-active-drill` | This proves the active-active protocol across non-loopback Kubernetes services and a physical zone outage in ephemeral CI. Independent remote regions are still required for strict active-active admission. | | Kubernetes serverless lifecycle | PVC-backed PostgreSQL, Qdrant, and Redis preserve `24/24` memories through two scale-to-zero cycles; three zone-spread workers achieve write/delete coherence at `3/3` within `1.14 s`, and burst p99 remains below `2 s`. | `benchmarks/kubernetes_serverless_lifecycle_smoke_results.json`, `benchmarks/kubernetes_serverless_lifecycle_smoke.py`, `.github/workflows/kubernetes-operator-smoke.yml` | This proves external-state lifecycle and bounded worker-cache convergence in ephemeral non-loopback Kubernetes. Managed Knative/KEDA endpoints and remote telemetry are still required for strict serverless admission. | | Kubernetes PostgreSQL/Qdrant DR | A checksummed PostgreSQL backup restores into an independent namespace with fresh PVCs and an empty Qdrant service; recall and index parity are `24/24`, including after recovery API replacement. | `benchmarks/kubernetes_postgres_qdrant_dr_smoke_results.json`, `benchmarks/kubernetes_postgres_qdrant_dr_smoke.py`, `.github/workflows/kubernetes-operator-smoke.yml` | This proves logical backup/restore and vector-index reconstruction in ephemeral Kubernetes. It is not managed PostgreSQL PITR, remote object-store recovery, or multi-region DR. | @@ -2170,7 +2207,7 @@ public claim boundaries stable: | pgvector tuning | Real PostgreSQL/pgvector service profile now separates baseline HNSW, exact recall floor, and iterative HNSW tuning. | `benchmarks/production_pgvector_tuning_results.json` | This is a 50k service-backed tuning profile, not yet the 100k/1M production load SLO artifact. | | Qdrant streaming | Real Qdrant streaming smoke exists, tuned 1M and strict single-service 10M runs pass quality/latency gates, and the real four-service sharded 10M run reaches recall@10 `0.9925`, p99 `71.28 ms`. | `benchmarks/production_streaming_load_qdrant_smoke_results.json`, `benchmarks/production_streaming_load_qdrant_1m_results.json`, `benchmarks/production_streaming_load_qdrant_1m_tuned_results.json`, `benchmarks/production_streaming_load_qdrant_sharded_smoke_results.json`, `benchmarks/production_streaming_load_qdrant_10m_plan.json`, `benchmarks/production_streaming_load_qdrant_10m_results.json`, `benchmarks/production_streaming_load_qdrant_sharded_10m_plan.json`, `benchmarks/production_streaming_load_qdrant_sharded_10m_results.json`, `benchmarks/production_streaming_load_qdrant_sharded_100m_plan.json` | The 100M sharded Qdrant result remains locked until its real run artifact is produced. | | pgvector streaming | Real PostgreSQL/pgvector streaming smoke exists, and a 10M plan-only service contract is checked in. | `benchmarks/production_streaming_load_pgvector_smoke_results.json`, `benchmarks/production_streaming_load_pgvector_10m_plan.json` | The 10M pgvector service result is not claimed until `production_streaming_load_pgvector_10m_results.json` is produced by a real run. | -| HTTP cluster load | The checked artifact runs the mixed workload from inside Kubernetes against four pod-DNS API nodes: success/failover/delete suppression `1.00`, repaired replica `1`, query p99 `84.80 ms`, batch p99 `148.82 ms`, and external batch requests `24 -> 1`. Bulk lifecycle batch p99 is reported separately at `6694.76 ms`. | `benchmarks/http_cluster_load_results.json`, `.github/workflows/kubernetes-operator-smoke.yml`, `.github/workflows/external-http-cluster-load.yml` | This is ephemeral non-loopback Kubernetes evidence, not managed multi-region or million-scale service evidence. | +| HTTP cluster load | The checked artifact runs the mixed workload from inside Kubernetes against four pod-DNS API nodes: success/failover/delete suppression `1.00`, repaired replica `1`, query p99 `79.44 ms`, batch p99 `186.78 ms`, and external batch requests `24 -> 1`. Bulk lifecycle batch p99 is reported separately at `8351.04 ms`. | `benchmarks/http_cluster_load_results.json`, `.github/workflows/kubernetes-operator-smoke.yml`, `.github/workflows/external-http-cluster-load.yml` | This is ephemeral non-loopback Kubernetes evidence, not managed multi-region or million-scale service evidence. | | HTTP active-active regions | Local multi-process API-region evidence exists, and the external URL-based contract now has a loopback artifact. The external workflow can run the same namespace-delta contract against real regional API URLs. | `benchmarks/local_http_active_active_smoke_results.json`, `benchmarks/external_http_active_active_loopback_results.json`, `.github/workflows/external-http-active-active.yml` | Local/loopback active-active evidence is not a remote Kubernetes/serverless multi-region result until `benchmarks/external_http_active_active_results.json` is produced by a real run. | | Serverless telemetry | Loopback replica telemetry exists; remote telemetry has a dedicated manual workflow and artifact path. | `deploy/serverless/observed-telemetry.loopback.json`, `.github/workflows/serverless-observed-telemetry.yml` | Loopback evidence is not a hosted managed-serverless claim until `observed-telemetry.remote.json` is committed. | | Competitor adapters | Local Mem0/LangGraph/GraphRAG-style adapters run; optional Zep evidence is skipped until configured. | `benchmarks/memory_competitor_results.json` | Not a full independent Mem0/Zep/Letta leaderboard without live service credentials and public runner parity. | @@ -2919,7 +2956,7 @@ If you already use Chroma for local memory, see the practical migration guide: non-loopback CI evidence, but it is still ephemeral kind evidence rather than remote multi-region production admission. See `benchmarks/kubernetes_cluster_network_smoke_results.json` and its traceable - [GitHub Actions run](https://github.com/CaspianG/wavemind/actions/runs/29057247023). + [GitHub Actions run](https://github.com/CaspianG/wavemind/actions/runs/29165761261). - The checked active-active Kubernetes drill uses three separate replicated region APIs with persistent volumes in three worker zones. During a physical zone-B worker pause, regions A and C continue writes and delete propagation at @@ -2927,7 +2964,7 @@ If you already use Chroma for local memory, see the practical migration guide: deleted memory and the final sync is a no-op. This remains ephemeral kind evidence rather than independent remote-region admission. See `benchmarks/kubernetes_active_active_region_smoke_results.json` and its - [GitHub Actions run](https://github.com/CaspianG/wavemind/actions/runs/29058433643). + [GitHub Actions run](https://github.com/CaspianG/wavemind/actions/runs/29165761261). - pgvector is a candidate-index backend. PostgreSQL source-of-truth storage is also available separately. A Postgres-native PITR runbook/preflight now exists through `wavemind postgres-pitr-plan` and diff --git a/benchmarks/AGENT_IMPACT.md b/benchmarks/AGENT_IMPACT.md index 49a7821..41bf491 100644 --- a/benchmarks/AGENT_IMPACT.md +++ b/benchmarks/AGENT_IMPACT.md @@ -1,6 +1,6 @@ # WaveMind Agent Impact Leaderboard -Generated: `2026-07-11T05:56:44Z`. +Generated: `2026-07-11T19:56:03Z`. Agent-impact rows come from checked-in benchmark artifacts. They show behavioral lift on the configured tasks; they do not claim general agent success outside the listed scenarios. diff --git a/benchmarks/BENCHMARK_LEADERBOARD.md b/benchmarks/BENCHMARK_LEADERBOARD.md index 2ea1968..bdd3300 100644 --- a/benchmarks/BENCHMARK_LEADERBOARD.md +++ b/benchmarks/BENCHMARK_LEADERBOARD.md @@ -1,7 +1,7 @@ # WaveMind Benchmark Leaderboard Generated from `benchmarks/benchmark_matrix_results.json`. -Last refresh: `2026-07-11T05:56:44Z` from `c4f786e131c8`. +Last refresh: `2026-07-11T19:56:03Z` from `d0e3446be447`. This is a compact reader-facing view of checked-in benchmark results. It is not a universal vector-database leaderboard: each row uses the primary quality metric for that benchmark, and latency is shown separately so quality wins are not confused with speed wins. @@ -26,7 +26,7 @@ This is a compact reader-facing view of checked-in benchmark results. It is not | Qdrant 1M HNSW ef sweep | production-scale | Recall@k | - | hnsw_ef=2048: 0.977 / 64.8 ms | No WaveMind result; production SLO miss; cost if SLO fixed: hnsw_ef=512 $4.86/1M queries | | Production streaming load runner | production-scale | Recall@k | 10k smoke / WaveMind numpy-streaming: 1 / 0.098 ms | Qdrant sharded smoke / Qdrant sharded service streaming: 1 / 9.103 ms | Quality tie; WaveMind faster; production SLO pass: 10k smoke / WaveMind numpy-streaming; cost: 10k smoke / WaveMind numpy-streaming $0.69/1M queries | | Scale readiness profile | production-scale | precision@1 | WaveMind structured payloads: 1 / 1.959 ms | - | WaveMind-only check | -| Production readiness gate | production-scale | readiness score | WaveMind production readiness: 1 / - | - | WaveMind-only check | +| Production readiness gate | production-scale | readiness score | WaveMind production readiness: 0.974 / - | - | WaveMind-only check | | Memory competitor adapter profile | agent-memory | precision@1 | WaveMind: 0.8 / 14.5 ms | GraphRAG static graph: 0.852 / 0.079 ms | Baseline leads on quality | | [LongMemEval answer generation](https://github.com/xiaowu0162/LongMemEval) | long-term-agent-memory | token F1 | WaveMind + qwen2.5:1.5b: 0.333 / - | Chroma static + qwen2.5:1.5b: 0.17 / - | WaveMind leads on quality | @@ -34,7 +34,7 @@ This is a compact reader-facing view of checked-in benchmark results. It is not | area | current source | claim status | next action | |---|---|---|---| -| Artifact freshness | local matrix refresh at `2026-07-11T05:56:44Z` | source `c4f786e131c8`; audit gate enforced by `validate_benchmark_artifacts.py` | Keep weekly refresh green before public claims. | +| Artifact freshness | local matrix refresh at `2026-07-11T19:56:03Z` | source `d0e3446be447`; audit gate enforced by `validate_benchmark_artifacts.py` | Keep weekly refresh green before public claims. | | Serverless telemetry | loopback API pool; `loopback-api-capacity-estimate`; 4 measured replicas | observed SLO `True`; loopback evidence, not a managed-serverless claim | Run `.github/workflows/serverless-observed-telemetry.yml` against deployed API nodes. | | External HTTP cluster load | kubernetes-kind-non-loopback-ci; `kubernetes-pod-dns-physical-node-drill`; 4 nodes | SLO `True`; non-loopback Kubernetes pod-DNS evidence | Run `.github/workflows/external-http-cluster-load.yml` with a remote node manifest. | | External HTTP active-active loopback | local-loopback; `loopback-api-regions`; 3 regions | SLO `True`; external URL contract over local API regions | Run `.github/workflows/external-http-active-active.yml` with remote regions for production evidence. | @@ -46,7 +46,7 @@ This is a compact reader-facing view of checked-in benchmark results. It is not | Qdrant sharded streaming | real fanout smoke plus measured four-service 10M profile | smoke recall `1`, smoke p99 `16.0 ms`; 10M recall `0.993`, 10M p99 `71.3 ms`, shards `4`; 100M preflight `action_required`; planned shards `4`; blockers `none (measured artifact passes)` | Keep the measured 10M sharded profile green and run the strict 100M sharded profile next. | | Qdrant 1M streaming | real Qdrant service run before and after warmup/chunking tuning | cold p99 `3014.0 ms`; tuned recall `1`, tuned p99 `26.4 ms`, SLO `pass` | Use the tuned warmup/chunking profile for the 10M Qdrant service run. | | pgvector streaming | real PostgreSQL/pgvector service smoke plus 10M preflight | smoke recall `1`, smoke p99 `7.624 ms`; 10M preflight `action_required` | Run `.github/workflows/production-streaming-load.yml` with `pgvector-service` against sized Postgres storage. | -| Production readiness gate | checked-in benchmark artifacts | `pass`; 39/39 pass | Keep the gate at readiness_score 1.0 while repeating larger service-backed runs and moving external competitor evidence into the separate adapter profile. | +| Production readiness gate | checked-in benchmark artifacts | `fail`; 38/39 pass | Keep the gate at readiness_score 1.0 while repeating larger service-backed runs and moving external competitor evidence into the separate adapter profile. | | Competitor adapters | checked local adapters plus optional external services | configured `4`; skipped `Zep` | Configure skipped external services before claiming full competitor coverage. | ## Reading Rules diff --git a/benchmarks/BENCHMARK_REPORT.md b/benchmarks/BENCHMARK_REPORT.md index ef5de76..8d6225b 100644 --- a/benchmarks/BENCHMARK_REPORT.md +++ b/benchmarks/BENCHMARK_REPORT.md @@ -1,7 +1,7 @@ # WaveMind Benchmark Report This report is generated from `benchmarks/benchmark_matrix_results.json`. -Last refresh: `2026-07-11T05:56:44Z` from `c4f786e131c8`. +Last refresh: `2026-07-11T19:56:03Z` from `d0e3446be447`. It separates completed local runs from runner-ready public benchmarks and planned external evaluations. Planned rows are not claimed wins. They are the public proof path WaveMind must complete before stronger production claims. @@ -27,20 +27,20 @@ Planned rows are not claimed wins. They are the public proof path WaveMind must | Production load profile 100k | production-scale | implemented | Qdrant service: Recall@k 1.00, avg latency 10.3, p95 latency 19.0, p99 latency 21.3, build ms 27439.3, SLO pass, required replicas 2, autoscaled QPS 1635.0, cost status valid_slo, cost / 1M queries 1.39, monthly target cost 365.0, storage 0.24
WaveMind pgvector: Recall@k 0.74, avg latency 17.8, p95 latency 23.5, build ms 455703.7, SLO fail, required replicas 3, autoscaled QPS 945.9, cost status invalid_slo, cost / 1M queries 2.08, monthly target cost 547.5, storage 0.24
WaveMind faiss-persisted: skipped - Set WAVEMIND_FAISS_PATH to use the persisted FAISS backend | Keep at least one service backend at SLO pass while adding persisted FAISS from the Linux benchmark container and raising the target QPS. | | Production load profile 1M | production-scale | implemented | Qdrant service: Recall@k 0.98, avg latency 82.6, p95 latency 126.0, p99 latency 137.9, build ms 441775.0, SLO fail, required replicas 12, autoscaled QPS 203.5, cost status invalid_slo, cost / 1M queries 8.33, monthly target cost 2190.2, storage 2.38
WaveMind faiss-persisted: Recall@k 1.00, avg latency 39.1, p95 latency 45.3, p99 latency 57.7, build ms 20788.1, SLO scale_required, required replicas 6, autoscaled QPS 429.5, cost status valid_slo, cost / 1M queries 4.17, monthly target cost 1095.2, storage 2.38 | Keep the FAISS persisted 1M profile green and tune Qdrant/pgvector so service-mode vector DB backends also pass recall and p99. | | Qdrant 1M HNSW ef sweep | production-scale | implemented | hnsw_ef=512: Recall@k 0.75, avg latency 47.2, p95 latency 68.5, p99 latency 68.5, max latency ms 68.5, SLO fail, required replicas 7, autoscaled QPS 356.2, cost status invalid_slo, cost / 1M queries 4.86, monthly target cost 1277.7, storage 2.38
hnsw_ef=768: Recall@k 0.85, avg latency 44.0, p95 latency 69.1, p99 latency 69.8, max latency ms 69.8, SLO fail, required replicas 7, autoscaled QPS 381.5, cost status invalid_slo, cost / 1M queries 4.86, monthly target cost 1277.7, storage 2.38
hnsw_ef=1024: Recall@k 0.88, avg latency 62.9, p95 latency 81.1, p99 latency 85.5, max latency ms 85.5, SLO fail, required replicas 9, autoscaled QPS 267.0, cost status invalid_slo, cost / 1M queries 6.25, monthly target cost 1642.7, storage 2.38
hnsw_ef=1536: Recall@k 0.94, avg latency 65.6, p95 latency 111.2, p99 latency 119.7, max latency ms 119.7, SLO fail, required replicas 10, autoscaled QPS 256.3, cost status invalid_slo, cost / 1M queries 6.94, monthly target cost 1825.2, storage 2.38
hnsw_ef=2048: Recall@k 0.98, avg latency 64.8, p95 latency 91.2, p99 latency 103.8, max latency ms 103.8, SLO fail, required replicas 10, autoscaled QPS 259.4, cost status invalid_slo, cost / 1M queries 6.94, monthly target cost 1825.2, storage 2.38 | Repeat with 100+ queries and collection-level HNSW build parameters; the current best recall setting still misses the p99 SLO. | -| Production streaming load runner | production-scale | implemented | 10k smoke / WaveMind numpy-streaming: Recall@k 1, target recall@k 1, target recall@1 1, avg latency 0.10, p95 latency 0.12, p99 latency 0.46, build ms 27.6, SLO pass, required replicas 1, autoscaled QPS 170611.0, cost status valid_slo, cost / 1M queries 0.69, monthly target cost 182.5, storage 0.02, memory mode stores full matrix; smoke/testing only
100k compressed / WaveMind faiss-ivfpq-persisted streaming: Recall@k 0.96, target recall@k 0.96, target recall@1 0.96, avg latency 0.47, p95 latency 0.54, p99 latency 1.10, build ms 27322.1, SLO pass, required replicas 1, autoscaled QPS 36063.6, cost status valid_slo, cost / 1M queries 0.69, monthly target cost 182.5, storage 0.24, memory mode streaming IVF-PQ; compressed codes plus query source vectors only
1M compressed / WaveMind faiss-ivfpq-persisted streaming: Recall@k 0.99, target recall@k 0.99, target recall@1 0.98, avg latency 3.18, p95 latency 4.13, p99 latency 4.99, build ms 67255.7, SLO pass, required replicas 1, autoscaled QPS 5287.9, cost status valid_slo, cost / 1M queries 0.69, monthly target cost 182.7, storage 2.38, memory mode streaming IVF-PQ; compressed codes plus query source vectors only
10M compressed / WaveMind faiss-ivfpq-persisted streaming: Recall@k 0.99, target recall@k 0.99, target recall@1 0.95, avg latency 45.8, p95 latency 57.5, p99 latency 60.1, build ms 249349.6, SLO scale_required, required replicas 7, autoscaled QPS 366.8, cost status valid_slo, cost / 1M queries 4.86, monthly target cost 1279.9, storage 23.8, memory mode streaming IVF-PQ; compressed codes plus query source vectors only
50M preflight / WaveMind faiss-ivfpq-persisted streaming: status action_required, vectors 50000000, vector dim 128, queries 2000, estimated index 1.12, transient runner 0.57, application storage 119.2, required local free 2.12, disk free 0.00, index mode persisted FAISS IVF-PQ compressed codes plus int64 ids, required env WAVEMIND_FAISS_IVFPQ_PATH, missing env WAVEMIND_FAISS_IVFPQ_PATH, blockers missing_env:WAVEMIND_FAISS_IVFPQ_PATH, insufficient_local_disk_for_index_and_transient_batches
Qdrant smoke / Qdrant service streaming: Recall@k 1, target recall@k 1, target recall@1 1, avg latency 9.73, p95 latency 17.7, p99 latency 17.9, build ms 1002.5, SLO pass, required replicas 2, autoscaled QPS 1726.3, cost status valid_slo, cost / 1M queries 1.39, monthly target cost 365.0, storage 0.00, memory mode streaming upsert; query source vectors only
1M Qdrant cold / Qdrant service streaming: Recall@k 0.99, target recall@k 0.99, target recall@1 0.99, avg latency 161.9, p95 latency 382.5, p99 latency 3014.0, build ms 361950.1, SLO fail, required replicas 24, autoscaled QPS 103.8, cost status invalid_slo, cost / 1M queries 16.7, monthly target cost 4380.2, storage 2.38, memory mode streaming upsert; query source vectors only
1M Qdrant tuned / Qdrant service streaming: Recall@k 1, target recall@k 1, target recall@1 1, avg latency 16.2, p95 latency 24.4, p99 latency 26.4, build ms 373952.8, SLO pass, required replicas 3, autoscaled QPS 1039.3, cost status valid_slo, cost / 1M queries 2.08, monthly target cost 547.7, storage 2.38, memory mode streaming upsert; query source vectors only
10M Qdrant measured / Qdrant service streaming: Recall@k 0.97, target recall@k 0.97, target recall@1 0.97, avg latency 30.3, p95 latency 37.8, p99 latency 43.3, build ms 10214.9, SLO scale_required, required replicas 5, autoscaled QPS 554.5, cost status valid_slo, cost / 1M queries 3.47, monthly target cost 914.9, storage 23.8, memory mode streaming upsert; query source vectors only
Qdrant sharded smoke / Qdrant sharded service streaming: Recall@k 1, target recall@k 1, target recall@1 1, avg latency 9.10, p95 latency 15.6, p99 latency 16.0, build ms 3681.3, SLO pass, required replicas 2, autoscaled QPS 1845.5, cost status valid_slo, cost / 1M queries 1.39, monthly target cost 365.0, storage 0.01, shard count 2, fanout workers 2, memory mode horizontally sharded streaming upsert; parallel fanout query merge
10M Qdrant sharded measured / Qdrant sharded service streaming: Recall@k 0.99, target recall@k 0.99, target recall@1 0.99, avg latency 46.6, p95 latency 60.5, p99 latency 71.3, build ms 5520602.2, SLO scale_required, required replicas 17, autoscaled QPS 720.9, cost status valid_slo, cost / 1M queries 4.72, monthly target cost 3104.9, storage 23.8, shard count 4, fanout workers 4, memory mode horizontally sharded streaming upsert; parallel fanout query merge
10M Qdrant preflight / Qdrant service streaming: status action_required, vectors 10000000, vector dim 128, queries 2000, estimated index 0.00, transient runner 0.00, application storage 23.8, required local free 0.00, disk free 0.00, index mode remote Qdrant service storage; local runner stores only generated batches, required env WAVEMIND_QDRANT_URL, missing env WAVEMIND_QDRANT_URL, blockers missing_env:WAVEMIND_QDRANT_URL, insufficient_local_disk_for_index_and_transient_batches
10M Qdrant sharded preflight / Qdrant sharded service streaming: status action_required, vectors 10000000, vector dim 128, queries 2000, estimated index 0.00, transient runner 0.00, application storage 23.8, required local free 0.00, disk free 10.8, index mode remote horizontally sharded Qdrant storage; local runner routes ids across service URLs and fanout-merges top-k, required env WAVEMIND_QDRANT_URLS, missing env WAVEMIND_QDRANT_URLS, blockers missing_env:WAVEMIND_QDRANT_URLS
100M Qdrant sharded preflight / Qdrant sharded service streaming: status action_required, vectors 100000000, vector dim 128, queries 5000, estimated index 0.00, transient runner 0.01, application storage 238.4, required local free 0.01, disk free 10.8, index mode remote horizontally sharded Qdrant storage; local runner routes ids across service URLs and fanout-merges top-k, required env WAVEMIND_QDRANT_URLS, missing env WAVEMIND_QDRANT_URLS, blockers missing_env:WAVEMIND_QDRANT_URLS
pgvector smoke / WaveMind pgvector streaming: Recall@k 1, target recall@k 1, target recall@1 1, avg latency 2.59, p95 latency 4.45, p99 latency 7.62, build ms 763.6, SLO pass, required replicas 1, autoscaled QPS 6476.7, cost status valid_slo, cost / 1M queries 0.69, monthly target cost 182.5, storage 0.00, memory mode streaming PostgreSQL insert; query source vectors only
10M pgvector preflight / WaveMind pgvector streaming: status action_required, vectors 10000000, vector dim 128, queries 2000, estimated index 0.00, transient runner 0.00, application storage 23.8, required local free 0.00, disk free 0.00, index mode remote PostgreSQL/pgvector storage; local runner stores only generated batches, required env WAVEMIND_PGVECTOR_DSN, missing env WAVEMIND_PGVECTOR_DSN, blockers missing_env:WAVEMIND_PGVECTOR_DSN, insufficient_local_disk_for_index_and_transient_batches | Run .github/workflows/production-streaming-load.yml with sized pgvector infrastructure for 10M, then execute the 100M sharded Qdrant profile. | +| Production streaming load runner | production-scale | implemented | 10k smoke / WaveMind numpy-streaming: Recall@k 1, target recall@k 1, target recall@1 1, avg latency 0.10, p95 latency 0.12, p99 latency 0.46, build ms 27.6, SLO pass, required replicas 1, autoscaled QPS 170611.0, cost status valid_slo, cost / 1M queries 0.69, monthly target cost 182.5, storage 0.02, memory mode stores full matrix; smoke/testing only
100k compressed / WaveMind faiss-ivfpq-persisted streaming: Recall@k 0.96, target recall@k 0.96, target recall@1 0.96, avg latency 0.47, p95 latency 0.54, p99 latency 1.10, build ms 27322.1, SLO pass, required replicas 1, autoscaled QPS 36063.6, cost status valid_slo, cost / 1M queries 0.69, monthly target cost 182.5, storage 0.24, memory mode streaming IVF-PQ; compressed codes plus query source vectors only
1M compressed / WaveMind faiss-ivfpq-persisted streaming: Recall@k 0.99, target recall@k 0.99, target recall@1 0.98, avg latency 3.18, p95 latency 4.13, p99 latency 4.99, build ms 67255.7, SLO pass, required replicas 1, autoscaled QPS 5287.9, cost status valid_slo, cost / 1M queries 0.69, monthly target cost 182.7, storage 2.38, memory mode streaming IVF-PQ; compressed codes plus query source vectors only
10M compressed / WaveMind faiss-ivfpq-persisted streaming: Recall@k 0.99, target recall@k 0.99, target recall@1 0.95, avg latency 45.8, p95 latency 57.5, p99 latency 60.1, build ms 249349.6, SLO scale_required, required replicas 7, autoscaled QPS 366.8, cost status valid_slo, cost / 1M queries 4.86, monthly target cost 1279.9, storage 23.8, memory mode streaming IVF-PQ; compressed codes plus query source vectors only
50M preflight / WaveMind faiss-ivfpq-persisted streaming: status action_required, vectors 50000000, vector dim 128, queries 2000, estimated index 1.12, transient runner 0.57, application storage 119.2, required local free 2.12, disk free 0.00, index mode persisted FAISS IVF-PQ compressed codes plus int64 ids, required env WAVEMIND_FAISS_IVFPQ_PATH, missing env WAVEMIND_FAISS_IVFPQ_PATH, blockers missing_env:WAVEMIND_FAISS_IVFPQ_PATH, insufficient_local_disk_for_index_and_transient_batches
Qdrant smoke / Qdrant service streaming: Recall@k 1, target recall@k 1, target recall@1 1, avg latency 9.73, p95 latency 17.7, p99 latency 17.9, build ms 1002.5, SLO pass, required replicas 2, autoscaled QPS 1726.3, cost status valid_slo, cost / 1M queries 1.39, monthly target cost 365.0, storage 0.00, memory mode streaming upsert; query source vectors only
1M Qdrant cold / Qdrant service streaming: Recall@k 0.99, target recall@k 0.99, target recall@1 0.99, avg latency 161.9, p95 latency 382.5, p99 latency 3014.0, build ms 361950.1, SLO fail, required replicas 24, autoscaled QPS 103.8, cost status invalid_slo, cost / 1M queries 16.7, monthly target cost 4380.2, storage 2.38, memory mode streaming upsert; query source vectors only
1M Qdrant tuned / Qdrant service streaming: Recall@k 1, target recall@k 1, target recall@1 1, avg latency 16.2, p95 latency 24.4, p99 latency 26.4, build ms 373952.8, SLO pass, required replicas 3, autoscaled QPS 1039.3, cost status valid_slo, cost / 1M queries 2.08, monthly target cost 547.7, storage 2.38, memory mode streaming upsert; query source vectors only
10M Qdrant measured / Qdrant service streaming: Recall@k 0.97, target recall@k 0.97, target recall@1 0.97, avg latency 30.3, p95 latency 37.8, p99 latency 43.3, build ms 10214.9, SLO scale_required, required replicas 5, autoscaled QPS 554.5, cost status valid_slo, cost / 1M queries 3.47, monthly target cost 914.9, storage 23.8, memory mode streaming upsert; query source vectors only
Qdrant sharded smoke / Qdrant sharded service streaming: Recall@k 1, target recall@k 1, target recall@1 1, avg latency 9.10, p95 latency 15.6, p99 latency 16.0, build ms 3681.3, SLO pass, required replicas 2, autoscaled QPS 1845.5, cost status valid_slo, cost / 1M queries 1.39, monthly target cost 365.0, storage 0.01, shard count 2, fanout workers 2, memory mode horizontally sharded streaming upsert; parallel fanout query merge
10M Qdrant sharded measured / Qdrant sharded service streaming: Recall@k 0.99, target recall@k 0.99, target recall@1 0.99, avg latency 46.6, p95 latency 60.5, p99 latency 71.3, build ms 5520602.2, SLO scale_required, required replicas 17, autoscaled QPS 720.9, cost status valid_slo, cost / 1M queries 4.72, monthly target cost 3104.9, storage 23.8, shard count 4, fanout workers 4, memory mode horizontally sharded streaming upsert; parallel fanout query merge
10M Qdrant preflight / Qdrant service streaming: status action_required, vectors 10000000, vector dim 128, queries 2000, estimated index 0.00, transient runner 0.00, application storage 23.8, required local free 0.00, disk free 0.00, index mode remote Qdrant service storage; local runner stores only generated batches, required env WAVEMIND_QDRANT_URL, missing env WAVEMIND_QDRANT_URL, blockers missing_env:WAVEMIND_QDRANT_URL, insufficient_local_disk_for_index_and_transient_batches
10M Qdrant sharded preflight / Qdrant sharded service streaming: status action_required, vectors 10000000, vector dim 128, queries 2000, estimated index 0.00, transient runner 0.00, application storage 23.8, required local free 0.00, disk free 10.8, index mode remote horizontally sharded Qdrant storage; local runner routes ids across service URLs and fanout-merges top-k, required env WAVEMIND_QDRANT_URLS, missing env WAVEMIND_QDRANT_URLS, blockers missing_env:WAVEMIND_QDRANT_URLS
100M Qdrant sharded preflight / Qdrant sharded service streaming: status action_required, vectors 100000000, vector dim 128, queries 5000, estimated index 0.00, transient runner 0.01, application storage 238.4, required local free 0.01, disk free 10.8, index mode remote horizontally sharded Qdrant storage; local runner routes ids across service URLs and fanout-merges top-k, required env WAVEMIND_QDRANT_URLS, missing env WAVEMIND_QDRANT_URLS, blockers missing_env:WAVEMIND_QDRANT_URLS
pgvector smoke / WaveMind pgvector streaming: Recall@k 1, target recall@k 1, target recall@1 1, avg latency 2.59, p95 latency 4.45, p99 latency 7.62, build ms 763.6, SLO pass, required replicas 1, autoscaled QPS 6476.7, cost status valid_slo, cost / 1M queries 0.69, monthly target cost 182.5, storage 0.00, memory mode streaming PostgreSQL insert; query source vectors only
10M pgvector preflight / WaveMind pgvector streaming: status action_required, vectors 10000000, vector dim 128, queries 2000, estimated index 0.00, transient runner 0.00, application storage 23.8, required local free 0.00, disk free 0.00, index mode modulo-sharded PostgreSQL/pgvector services; local runner stores only generated batches, required env WAVEMIND_PGVECTOR_DSNS, missing env WAVEMIND_PGVECTOR_DSNS, blockers missing_env:WAVEMIND_PGVECTOR_DSNS, insufficient_local_disk_for_index_and_transient_batches | Run .github/workflows/production-streaming-load.yml with sized pgvector infrastructure for 10M, then execute the 100M sharded Qdrant profile. | | Scale readiness profile | production-scale | implemented | WaveMind cluster planner: simulated memories 1000000, namespaces 4096, nodes 4, replication factor 2, node loss min availability 1.00, zone loss min availability 1.00, read quorum 1, write quorum 2, kubernetes manifest kind StatefulSet, kubernetes repair cronjob kind CronJob, kubernetes repair cronjob namespaces 4096, placement ms 85.9
WaveMind cluster autoscaler: status scale_required, namespace count 4096, current nodes 4, required nodes 50, additional nodes 46, replication factor 3, target memories 10000000, max memories per node 1000000, headroom 0.70, current max node memories 10000000, target max node memories 678711, target within headroom True, move sample 4094, omitted moves 0, has scale action True, rebalance status ready, rebalance full plan True, rebalance batches 82, rebalance move count 4094, rebalance write quorum 2, rebalance read quorum 1, rebalance estimated steps 328, rebalance max batch node pressure 50, rebalance all batches checkpointed True, rebalance all batches repaired True, rebalance all batches validated True, plan ms 7267.0
WaveMind hot cache: queries 2000, capacity 512, hit rate 0.92, evictions 0, prewarm warmed 1, prewarm hit True, p99 lookup ms 0.01
WaveMind query vector cache: queries 200, local encode calls 1, local hit rate 0.99, Redis shared True, Redis encode calls 1, Redis reader hits 1, p99 local query ms 4.44, service boundary FastAPI TestClient, service queries 200, service encode calls 1, service saved encode calls 199, service hit rate 0.99, service metrics exposed True, service p99 latency 11.9
WaveMind API batch query: queries 100, individual HTTP requests 100, batch HTTP requests 1, request reduction 0.99, individual success True, batch success True, individual encode calls 1, batch encode calls 1, batch hit rate 0.99, batch metrics exposed True, batch total speedup 6.30, batch request latency 97.0
WaveMind shared rate limiter: backend redis-compatible fixed window, workers 2, limit per minute 4, allowed 4, limited 1, shared across workers True, expire seconds 120, p99 check ms 0.07
WaveMind Memory OS: ok True, hot queries 2, prewarm warmed 2, prewarm hit True, predictive prefetch generated 6, predictive prefetch warmed 6, transition-prefetch queries risk limits, transition-prefetch edges {'namespace': 'tenant:os', 'from_query': 'budget recall', 'to_query': 'risk limits', 'count': 1, 'probability': 1.0, 'last_seen': 1783637014.8730922}, transition-prefetch hit True, expired purged 1, concepts created 1, concept recall True, user feedback events 8, positive feedback priority delta 0.40, negative feedback priority delta -0.30, priority predictions 2, forgetting demotions 4, architecture advice architecture_required, architecture ids scale-plan, service-index, namespace-sharding, production-controls, replication-capacity, load-test, multimodal-payloads, architecture commands 10, run ms 77.4
WaveMind Redis hot cache: client redis-compatible, shared cache visible across clients True, cache prewarm warmed 1, cache prewarm cross worker hit True, memory os ok True, memory os hot queries 2, memory os prewarm warmed 2, memory os predictive generated 6, memory os predictive warmed 6, Memory OS transition-prefetch queries risk limits, Memory OS transition-prefetch edges {'namespace': 'tenant:redis-os', 'from_query': 'budget recall', 'to_query': 'risk limits', 'count': 1, 'probability': 1.0, 'last_seen': 1783637014.1264112}, Memory OS transition-prefetch hit True, memory os concepts created 1, memory os user feedback events 8, memory os positive feedback priority delta 0.40, memory os negative feedback priority delta -0.30, memory os priority predictions 2, memory os forgetting demotions 4, Memory OS architecture advice architecture_required, Memory OS architecture ids scale-plan, service-index, namespace-sharding, production-controls, replication-capacity, load-test, multimodal-payloads, memory os cross worker hit True, namespace invalidation removed True, redis keys 8, avg lookup ms 0.09, p99 lookup ms 0.10
WaveMind sustained HTTP cluster load: nodes 4, namespaces 4, replication factor 3, writes 8, queries 8, failover queries 8, write batch http requests 4, write batch individual http requests 24, write batch request reduction ratio 0.83, forget batch http requests 4, forget batch individual http requests 12, tombstone batch http requests 4, tombstone batch individual http requests 12, forget tombstone batch http requests 8, forget tombstone batch individual http requests 24, forget tombstone batch request reduction ratio 0.67, query batch http requests 3, query batch individual http requests 8, query batch request reduction ratio 0.62, failover batch http requests 2, failover batch individual http requests 8, failover batch request reduction ratio 0.75, write success rate 1.00, query hit rate 1.00, failover hit rate 1.00, delete suppression rate 1.00, repair repaired 1, success rate 1.00, p99 operation ms 891.7
WaveMind API cache mutation safety: client fastapi+redis-compatible-cache, first query cached True, cache invalidated on remember True, stale prevented after remember True, cache invalidated on feedback True, feedback demoted rejected memory True, cache invalidated on forget True, stale prevented after forget True, old recall after forget True, avg api ms 7.16, p99 api ms 8.62
WaveMind batch feedback: client fastapi+redis-compatible-cache, items 3, accepted 2, rejected 1, ok True, cache was warmed True, cache invalidated True, audit events 2, positive feedback priority delta 0.55, negative feedback priority delta -0.25, avg api ms 1.84, p99 api ms 1.84
WaveMind Kubernetes operator: bundle has crd True, bundle has operator deployment True, has service True, has statefulset True, has hpa True, has repair cronjob True, has memory os cronjob True, autoscaling min replicas 34, autoscaling max replicas 34, operator status ready True, operator status phase Ready, operator ready replicas 34, operator required replicas 34, operator capacity within headroom True, status memory os ready True, status memory os redis required True, status memory os redis configured True, operator true conditions AutoscalingReady, CapacityPlanned, ControlPlaneReady, MemoryOSReady, ProductionAdmissionReady, RebalancePlanned, RepairScheduled, ResourcesReady, autoscaling metrics cpu, memory, repair namespaces 4096, memory os calls plan True, memory os calls run True, memory os applies plan lock True, memory os blocks missing redis True
WaveMind serverless plan: has knative service True, has keda scaled object True, scale to zero True, max scale 256, target concurrency 80, uses postgres True, uses external qdrant True, uses shared cache True, safe for pod eviction True, keda scale target kind Deployment, valid keda scale target True, env has postgres dsn True, env has qdrant url True, env has redis url True
WaveMind serverless operational profile: slo pass True, requests per second 3200.0, avg request ms 80.0, p99 request ms 320.0, target p99 ms 500.0, cold start ms 900.0, cold start total ms 1220.0, cold start budget ms 3500.0, cold start budget ok True, required replicas 4, warm replicas 4, max scale 256, target concurrency 80, burst capacity rps 256000.0, scale out possible True, scale to zero safe True, external state ok True, uses postgres True, uses external qdrant True, uses shared cache True, has auth secret True, safe for pod eviction True, monthly compute cost usd 81.8, monthly budget usd 750.0, cost ok True, observed telemetry present True, observed telemetry source loopback-api-capacity-estimate, observed requests per second 37328.6, observed measured pool requests per second 583.3, observed per replica requests per second 145.8, observed measured replicas 4, observed p99 request ms 17.0, observed cold start total ms 2075.9, observed error rate 0.00, observed max replicas 22, observed scale out seconds 18.0, observed monthly compute cost usd -, observed slo pass True
WaveMind distributed sharding: nodes 3, replication factor 2, write quorum 2, read quorum 1, writes 2, recalled after primary loss True, repair repaired 1, repair ok True, recall after repair True, forget replicated deletes 2, tombstone RF 3, tombstone suppress before repair True, tombstone deleted 1, tombstone suppress after repair True, anti-entropy worker ok True, anti-entropy repaired 1, anti-entropy tombstone deleted 1, query after primary loss ms 1.49
WaveMind distributed HTTP sharding: nodes 3, replication factor 3, write quorum 2, read quorum 1, proxy bypass default True, writes 2, recalled after primary loss True, repair repaired 1, repair ok True, recall after repair True, tombstone missed delete replica records 1, tombstone suppress before repair True, tombstone deleted 1, tombstone stale records after repair 0, tombstone suppress after repair True, concurrent writes 12, concurrent write ok True, concurrent query hit rate 1.00, query after primary loss ms 16.2, concurrent ms 1311.0, repair ms 53.8
WaveMind replicated runtime: nodes 3, replication factor 3, write quorum 2, read quorum 1, recalled after node loss True, repair copied records 1, tombstone deleted 1, concurrent writes 12, concurrent write ok True, concurrent query hit rate 1.00, concurrent ms 365.7, p99 query after loss ms 3.50
WaveMind active-active delta sync: regions 2, replication factor per region 3, records imported 6, converged after bidirectional sync True, suppressed stale import after delete True, tombstone converged True, sync ms 42.5
WaveMind sustained active-active sync: regions 3, namespaces 3, replication factor per region 3, writes 18, sync cycles 5, pair syncs 90, cursor count 18, records imported 108, tombstones imported 6, deleted records 6, field keys exported 348, final noop records imported 0, final noop failed pairs 0, convergence rate 1.00, delete suppression rate 1.00, success rate 1.00, failed pairs 0, has more pairs 0, p99 sync ms 403.0, avg sync ms 216.5
WaveMind HTTP active-active service-region sync: service boundary FastAPI TestClient, api export endpoint /namespace-delta/export, api import endpoint /namespace-delta/import, regions 3, namespaces 2, replication factor per region 3, writes 6, sync cycles 4, pair syncs 48, cursor count 12, export calls 48, import calls 48, records imported 36, tombstones imported 6, deleted records 6, field keys exported 122, final noop records imported 0, final noop failed pairs 0, convergence rate 1.00, delete suppression rate 1.00, success rate 1.00, failed pairs 0, has more pairs 0, p99 sync ms 465.9, avg sync ms 342.7
WaveMind field-state CRDT: regions 3, commutative convergence True, idempotent remerge True, tombstone wins True, top key converged True, watermark convergence True, watermark actors 3, watermark health ok True, watermark health status pass, watermark missing detected True, watermark lag detected True, budget activation 5.00, merge ms 0.18
WaveMind replicated snapshot: nodes 3, manifest healthy True, offsite verified True, archive verified True, object store verified True, object store latest verified True, object store pruned 2, object store download verified True, object store drill ok True, restored files 3, recalled after restore node loss True, snapshot ms 269.9, restore ms 162.2
WaveMind structured payloads: queries 7, precision@1 1.00, cross-modal queries 7, cross-modal precision@1 1.00, cross-modal dim 64, cross-modal vectors persisted 1.00, cross-modal provenance 1.00, precomputed-vector queries 4, precomputed-vector precision@1 1.00, precomputed-vector dim 4, precomputed-vector persisted 1.00, encoder contract True, encoder contract payloads 7, encoder target precision@1 1.00, encoder global precision@1 1.00, encoder routing 1.00, encoder vectors persisted 1.00, encoder vectors normalized 1.00, encoder vectors finite 1.00, encoder provenance 1.00, encoder min margin 0.81, encoder health ok True, encoder health encoder descriptor, encoder health payloads 7, encoder health queries 7, encoder health global precision at 1 1.00, encoder health target modality routing rate 1.00, encoder health finite payload vector rate 1.00, encoder health normalized payload vector rate 1.00, encoder health finite query vector rate 1.00, encoder health normalized query vector rate 1.00, encoder health dimension match rate 1.00, encoder health payload encode p95 ms 8.33, encoder health query encode p95 ms 1.52, encoder health min global margin 0.25, encoder health min required margin 0.01, temporal-event queries 4, temporal-event precision@1 1.00, temporal around@1 1, temporal window@1 1, temporal recency@1 1, temporal interval@1 1, temporal persistence 1.00, temporal provenance 1.00, knowledge-graph queries 4, knowledge-graph precision@1 1.00, knowledge-graph path precision@1 1.00, knowledge-graph direct@1 1, knowledge-graph two-hop@1 1, knowledge-graph three-hop@1 1, knowledge-graph predicate@1 1, knowledge-graph persistence 1.00, knowledge-graph provenance 1.00, avg latency 1.96, p99 latency 4.91, cross-modal avg latency 4.71, cross-modal p99 latency 6.71, precomputed-vector avg latency 1.87, precomputed-vector p99 latency 2.08, temporal avg latency 2.57, temporal p99 latency 5.28, knowledge-graph avg latency 2.34, knowledge-graph p99 latency 4.36
WaveMind 100M capacity envelope: target memories 100000000, namespace count 32768, node count 128, zones 8, replication factor 3, write quorum 2, node loss min availability 1.00, zone loss min availability 1.00, replica load skew 1.09, primary load skew 1.18, max storage per node gb 5.81, recommended autoscaling max replicas 192, valid capacity plan True, placement ms 53699.9 | Move from deterministic 100M capacity planning to service-backed 100M Qdrant/pgvector/FAISS load tests on sized hardware. | | Postgres PITR runbook/preflight | production-ops | implemented | WaveMind Postgres PITR preflight: status ready, environment status missing_env, command count 7, retention hours 72, missing env WAVEMIND_POSTGRES_DSN, WAVEMIND_POSTGRES_BASEBACKUP_DIR, WAVEMIND_POSTGRES_WAL_ARCHIVE_DIR, WAVEMIND_POSTGRES_RESTORE_DATA_DIR, WAVEMIND_POSTGRES_RESTORE_TARGET_TIME, ok True | Execute the same runbook against staging or managed Postgres and commit a real PITR drill report with replay LSN, target timestamp, restore duration, and post-restore row/index checks. | -| Production readiness gate | production-scale | implemented | WaveMind production readiness: readiness score 1.00, overall status pass, passed criteria 39, action required 0, failed criteria 0, total criteria 39 | Keep the gate at readiness_score 1.0 while repeating larger service-backed runs and moving external competitor evidence into the separate adapter profile. | +| Production readiness gate | production-scale | implemented | WaveMind production readiness: readiness score 0.97, overall status fail, passed criteria 38, action required 0, failed criteria 1, total criteria 39 | Keep the gate at readiness_score 1.0 while repeating larger service-backed runs and moving external competitor evidence into the separate adapter profile. | | Production evidence environment contract | production-ops | implemented | WaveMind production evidence env: overall status action_required, required env count 9, configured required count 0, missing required count 9, recommended missing count 3, workflow count 4, dispatch job count 8, scale gap profile count 5 | Set the missing environment-scoped GitHub secrets, run the safe dispatch commands, and ingest downloaded artifacts through the strict evidence gate. | | Strict evidence readiness runbook | production-scale | implemented | WaveMind strict evidence readiness: status pass, readiness status action_required, claim status claims_limited, total requirements 8, action required 4, ready for safe dispatch count 0, can auto run now count 0, target memories total 180000000 | Provision the missing remote/service environments, run safe dispatch commands with commit_results=false, ingest downloaded artifacts, then rerun strict evidence validation before changing release claims. | | Local HTTP cluster smoke | production-scale | implemented | WaveMind local HTTP cluster smoke: nodes 4, namespaces 4, memories per namespace 2, replication factor 3, read fanout 1, workers 4, success rate 1.00, write success rate 1.00, query hit rate 1.00, failover hit rate 1.00, delete suppression rate 1.00, repair repaired 1, p99 operation ms 348.8, slo pass True | Run the same workload against external service nodes and then increase namespace count and payload size on sized hardware. | | Local HTTP active-active service-region smoke | production-scale | implemented | WaveMind real HTTP active-active service-region sync: region count 3, namespaces 2, writes 6, sync cycles 3, pair syncs 36, cursor count 12, records imported 36, tombstones imported 6, deleted records 6, field keys exported 100, final noop records imported 0, final noop failed pairs 0, convergence rate 1.00, delete suppression rate 1.00, success rate 1.00, failed pairs 0, has more pairs 0, avg sync ms 525.4, p99 sync ms 925.2, avg operation ms 53.5, p99 operation ms 347.6, slo pass True | Run the same active-active service-region workload against remote Kubernetes/serverless API nodes with external Postgres/Qdrant/Redis state. | -| External HTTP cluster load runner | production-scale | implemented | WaveMind external HTTP cluster load: nodes 4, namespaces 32, memories per namespace 8, replication factor 3, read fanout 1, workers 8, success rate 1.00, write success rate 1.00, query hit rate 1.00, failover hit rate 1.00, delete suppression rate 1.00, repair repaired 1, p99 operation ms 6694.8, slo pass True, batch query success True, batch query size 24, batch query http requests 24 -> 1, batch query request reduction ratio 0.96, batch query p99 ms 148.8, batch query total speedup 1.26 | Replace the current loopback service-node artifact with a remote node manifest run from a multi-node deployment. | -| Kubernetes operator leader failover smoke | production-scale | implemented | WaveMind Kubernetes operator failover: status pass, environment kind-multinode-ci, evidence source github-actions-kind, source ref 9956e3112dc575d0f12ff99ec59326dd8e035f70, workflow run id 29057247023, workflow run url https://github.com/CaspianG/wavemind/actions/runs/29057247023, node count 4, operator pod count 2, operator node count 2, lease transitions after 1, desired replicas after scale 4, ready replicas after scale 4, operator status tracks leader True, topology spread constraint count 2, pdb min available 3, pdb disruptions allowed 1, data pod uid changed True, api healthy after recovery True, rolling upgrade revision changed True, rolling upgrade replaced pods 4, api healthy after upgrade True, passed checks 14, check count 14, claim boundary Ephemeral multi-node Kubernetes CI evidence. It proves real Kubernetes API reconciliation, Lease/etcd-backed operator failover, StatefulSet reconciliation, and pod recovery. It does not unlock remote production, multi-region, managed-serverless, or 100M admission claims. | Run the same failure drill against a non-ephemeral remote Kubernetes staging cluster and feed the resulting endpoints into cluster admission. | -| Kubernetes service-network physical worker failure smoke | production-scale | implemented | WaveMind Kubernetes physical worker failure: status pass, environment kind-multinode-network-ci, evidence source github-actions-kind-physical-node-pause, source ref 3d40e894b493253a7d263fa2bf6d67bbc8f4019a, workflow run id 29070578441, workflow run url https://github.com/CaspianG/wavemind/actions/runs/29070578441, service node count 4, zone count 3, failure method docker-pause-kind-worker, target worker wavemind-ci-worker, target zone zone-a, target data pods wavemind-ci-2, outage duration ms 8911.6, seed memories 256, outage hit rate 1.00, failed nodes during outage wavemind-ci-2, recovery hit rate 1.00, failed nodes after recovery -, node ready after recovery True, pod uids preserved True, passed checks 13, check count 13, claim boundary Ephemeral non-loopback Kubernetes service-network and physical kind-worker failure evidence; not remote multi-region production admission. | Repeat the same protocol on a non-ephemeral remote Kubernetes staging cluster, then run node, zone, and region drills through strict cluster admission. | -| Kubernetes active-active region failure and recovery smoke | production-scale | implemented | WaveMind Kubernetes active-active region recovery: status pass, environment kind-multizone-active-active-ci, evidence source github-actions-kind-physical-region-worker-pause, source ref b6392353668203196007e1cac695e178752d34d1, workflow run id 29058433643, workflow run url https://github.com/CaspianG/wavemind/actions/runs/29058433643, region count 3, zone count 3, all regions use pvc True, failure method docker-pause-kind-worker, target region region-b, outage duration ms 9900.1, seed writes 48, seed convergence rate 1.00, outage unavailable regions region-b, outage writes 32, outage convergence rate 1.00, outage delete suppression rate 1.00, recovery convergence rate 1.00, recovery delete suppression rate 1.00, final noop records imported 0, final noop tombstones imported 0, passed checks 17, check count 17, claim boundary Ephemeral non-loopback Kubernetes three-zone active-active region failure evidence; not remote multi-region production admission. | Run this protocol across independent remote Kubernetes regions and use that external artifact for strict active-active admission. | -| Kubernetes serverless external-state lifecycle smoke | production-scale | implemented | WaveMind Kubernetes serverless lifecycle: status pass, environment kind-multizone-serverless-lifecycle-ci, evidence source github-actions-kind-external-state-manual-scale-lifecycle, source ref 880cafdf5ce5f7c9da878bd8e82ada39ca2d02cc, workflow run id 29064934749, workflow run url https://github.com/CaspianG/wavemind/actions/runs/29064934749, passed checks 13, check count 13, persistent volume claims 3, cold start ms 5086.2, restored after zero rate 1.00, ready replicas 3, zone count 3, visible replicas 3, suppressed replicas 3, write propagation ms 1130.7, delete propagation ms 915.0, burst requests per second 41.4, burst p99 ms 1461.5, final restore rate 1.00, claim boundary Ephemeral non-loopback Kubernetes lifecycle evidence with external durable state. It proves scale-to-zero state safety and multi-replica behavior, but does not unlock remote managed Knative/KEDA production admission. | Repeat the same lifecycle on managed Knative/KEDA endpoints and ingest remote telemetry through strict serverless admission. | -| Kubernetes PostgreSQL backup and Qdrant rebuild DR smoke | production-ops | implemented | WaveMind Kubernetes PostgreSQL/Qdrant DR: status pass, environment kind-independent-namespace-postgres-qdrant-dr-ci, evidence source github-actions-kind-pg-dump-independent-restore, source ref 880cafdf5ce5f7c9da878bd8e82ada39ca2d02cc, workflow run id 29064934749, workflow run url https://github.com/CaspianG/wavemind/actions/runs/29064934749, passed checks 10, check count 10, backup format pg_dump-custom, backup bytes 1016635, source state stopped True, recovery pvcs 3, restored rate 1.00, index healthy True, index expected records 24, index vector records 24, restored after api replacement rate 1.00, restore elapsed ms 20449.1, claim boundary Ephemeral non-loopback Kubernetes disaster-recovery evidence. It proves logical PostgreSQL backup/restore and Qdrant rebuild in an independent namespace, not managed-cloud PITR or multi-region DR. | Run the same protocol against managed PostgreSQL backups, object storage, and a remote Kubernetes recovery cluster before claiming managed-cloud PITR or multi-region DR. | +| External HTTP cluster load runner | production-scale | implemented | WaveMind external HTTP cluster load: nodes 4, namespaces 32, memories per namespace 8, replication factor 3, read fanout 1, workers 8, success rate 1.00, write success rate 1.00, query hit rate 1.00, failover hit rate 1.00, delete suppression rate 1.00, repair repaired 1, p99 operation ms 8351.0, slo pass True, batch query success True, batch query size 24, batch query http requests 24 -> 1, batch query request reduction ratio 0.96, batch query p99 ms 186.8, batch query total speedup 1.08 | Replace the current loopback service-node artifact with a remote node manifest run from a multi-node deployment. | +| Kubernetes operator leader failover smoke | production-scale | implemented | WaveMind Kubernetes operator failover: status pass, environment kind-multinode-ci, evidence source github-actions-kind, source ref 29f84dc226c5fd11ae9696d04fa48ffb1ae371e3, workflow run id 29165761261, workflow run url https://github.com/CaspianG/wavemind/actions/runs/29165761261, node count 4, operator pod count 2, operator node count 2, lease transitions after 1, desired replicas after scale 4, ready replicas after scale 4, operator status tracks leader True, topology spread constraint count 2, pdb min available 3, pdb disruptions allowed 1, data pod uid changed True, api healthy after recovery True, rolling upgrade revision changed True, rolling upgrade replaced pods 4, api healthy after upgrade True, passed checks 14, check count 14, claim boundary Ephemeral multi-node Kubernetes CI evidence. It proves real Kubernetes API reconciliation, Lease/etcd-backed operator failover, StatefulSet reconciliation, and pod recovery. It does not unlock remote production, multi-region, managed-serverless, or 100M admission claims. | Run the same failure drill against a non-ephemeral remote Kubernetes staging cluster and feed the resulting endpoints into cluster admission. | +| Kubernetes service-network physical worker failure smoke | production-scale | implemented | WaveMind Kubernetes physical worker failure: status pass, environment kind-multinode-network-ci, evidence source github-actions-kind-physical-node-pause, source ref 29f84dc226c5fd11ae9696d04fa48ffb1ae371e3, workflow run id 29165761261, workflow run url https://github.com/CaspianG/wavemind/actions/runs/29165761261, service node count 4, zone count 3, failure method docker-pause-kind-worker, target worker wavemind-ci-worker2, target zone zone-b, target data pods wavemind-ci-2, outage duration ms 9431.1, seed memories 256, outage hit rate 1.00, failed nodes during outage wavemind-ci-2, recovery hit rate 1.00, failed nodes after recovery -, node ready after recovery True, pod uids preserved True, passed checks 13, check count 13, claim boundary Ephemeral non-loopback Kubernetes service-network and physical kind-worker failure evidence; not remote multi-region production admission. | Repeat the same protocol on a non-ephemeral remote Kubernetes staging cluster, then run node, zone, and region drills through strict cluster admission. | +| Kubernetes active-active region failure and recovery smoke | production-scale | implemented | WaveMind Kubernetes active-active region recovery: status pass, environment kind-multizone-active-active-ci, evidence source github-actions-kind-physical-region-worker-pause, source ref 29f84dc226c5fd11ae9696d04fa48ffb1ae371e3, workflow run id 29165761261, workflow run url https://github.com/CaspianG/wavemind/actions/runs/29165761261, region count 3, zone count 3, all regions use pvc True, failure method docker-pause-kind-worker, target region region-b, outage duration ms 9658.9, seed writes 48, seed convergence rate 1.00, outage unavailable regions region-b, outage writes 32, outage convergence rate 1.00, outage delete suppression rate 1.00, recovery convergence rate 1.00, recovery delete suppression rate 1.00, final noop records imported 0, final noop tombstones imported 0, passed checks 17, check count 17, claim boundary Ephemeral non-loopback Kubernetes three-zone active-active region failure evidence; not remote multi-region production admission. | Run this protocol across independent remote Kubernetes regions and use that external artifact for strict active-active admission. | +| Kubernetes serverless external-state lifecycle smoke | production-scale | implemented | WaveMind Kubernetes serverless lifecycle: status pass, environment kind-multizone-serverless-lifecycle-ci, evidence source github-actions-kind-external-state-manual-scale-lifecycle, source ref 29f84dc226c5fd11ae9696d04fa48ffb1ae371e3, workflow run id 29165761261, workflow run url https://github.com/CaspianG/wavemind/actions/runs/29165761261, passed checks 13, check count 13, persistent volume claims 3, cold start ms 5055.0, restored after zero rate 1.00, ready replicas 3, zone count 3, visible replicas 3, suppressed replicas 3, write propagation ms 1136.9, delete propagation ms 884.8, burst requests per second 40.7, burst p99 ms 1891.9, final restore rate 1.00, claim boundary Ephemeral non-loopback Kubernetes lifecycle evidence with external durable state. It proves scale-to-zero state safety and multi-replica behavior, but does not unlock remote managed Knative/KEDA production admission. | Repeat the same lifecycle on managed Knative/KEDA endpoints and ingest remote telemetry through strict serverless admission. | +| Kubernetes PostgreSQL backup and Qdrant rebuild DR smoke | production-ops | implemented | WaveMind Kubernetes PostgreSQL/Qdrant DR: status pass, environment kind-independent-namespace-postgres-qdrant-dr-ci, evidence source github-actions-kind-pg-dump-independent-restore, source ref 29f84dc226c5fd11ae9696d04fa48ffb1ae371e3, workflow run id 29165761261, workflow run url https://github.com/CaspianG/wavemind/actions/runs/29165761261, passed checks 10, check count 10, backup format pg_dump-custom, backup bytes 1016642, source state stopped True, recovery pvcs 3, restored rate 1.00, index healthy True, index expected records 24, index vector records 24, restored after api replacement rate 1.00, restore elapsed ms 23338.1, claim boundary Ephemeral non-loopback Kubernetes disaster-recovery evidence. It proves logical PostgreSQL backup/restore and Qdrant rebuild in an independent namespace, not managed-cloud PITR or multi-region DR. | Run the same protocol against managed PostgreSQL backups, object storage, and a remote Kubernetes recovery cluster before claiming managed-cloud PITR or multi-region DR. | | External HTTP active-active loopback | production-scale | implemented | WaveMind real HTTP active-active service-region sync: region count 3, namespaces 16, writes 48, sync cycles 3, pair syncs 288, cursor count 96, records imported 288, tombstones imported 6, deleted records 6, final noop records imported 0, final noop failed pairs 0, convergence rate 1.00, delete suppression rate 1.00, success rate 1.00, failed pairs 0, p99 operation ms 349.2, slo pass True | Use the same runner path against real remote API regions through `.github/workflows/external-http-active-active.yml`. | | External HTTP active-active region runner | production-scale | implemented | WaveMind real HTTP active-active service-region sync: status action_required, reason Run external-http-active-active with real region URLs or a regions manifest. | Run workflow_dispatch external-http-active-active against at least three remote API regions backed by external Postgres/Qdrant/Redis state. | | Memory OS policy evolution | production-scale | implemented | WaveMind Memory OS policy evolution: status pass, cycles 3, target memories 2000000, namespace count 4096, replayed query count 24, decision coverage rate 1.00, repeated required cycle count 2, history suggestion count 4, escalation action count 2, scheduler history trend repeated_architecture_required, scheduler history previous runs 3, scheduler policy escalation ids scale-policy, stable ok ids forgetting-policy, prefetch-policy, priority-policy, prewarm warmed 16, predictive prefetch warmed 30, priority predictions 14 | Run the same policy-evolution benchmark against a real Redis-backed staging namespace and promote the artifact only after memory-os-admission remains plan-limited by external production evidence rather than local worker behavior. | diff --git a/benchmarks/CLUSTER_ADMISSION.md b/benchmarks/CLUSTER_ADMISSION.md index 5cdb261..9b0f639 100644 --- a/benchmarks/CLUSTER_ADMISSION.md +++ b/benchmarks/CLUSTER_ADMISSION.md @@ -28,13 +28,13 @@ stay useful for development, but do not unlock this gate. | requirement | status | artifact | evidence | |---|---|---|---| -| Non-loopback Kubernetes or external HTTP service-node load | `pass` | `benchmarks/http_cluster_load_results.json` | nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms | +| Non-loopback Kubernetes or external HTTP service-node load | `pass` | `benchmarks/http_cluster_load_results.json` | nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms | ## Requested Evidence | status | min nodes | namespaces | RF | read quorum | read fanout | batch size | p99 SLO ms | evidence | |---|---:|---:|---:|---:|---:|---:|---:|---| -| `pass` | `4` | `32` | `3` | `1` | `1` | `24` | `1000.0` | nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms | +| `pass` | `4` | `32` | `3` | `1` | `1` | `24` | `1000.0` | nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms | ## Preflight diff --git a/benchmarks/COST_EFFICIENCY.md b/benchmarks/COST_EFFICIENCY.md index 59887fe..3d72feb 100644 --- a/benchmarks/COST_EFFICIENCY.md +++ b/benchmarks/COST_EFFICIENCY.md @@ -1,6 +1,6 @@ # WaveMind Cost Efficiency Leaderboard -Generated: `2026-07-11T05:56:44Z`. +Generated: `2026-07-11T19:56:03Z`. Measured rows come from checked-in load artifacts. Planned rows are capacity and cost contracts only; they do not unlock production latency or recall claims until the matching benchmark result exists. diff --git a/benchmarks/PRODUCTION_EVIDENCE.md b/benchmarks/PRODUCTION_EVIDENCE.md index 7143124..97e467d 100644 --- a/benchmarks/PRODUCTION_EVIDENCE.md +++ b/benchmarks/PRODUCTION_EVIDENCE.md @@ -14,7 +14,7 @@ multi-region, managed-serverless, 50M, or 100M production scale cannot. | requirement | status | evidence | artifact | command | unlocks | |---|---|---|---|---|---| -| Non-loopback Kubernetes or external HTTP service-node load | `pass` | nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms | `benchmarks/http_cluster_load_results.json` | `gh workflow run external-http-cluster-load.yml -f nodes="node-a=https://wm-a.example.com,node-b=https://wm-b.example.com,node-c=https://wm-c.example.com,node-d=https://wm-d.example.com" -f replication_factor=3 -f read_quorum=1 -f read_fanout=1 -f batch_query_size=24 -f fail_on_slo=true -f commit_results=true` | Non-loopback Kubernetes service-node cluster load SLO. | +| Non-loopback Kubernetes or external HTTP service-node load | `pass` | nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms | `benchmarks/http_cluster_load_results.json` | `gh workflow run external-http-cluster-load.yml -f nodes="node-a=https://wm-a.example.com,node-b=https://wm-b.example.com,node-c=https://wm-c.example.com,node-d=https://wm-d.example.com" -f replication_factor=3 -f read_quorum=1 -f read_fanout=1 -f batch_query_size=24 -f fail_on_slo=true -f commit_results=true` | Non-loopback Kubernetes service-node cluster load SLO. | | External HTTP active-active regions | `action_required` | no checked-in external HTTP active-active region result; issues: missing artifact | `benchmarks/external_http_active_active_results.json` | `gh workflow run external-http-active-active.yml -f regions="us-east=https://wm-us.example.com,eu-west=https://wm-eu.example.com,ap-south=https://wm-ap.example.com" -f namespace_count=16 -f p99_slo_ms=1500 -f fail_on_slo=true -f commit_results=true` | Remote multi-region active-active memory convergence. | | Managed/serverless remote telemetry | `action_required` | missing remote serverless telemetry; issues: missing artifact | `deploy/serverless/observed-telemetry.remote.json` | `gh workflow run serverless-observed-telemetry.yml -f nodes="https://wm-a.example.com,https://wm-b.example.com" -f seed_mode=first -f commit_results=true` | Hosted/serverless p99, cold-start, error-rate, and scale-out SLO. | | 10M Qdrant service load | `pass` | Qdrant service streaming: vectors 10000000, recall 0.975, p99 43.266599997878075 ms, cost valid_slo, source be2cebfd776b, run local-qdrant10m-quantized-ef1024-v1182-20260711 | `benchmarks/production_streaming_load_qdrant_10m_results.json` | `python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines qdrant-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks/production_streaming_load_qdrant_10m_results.json --checkpoint-path state/production-runs/qdrant-service-10000000.checkpoint.json` | 10M Qdrant service-backed candidate index SLO. | diff --git a/benchmarks/PRODUCTION_EVIDENCE_BUNDLE.md b/benchmarks/PRODUCTION_EVIDENCE_BUNDLE.md index c463035..9587275 100644 --- a/benchmarks/PRODUCTION_EVIDENCE_BUNDLE.md +++ b/benchmarks/PRODUCTION_EVIDENCE_BUNDLE.md @@ -8,7 +8,7 @@ claim boundaries, and the exact next actions required to unlock blocked claims. |---|---:| | claim status | `claims_limited` | | strict evidence | `4/8` | -| preflight ready | `0/8` | +| preflight ready | `1/8` | | production readiness | `pass` | | readiness score | `1.0` | | artifact audit | `pass` | @@ -34,7 +34,7 @@ claim boundaries, and the exact next actions required to unlock blocked claims. |---|---|---|---:|---|---| | qdrant-10m | `action_required` | `qdrant-service` | 10000000 | `benchmarks/production_streaming_load_qdrant_10m_results.json` | `WAVEMIND_QDRANT_URL` | | qdrant-sharded-10m | `action_required` | `qdrant-sharded-service` | 10000000 | `benchmarks/production_streaming_load_qdrant_sharded_10m_results.json` | `WAVEMIND_QDRANT_URLS` | -| pgvector-10m | `action_required` | `pgvector-service` | 10000000 | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `WAVEMIND_PGVECTOR_DSN` | +| pgvector-10m | `action_required` | `pgvector-service` | 10000000 | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `WAVEMIND_PGVECTOR_DSNS` | | faiss-ivfpq-50m | `action_required` | `faiss-ivfpq-persisted` | 50000000 | `benchmarks/production_streaming_load_ivfpq_50m_results.json` | `WAVEMIND_FAISS_IVFPQ_PATH` | | qdrant-sharded-100m | `action_required` | `qdrant-sharded-service` | 100000000 | `benchmarks/production_streaming_load_qdrant_sharded_100m_results.json` | `WAVEMIND_QDRANT_URLS` | @@ -44,5 +44,5 @@ claim boundaries, and the exact next actions required to unlock blocked claims. |---|---|---|---|---|---| | External HTTP active-active regions | `action_required` | `action_required` | `benchmarks/external_http_active_active_results.json` | `WAVEMIND_ACTIVE_ACTIVE_REGIONS, WAVEMIND_ACTIVE_ACTIVE_REGIONS_MANIFEST_JSON; issues: missing artifact` | `gh workflow run external-http-active-active.yml -f regions="$WAVEMIND_ACTIVE_ACTIVE_REGIONS" -f commit_results=true` | | Managed/serverless remote telemetry | `action_required` | `action_required` | `deploy/serverless/observed-telemetry.remote.json` | `WAVEMIND_SERVERLESS_NODES; issues: missing artifact` | `gh workflow run serverless-observed-telemetry.yml -f nodes="$WAVEMIND_SERVERLESS_NODES" -f seed_mode=first -f commit_results=true` | -| 10M pgvector service load | `action_required` | `action_required` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `WAVEMIND_PGVECTOR_DSN; issues: missing artifact` | `python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks/production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json` | +| 10M pgvector service load | `action_required` | `ready` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `issues: missing artifact` | `gh workflow run production-streaming-load.yml --ref main -f engine=pgvector-service -f size=10000000 -f dim=128 -f queries=2000 -f top_k=10 -f batch_size=5000 -f target_recall=0.95 -f target_p99_ms=100 -f target_qps=100 -f replicas=3 -f autoscaling_max_replicas=24 -f capacity_headroom=0.7 -f runner_label=ubuntu-latest -f provision_pgvector_shards=true -f pgvector_shard_count=4 -f runner_storage_root=state -f commit_results=true` | | 100M remote load result | `action_required` | `action_required` | `benchmarks/production_streaming_load_qdrant_sharded_100m_results.json` | `WAVEMIND_QDRANT_URLS; issues: missing artifact` | `python benchmarks/production_streaming_load_benchmark.py --sizes 100000000 --dim 128 --queries 5000 --top-k 10 --seed 42 --noise 0.08 --batch-size 10000 --engines qdrant-sharded-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 500.0 --replicas 8 --autoscaling-max-replicas 128 --capacity-headroom 0.7 --output benchmarks/production_streaming_load_qdrant_sharded_100m_results.json --checkpoint-path state/production-runs/qdrant-sharded-service-100000000.checkpoint.json` | diff --git a/benchmarks/PRODUCTION_EVIDENCE_DISPATCH.md b/benchmarks/PRODUCTION_EVIDENCE_DISPATCH.md index e0d95b5..35ab074 100644 --- a/benchmarks/PRODUCTION_EVIDENCE_DISPATCH.md +++ b/benchmarks/PRODUCTION_EVIDENCE_DISPATCH.md @@ -8,8 +8,8 @@ strict production-evidence validation. | metric | value | |---|---:| | overall status | `action_required` | -| ready to dispatch | `0` | -| blocked by preflight | `4` | +| ready to dispatch | `1` | +| blocked by preflight | `3` | | complete | `4` | | total jobs | `8` | | runner label | `self-hosted-large` | @@ -24,7 +24,7 @@ strict production-evidence validation. | Managed/serverless remote telemetry | `blocked_by_preflight` | `remote-service` | `serverless-observed-telemetry.yml` | `deploy/serverless/observed-telemetry.remote.json` | `WAVEMIND_SERVERLESS_NODES` | | 10M Qdrant service load | `complete` | `service-scale-10m` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_qdrant_10m_results.json` | `WAVEMIND_QDRANT_URL` | | 10M sharded Qdrant service load | `complete` | `service-scale-10m` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_qdrant_sharded_10m_results.json` | `WAVEMIND_QDRANT_URLS` | -| 10M pgvector service load | `blocked_by_preflight` | `service-scale-10m` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `WAVEMIND_PGVECTOR_DSN` | +| 10M pgvector service load | `ready_to_dispatch` | `service-scale-10m` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `` | | 50M FAISS IVF-PQ streaming load | `complete` | `large-local-index` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_ivfpq_50m_results.json` | `WAVEMIND_FAISS_IVFPQ_PATH` | | 100M remote load result | `blocked_by_preflight` | `hundred-million-service` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_qdrant_sharded_100m_results.json` | `WAVEMIND_QDRANT_URLS` | @@ -35,7 +35,7 @@ strict production-evidence validation. - `serverless_remote_telemetry`: `gh workflow run serverless-observed-telemetry.yml -f nodes="$WAVEMIND_SERVERLESS_NODES" -f requests="240" -f workers="4" -f seed_memories="24" -f seed_mode="first" -f max_scale="256" -f target_rps="3200" -f target_p99_ms="500" -f external_cold_start_ms="900" -f estimated_scale_out_seconds="18" -f commit_results="false"` - `qdrant_10m_service`: `gh workflow run production-streaming-load.yml -f engine="qdrant-service" -f size="10000000" -f dim="128" -f queries="2000" -f top_k="10" -f batch_size="5000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="100.0" -f replicas="3" -f autoscaling_max_replicas="24" -f capacity_headroom="0.7" -f runner_label="self-hosted-large" -f runner_storage_root="state/production-runs" -f commit_results="false" -f qdrant_url="$WAVEMIND_QDRANT_URL"` - `qdrant_sharded_10m_service`: `gh workflow run production-streaming-load.yml -f engine="qdrant-sharded-service" -f size="10000000" -f dim="128" -f queries="2000" -f top_k="10" -f batch_size="5000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="250.0" -f replicas="4" -f autoscaling_max_replicas="48" -f capacity_headroom="0.7" -f runner_label="self-hosted-large" -f runner_storage_root="state/production-runs" -f commit_results="false" -f qdrant_urls="$WAVEMIND_QDRANT_URLS"` -- `pgvector_10m_service`: `gh workflow run production-streaming-load.yml -f engine="pgvector-service" -f size="10000000" -f dim="128" -f queries="2000" -f top_k="10" -f batch_size="5000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="100.0" -f replicas="3" -f autoscaling_max_replicas="24" -f capacity_headroom="0.7" -f runner_label="self-hosted-large" -f runner_storage_root="state/production-runs" -f commit_results="false" -f pgvector_dsn="$WAVEMIND_PGVECTOR_DSN"` +- `pgvector_10m_service`: `gh workflow run production-streaming-load.yml -f engine="pgvector-service" -f size="10000000" -f dim="128" -f queries="2000" -f top_k="10" -f batch_size="5000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="100.0" -f replicas="3" -f autoscaling_max_replicas="24" -f capacity_headroom="0.7" -f runner_label="ubuntu-latest" -f runner_storage_root="state/production-runs" -f commit_results="false" -f provision_pgvector_shards="true" -f pgvector_shard_count="4"` - `faiss_ivfpq_50m`: `gh workflow run production-streaming-load.yml -f engine="faiss-ivfpq-persisted" -f size="50000000" -f dim="128" -f queries="2000" -f top_k="10" -f batch_size="1000000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="100.0" -f replicas="3" -f autoscaling_max_replicas="24" -f capacity_headroom="0.7" -f runner_label="self-hosted-large" -f runner_storage_root="state/production-runs" -f commit_results="false" -f faiss_ivfpq_path="$WAVEMIND_FAISS_IVFPQ_PATH"` - `hundred_million_remote_load`: `gh workflow run production-streaming-load.yml -f engine="qdrant-sharded-service" -f size="100000000" -f dim="128" -f queries="5000" -f top_k="10" -f batch_size="10000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="500.0" -f replicas="8" -f autoscaling_max_replicas="128" -f capacity_headroom="0.7" -f runner_label="self-hosted-large" -f runner_storage_root="state/production-runs" -f commit_results="false" -f qdrant_urls="$WAVEMIND_QDRANT_URLS"` diff --git a/benchmarks/PRODUCTION_EVIDENCE_ENV.md b/benchmarks/PRODUCTION_EVIDENCE_ENV.md index f82886c..5562d2a 100644 --- a/benchmarks/PRODUCTION_EVIDENCE_ENV.md +++ b/benchmarks/PRODUCTION_EVIDENCE_ENV.md @@ -28,7 +28,7 @@ Environment contract only. It stores placeholders and secret names, never creden | `WAVEMIND_CLUSTER_NODES_MANIFEST_JSON` | `missing` | `api-node-manifest-json` | `external_http_cluster` | `external-http-cluster-load.yml` | `benchmarks/http_cluster_load_results.json` | | `WAVEMIND_FAISS_IVFPQ_FREE_GB` | `optional` | `disk-free-override-gb` | | | | | `WAVEMIND_FAISS_IVFPQ_PATH` | `missing` | `filesystem-path` | `faiss_ivfpq_50m` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_ivfpq_50m_results.json` | -| `WAVEMIND_PGVECTOR_DSN` | `missing` | `postgres-dsn` | `pgvector_10m_service` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | +| `WAVEMIND_PGVECTOR_DSNS` | `missing` | `postgres-dsn-list` | `pgvector_10m_service` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | | `WAVEMIND_QDRANT_API_KEY` | `recommended` | `qdrant-api-secret` | `qdrant_10m_service` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_qdrant_10m_results.json` | | `WAVEMIND_QDRANT_API_KEYS` | `recommended` | `qdrant-api-secret-list` | `hundred_million_remote_load`, `qdrant_sharded_10m_service` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_qdrant_sharded_100m_results.json`, `benchmarks/production_streaming_load_qdrant_sharded_10m_results.json` | | `WAVEMIND_QDRANT_URL` | `missing` | `qdrant-url` | `qdrant_10m_service` | `production-streaming-load.yml` | `benchmarks/production_streaming_load_qdrant_10m_results.json` | @@ -48,7 +48,7 @@ the repository. - `printf '%s' "$WAVEMIND_CLUSTER_NODES_MANIFEST_JSON" | gh secret set WAVEMIND_CLUSTER_NODES_MANIFEST_JSON --repo CaspianG/wavemind --body-file -` - `printf '%s' "$WAVEMIND_FAISS_IVFPQ_FREE_GB" | gh secret set WAVEMIND_FAISS_IVFPQ_FREE_GB --repo CaspianG/wavemind --body-file -` - `printf '%s' "$WAVEMIND_FAISS_IVFPQ_PATH" | gh secret set WAVEMIND_FAISS_IVFPQ_PATH --repo CaspianG/wavemind --body-file -` -- `printf '%s' "$WAVEMIND_PGVECTOR_DSN" | gh secret set WAVEMIND_PGVECTOR_DSN --repo CaspianG/wavemind --body-file -` +- `printf '%s' "$WAVEMIND_PGVECTOR_DSNS" | gh secret set WAVEMIND_PGVECTOR_DSNS --repo CaspianG/wavemind --body-file -` - `printf '%s' "$WAVEMIND_QDRANT_API_KEY" | gh secret set WAVEMIND_QDRANT_API_KEY --repo CaspianG/wavemind --body-file -` - `printf '%s' "$WAVEMIND_QDRANT_API_KEYS" | gh secret set WAVEMIND_QDRANT_API_KEYS --repo CaspianG/wavemind --body-file -` - `printf '%s' "$WAVEMIND_QDRANT_URL" | gh secret set WAVEMIND_QDRANT_URL --repo CaspianG/wavemind --body-file -` diff --git a/benchmarks/PRODUCTION_EVIDENCE_PREFLIGHT.md b/benchmarks/PRODUCTION_EVIDENCE_PREFLIGHT.md index 588d7a4..cdc25ba 100644 --- a/benchmarks/PRODUCTION_EVIDENCE_PREFLIGHT.md +++ b/benchmarks/PRODUCTION_EVIDENCE_PREFLIGHT.md @@ -7,8 +7,8 @@ the strict production evidence gate; it only verifies prerequisites. | metric | value | |---|---:| | overall status | `action_required` | -| ready checks | `0` | -| action required | `8` | +| ready checks | `1` | +| action required | `7` | | total checks | `8` | | check | status | evidence | missing env | output | command | @@ -18,6 +18,6 @@ the strict production evidence gate; it only verifies prerequisites. | Managed/serverless remote telemetry preflight | `action_required` | 0 node URLs configured; issues: serverless node requires at least 1 URLs, set WAVEMIND_SERVERLESS_NODES; warnings: WAVEMIND_API_KEY is not set; only use this if the target endpoints are intentionally unauthenticated | `WAVEMIND_SERVERLESS_NODES` | `deploy/serverless/observed-telemetry.remote.json` | `gh workflow run serverless-observed-telemetry.yml -f nodes="$WAVEMIND_SERVERLESS_NODES" -f seed_mode=first -f commit_results=true` | | 10M Qdrant service load preflight | `action_required` | plan benchmarks/production_streaming_load_qdrant_10m_plan.json, vectors 10000000, required env 1, missing env 1, required local free 0.004 GB; issues: set WAVEMIND_QDRANT_URL | `WAVEMIND_QDRANT_URL` | `benchmarks/production_streaming_load_qdrant_10m_results.json` | `python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines qdrant-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks\production_streaming_load_qdrant_10m_results.json --checkpoint-path state/production-runs/qdrant-service-10000000.checkpoint.json` | | 10M sharded Qdrant service load preflight | `action_required` | plan benchmarks/production_streaming_load_qdrant_sharded_10m_plan.json, vectors 10000000, required env 1, missing env 1, required local free 0.004 GB; issues: set WAVEMIND_QDRANT_URLS | `WAVEMIND_QDRANT_URLS` | `benchmarks/production_streaming_load_qdrant_sharded_10m_results.json` | `python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines qdrant-sharded-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 250.0 --replicas 4 --autoscaling-max-replicas 48 --capacity-headroom 0.7 --output benchmarks\production_streaming_load_qdrant_sharded_10m_results.json --checkpoint-path state/production-runs/qdrant-sharded-service-10000000.checkpoint.json` | -| 10M pgvector service load preflight | `action_required` | plan benchmarks/production_streaming_load_pgvector_10m_plan.json, vectors 10000000, required env 1, missing env 1, required local free 0.004 GB; issues: set WAVEMIND_PGVECTOR_DSN | `WAVEMIND_PGVECTOR_DSN` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks\production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json` | +| 10M pgvector service load preflight | `ready` | workflow-provisioned four-service pgvector topology; external DSNs optional; plan benchmarks/production_streaming_load_pgvector_10m_plan.json; warnings: Ephemeral isolated service processes prove the 10M candidate-index SLO, not PostgreSQL HA or independent-node failure tolerance. | `` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `gh workflow run production-streaming-load.yml --ref main -f engine=pgvector-service -f size=10000000 -f dim=128 -f queries=2000 -f top_k=10 -f batch_size=5000 -f target_recall=0.95 -f target_p99_ms=100 -f target_qps=100 -f replicas=3 -f autoscaling_max_replicas=24 -f capacity_headroom=0.7 -f runner_label=ubuntu-latest -f provision_pgvector_shards=true -f pgvector_shard_count=4 -f runner_storage_root=state -f commit_results=true` | | 50M FAISS IVF-PQ streaming load preflight | `action_required` | plan benchmarks/production_streaming_load_50m_plan.json, vectors 50000000, required env 1, missing env 1, required local free 2.116 GB; issues: set WAVEMIND_FAISS_IVFPQ_PATH | `WAVEMIND_FAISS_IVFPQ_PATH` | `benchmarks/production_streaming_load_ivfpq_50m_results.json` | `python benchmarks/production_streaming_load_benchmark.py --sizes 50000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 1000000 --engines faiss-ivfpq-persisted --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks\production_streaming_load_ivfpq_50m_results.json --checkpoint-path state/production-runs/faiss-ivfpq-persisted-50000000.checkpoint.json` | | 100M sharded Qdrant service load preflight | `action_required` | plan benchmarks/production_streaming_load_qdrant_sharded_100m_plan.json, vectors 100000000, required env 1, missing env 1, required local free 0.009 GB; issues: set WAVEMIND_QDRANT_URLS | `WAVEMIND_QDRANT_URLS` | `benchmarks/production_streaming_load_qdrant_sharded_100m_results.json` | `python benchmarks/production_streaming_load_benchmark.py --sizes 100000000 --dim 128 --queries 5000 --top-k 10 --seed 42 --noise 0.08 --batch-size 10000 --engines qdrant-sharded-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 500.0 --replicas 8 --autoscaling-max-replicas 128 --capacity-headroom 0.7 --output benchmarks\production_streaming_load_qdrant_sharded_100m_results.json --checkpoint-path state/production-runs/qdrant-sharded-service-100000000.checkpoint.json` | diff --git a/benchmarks/PRODUCTION_READINESS.md b/benchmarks/PRODUCTION_READINESS.md index 91ffc95..ddb2612 100644 --- a/benchmarks/PRODUCTION_READINESS.md +++ b/benchmarks/PRODUCTION_READINESS.md @@ -14,7 +14,7 @@ verdict, not a marketing claim. | criterion | status | evidence | next step | |---|---|---|---| -| Checked-in benchmark artifacts are synchronized | `pass` | audit status pass, generated_at 2026-07-11T05:53:28Z | Keep the benchmark refresh workflow green and block stale artifacts before release. | +| Checked-in benchmark artifacts are synchronized | `pass` | audit status pass, generated_at 2026-07-11T19:56:03Z | Keep the benchmark refresh workflow green and block stale artifacts before release. | | Agent coherence benchmark proves behavioral lift | `pass` | WaveMind success 0.917, Chroma static 0.333, Static vector 0.333, stale error 0.000, context saved 0.931, coherent turn rate 0.750, avg latency 2.647 ms | Keep agent-behavior quality gated in CI and extend it with LLM answer-quality runs on LoCoMo/LongMemEval. | | LongMemEval answer generation beats static RAG baselines | `pass` | ollama qwen2.5:1.5b, queries 50, exact 0.240, contains 0.380, token F1 0.333, answered 0.520, grounded 0.520, supported 1.000, unsupported 0.000, faithful 1.000, abstain 0.480, evidence recall 0.920, retrieval 36.586 ms, Chroma F1 0.170, Qdrant F1 0.170 | Scale this from the checked 50-query local run to full LongMemEval-S with stronger local/API models and faithfulness scoring. | | 100k service-backed load profile passes SLO and cost gate | `pass` | recall 1.0, p99 21.25629998045042 ms, cost $1.39/1M queries | Keep the 100k profile green while adding persisted FAISS and pgvector service runs. | @@ -24,14 +24,14 @@ verdict, not a marketing claim. | pgvector exact and iterative service profile passes 50k tuning gate | `pass` | size 50000, exact recall 1.0, exact p99 76.98170002549887 ms, iterative recall 0.97, iterative p99 55.18779996782541 ms, Qdrant reference recall 1.0 | Promote pgvector-iterative into the 100k and 1M production load SLO profiles after allocating enough disk/build time. | | Qdrant streaming runner has service smoke and 10M preflight | `pass` | smoke vectors 1000, smoke recall 1, smoke p99 17.90370000526309 ms, plan status action_required, plan required local free 0.004 GB, blockers missing_env:WAVEMIND_QDRANT_URL, insufficient_local_disk_for_index_and_transient_batches | Run the embedded 10M Qdrant command against a sized Qdrant service and commit production_streaming_load_qdrant_10m_results.json. | | Qdrant streaming 1M tuned profile passes recall, p99, and cost gate | `pass` | cold recall 0.99, cold p99 3013.9777000295 ms, tuned recall 1, tuned p99 26.373000000603497 ms, warmup 100, wait 30.0 s, upsert chunk 2000, SLO pass | Promote the same warmup/chunking profile into the checked 10M Qdrant service run on storage sized for the index. | -| pgvector streaming runner has service smoke and 10M preflight | `pass` | smoke vectors 1000, smoke recall 1, smoke p99 7.623799960128963 ms, plan status action_required, plan required local free 0.004 GB, blockers missing_env:WAVEMIND_PGVECTOR_DSN, insufficient_local_disk_for_index_and_transient_batches | Run the embedded 10M pgvector command against a sized PostgreSQL service and commit production_streaming_load_pgvector_10m_results.json. | +| pgvector streaming runner has service smoke and 10M preflight | `pass` | smoke vectors 1000, smoke recall 1, smoke p99 7.623799960128963 ms, plan status action_required, plan required local free 0.004 GB, blockers missing_env:WAVEMIND_PGVECTOR_DSNS, insufficient_local_disk_for_index_and_transient_batches | Run the embedded 10M pgvector command against a sized PostgreSQL service and commit production_streaming_load_pgvector_10m_results.json. | | VectorDBBench custom dataset export is reproducible | `pass` | status ready, vectors 10000, queries 100, files ['neighbors', 'scalar_labels', 'test', 'train'] | Run this custom dataset through official VectorDBBench targets for Qdrant, Milvus, pgvector, and WaveMind-backed FAISS/Qdrant profiles. | | Namespace placement survives node and zone loss | `pass` | node loss 1.0, zone loss 1.0, namespaces 4096 | Validate the same placement under live multi-node service load. | | Cluster autoscaler plans node additions within headroom | `pass` | current 4, required 50, target max 678711, moves 4094+0, rebalance ready, batches 82, write quorum 2 | Connect rolling rebalance execution to operator reconciliation status and real HPA/load metrics. | | Control-plane consensus blocks split-brain config changes | `pass` | voters 3 -> 5, term 1 -> 2, revision 2, minority blocked True | Wrap the same majority lease/revision contract around remote operator membership changes. | | 100M-memory capacity envelope is planned across a large cluster | `pass` | 100000000 memories, 128 nodes, RF 3, replica skew 1.09375, max storage/node 5.806214176118374 GB | Promote this envelope from deterministic planning to a real 100M service-backed Qdrant/pgvector/FAISS load run on sized hardware. | -| Kubernetes operator passes a real multi-node failover drill | `pass` | CRD True, HPA True, repair True, memory OS True, operator replicas 2, leader election True via coordination.k8s.io/v1, kind nodes 4, failure drill pass, Lease transitions 1, recovered API True, PDB min available 3, topology constraints 2, rolling pods replaced 4, upgraded API True, workflow https://github.com/CaspianG/wavemind/actions/runs/29057247023, network drill pass via docker-pause-kind-worker, service nodes 4, zones 3, outage recall 1.0, failed nodes ['wavemind-ci-2'], recovery recall 1.0, network workflow https://github.com/CaspianG/wavemind/actions/runs/29070578441, rebalance config True, rebalance ready 4048 moves/81 batches, replicas 34, required 34, target max 678711, status Ready, memory OS ready True, production admission ready True, control-plane True | Repeat the same drill against a non-ephemeral remote Kubernetes staging cluster and feed its service endpoints into strict cluster admission. | -| Serverless plan externalizes state and validates KEDA target | `pass` | Postgres True, Qdrant True, Redis True, required replicas 4, burst rps 256000, cold start 1220.0 ms, cost $81.76, observed source loopback-api-capacity-estimate, observed replicas 4, observed pool rps 583.26, observed p99 17.039 ms, observed errors 0.0, kind lifecycle pass 13/13, coherence 3/3, propagation 1130.6929430000991/915.0309450000123 ms, burst p99 1461.4564480000354 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29064934749 | Run the same profile against a real Knative/KEDA cluster and replace loopback telemetry with remote p95/p99/cold-start/error-rate/scale-out metrics. | +| Kubernetes operator passes a real multi-node failover drill | `pass` | CRD True, HPA True, repair True, memory OS True, operator replicas 2, leader election True via coordination.k8s.io/v1, kind nodes 4, failure drill pass, Lease transitions 1, recovered API True, PDB min available 3, topology constraints 2, rolling pods replaced 4, upgraded API True, workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261, network drill pass via docker-pause-kind-worker, service nodes 4, zones 3, outage recall 1.0, failed nodes ['wavemind-ci-2'], recovery recall 1.0, network workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261, rebalance config True, rebalance ready 4048 moves/81 batches, replicas 34, required 34, target max 678711, status Ready, memory OS ready True, production admission ready True, control-plane True | Repeat the same drill against a non-ephemeral remote Kubernetes staging cluster and feed its service endpoints into strict cluster admission. | +| Serverless plan externalizes state and validates KEDA target | `pass` | Postgres True, Qdrant True, Redis True, required replicas 4, burst rps 256000, cold start 1220.0 ms, cost $81.76, observed source loopback-api-capacity-estimate, observed replicas 4, observed pool rps 583.26, observed p99 17.039 ms, observed errors 0.0, kind lifecycle pass 13/13, coherence 3/3, propagation 1136.9370459999573/884.7577889999911 ms, burst p99 1891.9401950000747 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261 | Run the same profile against a real Knative/KEDA cluster and replace loopback telemetry with remote p95/p99/cold-start/error-rate/scale-out metrics. | | Hot cache and query-audit prewarm work | `pass` | hit rate 0.92, prewarm hit True, p99 0.014099998224992305 ms | Keep local cache prewarm green while Redis carries multi-worker production cache evidence. | | Query-vector cache avoids repeated encoder work | `pass` | local encode calls 1, local hit rate 0.995, Redis shared True, Redis encode calls 1, service encode calls 1, service hit rate 0.995, service p99 11.885699997947086 ms, service metrics True | Run the same service-mode vector-cache profile with a real sentence-transformer encoder on a larger API load test. | | Batch query API amortizes service recall overhead | `pass` | queries 100, HTTP requests 100 -> 1, batch encode calls 1, batch hit rate 0.99, speedup 6.296683035844507, metrics True | Keep the real Redis multi-process batch recall profile green, then repeat it with larger batches on remote Kubernetes/serverless nodes. | @@ -47,8 +47,8 @@ verdict, not a marketing claim. | HTTP shard transport handles failover, repair, and tombstones | `pass` | proxy bypass True, failover True, repair 1, tombstone deleted 1, concurrent hit rate 1.0 | Extend the same HTTP shard profile to remote service nodes and sustained load. | | Sustained HTTP cluster load survives failover and repair | `pass` | nodes 4, writes 8, queries 8, failover hit 1.0, success 1.0, p99 891.74 ms | Repeat this profile against remote service nodes and larger namespace counts before claiming full distributed production scale. | | Runtime replica quorum survives node loss | `pass` | recall after loss True, repair copied 1, p99 3.497999998216983 ms, concurrent hit rate 1.0 | Extend the same replicated runtime profile to remote service nodes and sustained load. | -| Active-active sync and field-state CRDT converge | `pass` | delta sync True, incremental records 1, field-only keys 1, sustained regions 3, sustained convergence 1.0, sustained delete suppression 1.0, sustained success 1.0, HTTP service-region convergence 1.0, HTTP final no-op imports 0, Kubernetes regions 3, zones 3, failure docker-pause-kind-worker, outage unavailable ['region-b'], outage writes 32, outage convergence 1.0, outage delete suppression 1.0, recovery convergence 1.0, recovery delete suppression 1.0, region workflow https://github.com/CaspianG/wavemind/actions/runs/29058433643, CRDT idempotent True, watermarks 3, watermark health pass, missing detected True, lag detected True | Repeat the passing kind region-failure protocol on independent remote Kubernetes regions and feed that artifact into strict active-active admission. | -| Snapshots, archives, offsite mirror, and object-store DR verify | `pass` | archive True, object-store DR True, restored files 3, PITR full True, PITR point True, journal entries 5, deleted 2, Postgres PITR ready, commands 7, env missing_env, Kubernetes DR pass 10/10, backup bytes 1016635, restore recall 1.0, Qdrant 24/24, replacement recall 1.0, restore 20449.092 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29064934749 | Repeat the passing independent-namespace Kubernetes restore with managed PostgreSQL PITR, remote object storage, and a separate recovery cluster before claiming managed-cloud or multi-region DR. | +| Active-active sync and field-state CRDT converge | `pass` | delta sync True, incremental records 1, field-only keys 1, sustained regions 3, sustained convergence 1.0, sustained delete suppression 1.0, sustained success 1.0, HTTP service-region convergence 1.0, HTTP final no-op imports 0, Kubernetes regions 3, zones 3, failure docker-pause-kind-worker, outage unavailable ['region-b'], outage writes 32, outage convergence 1.0, outage delete suppression 1.0, recovery convergence 1.0, recovery delete suppression 1.0, region workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261, CRDT idempotent True, watermarks 3, watermark health pass, missing detected True, lag detected True | Repeat the passing kind region-failure protocol on independent remote Kubernetes regions and feed that artifact into strict active-active admission. | +| Snapshots, archives, offsite mirror, and object-store DR verify | `pass` | archive True, object-store DR True, restored files 3, PITR full True, PITR point True, journal entries 5, deleted 2, Postgres PITR ready, commands 7, env missing_env, Kubernetes DR pass 10/10, backup bytes 1016642, restore recall 1.0, Qdrant 24/24, replacement recall 1.0, restore 23338.114 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261 | Repeat the passing independent-namespace Kubernetes restore with managed PostgreSQL PITR, remote object storage, and a separate recovery cluster before claiming managed-cloud or multi-region DR. | | Structured and multimodal payload retrieval works | `pass` | modalities image, audio, table, event, video, 3d, graph, precision@1 1.0, cross-modal precision@1 1.0, vectors persisted 1.0, precomputed precision@1 1.0, encoder contract True, encoder global@1 1.0, encoder target@1 1.0, encoder margin 0.8113164444259024, encoder health True, encoder health global@1 1.0, encoder health query p95 1.5152499952819198 ms, provenance 1.0, temporal precision@1 1.0, temporal around/window/recency/interval 1/1/1/1, temporal persisted 1.0, temporal provenance 1.0, knowledge graph precision@1 1.0, knowledge graph direct/two-hop/three-hop/predicate 1/1/1/1, knowledge graph paths 1.0, knowledge graph persisted 1.0, knowledge graph provenance 1.0, asset manifest verified True, asset provenance 1 | Run the same encoder contract and encoder-health check with real CLIP/audio/video/3D vectors from production encoders, then expand the multimodal, temporal-event, and knowledge-graph retrieval profiles against larger object-store-backed corpora. | | 10M-vector production load profile passes recall, p99, and cost gate | `pass` | WaveMind faiss-ivfpq-persisted streaming: recall 0.99, p99 60.12930005090311 ms, cost valid_slo | Keep the 10M compressed FAISS IVF-PQ profile green and repeat with Qdrant/pgvector service profiles when larger service hardware is available. | | 50M streaming load run has a checked preflight contract | `pass` | WaveMind faiss-ivfpq-persisted streaming: status action_required, index 1.12 GB, app storage 119.209 GB, required local free 2.116 GB, blockers missing_env:WAVEMIND_FAISS_IVFPQ_PATH, insufficient_local_disk_for_index_and_transient_batches | Set WAVEMIND_FAISS_IVFPQ_PATH on sized storage and run the embedded 50M command to produce production_streaming_load_ivfpq_50m_results.json. | @@ -62,5 +62,5 @@ Missing commercial API credentials or remote cluster URLs should not turn a core | evidence | status | result | next step | |---|---|---|---| | Mem0, Zep, and LangGraph adapter evidence | `action_required` | skipped: Zep | Configure ZEP_API_URL or ZEP_API_KEY for a real Zep service and check in the live Zep adapter result. | -| External HTTP service-node load evidence | `pass` | nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms | Keep the external service-node run current for release candidates. | +| External HTTP service-node load evidence | `pass` | nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms | Keep the external service-node run current for release candidates. | | External HTTP active-active region evidence | `action_required` | no checked-in external HTTP active-active region result | Run external-http-active-active against real API regions and upload or commit the resulting artifact. | diff --git a/benchmarks/RELEASE_CLAIMS.md b/benchmarks/RELEASE_CLAIMS.md index 27ff2ea..c04234f 100644 --- a/benchmarks/RELEASE_CLAIMS.md +++ b/benchmarks/RELEASE_CLAIMS.md @@ -36,5 +36,5 @@ until strict external evidence artifacts pass. |---|---|---|---|---|---| | External HTTP active-active regions | `action_required` | `action_required` | `benchmarks/external_http_active_active_results.json` | `WAVEMIND_ACTIVE_ACTIVE_REGIONS, WAVEMIND_ACTIVE_ACTIVE_REGIONS_MANIFEST_JSON` | `gh workflow run external-http-active-active.yml -f regions="$WAVEMIND_ACTIVE_ACTIVE_REGIONS" -f commit_results=true` | | Managed/serverless remote telemetry | `action_required` | `action_required` | `deploy/serverless/observed-telemetry.remote.json` | `WAVEMIND_SERVERLESS_NODES` | `gh workflow run serverless-observed-telemetry.yml -f nodes="$WAVEMIND_SERVERLESS_NODES" -f seed_mode=first -f commit_results=true` | -| 10M pgvector service load | `action_required` | `action_required` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `WAVEMIND_PGVECTOR_DSN` | `python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks/production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json` | +| 10M pgvector service load | `action_required` | `action_required` | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `WAVEMIND_PGVECTOR_DSNS` | `python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks/production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json` | | 100M remote load result | `action_required` | `action_required` | `benchmarks/production_streaming_load_qdrant_sharded_100m_results.json` | `WAVEMIND_QDRANT_URLS` | `python benchmarks/production_streaming_load_benchmark.py --sizes 100000000 --dim 128 --queries 5000 --top-k 10 --seed 42 --noise 0.08 --batch-size 10000 --engines qdrant-sharded-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 500.0 --replicas 8 --autoscaling-max-replicas 128 --capacity-headroom 0.7 --output benchmarks/production_streaming_load_qdrant_sharded_100m_results.json --checkpoint-path state/production-runs/qdrant-sharded-service-100000000.checkpoint.json` | diff --git a/benchmarks/SCALE_GAP.md b/benchmarks/SCALE_GAP.md index 2c7e91d..8d17cbb 100644 --- a/benchmarks/SCALE_GAP.md +++ b/benchmarks/SCALE_GAP.md @@ -19,7 +19,7 @@ are proven, which are plan-only, and what must run next. |---|---|---:|---:|---:|---|---| | qdrant-10m | `complete` | 10000000 | 1000000 | 10.0 | `benchmarks/production_streaming_load_qdrant_10m_results.json` | `WAVEMIND_QDRANT_URL` | | qdrant-sharded-10m | `complete` | 10000000 | 1000000 | 10.0 | `benchmarks/production_streaming_load_qdrant_sharded_10m_results.json` | `WAVEMIND_QDRANT_URLS` | -| pgvector-10m | `blocked_by_env` | 10000000 | 50000 | 200.0 | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `WAVEMIND_PGVECTOR_DSN` | +| pgvector-10m | `blocked_by_env` | 10000000 | 50000 | 200.0 | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `WAVEMIND_PGVECTOR_DSNS` | | faiss-ivfpq-50m | `complete` | 50000000 | 10000000 | 5.0 | `benchmarks/production_streaming_load_ivfpq_50m_results.json` | `WAVEMIND_FAISS_IVFPQ_PATH` | | qdrant-sharded-100m | `blocked_by_env` | 100000000 | 1000000 | 100.0 | `benchmarks/production_streaming_load_qdrant_sharded_100m_results.json` | `WAVEMIND_QDRANT_URLS` | diff --git a/benchmarks/STRICT_EVIDENCE_READINESS.md b/benchmarks/STRICT_EVIDENCE_READINESS.md index 54f4322..a149691 100644 --- a/benchmarks/STRICT_EVIDENCE_READINESS.md +++ b/benchmarks/STRICT_EVIDENCE_READINESS.md @@ -12,8 +12,8 @@ production evidence by itself. | claim status | `claims_limited` | | total requirements | `8` | | action required | `4` | -| ready for safe dispatch | `0` | -| can auto-run now | `0` | +| ready for safe dispatch | `1` | +| can auto-run now | `1` | | planned target memories | `180000000` | ## Integrity Checks @@ -38,7 +38,7 @@ production evidence by itself. | Managed/serverless remote telemetry | `missing_env` | `blocked_by_preflight` | | `deploy/serverless/observed-telemetry.remote.json` | `WAVEMIND_SERVERLESS_NODES` | Hosted/serverless p99, cold-start, error-rate, and scale-out SLO. | | 10M Qdrant service load | `complete` | `complete` | 10000000 | `benchmarks/production_streaming_load_qdrant_10m_results.json` | `WAVEMIND_QDRANT_URL` | 10M-100M service-backed production scale | | 10M sharded Qdrant service load | `complete` | `complete` | 10000000 | `benchmarks/production_streaming_load_qdrant_sharded_10m_results.json` | `WAVEMIND_QDRANT_URLS` | 10M-100M service-backed production scale | -| 10M pgvector service load | `missing_env` | `blocked_by_preflight` | 10000000 | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `WAVEMIND_PGVECTOR_DSN` | 10M-100M service-backed production scale | +| 10M pgvector service load | `missing_artifact` | `ready_to_dispatch` | 10000000 | `benchmarks/production_streaming_load_pgvector_10m_results.json` | `` | 10M-100M service-backed production scale | | 50M FAISS IVF-PQ streaming load | `complete` | `complete` | 50000000 | `benchmarks/production_streaming_load_ivfpq_50m_results.json` | `WAVEMIND_FAISS_IVFPQ_PATH` | 10M-100M service-backed production scale | | 100M remote load result | `missing_env` | `blocked_by_preflight` | 100000000 | `benchmarks/production_streaming_load_qdrant_sharded_100m_results.json` | `WAVEMIND_QDRANT_URLS` | 10M-100M service-backed production scale | @@ -49,7 +49,7 @@ production evidence by itself. - `serverless_remote_telemetry`: `gh workflow run serverless-observed-telemetry.yml -f nodes="$WAVEMIND_SERVERLESS_NODES" -f requests="240" -f workers="4" -f seed_memories="24" -f seed_mode="first" -f max_scale="256" -f target_rps="3200" -f target_p99_ms="500" -f external_cold_start_ms="900" -f estimated_scale_out_seconds="18" -f commit_results="false"` - `qdrant_10m_service`: `gh workflow run production-streaming-load.yml -f engine="qdrant-service" -f size="10000000" -f dim="128" -f queries="2000" -f top_k="10" -f batch_size="5000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="100.0" -f replicas="3" -f autoscaling_max_replicas="24" -f capacity_headroom="0.7" -f runner_label="self-hosted-large" -f runner_storage_root="state/production-runs" -f commit_results="false" -f qdrant_url="$WAVEMIND_QDRANT_URL"` - `qdrant_sharded_10m_service`: `gh workflow run production-streaming-load.yml -f engine="qdrant-sharded-service" -f size="10000000" -f dim="128" -f queries="2000" -f top_k="10" -f batch_size="5000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="250.0" -f replicas="4" -f autoscaling_max_replicas="48" -f capacity_headroom="0.7" -f runner_label="self-hosted-large" -f runner_storage_root="state/production-runs" -f commit_results="false" -f qdrant_urls="$WAVEMIND_QDRANT_URLS"` -- `pgvector_10m_service`: `gh workflow run production-streaming-load.yml -f engine="pgvector-service" -f size="10000000" -f dim="128" -f queries="2000" -f top_k="10" -f batch_size="5000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="100.0" -f replicas="3" -f autoscaling_max_replicas="24" -f capacity_headroom="0.7" -f runner_label="self-hosted-large" -f runner_storage_root="state/production-runs" -f commit_results="false" -f pgvector_dsn="$WAVEMIND_PGVECTOR_DSN"` +- `pgvector_10m_service`: `gh workflow run production-streaming-load.yml -f engine="pgvector-service" -f size="10000000" -f dim="128" -f queries="2000" -f top_k="10" -f batch_size="5000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="100.0" -f replicas="3" -f autoscaling_max_replicas="24" -f capacity_headroom="0.7" -f runner_label="ubuntu-latest" -f runner_storage_root="state/production-runs" -f commit_results="false" -f provision_pgvector_shards="true" -f pgvector_shard_count="4"` - `faiss_ivfpq_50m`: `gh workflow run production-streaming-load.yml -f engine="faiss-ivfpq-persisted" -f size="50000000" -f dim="128" -f queries="2000" -f top_k="10" -f batch_size="1000000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="100.0" -f replicas="3" -f autoscaling_max_replicas="24" -f capacity_headroom="0.7" -f runner_label="self-hosted-large" -f runner_storage_root="state/production-runs" -f commit_results="false" -f faiss_ivfpq_path="$WAVEMIND_FAISS_IVFPQ_PATH"` - `hundred_million_remote_load`: `gh workflow run production-streaming-load.yml -f engine="qdrant-sharded-service" -f size="100000000" -f dim="128" -f queries="5000" -f top_k="10" -f batch_size="10000" -f target_recall="0.95" -f target_p99_ms="100.0" -f target_qps="500.0" -f replicas="8" -f autoscaling_max_replicas="128" -f capacity_headroom="0.7" -f runner_label="self-hosted-large" -f runner_storage_root="state/production-runs" -f commit_results="false" -f qdrant_urls="$WAVEMIND_QDRANT_URLS"` diff --git a/benchmarks/agent_impact_results.json b/benchmarks/agent_impact_results.json index cedacca..2278332 100644 --- a/benchmarks/agent_impact_results.json +++ b/benchmarks/agent_impact_results.json @@ -1,7 +1,7 @@ { "schema": "wavemind.agent_impact_leaderboard.v1", - "generated_at": "2026-07-11T05:56:44Z", - "source_ref": "c4f786e131c8", + "generated_at": "2026-07-11T19:56:03Z", + "source_ref": "d0e3446be447", "claim_boundary": "Agent-impact rows come from checked-in benchmark artifacts. They show behavioral lift on the configured tasks; they do not claim general agent success outside the listed scenarios.", "summary": { "benchmark_count": 6, diff --git a/benchmarks/benchmark_artifact_audit.json b/benchmarks/benchmark_artifact_audit.json index 57e91dd..c400ac8 100644 --- a/benchmarks/benchmark_artifact_audit.json +++ b/benchmarks/benchmark_artifact_audit.json @@ -1,13 +1,13 @@ { "schema": "wavemind.benchmark_artifact_audit.v1", "status": "pass", - "checked_at": "2026-07-11T05:56:54Z", - "generated_at": "2026-07-11T05:56:44Z", - "source_ref": "c4f786e131c8", + "checked_at": "2026-07-11T20:56:13Z", + "generated_at": "2026-07-11T19:56:03Z", + "source_ref": "d0e3446be447", "workflow_run_id": null, "refresh_profile": "local", "max_age_days": 8.0, - "age_days": 0.00012261520833333332, + "age_days": 0.04178398337962963, "implemented_count": 37, "runner_ready_count": 3, "planned_count": 6, diff --git a/benchmarks/benchmark_matrix_results.json b/benchmarks/benchmark_matrix_results.json index d0be8ae..fbafbcb 100644 --- a/benchmarks/benchmark_matrix_results.json +++ b/benchmarks/benchmark_matrix_results.json @@ -1,7 +1,7 @@ { "schema": "wavemind.benchmark_matrix.v1", - "generated_at": "2026-07-11T05:56:44Z", - "source_ref": "c4f786e131c8", + "generated_at": "2026-07-11T19:56:03Z", + "source_ref": "d0e3446be447", "workflow_run_id": null, "refresh_profile": "local", "note": "Implemented entries are runnable from this repository. Planned entries are public benchmarks that require optional datasets, services, or heavier dependencies.", @@ -1281,15 +1281,15 @@ "estimated_application_storage_gb": 23.842, "required_local_free_gb": 0.004, "disk_free_gb": 0.0, - "index_mode": "remote PostgreSQL/pgvector storage; local runner stores only generated batches", + "index_mode": "modulo-sharded PostgreSQL/pgvector services; local runner stores only generated batches", "required_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "blockers": [ - "missing_env:WAVEMIND_PGVECTOR_DSN", + "missing_env:WAVEMIND_PGVECTOR_DSNS", "insufficient_local_disk_for_index_and_transient_batches" ] } @@ -1963,11 +1963,11 @@ ], "current": { "WaveMind production readiness": { - "readiness_score": 1.0, - "overall_status": "pass", - "pass_count": 39, + "readiness_score": 0.9743589743589743, + "overall_status": "fail", + "pass_count": 38, "action_required_count": 0, - "fail_count": 0, + "fail_count": 1, "total_criteria": 39 } }, @@ -2165,14 +2165,14 @@ "failover_hit_rate": 1.0, "delete_suppression_rate": 1.0, "repair_repaired_total": 1, - "p99_operation_ms": 6694.75712100001, + "p99_operation_ms": 8351.044338999998, "slo_pass": true, "batch_query_success": true, "batch_query_size": 24, "batch_query_http_requests": "24 -> 1", "batch_query_request_reduction_ratio": 0.9583333333333334, - "batch_query_p99_ms": 148.8199839999993, - "batch_query_total_speedup": 1.2556579632494516 + "batch_query_p99_ms": 186.78031600001077, + "batch_query_total_speedup": 1.0791519808749621 } }, "target": "Keep the service-node workload and external batch recall green against real API processes, then repeat it against remote Kubernetes or serverless nodes before claiming external-cluster production readiness.", @@ -2201,9 +2201,9 @@ "status": "pass", "environment": "kind-multinode-ci", "evidence_source": "github-actions-kind", - "source_ref": "9956e3112dc575d0f12ff99ec59326dd8e035f70", - "workflow_run_id": "29057247023", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29057247023", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "node_count": 4, "operator_pod_count": 2, "operator_node_count": 2, @@ -2251,18 +2251,18 @@ "status": "pass", "environment": "kind-multinode-network-ci", "evidence_source": "github-actions-kind-physical-node-pause", - "source_ref": "3d40e894b493253a7d263fa2bf6d67bbc8f4019a", - "workflow_run_id": "29070578441", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29070578441", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "service_node_count": 4, "zone_count": 3, "failure_method": "docker-pause-kind-worker", - "target_worker": "wavemind-ci-worker", - "target_zone": "zone-a", + "target_worker": "wavemind-ci-worker2", + "target_zone": "zone-b", "target_data_pods": [ "wavemind-ci-2" ], - "outage_duration_ms": 8911.648, + "outage_duration_ms": 9431.139, "seed_memories": 256, "outage_hit_rate": 1.0, "failed_nodes_during_outage": [ @@ -2305,15 +2305,15 @@ "status": "pass", "environment": "kind-multizone-active-active-ci", "evidence_source": "github-actions-kind-physical-region-worker-pause", - "source_ref": "b6392353668203196007e1cac695e178752d34d1", - "workflow_run_id": "29058433643", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29058433643", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "region_count": 3, "zone_count": 3, "all_regions_use_pvc": true, "failure_method": "docker-pause-kind-worker", "target_region": "region-b", - "outage_duration_ms": 9900.072, + "outage_duration_ms": 9658.868, "seed_writes": 48, "seed_convergence_rate": 1.0, "outage_unavailable_regions": [ @@ -2359,22 +2359,22 @@ "status": "pass", "environment": "kind-multizone-serverless-lifecycle-ci", "evidence_source": "github-actions-kind-external-state-manual-scale-lifecycle", - "source_ref": "880cafdf5ce5f7c9da878bd8e82ada39ca2d02cc", - "workflow_run_id": "29064934749", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29064934749", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "passed_checks": 13, "check_count": 13, "persistent_volume_claims": 3, - "cold_start_ms": 5086.153, + "cold_start_ms": 5055.034, "restored_after_zero_rate": 1.0, "ready_replicas": 3, "zone_count": 3, "visible_replicas": 3, "suppressed_replicas": 3, - "write_propagation_ms": 1130.6929430000991, - "delete_propagation_ms": 915.0309450000123, - "burst_requests_per_second": 41.39356985869663, - "burst_p99_ms": 1461.4564480000354, + "write_propagation_ms": 1136.9370459999573, + "delete_propagation_ms": 884.7577889999911, + "burst_requests_per_second": 40.74436448541339, + "burst_p99_ms": 1891.9401950000747, "final_restore_rate": 1.0, "claim_boundary": "Ephemeral non-loopback Kubernetes lifecycle evidence with external durable state. It proves scale-to-zero state safety and multi-replica behavior, but does not unlock remote managed Knative/KEDA production admission." } @@ -2405,13 +2405,13 @@ "status": "pass", "environment": "kind-independent-namespace-postgres-qdrant-dr-ci", "evidence_source": "github-actions-kind-pg-dump-independent-restore", - "source_ref": "880cafdf5ce5f7c9da878bd8e82ada39ca2d02cc", - "workflow_run_id": "29064934749", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29064934749", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "passed_checks": 10, "check_count": 10, "backup_format": "pg_dump-custom", - "backup_bytes": 1016635, + "backup_bytes": 1016642, "source_state_stopped": true, "recovery_pvcs": 3, "restored_rate": 1.0, @@ -2419,7 +2419,7 @@ "index_expected_records": 24, "index_vector_records": 24, "restored_after_api_replacement_rate": 1.0, - "restore_elapsed_ms": 20449.092, + "restore_elapsed_ms": 23338.114, "claim_boundary": "Ephemeral non-loopback Kubernetes disaster-recovery evidence. It proves logical PostgreSQL backup/restore and Qdrant rebuild in an independent namespace, not managed-cloud PITR or multi-region DR." } }, diff --git a/benchmarks/cluster_admission_results.json b/benchmarks/cluster_admission_results.json index af0b1cf..753a08c 100644 --- a/benchmarks/cluster_admission_results.json +++ b/benchmarks/cluster_admission_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.cluster_admission.v1", - "generated_at": "2026-07-10T05:23:44Z", + "generated_at": "2026-07-11T20:01:43Z", "status": "admitted", "admitted": true, "deployment": "kind-non-loopback-ci", @@ -33,7 +33,7 @@ "title": "Non-loopback Kubernetes or external HTTP service-node load", "status": "pass", "artifact": "benchmarks/http_cluster_load_results.json", - "evidence": "nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms", + "evidence": "nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms", "issues": [], "command": "gh workflow run external-http-cluster-load.yml -f nodes=\"node-a=https://wm-a.example.com,node-b=https://wm-b.example.com,node-c=https://wm-c.example.com,node-d=https://wm-d.example.com\" -f replication_factor=3 -f read_quorum=1 -f read_fanout=1 -f batch_query_size=24 -f fail_on_slo=true -f commit_results=true", "claim_unlocked": "Non-loopback Kubernetes service-node cluster load SLO." @@ -41,7 +41,7 @@ "requested_evidence": { "status": "pass", "artifact": "benchmarks/http_cluster_load_results.json", - "evidence": "nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms", + "evidence": "nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms", "issues": [], "min_nodes": 4, "namespace_count": 32, diff --git a/benchmarks/cluster_autoscale_results.json b/benchmarks/cluster_autoscale_results.json index cb7f3ab..f9ff649 100644 --- a/benchmarks/cluster_autoscale_results.json +++ b/benchmarks/cluster_autoscale_results.json @@ -1,7 +1,7 @@ { "schema": "wavemind.cluster_autoscale_report.v1", "generated_at": "2026-07-09T22:45:10Z", - "source_ref": "c4f786e131c8", + "source_ref": "d0e3446be447", "source_file": "benchmarks/scale_readiness_results.json", "claim_boundary": "Cluster autoscale evidence is extracted from the checked-in scale-readiness artifact. It proves deterministic shard placement, failure-domain availability, autoscale planning, rebalance planning, operator reconciliation, quorum safety, active-active convergence, field-state CRDT behavior, and the 100M capacity envelope on these fixtures. It is not a real 100M vector-query latency benchmark, managed Kubernetes production run, or independent multi-region SLO.", "summary": { diff --git a/benchmarks/cost_efficiency_results.json b/benchmarks/cost_efficiency_results.json index 9f2b7d5..f2b4d64 100644 --- a/benchmarks/cost_efficiency_results.json +++ b/benchmarks/cost_efficiency_results.json @@ -1,7 +1,7 @@ { "schema": "wavemind.cost_efficiency_leaderboard.v1", - "generated_at": "2026-07-11T05:56:44Z", - "source_ref": "c4f786e131c8", + "generated_at": "2026-07-11T19:56:03Z", + "source_ref": "d0e3446be447", "claim_boundary": "Measured rows come from checked-in load artifacts. Planned rows are capacity and cost contracts only; they do not unlock production latency or recall claims until the matching benchmark result exists.", "summary": { "measured_row_count": 22, @@ -664,7 +664,7 @@ "claim_status": "plan_only", "output_artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", "blockers": [ - "missing_env:WAVEMIND_PGVECTOR_DSN", + "missing_env:WAVEMIND_PGVECTOR_DSNS", "insufficient_local_disk_for_index_and_transient_batches" ], "slo_pass": false, @@ -725,6 +725,7 @@ "output_artifact": "benchmarks/production_streaming_load_ivfpq_50m_results.json", "blockers": [ "missing_env:WAVEMIND_FAISS_IVFPQ_PATH", + "missing_module:faiss", "insufficient_local_disk_for_index_and_transient_batches" ], "slo_pass": false, @@ -1345,6 +1346,7 @@ "output_artifact": "benchmarks/production_streaming_load_ivfpq_50m_results.json", "blockers": [ "missing_env:WAVEMIND_FAISS_IVFPQ_PATH", + "missing_module:faiss", "insufficient_local_disk_for_index_and_transient_batches" ], "slo_pass": false, @@ -1373,7 +1375,7 @@ "claim_status": "plan_only", "output_artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", "blockers": [ - "missing_env:WAVEMIND_PGVECTOR_DSN", + "missing_env:WAVEMIND_PGVECTOR_DSNS", "insufficient_local_disk_for_index_and_transient_batches" ], "slo_pass": false, diff --git a/benchmarks/http_cluster_load_results.json b/benchmarks/http_cluster_load_results.json index 847b74f..4b5267c 100644 --- a/benchmarks/http_cluster_load_results.json +++ b/benchmarks/http_cluster_load_results.json @@ -23,28 +23,28 @@ "write_quorum": 2, "read_quorum": 1, "read_fanout": 1, - "namespace_prefix": "kind-external:29070578441", + "namespace_prefix": "kind-external:29165761261", "namespace_count": 32, "memories_per_namespace": 8, "workers": 8, "batch_query_size": 24, - "deployment_id": "github-actions-29070578441-wavemind-ci-wavemind-system", + "deployment_id": "github-actions-29165761261-wavemind-ci-wavemind-system", "environment": "kubernetes-kind-non-loopback-ci", "source": "kubernetes-pod-dns-physical-node-drill", - "source_ref": "3d40e894b493253a7d263fa2bf6d67bbc8f4019a", - "workflow_run_id": "29070578441", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29070578441", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "description": "External WaveMind API-node sustained cluster benchmark for production service deployments.", "kubernetes_attestation": { "schema": "wavemind.kubernetes_external_http_cluster_attestation.v1", - "generated_at": "2026-07-10T05:13:58.888058+00:00", + "generated_at": "2026-07-11T19:49:51.279909+00:00", "status": "pass", "evidence_source": "github-actions-kind-physical-node-pause", "network_evidence_artifact": "benchmarks/kubernetes_cluster_network_smoke_results.json", - "network_evidence_sha256": "8f7a2e194c77b68c3ff5a3fa4bafe18c72b378bc913df14087ed25ce5bf8933b", - "source_ref": "3d40e894b493253a7d263fa2bf6d67bbc8f4019a", - "workflow_run_id": "29070578441", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29070578441", + "network_evidence_sha256": "aa11cb6d54a4a0973d17eeb9e7debcecb486326460877a8724bff5f4539a69ad", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "service_addresses": [ "http://wavemind-ci-0.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", "http://wavemind-ci-1.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", @@ -58,11 +58,11 @@ ], "physical_failure": { "method": "docker-pause-kind-worker", - "target_worker": "wavemind-ci-worker", + "target_worker": "wavemind-ci-worker2", "target_pods": [ "wavemind-ci-2" ], - "outage_duration_ms": 8911.648 + "outage_duration_ms": 9431.139 }, "summary": { "check_count": 10, @@ -76,7 +76,7 @@ "observed": { "slo_pass": true, "success_rate": 1.0, - "p99_operation_ms": 6694.75712100001 + "p99_operation_ms": 8351.044338999998 }, "required": "mixed workload SLO pass" }, @@ -132,7 +132,7 @@ "passed": true, "observed": { "method": "docker-pause-kind-worker", - "target_worker": "wavemind-ci-worker", + "target_worker": "wavemind-ci-worker2", "target_pods": [ "wavemind-ci-2" ], @@ -171,26 +171,26 @@ "current_pods": { "wavemind-ci-0": { "id": "wavemind-ci-0", - "uid": "51561740-d801-4eba-a58e-6ef49da6641b", - "worker": "wavemind-ci-worker2", + "uid": "7911e75d-25f8-492b-b960-49e6b30e0aee", + "worker": "wavemind-ci-worker", "ready": true }, "wavemind-ci-1": { "id": "wavemind-ci-1", - "uid": "a45e58c6-fe85-4dc1-ae4f-ec5e12191205", + "uid": "7e6b108e-c9b0-4cf4-97cb-41fd20edd6ec", "worker": "wavemind-ci-worker3", "ready": true }, "wavemind-ci-2": { "id": "wavemind-ci-2", - "uid": "6e6bd316-d273-4434-bac3-0b5784ebdfce", - "worker": "wavemind-ci-worker", + "uid": "caf4f583-344d-4ceb-917e-f47768ca1ecc", + "worker": "wavemind-ci-worker2", "ready": true }, "wavemind-ci-3": { "id": "wavemind-ci-3", - "uid": "2d3cc95a-eede-4192-bdb2-59e71a220d52", - "worker": "wavemind-ci-worker3", + "uid": "8e30fe6f-1693-467a-a3b0-5bb8709a3539", + "worker": "wavemind-ci-worker", "ready": true } } @@ -201,9 +201,9 @@ "id": "github_workflow_provenance", "passed": true, "observed": { - "source_ref": "3d40e894b493253a7d263fa2bf6d67bbc8f4019a", - "workflow_run_id": "29070578441", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29070578441" + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261" }, "required": "traceable GitHub Actions source SHA and workflow run" } @@ -264,22 +264,22 @@ "total_checks": 836, "errors": [], "error_count": 0, - "elapsed_ms": 8896.252176000018, - "avg_operation_ms": 1480.0015598333498, - "p95_operation_ms": 6694.75712100001, - "p99_operation_ms": 6694.75712100001, - "write_avg_ms": 6694.75712100001, - "write_p99_ms": 6694.75712100001, - "query_batch_avg_ms": 539.4693655000253, - "query_batch_p99_ms": 682.5198170000135, - "failover_batch_avg_ms": 333.70055400001775, - "failover_batch_p99_ms": 333.70055400001775, - "repair_avg_ms": 23.42367300002479, - "repair_p99_ms": 23.42367300002479, - "forget_avg_ms": 749.189279999996, - "forget_p99_ms": 749.189279999996, + "elapsed_ms": 11038.628927000047, + "avg_operation_ms": 1836.9360310000125, + "p95_operation_ms": 8351.044338999998, + "p99_operation_ms": 8351.044338999998, + "write_avg_ms": 8351.044338999998, + "write_p99_ms": 8351.044338999998, + "query_batch_avg_ms": 672.4865545000114, + "query_batch_p99_ms": 859.8281710000037, + "failover_batch_avg_ms": 455.4705230000309, + "failover_batch_p99_ms": 455.4705230000309, + "repair_avg_ms": 40.554676999988715, + "repair_p99_ms": 40.554676999988715, + "forget_avg_ms": 829.5735380000337, + "forget_p99_ms": 829.5735380000337, "batch_query": { - "namespace": "kind-external:29070578441:batch-query", + "namespace": "kind-external:29165761261:batch-query", "write_node_count": 4, "individual_node": "wavemind-ci-0", "batch_node": "wavemind-ci-1", @@ -290,16 +290,16 @@ "individual_success": true, "batch_success": true, "success": true, - "write_ms": 3919.7300380000115, - "individual_total_ms": 186.86699800025508, - "individual_p95_ms": 3.9425413999651933, - "individual_p99_ms": 84.79942335998312, - "batch_ms": 148.8199839999993, - "batch_p99_ms": 148.8199839999993, - "total_speedup": 1.2556579632494516 + "write_ms": 4103.411026999993, + "individual_total_ms": 201.564347999863, + "individual_p95_ms": 5.628512749973423, + "individual_p99_ms": 79.44286219996737, + "batch_ms": 186.78031600001077, + "batch_p99_ms": 186.78031600001077, + "total_speedup": 1.0791519808749621 }, - "lifecycle_batch_p99_ms": 6694.75712100001, - "query_p99_ms": 84.79942335998312, + "lifecycle_batch_p99_ms": 8351.044338999998, + "query_p99_ms": 79.44286219996737, "slo_min_success_rate": 1.0, "slo_min_failover_hit_rate": 0.95, "slo_p99_ms": 1000.0, diff --git a/benchmarks/kubernetes_active_active_region_smoke_results.json b/benchmarks/kubernetes_active_active_region_smoke_results.json index 573cc95..5ae8104 100644 --- a/benchmarks/kubernetes_active_active_region_smoke_results.json +++ b/benchmarks/kubernetes_active_active_region_smoke_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.kubernetes_active_active_region_smoke.v1", - "generated_at": "2026-07-09T23:55:54.133679+00:00", + "generated_at": "2026-07-11T19:50:19.980112+00:00", "environment": "kind-multizone-active-active-ci", "evidence_source": "github-actions-kind-physical-region-worker-pause", "claim_boundary": "Ephemeral non-loopback Kubernetes three-zone active-active region failure evidence; not remote multi-region production admission.", @@ -147,21 +147,21 @@ "region_placement": { "region-a": { "pod": "region-a-0", - "uid": "be825835-7662-4dcf-8e4c-2a2631dd727f", + "uid": "a767f282-7136-499c-930e-b5015d613285", "worker": "wavemind-ci-worker", "zone": "zone-a", "uses_pvc": "true" }, "region-b": { "pod": "region-b-0", - "uid": "527357d6-37aa-48b9-867c-ca884a206afc", + "uid": "ba1ad34b-caf4-4684-84e5-ac6dd3801124", "worker": "wavemind-ci-worker2", "zone": "zone-b", "uses_pvc": "true" }, "region-c": { "pod": "region-c-0", - "uid": "f1bd06d4-0ac3-42e7-a9a4-2eba09c76f4d", + "uid": "30732c56-e036-4ef7-82e2-d5de1e41c841", "worker": "wavemind-ci-worker3", "zone": "zone-c", "uses_pvc": "true" @@ -177,13 +177,13 @@ "target_zone": "zone-b", "failure_method": "docker-pause-kind-worker", "failure_settle_seconds": 2.0, - "outage_duration_ms": 9900.072, + "outage_duration_ms": 9658.868, "worker_unpaused": true, "target_region_ready_after_recovery": true, "target_region_pod_uid_preserved": true, "seed": { "schema": "wavemind.active_active_drill.v1", - "generated_at": "2026-07-09T23:55:36.518023+00:00", + "generated_at": "2026-07-11T19:50:03.070474+00:00", "mode": "seed", "namespace_prefix": "kind-region-drill", "namespace_count": 16, @@ -206,17 +206,17 @@ "region-a": { "status": "healthy", "error": null, - "latency_ms": 18.277 + "latency_ms": 39.389 }, "region-b": { "status": "healthy", "error": null, - "latency_ms": 13.729 + "latency_ms": 13.479 }, "region-c": { "status": "healthy", "error": null, - "latency_ms": 12.264 + "latency_ms": 12.674 } }, "unavailable_regions": [], @@ -230,7 +230,7 @@ "failed_pairs": 0, "records_imported": 288, "tombstones_imported": 0, - "duration_ms": 1520.8288820000462 + "duration_ms": 1441.4634409999962 }, "verification": { "expected_checks": 144, @@ -240,11 +240,11 @@ "delete_suppression_rate": 1.0 }, "status": "pass", - "elapsed_ms": 3076.452 + "elapsed_ms": 2848.907 }, "outage": { "schema": "wavemind.active_active_drill.v1", - "generated_at": "2026-07-09T23:55:47.464356+00:00", + "generated_at": "2026-07-11T19:50:13.731626+00:00", "mode": "outage", "namespace_prefix": "kind-region-drill", "namespace_count": 16, @@ -267,17 +267,17 @@ "region-a": { "status": "healthy", "error": null, - "latency_ms": 15.669 + "latency_ms": 15.901 }, "region-b": { "status": "unavailable", "error": "timed out", - "latency_ms": 5004.815 + "latency_ms": 5007.26 }, "region-c": { "status": "healthy", "error": null, - "latency_ms": 16.324 + "latency_ms": 15.081 } }, "unavailable_regions": [ @@ -299,7 +299,7 @@ "failed_pairs": 0, "records_imported": 96, "tombstones_imported": 3, - "duration_ms": 528.2183379999879 + "duration_ms": 498.5052670000414 }, "verification": { "expected_checks": 158, @@ -309,11 +309,11 @@ "delete_suppression_rate": 1.0 }, "status": "pass", - "elapsed_ms": 6985.804 + "elapsed_ms": 6779.202 }, "recovered": { "schema": "wavemind.active_active_drill.v1", - "generated_at": "2026-07-09T23:55:50.375222+00:00", + "generated_at": "2026-07-11T19:50:16.462216+00:00", "mode": "recover", "namespace_prefix": "kind-region-drill", "namespace_count": 16, @@ -336,17 +336,17 @@ "region-a": { "status": "healthy", "error": null, - "latency_ms": 19.429 + "latency_ms": 45.71 }, "region-b": { "status": "healthy", "error": null, - "latency_ms": 14.38 + "latency_ms": 19.445 }, "region-c": { "status": "healthy", "error": null, - "latency_ms": 13.506 + "latency_ms": 35.405 } }, "unavailable_regions": [], @@ -360,7 +360,7 @@ "final_noop_records_imported": 0, "final_noop_tombstones_imported": 0, "final_noop_failed_pairs": 0, - "duration_ms": 2046.9371570000021 + "duration_ms": 1970.3422709999927 }, "verification": { "expected_checks": 237, @@ -370,11 +370,11 @@ "delete_suppression_rate": 1.0 }, "status": "pass", - "elapsed_ms": 3676.424 + "elapsed_ms": 3491.353 }, - "elapsed_ms": 29733.65 + "elapsed_ms": 27985.205 }, - "source_ref": "b6392353668203196007e1cac695e178752d34d1", - "workflow_run_id": "29058433643", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29058433643" + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261" } diff --git a/benchmarks/kubernetes_cluster_network_smoke_results.json b/benchmarks/kubernetes_cluster_network_smoke_results.json index cdab034..adc5f41 100644 --- a/benchmarks/kubernetes_cluster_network_smoke_results.json +++ b/benchmarks/kubernetes_cluster_network_smoke_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.kubernetes_cluster_network_smoke.v1", - "generated_at": "2026-07-10T05:13:44.475824+00:00", + "generated_at": "2026-07-11T19:49:34.107653+00:00", "environment": "kind-multinode-network-ci", "evidence_source": "github-actions-kind-physical-node-pause", "claim_boundary": "Ephemeral non-loopback Kubernetes service-network and physical kind-worker failure evidence; not remote multi-region production admission.", @@ -49,7 +49,7 @@ "passed": true, "observed": { "method": "docker-pause-kind-worker", - "worker": "wavemind-ci-worker", + "worker": "wavemind-ci-worker2", "pods": [ "wavemind-ci-2" ] @@ -60,8 +60,8 @@ "id": "runner_survived_other_node", "passed": true, "observed": { - "runner": "wavemind-ci-worker2", - "target": "wavemind-ci-worker" + "runner": "wavemind-ci-worker", + "target": "wavemind-ci-worker2" }, "required": "workload runner on a surviving worker" }, @@ -69,8 +69,8 @@ "id": "zone_failure_isolated", "passed": true, "observed": { - "runner": "zone-b", - "target": "zone-a" + "runner": "zone-a", + "target": "zone-b" }, "required": "runner and failed worker in distinct zones" }, @@ -131,50 +131,50 @@ "pod_placement": [ { "id": "wavemind-ci-0", - "worker": "wavemind-ci-worker2", - "zone": "zone-b", + "worker": "wavemind-ci-worker", + "zone": "zone-a", "address": "http://wavemind-ci-0.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "uid": "51561740-d801-4eba-a58e-6ef49da6641b" + "uid": "7911e75d-25f8-492b-b960-49e6b30e0aee" }, { "id": "wavemind-ci-1", "worker": "wavemind-ci-worker3", "zone": "zone-c", "address": "http://wavemind-ci-1.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "uid": "a45e58c6-fe85-4dc1-ae4f-ec5e12191205" + "uid": "7e6b108e-c9b0-4cf4-97cb-41fd20edd6ec" }, { "id": "wavemind-ci-2", - "worker": "wavemind-ci-worker", - "zone": "zone-a", + "worker": "wavemind-ci-worker2", + "zone": "zone-b", "address": "http://wavemind-ci-2.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "uid": "6e6bd316-d273-4434-bac3-0b5784ebdfce" + "uid": "caf4f583-344d-4ceb-917e-f47768ca1ecc" }, { "id": "wavemind-ci-3", - "worker": "wavemind-ci-worker3", - "zone": "zone-c", + "worker": "wavemind-ci-worker", + "zone": "zone-a", "address": "http://wavemind-ci-3.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "uid": "2d3cc95a-eede-4192-bdb2-59e71a220d52" + "uid": "8e30fe6f-1693-467a-a3b0-5bb8709a3539" } ], "runner_pod": "wavemind-ci-0", - "runner_worker": "wavemind-ci-worker2", - "runner_zone": "zone-b", - "target_worker": "wavemind-ci-worker", - "target_zone": "zone-a", + "runner_worker": "wavemind-ci-worker", + "runner_zone": "zone-a", + "target_worker": "wavemind-ci-worker2", + "target_zone": "zone-b", "target_data_pods": [ "wavemind-ci-2" ], "failure_method": "docker-pause-kind-worker", "failure_settle_seconds": 2.0, - "outage_duration_ms": 8911.648, + "outage_duration_ms": 9431.139, "worker_unpaused": true, "node_ready_after_recovery": true, "pod_uids_preserved": true, "seed": { "schema": "wavemind.cluster_drill.v1", - "generated_at": "2026-07-10T05:13:26.557804+00:00", + "generated_at": "2026-07-11T19:49:13.799749+00:00", "mode": "seed", "namespace_prefix": "kind-network-drill", "namespace_count": 32, @@ -187,7 +187,7 @@ { "id": "wavemind-ci-0", "address": "http://wavemind-ci-0.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-b", + "zone": "zone-a", "weight": 1.0 }, { @@ -199,13 +199,13 @@ { "id": "wavemind-ci-2", "address": "http://wavemind-ci-2.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-a", + "zone": "zone-b", "weight": 1.0 }, { "id": "wavemind-ci-3", "address": "http://wavemind-ci-3.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-c", + "zone": "zone-a", "weight": 1.0 } ], @@ -220,17 +220,17 @@ "individual_write_http_requests": 768, "request_reduction_ratio": 0.9947916666666666, "node_writes": { - "wavemind-ci-0": 256, - "wavemind-ci-1": 120, + "wavemind-ci-0": 88, + "wavemind-ci-1": 256, "wavemind-ci-2": 256, - "wavemind-ci-3": 136 + "wavemind-ci-3": 168 }, "failed_nodes_seen": [], "node_health": { "wavemind-ci-0": { "id": "wavemind-ci-0", "address": "http://wavemind-ci-0.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-b", + "zone": "zone-a", "available": true, "status": "healthy", "successes": 1, @@ -250,7 +250,7 @@ "wavemind-ci-2": { "id": "wavemind-ci-2", "address": "http://wavemind-ci-2.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-a", + "zone": "zone-b", "available": true, "status": "healthy", "successes": 1, @@ -260,7 +260,7 @@ "wavemind-ci-3": { "id": "wavemind-ci-3", "address": "http://wavemind-ci-3.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-c", + "zone": "zone-a", "available": true, "status": "healthy", "successes": 1, @@ -268,11 +268,11 @@ "last_error": null } }, - "elapsed_ms": 7417.607 + "elapsed_ms": 8770.257 }, "outage": { "schema": "wavemind.cluster_drill.v1", - "generated_at": "2026-07-10T05:13:36.643701+00:00", + "generated_at": "2026-07-11T19:49:25.453740+00:00", "mode": "verify", "namespace_prefix": "kind-network-drill", "namespace_count": 32, @@ -285,7 +285,7 @@ { "id": "wavemind-ci-0", "address": "http://wavemind-ci-0.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-b", + "zone": "zone-a", "weight": 1.0 }, { @@ -297,13 +297,13 @@ { "id": "wavemind-ci-2", "address": "http://wavemind-ci-2.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-a", + "zone": "zone-b", "weight": 1.0 }, { "id": "wavemind-ci-3", "address": "http://wavemind-ci-3.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-c", + "zone": "zone-a", "weight": 1.0 } ], @@ -326,7 +326,7 @@ "wavemind-ci-0": { "id": "wavemind-ci-0", "address": "http://wavemind-ci-0.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-b", + "zone": "zone-a", "available": true, "status": "healthy", "successes": 1, @@ -346,7 +346,7 @@ "wavemind-ci-2": { "id": "wavemind-ci-2", "address": "http://wavemind-ci-2.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-a", + "zone": "zone-b", "available": true, "status": "degraded", "successes": 0, @@ -356,7 +356,7 @@ "wavemind-ci-3": { "id": "wavemind-ci-3", "address": "http://wavemind-ci-3.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-c", + "zone": "zone-a", "available": true, "status": "healthy", "successes": 1, @@ -369,10 +369,10 @@ "wavemind-ci-0": { "id": "wavemind-ci-0", "address": "http://wavemind-ci-0.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-b", + "zone": "zone-a", "available": true, "status": "healthy", - "successes": 34, + "successes": 13, "failures": 0, "last_error": null }, @@ -382,14 +382,14 @@ "zone": "zone-c", "available": true, "status": "healthy", - "successes": 17, + "successes": 34, "failures": 0, "last_error": null }, "wavemind-ci-2": { "id": "wavemind-ci-2", "address": "http://wavemind-ci-2.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-a", + "zone": "zone-b", "available": false, "status": "unavailable", "successes": 0, @@ -399,19 +399,19 @@ "wavemind-ci-3": { "id": "wavemind-ci-3", "address": "http://wavemind-ci-3.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-c", + "zone": "zone-a", "available": true, "status": "healthy", - "successes": 19, + "successes": 23, "failures": 0, "last_error": null } }, - "elapsed_ms": 6178.459 + "elapsed_ms": 6478.544 }, "recovered": { "schema": "wavemind.cluster_drill.v1", - "generated_at": "2026-07-10T05:13:43.580057+00:00", + "generated_at": "2026-07-11T19:49:32.922232+00:00", "mode": "verify", "namespace_prefix": "kind-network-drill", "namespace_count": 32, @@ -424,7 +424,7 @@ { "id": "wavemind-ci-0", "address": "http://wavemind-ci-0.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-b", + "zone": "zone-a", "weight": 1.0 }, { @@ -436,13 +436,13 @@ { "id": "wavemind-ci-2", "address": "http://wavemind-ci-2.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-a", + "zone": "zone-b", "weight": 1.0 }, { "id": "wavemind-ci-3", "address": "http://wavemind-ci-3.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-c", + "zone": "zone-a", "weight": 1.0 } ], @@ -463,7 +463,7 @@ "wavemind-ci-0": { "id": "wavemind-ci-0", "address": "http://wavemind-ci-0.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-b", + "zone": "zone-a", "available": true, "status": "healthy", "successes": 1, @@ -483,7 +483,7 @@ "wavemind-ci-2": { "id": "wavemind-ci-2", "address": "http://wavemind-ci-2.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-a", + "zone": "zone-b", "available": true, "status": "healthy", "successes": 1, @@ -493,7 +493,7 @@ "wavemind-ci-3": { "id": "wavemind-ci-3", "address": "http://wavemind-ci-3.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-c", + "zone": "zone-a", "available": true, "status": "healthy", "successes": 1, @@ -506,10 +506,10 @@ "wavemind-ci-0": { "id": "wavemind-ci-0", "address": "http://wavemind-ci-0.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-b", + "zone": "zone-a", "available": true, "status": "healthy", - "successes": 34, + "successes": 13, "failures": 0, "last_error": null }, @@ -519,14 +519,14 @@ "zone": "zone-c", "available": true, "status": "healthy", - "successes": 17, + "successes": 34, "failures": 0, "last_error": null }, "wavemind-ci-2": { "id": "wavemind-ci-2", "address": "http://wavemind-ci-2.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-a", + "zone": "zone-b", "available": true, "status": "healthy", "successes": 34, @@ -536,19 +536,19 @@ "wavemind-ci-3": { "id": "wavemind-ci-3", "address": "http://wavemind-ci-3.wavemind-ci-headless.wavemind-system.svc.cluster.local:8000", - "zone": "zone-c", + "zone": "zone-a", "available": true, "status": "healthy", - "successes": 19, + "successes": 23, "failures": 0, "last_error": null } }, - "elapsed_ms": 822.64 + "elapsed_ms": 1067.914 }, - "elapsed_ms": 18612.326 + "elapsed_ms": 21261.372 }, - "source_ref": "3d40e894b493253a7d263fa2bf6d67bbc8f4019a", - "workflow_run_id": "29070578441", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29070578441" + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261" } diff --git a/benchmarks/kubernetes_operator_smoke_results.json b/benchmarks/kubernetes_operator_smoke_results.json index 162bddf..9916574 100644 --- a/benchmarks/kubernetes_operator_smoke_results.json +++ b/benchmarks/kubernetes_operator_smoke_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.kubernetes_operator_smoke.v1", - "generated_at": "2026-07-09T23:28:34.759347Z", + "generated_at": "2026-07-11T19:49:12.785845Z", "environment": "kind-multinode-ci", "evidence_source": "github-actions-kind", "claim_boundary": "Ephemeral multi-node Kubernetes CI evidence. It proves real Kubernetes API reconciliation, Lease/etcd-backed operator failover, StatefulSet reconciliation, and pod recovery. It does not unlock remote production, multi-region, managed-serverless, or 100M admission claims.", @@ -11,15 +11,15 @@ "node_count": 4, "operator_pod_count": 2, "operator_node_count": 2, - "initial_holder": "wavemind-operator-78944944b8-m69rk", - "next_holder": "wavemind-operator-78944944b8-bx88m", + "initial_holder": "wavemind-operator-78944944b8-b568n", + "next_holder": "wavemind-operator-78944944b8-jllkt", "lease_transitions_before": 0, "lease_transitions_after": 1, - "lease_resource_version_after": "994", + "lease_resource_version_after": "1096", "original_replicas": 3, "desired_replicas_after_scale": 4, "ready_replicas_after_scale": 4, - "cluster_status_holder": "wavemind-operator-78944944b8-bx88m", + "cluster_status_holder": "wavemind-operator-78944944b8-jllkt", "data_pod_uid_changed": true, "api_healthy_after_recovery": true, "topology_spread_constraint_count": 2, @@ -29,7 +29,7 @@ "rolling_upgrade_revision_changed": true, "rolling_upgrade_replaced_pods": 4, "api_healthy_after_upgrade": true, - "elapsed_ms": 164226.854 + "elapsed_ms": 162586.254 }, "checks": [ { @@ -68,7 +68,7 @@ { "id": "leader_failover", "passed": true, - "value": "wavemind-operator-78944944b8-m69rk -> wavemind-operator-78944944b8-bx88m", + "value": "wavemind-operator-78944944b8-b568n -> wavemind-operator-78944944b8-jllkt", "target": "holder changes" }, { @@ -86,8 +86,8 @@ { "id": "operator_status_tracks_leader", "passed": true, - "value": "wavemind-operator-78944944b8-bx88m", - "target": "wavemind-operator-78944944b8-bx88m" + "value": "wavemind-operator-78944944b8-jllkt", + "target": "wavemind-operator-78944944b8-jllkt" }, { "id": "data_pod_recovered", @@ -120,7 +120,7 @@ "target": true } ], - "source_ref": "9956e3112dc575d0f12ff99ec59326dd8e035f70", - "workflow_run_id": "29057247023", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29057247023" + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261" } diff --git a/benchmarks/kubernetes_postgres_qdrant_dr_smoke_results.json b/benchmarks/kubernetes_postgres_qdrant_dr_smoke_results.json index ff5c790..55d5952 100644 --- a/benchmarks/kubernetes_postgres_qdrant_dr_smoke_results.json +++ b/benchmarks/kubernetes_postgres_qdrant_dr_smoke_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.kubernetes_postgres_qdrant_dr_smoke.v1", - "generated_at": "2026-07-10T02:42:53.084317+00:00", + "generated_at": "2026-07-11T19:52:28.621094+00:00", "environment": "kind-independent-namespace-postgres-qdrant-dr-ci", "evidence_source": "github-actions-kind-pg-dump-independent-restore", "claim_boundary": "Ephemeral non-loopback Kubernetes disaster-recovery evidence. It proves logical PostgreSQL backup/restore and Qdrant rebuild in an independent namespace, not managed-cloud PITR or multi-region DR.", @@ -25,8 +25,8 @@ "passed": true, "observed": { "format": "pg_dump-custom", - "bytes": 1016635, - "sha256": "34a7718692967a0191a11db9f34bcfe8b532060c2483cbbc3f2b8b51dc231fb1" + "bytes": 1016642, + "sha256": "3ca5565e2add8a8a640ef5174dbf020195fafb4d4a188a4f252de14edf749cc9" }, "required": "non-empty checksummed pg_dump custom archive" }, @@ -84,8 +84,8 @@ "id": "recovery_api_replaced", "passed": true, "observed": { - "before": "b3a78af0-35ed-4c48-9de8-7faecf7b07dd", - "after": "5ad14bb8-5ccc-435c-8517-e26c795708d0" + "before": "df0d2224-6db2-4683-920a-ffe1ef787e1a", + "after": "7cd8717c-1064-4353-bf86-1c5a858284f5" }, "required": "replacement pod UID differs" }, @@ -98,7 +98,7 @@ { "id": "restore_time_budget", "passed": true, - "observed": 20449.092, + "observed": 23338.114, "required": "<= 180000.0 ms" } ], @@ -108,8 +108,8 @@ "source_service": "http://wavemind-serverless-keda.wavemind-serverless.svc.cluster.local:8000", "recovery_service": "http://wavemind-serverless-keda.wavemind-serverless-dr.svc.cluster.local:8000", "backup_format": "pg_dump-custom", - "backup_bytes": 1016635, - "backup_sha256": "34a7718692967a0191a11db9f34bcfe8b532060c2483cbbc3f2b8b51dc231fb1", + "backup_bytes": 1016642, + "backup_sha256": "3ca5565e2add8a8a640ef5174dbf020195fafb4d4a188a4f252de14edf749cc9", "source_state_stopped": true, "recovery_services": [ "postgres", @@ -123,30 +123,30 @@ "successes": 24, "count": 24, "latencies_ms": [ - 48.89864000006128, - 14.456939999945462, - 15.217296000059832, - 17.119446000037897, - 26.746441999989656, - 21.148957000036717, - 13.138636999997289, - 28.50698600002488, - 12.80256800009738, - 23.791160999962813, - 13.051663999931407, - 12.060719000032805, - 36.20166599989716, - 12.906894000025204, - 25.377370000001065, - 15.448449000018627, - 13.92912599999363, - 39.289133000011134, - 33.41630099998838, - 25.914307999983066, - 36.09424399996897, - 29.55166099991402, - 12.935454000057689, - 10.89906599997903 + 53.30341100000169, + 13.264843000001747, + 13.382179000018368, + 11.844257000007019, + 15.191383000001224, + 47.106395000014345, + 12.721317999989878, + 14.088360999949145, + 12.783281999986684, + 10.26981200004684, + 53.37585400002354, + 11.981847000015478, + 11.148251000008713, + 11.820545000091442, + 11.094059000015477, + 48.81516800003283, + 12.758286999996926, + 10.591905000069346, + 10.34034100007375, + 10.191953000003195, + 53.49227499993958, + 11.104881999926874, + 10.64273300005425, + 10.360217999959787 ], "rate": 1.0 }, @@ -155,7 +155,7 @@ "expired_memories": 0, "total_memories": 24, "audit_events": 101, - "field_energy": 503.720306, + "field_energy": 503.506165, "clusters": 1, "field_shape": "128x128x6", "index": "qdrant-cosine", @@ -188,44 +188,44 @@ "graph_negative_edges": 0, "graph_energy": 0.0 }, - "recovery_api_uid_before": "b3a78af0-35ed-4c48-9de8-7faecf7b07dd", - "recovery_api_uid_after": "5ad14bb8-5ccc-435c-8517-e26c795708d0", + "recovery_api_uid_before": "df0d2224-6db2-4683-920a-ffe1ef787e1a", + "recovery_api_uid_after": "7cd8717c-1064-4353-bf86-1c5a858284f5", "restored_after_api_replacement": { "successes": 24, "count": 24, "latencies_ms": [ - 17.16054299993175, - 3.6655380000638615, - 3.669010999942657, - 4.605489000027774, - 3.8615660000687058, - 3.7763429999131404, - 3.76651099998071, - 3.852449999953933, - 3.7426740000228165, - 3.6249739999902886, - 3.6626419999947757, - 3.735196999969048, - 3.586227000027975, - 3.508641999928841, - 3.7032230000022537, - 3.622644999950353, - 3.557247000003372, - 3.5525629999710873, - 3.6006450000058976, - 3.705807999949684, - 3.4992890000467014, - 3.5462220000681555, - 3.574258999947233, - 3.5155719999693247 + 14.602483999965443, + 4.4185240000160775, + 3.9640150000650465, + 4.780300999982501, + 4.02783900005943, + 3.7432699999726537, + 3.877389999956904, + 3.8206149999950867, + 3.638992999981383, + 3.914000000008855, + 3.660715000023629, + 3.329704999941896, + 3.5417819999565836, + 4.632454999978108, + 3.7015909999809082, + 3.6497769999641605, + 3.8059659999589712, + 3.5994879999634577, + 3.656796999962353, + 4.599696000013864, + 3.76703099993847, + 3.661375999968186, + 4.543857999919965, + 3.5227970000732967 ], "rate": 1.0 }, - "restore_elapsed_ms": 20449.092, + "restore_elapsed_ms": 23338.114, "restore_budget_ms": 180000.0, - "elapsed_ms": 22461.369 + "elapsed_ms": 25346.265 }, - "source_ref": "880cafdf5ce5f7c9da878bd8e82ada39ca2d02cc", - "workflow_run_id": "29064934749", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29064934749" + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261" } diff --git a/benchmarks/kubernetes_serverless_lifecycle_smoke_results.json b/benchmarks/kubernetes_serverless_lifecycle_smoke_results.json index a9ea047..f7576bd 100644 --- a/benchmarks/kubernetes_serverless_lifecycle_smoke_results.json +++ b/benchmarks/kubernetes_serverless_lifecycle_smoke_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.kubernetes_serverless_lifecycle_smoke.v1", - "generated_at": "2026-07-10T02:42:30.318307+00:00", + "generated_at": "2026-07-11T19:52:02.978556+00:00", "environment": "kind-multizone-serverless-lifecycle-ci", "evidence_source": "github-actions-kind-external-state-manual-scale-lifecycle", "claim_boundary": "Ephemeral non-loopback Kubernetes lifecycle evidence with external durable state. It proves scale-to-zero state safety and multi-replica behavior, but does not unlock remote managed Knative/KEDA production admission.", @@ -49,7 +49,7 @@ { "id": "cold_start_ready", "passed": true, - "observed": 5086.153, + "observed": 5055.034, "required": "<= 120000.0 ms" }, { @@ -112,8 +112,8 @@ "id": "cross_replica_coherence_budget", "passed": true, "observed": { - "write_ms": 1130.6929430000991, - "delete_ms": 915.0309450000123 + "write_ms": 1136.9370459999573, + "delete_ms": 884.7577889999911 }, "required": "both <= 2000.0 ms" }, @@ -130,7 +130,7 @@ { "id": "burst_p99_budget", "passed": true, - "observed": 1461.4564480000354, + "observed": 1891.9401950000747, "required": "<= 2000.0 ms" }, { @@ -150,8 +150,8 @@ "persistent_volume_claims": 3, "zero_replicas": true, "zero_endpoints": true, - "first_cold_start_ms": 4734.679, - "cold_start_ms": 5086.153, + "first_cold_start_ms": 4776.095, + "cold_start_ms": 5055.034, "cold_start_budget_ms": 120000.0, "seed": { "ids": [ @@ -181,85 +181,85 @@ 24 ], "latencies_ms": [ - 56.52392100000725, - 46.770147999950495, - 50.719106999963515, - 27.228246000049694, - 47.72995399991942, - 46.33577399999922, - 50.71650200000022, - 50.50591399992754, - 49.90288999999848, - 40.45117199996184, - 50.86713200000759, - 44.6864350000169, - 25.9589339999593, - 72.98723099995641, - 30.95640399999411, - 68.73927200001617, - 30.865014000028168, - 70.24466600000778, - 29.7877379999818, - 69.75486199996794, - 26.901977999955307, - 72.78671599999598, - 26.975929000059296, - 65.82130300000699 + 47.93942899993908, + 34.977969999999914, + 53.0622129999756, + 26.916568000046937, + 66.83188199997403, + 33.74368699996921, + 68.20604500001082, + 27.495722000026035, + 70.23644600008083, + 27.077748000010615, + 69.51900299998215, + 28.0605090000563, + 71.63755999999921, + 26.36115999996491, + 78.2818899999711, + 28.717959000005067, + 65.5832620000183, + 29.398104000051717, + 78.86673499990593, + 27.743474999965656, + 64.74624700001641, + 29.17586700004904, + 70.0907970000344, + 26.655686000026435 ] }, "restored_after_zero": { "successes": 24, "count": 24, "latencies_ms": [ - 58.135491000030015, - 13.491078000015477, - 14.206270000045151, - 12.460715999964123, - 10.449793999896428, - 30.181994999907147, - 11.396295000054124, - 11.255570999992415, - 12.769344000048477, - 12.322426999958225, - 49.62267099995188, - 12.23055299999487, - 11.303560000101243, - 14.06431700002031, - 55.21566700008407, - 14.865946999975677, - 18.966669999940677, - 11.586820999923475, - 52.63476899995112, - 14.15218099998583, - 10.54465100003199, - 11.347214000011263, - 10.539690999962659, - 11.853790999907687 + 46.9567970000071, + 14.016913999967073, + 11.078535999899941, + 11.07806199991046, + 11.179693000030966, + 30.458027999998194, + 25.483965000034914, + 18.615604999922652, + 11.612022000008437, + 40.6359449999627, + 10.331725000014558, + 11.664179000035801, + 10.485463000009076, + 10.277052000105869, + 51.319888999955765, + 10.53203999993002, + 10.788249000029282, + 10.178598000038619, + 10.317759000031401, + 10.591792000013811, + 51.31542100002662, + 10.905578999995669, + 11.818959000038376, + 11.242460000062238 ], "rate": 1.0, - "p99_ms": 58.135491000030015 + "p99_ms": 51.319888999955765 }, "ready_replicas": 3, "endpoint_count": 3, "zone_count": 3, "placements": [ { - "pod": "wavemind-serverless-keda-57664cc977-jn55k", + "pod": "wavemind-serverless-keda-57664cc977-8xbk8", + "worker": "wavemind-ci-worker", + "zone": "zone-a", + "pod_ip": "10.244.2.18" + }, + { + "pod": "wavemind-serverless-keda-57664cc977-kz946", "worker": "wavemind-ci-worker3", "zone": "zone-c", - "pod_ip": "10.244.2.14" + "pod_ip": "10.244.3.12" }, { - "pod": "wavemind-serverless-keda-57664cc977-mkb25", + "pod": "wavemind-serverless-keda-57664cc977-skz7w", "worker": "wavemind-ci-worker2", "zone": "zone-b", - "pod_ip": "10.244.3.12" - }, - { - "pod": "wavemind-serverless-keda-57664cc977-xngzt", - "worker": "wavemind-ci-worker", - "zone": "zone-a", - "pod_ip": "10.244.1.17" + "pod_ip": "10.244.1.13" } ], "cross_replica": { @@ -279,12 +279,12 @@ ], "write_replicas": [ { - "base": "http://10.244.2.14:8000", + "base": "http://10.244.2.18:8000", "result_ids": [ 25 ], "result_texts": [ - "serverless shared mutation cross-replica-1783651339312" + "serverless shared mutation cross-replica-1783799510950" ], "result_tags": [ [ @@ -304,7 +304,7 @@ 25 ], "result_texts": [ - "serverless shared mutation cross-replica-1783651339312" + "serverless shared mutation cross-replica-1783799510950" ], "result_tags": [ [ @@ -319,12 +319,12 @@ "index_extra_records": 0 }, { - "base": "http://10.244.1.17:8000", + "base": "http://10.244.1.13:8000", "result_ids": [ 25 ], "result_texts": [ - "serverless shared mutation cross-replica-1783651339312" + "serverless shared mutation cross-replica-1783799510950" ], "result_tags": [ [ @@ -341,7 +341,7 @@ ], "delete_replicas": [ { - "base": "http://10.244.2.14:8000", + "base": "http://10.244.2.18:8000", "result_ids": [], "result_texts": [], "result_tags": [], @@ -363,7 +363,7 @@ "index_extra_records": 0 }, { - "base": "http://10.244.1.17:8000", + "base": "http://10.244.1.13:8000", "result_ids": [], "result_texts": [], "result_tags": [], @@ -374,66 +374,66 @@ "index_extra_records": 0 } ], - "write_propagation_ms": 1130.6929430000991, - "delete_propagation_ms": 915.0309450000123, + "write_propagation_ms": 1136.9370459999573, + "delete_propagation_ms": 884.7577889999911, "coherence_budget_ms": 2000.0, "seed_count": 24, "deleted": 1, - "write_ms": 47.8542619999871, + "write_ms": 41.472491999911654, "read_ms": [ - 38.45003799995084, - 40.01732000006086, - 45.524652999915816 + 31.152385999916987, + 34.83105299994804, + 38.3418779999829 ], - "delete_ms": 22.87207300003047 + "delete_ms": 20.884912000042277 }, "burst": { "requests": 120, "successes": 120, "errors": 0, "error_sample": [], - "wall_seconds": 2.899000990000104, - "requests_per_second": 41.39356985869663, - "avg_ms": 192.10381628333363, - "p95_ms": 1043.9507769999636, - "p99_ms": 1461.4564480000354 + "wall_seconds": 2.945192581000015, + "requests_per_second": 40.74436448541339, + "avg_ms": 195.23711467500678, + "p95_ms": 1023.5760660000324, + "p99_ms": 1891.9401950000747 }, "burst_p99_budget_ms": 2000.0, "final_restore": { "successes": 24, "count": 24, "latencies_ms": [ - 17.159633999995094, - 3.8506440000674047, - 3.698373000020183, - 3.6452719999715555, - 3.6397009999973307, - 3.8230229999953735, - 4.421815000000606, - 3.634207000004608, - 3.3807180000167136, - 3.98342899995896, - 4.091241999958584, - 3.6701969999057837, - 3.713881999942714, - 3.5456179999755477, - 3.9846570000463544, - 3.8972700000385885, - 4.3008000000099855, - 3.493462000051295, - 3.366497999991225, - 3.4199210000451785, - 3.264670000021397, - 3.488211999979285, - 3.2832639999469393, - 3.3759530000452287 + 18.581979000032334, + 3.665214999955424, + 6.7435260000365815, + 3.6489320000328007, + 3.5716070000262334, + 3.773433000105797, + 3.6847490000582184, + 3.7224100000230465, + 3.4003900000243448, + 3.504951999957484, + 3.50271500008148, + 3.693144999942888, + 3.5679009999967093, + 3.8319870000123046, + 3.5032769999361335, + 3.649031999998442, + 3.2745610000119996, + 3.6141799999995783, + 3.6570869999650313, + 3.390364999972917, + 3.9041730000235475, + 3.8251490000220656, + 3.6090999999487394, + 3.394516000071235 ], "rate": 1.0, - "p99_ms": 17.159633999995094 + "p99_ms": 18.581979000032334 }, - "elapsed_ms": 37018.135 + "elapsed_ms": 39058.312 }, - "source_ref": "880cafdf5ce5f7c9da878bd8e82ada39ca2d02cc", - "workflow_run_id": "29064934749", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29064934749" + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261" } diff --git a/benchmarks/memory_os_intelligence_results.json b/benchmarks/memory_os_intelligence_results.json index fdbdf08..fb31104 100644 --- a/benchmarks/memory_os_intelligence_results.json +++ b/benchmarks/memory_os_intelligence_results.json @@ -1,7 +1,7 @@ { "schema": "wavemind.memory_os_intelligence_report.v1", "generated_at": "2026-07-09T22:45:10Z", - "source_ref": "c4f786e131c8", + "source_ref": "d0e3446be447", "source_files": [ "benchmarks/scale_readiness_results.json", "benchmarks/agent_coherence_results.json", diff --git a/benchmarks/production_evidence_bundle_results.json b/benchmarks/production_evidence_bundle_results.json index 60a2706..f2d5f20 100644 --- a/benchmarks/production_evidence_bundle_results.json +++ b/benchmarks/production_evidence_bundle_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.production_evidence_bundle.v1", - "generated_at": "2026-07-11T05:56:51Z", + "generated_at": "2026-07-11T20:56:03Z", "claim_status": "claims_limited", "summary": { "claim_status": "claims_limited", @@ -8,7 +8,7 @@ "strict_pass_count": 4, "strict_total_requirements": 8, "preflight_overall_status": "action_required", - "preflight_ready_count": 0, + "preflight_ready_count": 1, "preflight_total_checks": 8, "production_readiness_status": "pass", "production_readiness_score": 1.0, @@ -21,7 +21,7 @@ }, "strict_production_evidence": { "schema": "wavemind.production_evidence.v1", - "generated_at": "2026-07-11T05:56:51Z", + "generated_at": "2026-07-11T20:56:03Z", "overall_status": "action_required", "summary": { "overall_status": "action_required", @@ -35,7 +35,7 @@ "id": "external_http_cluster", "title": "Non-loopback Kubernetes or external HTTP service-node load", "status": "pass", - "evidence": "nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms", + "evidence": "nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms", "artifact": "benchmarks/http_cluster_load_results.json", "command": "gh workflow run external-http-cluster-load.yml -f nodes=\"node-a=https://wm-a.example.com,node-b=https://wm-b.example.com,node-c=https://wm-c.example.com,node-d=https://wm-d.example.com\" -f replication_factor=3 -f read_quorum=1 -f read_fanout=1 -f batch_query_size=24 -f fail_on_slo=true -f commit_results=true", "claim_unlocked": "Non-loopback Kubernetes service-node cluster load SLO.", @@ -123,12 +123,12 @@ }, "production_evidence_preflight": { "schema": "wavemind.production_evidence_preflight.v1", - "generated_at": "2026-07-11T05:56:51Z", + "generated_at": "2026-07-11T20:56:03Z", "overall_status": "action_required", "summary": { "overall_status": "action_required", - "ready_count": 0, - "action_required_count": 8, + "ready_count": 1, + "action_required_count": 7, "total_checks": 8 }, "checks": [ @@ -243,21 +243,17 @@ { "id": "pgvector_10m_service", "title": "10M pgvector service load preflight", - "status": "action_required", - "ready": false, - "evidence": "plan benchmarks/production_streaming_load_pgvector_10m_plan.json, vectors 10000000, required env 1, missing env 1, required local free 0.004 GB", - "required_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], - "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], - "command": "python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks\\production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json", + "status": "ready", + "ready": true, + "evidence": "workflow-provisioned four-service pgvector topology; external DSNs optional; plan benchmarks/production_streaming_load_pgvector_10m_plan.json", + "required_env": [], + "missing_env": [], + "command": "gh workflow run production-streaming-load.yml --ref main -f engine=pgvector-service -f size=10000000 -f dim=128 -f queries=2000 -f top_k=10 -f batch_size=5000 -f target_recall=0.95 -f target_p99_ms=100 -f target_qps=100 -f replicas=3 -f autoscaling_max_replicas=24 -f capacity_headroom=0.7 -f runner_label=ubuntu-latest -f provision_pgvector_shards=true -f pgvector_shard_count=4 -f runner_storage_root=state -f commit_results=true", "output_artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", - "issues": [ - "set WAVEMIND_PGVECTOR_DSN" - ], - "warnings": [] + "issues": [], + "warnings": [ + "Ephemeral isolated service processes prove the 10M candidate-index SLO, not PostgreSQL HA or independent-node failure tolerance." + ] }, { "id": "faiss_ivfpq_50m", @@ -301,7 +297,7 @@ }, "production_readiness": { "schema": "wavemind.production_readiness.v1", - "generated_at": "2026-07-11T05:56:48Z", + "generated_at": "2026-07-11T20:18:10Z", "overall_status": "pass", "readiness_score": 1.0, "summary": { @@ -318,7 +314,7 @@ "title": "Checked-in benchmark artifacts are synchronized", "status": "pass", "requirement": "Benchmark matrix, report, and leaderboard must render from the same checked-in JSON.", - "evidence": "audit status pass, generated_at 2026-07-11T05:53:28Z", + "evidence": "audit status pass, generated_at 2026-07-11T19:56:03Z", "next_step": "Keep the benchmark refresh workflow green and block stale artifacts before release." }, { @@ -398,7 +394,7 @@ "title": "pgvector streaming runner has service smoke and 10M preflight", "status": "pass", "requirement": "PostgreSQL/pgvector must have a memory-bounded streaming runner that inserts vectors in batches, passes a real service smoke, and has a committed 10M plan-only contract with exact reproduction command.", - "evidence": "smoke vectors 1000, smoke recall 1, smoke p99 7.623799960128963 ms, plan status action_required, plan required local free 0.004 GB, blockers missing_env:WAVEMIND_PGVECTOR_DSN, insufficient_local_disk_for_index_and_transient_batches", + "evidence": "smoke vectors 1000, smoke recall 1, smoke p99 7.623799960128963 ms, plan status action_required, plan required local free 0.004 GB, blockers missing_env:WAVEMIND_PGVECTOR_DSNS, insufficient_local_disk_for_index_and_transient_batches", "next_step": "Run the embedded 10M pgvector command against a sized PostgreSQL service and commit production_streaming_load_pgvector_10m_results.json." }, { @@ -446,7 +442,7 @@ "title": "Kubernetes operator passes a real multi-node failover drill", "status": "pass", "requirement": "Operator output must include CRD, StatefulSet, Service, HPA, scheduled repair, capacity-aware replica reconciliation, a bounded rebalance ConfigMap with full rolling namespace-move plan metadata, Memory OS CronJob plan/run scheduling with Redis/shared-lock safety, production admission wiring for 10M+ targets, redundant operator replicas with Kubernetes Lease/etcd leader election, resourceVersion CAS, and cross-node anti-affinity, a traceable multi-node Kubernetes CI failure drill with leader takeover, post-failover reconcile, PDB/topology protection, pod replacement, CR-driven rolling upgrade, non-loopback pod-DNS quorum traffic through a physical worker outage, and API recovery, and status conditions for readiness/autoscaling/capacity/rebalance/repair/Memory OS/production admission plus control-plane consensus safety.", - "evidence": "CRD True, HPA True, repair True, memory OS True, operator replicas 2, leader election True via coordination.k8s.io/v1, kind nodes 4, failure drill pass, Lease transitions 1, recovered API True, PDB min available 3, topology constraints 2, rolling pods replaced 4, upgraded API True, workflow https://github.com/CaspianG/wavemind/actions/runs/29057247023, network drill pass via docker-pause-kind-worker, service nodes 4, zones 3, outage recall 1.0, failed nodes ['wavemind-ci-2'], recovery recall 1.0, network workflow https://github.com/CaspianG/wavemind/actions/runs/29070578441, rebalance config True, rebalance ready 4048 moves/81 batches, replicas 34, required 34, target max 678711, status Ready, memory OS ready True, production admission ready True, control-plane True", + "evidence": "CRD True, HPA True, repair True, memory OS True, operator replicas 2, leader election True via coordination.k8s.io/v1, kind nodes 4, failure drill pass, Lease transitions 1, recovered API True, PDB min available 3, topology constraints 2, rolling pods replaced 4, upgraded API True, workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261, network drill pass via docker-pause-kind-worker, service nodes 4, zones 3, outage recall 1.0, failed nodes ['wavemind-ci-2'], recovery recall 1.0, network workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261, rebalance config True, rebalance ready 4048 moves/81 batches, replicas 34, required 34, target max 678711, status Ready, memory OS ready True, production admission ready True, control-plane True", "next_step": "Repeat the same drill against a non-ephemeral remote Kubernetes staging cluster and feed its service endpoints into strict cluster admission." }, { @@ -454,7 +450,7 @@ "title": "Serverless plan externalizes state and validates KEDA target", "status": "pass", "requirement": "Serverless mode must use external durable state, external vector index, shared cache, valid KEDA scale target, scale-to-zero-safe workers, an operational SLO/cold-start/cost profile, and an observed-telemetry contract for real cluster load tests. A non-loopback multi-zone Kubernetes lifecycle must also prove scale-to-zero restore, bounded cross-worker coherence, and burst SLO.", - "evidence": "Postgres True, Qdrant True, Redis True, required replicas 4, burst rps 256000, cold start 1220.0 ms, cost $81.76, observed source loopback-api-capacity-estimate, observed replicas 4, observed pool rps 583.26, observed p99 17.039 ms, observed errors 0.0, kind lifecycle pass 13/13, coherence 3/3, propagation 1130.6929430000991/915.0309450000123 ms, burst p99 1461.4564480000354 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29064934749", + "evidence": "Postgres True, Qdrant True, Redis True, required replicas 4, burst rps 256000, cold start 1220.0 ms, cost $81.76, observed source loopback-api-capacity-estimate, observed replicas 4, observed pool rps 583.26, observed p99 17.039 ms, observed errors 0.0, kind lifecycle pass 13/13, coherence 3/3, propagation 1136.9370459999573/884.7577889999911 ms, burst p99 1891.9401950000747 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261", "next_step": "Run the same profile against a real Knative/KEDA cluster and replace loopback telemetry with remote p95/p99/cold-start/error-rate/scale-out metrics." }, { @@ -582,7 +578,7 @@ "title": "Active-active sync and field-state CRDT converge", "status": "pass", "requirement": "Multi-region memory deltas and field state must converge without duplicate amplification or full-namespace replay on incremental sync, and CRDT deltas must carry actor watermarks so regions can audit sync progress, missing actors, and replication lag. A non-loopback Kubernetes drill must also prove that surviving regions continue writes and tombstone convergence during a physical zone outage, then converge the recovered PVC-backed region without resurrecting deleted memory.", - "evidence": "delta sync True, incremental records 1, field-only keys 1, sustained regions 3, sustained convergence 1.0, sustained delete suppression 1.0, sustained success 1.0, HTTP service-region convergence 1.0, HTTP final no-op imports 0, Kubernetes regions 3, zones 3, failure docker-pause-kind-worker, outage unavailable ['region-b'], outage writes 32, outage convergence 1.0, outage delete suppression 1.0, recovery convergence 1.0, recovery delete suppression 1.0, region workflow https://github.com/CaspianG/wavemind/actions/runs/29058433643, CRDT idempotent True, watermarks 3, watermark health pass, missing detected True, lag detected True", + "evidence": "delta sync True, incremental records 1, field-only keys 1, sustained regions 3, sustained convergence 1.0, sustained delete suppression 1.0, sustained success 1.0, HTTP service-region convergence 1.0, HTTP final no-op imports 0, Kubernetes regions 3, zones 3, failure docker-pause-kind-worker, outage unavailable ['region-b'], outage writes 32, outage convergence 1.0, outage delete suppression 1.0, recovery convergence 1.0, recovery delete suppression 1.0, region workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261, CRDT idempotent True, watermarks 3, watermark health pass, missing detected True, lag detected True", "next_step": "Repeat the passing kind region-failure protocol on independent remote Kubernetes regions and feed that artifact into strict active-active admission." }, { @@ -590,7 +586,7 @@ "title": "Snapshots, archives, offsite mirror, and object-store DR verify", "status": "pass", "requirement": "Backups must be checksummed, restorable, offsite-capable, recover recall after restore, and support SQLite point-in-time recovery from an append-only mutation journal plus database-native Postgres PITR runbook/preflight coverage. Kubernetes external-state deployments must restore PostgreSQL into an independent namespace, rebuild Qdrant exactly, and preserve recall through API replacement.", - "evidence": "archive True, object-store DR True, restored files 3, PITR full True, PITR point True, journal entries 5, deleted 2, Postgres PITR ready, commands 7, env missing_env, Kubernetes DR pass 10/10, backup bytes 1016635, restore recall 1.0, Qdrant 24/24, replacement recall 1.0, restore 20449.092 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29064934749", + "evidence": "archive True, object-store DR True, restored files 3, PITR full True, PITR point True, journal entries 5, deleted 2, Postgres PITR ready, commands 7, env missing_env, Kubernetes DR pass 10/10, backup bytes 1016642, restore recall 1.0, Qdrant 24/24, replacement recall 1.0, restore 23338.114 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261", "next_step": "Repeat the passing independent-namespace Kubernetes restore with managed PostgreSQL PITR, remote object storage, and a separate recovery cluster before claiming managed-cloud or multi-region DR." }, { @@ -638,7 +634,7 @@ "id": "external_http_cluster_load", "title": "External HTTP service-node load evidence", "status": "pass", - "evidence": "nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms", + "evidence": "nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms", "next_step": "Keep the external service-node run current for release candidates." }, { @@ -653,13 +649,13 @@ "artifact_audit": { "schema": "wavemind.benchmark_artifact_audit.v1", "status": "pass", - "checked_at": "2026-07-11T05:53:37Z", - "generated_at": "2026-07-11T05:53:28Z", - "source_ref": "c4f786e131c8", + "checked_at": "2026-07-11T20:18:09Z", + "generated_at": "2026-07-11T19:56:03Z", + "source_ref": "d0e3446be447", "workflow_run_id": null, "refresh_profile": "local", "max_age_days": 8.0, - "age_days": 0.00011299333333333334, + "age_days": 0.015353437488425926, "implemented_count": 37, "runner_ready_count": 3, "planned_count": 6, @@ -715,13 +711,13 @@ "target_memories": 10000000, "output_artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", "required_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "blockers": [ - "missing_env:WAVEMIND_PGVECTOR_DSN", + "missing_env:WAVEMIND_PGVECTOR_DSNS", "insufficient_local_disk_for_index_and_transient_batches" ] }, @@ -739,6 +735,7 @@ ], "blockers": [ "missing_env:WAVEMIND_FAISS_IVFPQ_PATH", + "missing_module:faiss", "insufficient_local_disk_for_index_and_transient_batches" ] }, @@ -806,17 +803,17 @@ "id": "pgvector_10m_service", "title": "10M pgvector service load", "strict_status": "action_required", - "preflight_status": "action_required", + "preflight_status": "ready", "artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", "output_artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", "issues": [ "missing artifact" ], - "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" + "missing_env": [], + "warnings": [ + "Ephemeral isolated service processes prove the 10M candidate-index SLO, not PostgreSQL HA or independent-node failure tolerance." ], - "warnings": [], - "command": "python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks/production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json", + "command": "gh workflow run production-streaming-load.yml --ref main -f engine=pgvector-service -f size=10000000 -f dim=128 -f queries=2000 -f top_k=10 -f batch_size=5000 -f target_recall=0.95 -f target_p99_ms=100 -f target_qps=100 -f replicas=3 -f autoscaling_max_replicas=24 -f capacity_headroom=0.7 -f runner_label=ubuntu-latest -f provision_pgvector_shards=true -f pgvector_shard_count=4 -f runner_storage_root=state -f commit_results=true", "claim_unlocked": "10M PostgreSQL/pgvector service candidate-index SLO." }, { diff --git a/benchmarks/production_evidence_dispatch_results.json b/benchmarks/production_evidence_dispatch_results.json index 543c3b3..f55f88f 100644 --- a/benchmarks/production_evidence_dispatch_results.json +++ b/benchmarks/production_evidence_dispatch_results.json @@ -1,12 +1,12 @@ { "schema": "wavemind.production_evidence_dispatch.v1", - "generated_at": "2026-07-11T05:56:51Z", + "generated_at": "2026-07-11T20:56:02Z", "overall_status": "action_required", "summary": { "overall_status": "action_required", "total_jobs": 8, - "ready_to_dispatch_count": 0, - "blocked_by_preflight_count": 4, + "ready_to_dispatch_count": 1, + "blocked_by_preflight_count": 3, "complete_count": 4, "commit_results_default": false, "runner_label": "self-hosted-large", @@ -17,8 +17,9 @@ "service-scale-10m": 3 }, "status_counts": { - "blocked_by_preflight": 4, - "complete": 4 + "blocked_by_preflight": 3, + "complete": 4, + "ready_to_dispatch": 1 } }, "launch_policy": { @@ -279,11 +280,11 @@ { "id": "pgvector_10m_service", "title": "10M pgvector service load", - "status": "blocked_by_preflight", + "status": "ready_to_dispatch", "dispatch_required": true, - "ready": false, + "ready": true, "strict_status": "action_required", - "preflight_status": "action_required", + "preflight_status": "ready", "wave": "service-scale-10m", "workflow": "production-streaming-load.yml", "artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", @@ -301,27 +302,24 @@ "replicas": "3", "autoscaling_max_replicas": "24", "capacity_headroom": "0.7", - "runner_label": "self-hosted-large", + "runner_label": "ubuntu-latest", "runner_storage_root": "state/production-runs", "commit_results": false, - "pgvector_dsn": "$WAVEMIND_PGVECTOR_DSN" + "provision_pgvector_shards": true, + "pgvector_shard_count": "4" }, - "input_bindings": { - "pgvector_dsn": "$WAVEMIND_PGVECTOR_DSN" - }, - "required_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], - "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], + "input_bindings": {}, + "required_env": [], + "missing_env": [], "required_secrets": [], "issues": [ - "set WAVEMIND_PGVECTOR_DSN" + "missing artifact" ], - "warnings": [], - "safe_launch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"self-hosted-large\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"false\" -f pgvector_dsn=\"$WAVEMIND_PGVECTOR_DSN\"", - "publish_launch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"self-hosted-large\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"true\" -f pgvector_dsn=\"$WAVEMIND_PGVECTOR_DSN\"", + "warnings": [ + "Ephemeral isolated service processes prove the 10M candidate-index SLO, not PostgreSQL HA or independent-node failure tolerance." + ], + "safe_launch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"ubuntu-latest\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"false\" -f provision_pgvector_shards=\"true\" -f pgvector_shard_count=\"4\"", + "publish_launch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"ubuntu-latest\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"true\" -f provision_pgvector_shards=\"true\" -f pgvector_shard_count=\"4\"", "download_command": "gh run download --repo CaspianG/wavemind --dir state/production-evidence-downloads", "ingest_command": "wavemind ingest-production-evidence --artifact-dir state/production-evidence-downloads --refresh" }, diff --git a/benchmarks/production_evidence_env_contract.json b/benchmarks/production_evidence_env_contract.json index 5530c07..7706903 100644 --- a/benchmarks/production_evidence_env_contract.json +++ b/benchmarks/production_evidence_env_contract.json @@ -1,6 +1,6 @@ { "schema": "wavemind.production_evidence_env_contract.v1", - "generated_at": "2026-07-10T05:23:37Z", + "generated_at": "2026-07-11T19:47:54Z", "overall_status": "action_required", "repo": "CaspianG/wavemind", "claim_boundary": "Environment contract only. It stores placeholders and secret names, never credential values, and does not unlock production claims until strict evidence artifacts pass ingestion and validation.", @@ -22,7 +22,7 @@ "WAVEMIND_CLUSTER_NODES", "WAVEMIND_CLUSTER_NODES_MANIFEST_JSON", "WAVEMIND_FAISS_IVFPQ_PATH", - "WAVEMIND_PGVECTOR_DSN", + "WAVEMIND_PGVECTOR_DSNS", "WAVEMIND_QDRANT_URL", "WAVEMIND_QDRANT_URLS", "WAVEMIND_SERVERLESS_NODES" @@ -227,15 +227,15 @@ "github_secret_command": "printf '%s' \"$WAVEMIND_FAISS_IVFPQ_PATH\" | gh secret set WAVEMIND_FAISS_IVFPQ_PATH --repo CaspianG/wavemind --body-file -" }, { - "name": "WAVEMIND_PGVECTOR_DSN", + "name": "WAVEMIND_PGVECTOR_DSNS", "status": "missing", "configured": false, "required": true, "recommended": false, - "kind": "postgres-dsn", + "kind": "postgres-dsn-list", "sensitivity": "secret_or_internal", - "description": "PostgreSQL/pgvector DSN for 10M pgvector service streaming load.", - "example": "postgresql://USER:PASSWORD@pgvector.staging.example.com:5432/wavemind", + "description": "Comma-separated PostgreSQL/pgvector service DSNs for the namespace-sharded 10M streaming load.", + "example": "postgresql://USER:PASSWORD@pgvector-a.staging.example.com:5432/wavemind,postgresql://USER:PASSWORD@pgvector-b.staging.example.com:5432/wavemind", "used_by": [ "pgvector_10m_service" ], @@ -243,7 +243,7 @@ "production-streaming-load.yml" ], "workflow_inputs": [ - "pgvector_dsn" + "pgvector_dsns" ], "artifacts": [ "benchmarks/production_streaming_load_pgvector_10m_results.json" @@ -251,8 +251,8 @@ "claims": [ "10M PostgreSQL/pgvector service candidate-index SLO." ], - "github_secret": "WAVEMIND_PGVECTOR_DSN", - "github_secret_command": "printf '%s' \"$WAVEMIND_PGVECTOR_DSN\" | gh secret set WAVEMIND_PGVECTOR_DSN --repo CaspianG/wavemind --body-file -" + "github_secret": "WAVEMIND_PGVECTOR_DSNS", + "github_secret_command": "printf '%s' \"$WAVEMIND_PGVECTOR_DSNS\" | gh secret set WAVEMIND_PGVECTOR_DSNS --repo CaspianG/wavemind --body-file -" }, { "name": "WAVEMIND_QDRANT_API_KEY", @@ -407,7 +407,7 @@ "WAVEMIND_CLUSTER_NODES_MANIFEST_JSON", "WAVEMIND_FAISS_IVFPQ_FREE_GB", "WAVEMIND_FAISS_IVFPQ_PATH", - "WAVEMIND_PGVECTOR_DSN", + "WAVEMIND_PGVECTOR_DSNS", "WAVEMIND_QDRANT_API_KEY", "WAVEMIND_QDRANT_API_KEYS", "WAVEMIND_QDRANT_URL", @@ -422,7 +422,7 @@ "printf '%s' \"$WAVEMIND_CLUSTER_NODES_MANIFEST_JSON\" | gh secret set WAVEMIND_CLUSTER_NODES_MANIFEST_JSON --repo CaspianG/wavemind --body-file -", "printf '%s' \"$WAVEMIND_FAISS_IVFPQ_FREE_GB\" | gh secret set WAVEMIND_FAISS_IVFPQ_FREE_GB --repo CaspianG/wavemind --body-file -", "printf '%s' \"$WAVEMIND_FAISS_IVFPQ_PATH\" | gh secret set WAVEMIND_FAISS_IVFPQ_PATH --repo CaspianG/wavemind --body-file -", - "printf '%s' \"$WAVEMIND_PGVECTOR_DSN\" | gh secret set WAVEMIND_PGVECTOR_DSN --repo CaspianG/wavemind --body-file -", + "printf '%s' \"$WAVEMIND_PGVECTOR_DSNS\" | gh secret set WAVEMIND_PGVECTOR_DSNS --repo CaspianG/wavemind --body-file -", "printf '%s' \"$WAVEMIND_QDRANT_API_KEY\" | gh secret set WAVEMIND_QDRANT_API_KEY --repo CaspianG/wavemind --body-file -", "printf '%s' \"$WAVEMIND_QDRANT_API_KEYS\" | gh secret set WAVEMIND_QDRANT_API_KEYS --repo CaspianG/wavemind --body-file -", "printf '%s' \"$WAVEMIND_QDRANT_URL\" | gh secret set WAVEMIND_QDRANT_URL --repo CaspianG/wavemind --body-file -", @@ -430,7 +430,7 @@ "printf '%s' \"$WAVEMIND_SERVERLESS_NODES\" | gh secret set WAVEMIND_SERVERLESS_NODES --repo CaspianG/wavemind --body-file -" ] }, - "env_example": "# WaveMind strict production evidence environment\n# Fill these locally or as GitHub Actions environment-scoped secrets.\n# Do not commit real values.\n\n# Remote WaveMind API regions for active-active namespace sync evidence.\nWAVEMIND_ACTIVE_ACTIVE_REGIONS=us=https://wm-us.staging.example.com,eu=https://wm-eu.staging.example.com,ap=https://wm-ap.staging.example.com\n\n# JSON manifest alternative for remote active-active region evidence.\nWAVEMIND_ACTIVE_ACTIVE_REGIONS_MANIFEST_JSON={\"environment\":\"staging\",\"source\":\"kubernetes\",\"deployment_id\":\"wm-regions-2026-07\",\"regions\":[{\"id\":\"us\",\"url\":\"https://wm-us.staging.example.com\"},{\"id\":\"eu\",\"url\":\"https://wm-eu.staging.example.com\"},{\"id\":\"ap\",\"url\":\"https://wm-ap.staging.example.com\"}]}\n\n# Optional API key used when remote WaveMind endpoints require authentication.\nWAVEMIND_API_KEY=REDACTED_API_KEY\n\n# Remote WaveMind API nodes for the external service-node cluster workload.\nWAVEMIND_CLUSTER_NODES=node-a=https://wm-a.staging.example.com,node-b=https://wm-b.staging.example.com,node-c=https://wm-c.staging.example.com,node-d=https://wm-d.staging.example.com\n\n# JSON manifest alternative for external service-node cluster evidence.\nWAVEMIND_CLUSTER_NODES_MANIFEST_JSON={\"environment\":\"staging\",\"source\":\"kubernetes\",\"deployment_id\":\"wm-staging-2026-07\",\"nodes\":[{\"id\":\"node-a\",\"url\":\"https://wm-a.staging.example.com\"},{\"id\":\"node-b\",\"url\":\"https://wm-b.staging.example.com\"},{\"id\":\"node-c\",\"url\":\"https://wm-c.staging.example.com\"},{\"id\":\"node-d\",\"url\":\"https://wm-d.staging.example.com\"}]}\n\n# Optional deterministic free-disk override for FAISS IVF-PQ preflight.\nWAVEMIND_FAISS_IVFPQ_FREE_GB=8\n\n# Persisted FAISS IVF-PQ index path for the 50M streaming profile.\nWAVEMIND_FAISS_IVFPQ_PATH=/mnt/wavemind/faiss/wavemind-ivfpq-50m.faiss\n\n# PostgreSQL/pgvector DSN for 10M pgvector service streaming load.\nWAVEMIND_PGVECTOR_DSN=postgresql://USER:PASSWORD@pgvector.staging.example.com:5432/wavemind\n\n# Optional Qdrant API key for the single-service Qdrant production load job.\nWAVEMIND_QDRANT_API_KEY=REDACTED_QDRANT_API_KEY\n\n# Optional comma-separated Qdrant API keys for sharded Qdrant production load jobs.\nWAVEMIND_QDRANT_API_KEYS=REDACTED_QDRANT_API_KEY_A,REDACTED_QDRANT_API_KEY_B\n\n# Single Qdrant service URL for 10M service-backed streaming load.\nWAVEMIND_QDRANT_URL=https://qdrant-10m.staging.example.com:6333\n\n# Comma-separated Qdrant shard URLs for 10M sharded and 100M sharded service load.\nWAVEMIND_QDRANT_URLS=https://qdrant-a.staging.example.com:6333,https://qdrant-b.staging.example.com:6333\n\n# Deployed serverless/managed WaveMind API nodes for remote telemetry evidence.\nWAVEMIND_SERVERLESS_NODES=https://wm-serverless-a.example.com,https://wm-serverless-b.example.com\n", + "env_example": "# WaveMind strict production evidence environment\n# Fill these locally or as GitHub Actions environment-scoped secrets.\n# Do not commit real values.\n\n# Remote WaveMind API regions for active-active namespace sync evidence.\nWAVEMIND_ACTIVE_ACTIVE_REGIONS=us=https://wm-us.staging.example.com,eu=https://wm-eu.staging.example.com,ap=https://wm-ap.staging.example.com\n\n# JSON manifest alternative for remote active-active region evidence.\nWAVEMIND_ACTIVE_ACTIVE_REGIONS_MANIFEST_JSON={\"environment\":\"staging\",\"source\":\"kubernetes\",\"deployment_id\":\"wm-regions-2026-07\",\"regions\":[{\"id\":\"us\",\"url\":\"https://wm-us.staging.example.com\"},{\"id\":\"eu\",\"url\":\"https://wm-eu.staging.example.com\"},{\"id\":\"ap\",\"url\":\"https://wm-ap.staging.example.com\"}]}\n\n# Optional API key used when remote WaveMind endpoints require authentication.\nWAVEMIND_API_KEY=REDACTED_API_KEY\n\n# Remote WaveMind API nodes for the external service-node cluster workload.\nWAVEMIND_CLUSTER_NODES=node-a=https://wm-a.staging.example.com,node-b=https://wm-b.staging.example.com,node-c=https://wm-c.staging.example.com,node-d=https://wm-d.staging.example.com\n\n# JSON manifest alternative for external service-node cluster evidence.\nWAVEMIND_CLUSTER_NODES_MANIFEST_JSON={\"environment\":\"staging\",\"source\":\"kubernetes\",\"deployment_id\":\"wm-staging-2026-07\",\"nodes\":[{\"id\":\"node-a\",\"url\":\"https://wm-a.staging.example.com\"},{\"id\":\"node-b\",\"url\":\"https://wm-b.staging.example.com\"},{\"id\":\"node-c\",\"url\":\"https://wm-c.staging.example.com\"},{\"id\":\"node-d\",\"url\":\"https://wm-d.staging.example.com\"}]}\n\n# Optional deterministic free-disk override for FAISS IVF-PQ preflight.\nWAVEMIND_FAISS_IVFPQ_FREE_GB=8\n\n# Persisted FAISS IVF-PQ index path for the 50M streaming profile.\nWAVEMIND_FAISS_IVFPQ_PATH=/mnt/wavemind/faiss/wavemind-ivfpq-50m.faiss\n\n# Comma-separated PostgreSQL/pgvector service DSNs for the namespace-sharded 10M streaming load.\nWAVEMIND_PGVECTOR_DSNS=postgresql://USER:PASSWORD@pgvector-a.staging.example.com:5432/wavemind,postgresql://USER:PASSWORD@pgvector-b.staging.example.com:5432/wavemind\n\n# Optional Qdrant API key for the single-service Qdrant production load job.\nWAVEMIND_QDRANT_API_KEY=REDACTED_QDRANT_API_KEY\n\n# Optional comma-separated Qdrant API keys for sharded Qdrant production load jobs.\nWAVEMIND_QDRANT_API_KEYS=REDACTED_QDRANT_API_KEY_A,REDACTED_QDRANT_API_KEY_B\n\n# Single Qdrant service URL for 10M service-backed streaming load.\nWAVEMIND_QDRANT_URL=https://qdrant-10m.staging.example.com:6333\n\n# Comma-separated Qdrant shard URLs for 10M sharded and 100M sharded service load.\nWAVEMIND_QDRANT_URLS=https://qdrant-a.staging.example.com:6333,https://qdrant-b.staging.example.com:6333\n\n# Deployed serverless/managed WaveMind API nodes for remote telemetry evidence.\nWAVEMIND_SERVERLESS_NODES=https://wm-serverless-a.example.com,https://wm-serverless-b.example.com\n", "checks": [ { "name": "all_preflight_env_represented", diff --git a/benchmarks/production_evidence_preflight_results.json b/benchmarks/production_evidence_preflight_results.json index c95328f..864d3ef 100644 --- a/benchmarks/production_evidence_preflight_results.json +++ b/benchmarks/production_evidence_preflight_results.json @@ -1,11 +1,11 @@ { "schema": "wavemind.production_evidence_preflight.v1", - "generated_at": "2026-07-11T05:56:50Z", + "generated_at": "2026-07-11T20:54:43Z", "overall_status": "action_required", "summary": { "overall_status": "action_required", - "ready_count": 0, - "action_required_count": 8, + "ready_count": 1, + "action_required_count": 7, "total_checks": 8 }, "checks": [ @@ -120,21 +120,17 @@ { "id": "pgvector_10m_service", "title": "10M pgvector service load preflight", - "status": "action_required", - "ready": false, - "evidence": "plan benchmarks/production_streaming_load_pgvector_10m_plan.json, vectors 10000000, required env 1, missing env 1, required local free 0.004 GB", - "required_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], - "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], - "command": "python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks\\production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json", + "status": "ready", + "ready": true, + "evidence": "workflow-provisioned four-service pgvector topology; external DSNs optional; plan benchmarks/production_streaming_load_pgvector_10m_plan.json", + "required_env": [], + "missing_env": [], + "command": "gh workflow run production-streaming-load.yml --ref main -f engine=pgvector-service -f size=10000000 -f dim=128 -f queries=2000 -f top_k=10 -f batch_size=5000 -f target_recall=0.95 -f target_p99_ms=100 -f target_qps=100 -f replicas=3 -f autoscaling_max_replicas=24 -f capacity_headroom=0.7 -f runner_label=ubuntu-latest -f provision_pgvector_shards=true -f pgvector_shard_count=4 -f runner_storage_root=state -f commit_results=true", "output_artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", - "issues": [ - "set WAVEMIND_PGVECTOR_DSN" - ], - "warnings": [] + "issues": [], + "warnings": [ + "Ephemeral isolated service processes prove the 10M candidate-index SLO, not PostgreSQL HA or independent-node failure tolerance." + ] }, { "id": "faiss_ivfpq_50m", diff --git a/benchmarks/production_evidence_results.json b/benchmarks/production_evidence_results.json index 3427082..a0c20e9 100644 --- a/benchmarks/production_evidence_results.json +++ b/benchmarks/production_evidence_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.production_evidence.v1", - "generated_at": "2026-07-11T05:56:49Z", + "generated_at": "2026-07-11T19:56:06Z", "overall_status": "action_required", "summary": { "overall_status": "action_required", @@ -14,7 +14,7 @@ "id": "external_http_cluster", "title": "Non-loopback Kubernetes or external HTTP service-node load", "status": "pass", - "evidence": "nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms", + "evidence": "nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms", "artifact": "benchmarks/http_cluster_load_results.json", "command": "gh workflow run external-http-cluster-load.yml -f nodes=\"node-a=https://wm-a.example.com,node-b=https://wm-b.example.com,node-c=https://wm-c.example.com,node-d=https://wm-d.example.com\" -f replication_factor=3 -f read_quorum=1 -f read_fanout=1 -f batch_query_size=24 -f fail_on_slo=true -f commit_results=true", "claim_unlocked": "Non-loopback Kubernetes service-node cluster load SLO.", diff --git a/benchmarks/production_load_benchmark.py b/benchmarks/production_load_benchmark.py index 92e51eb..2959f21 100644 --- a/benchmarks/production_load_benchmark.py +++ b/benchmarks/production_load_benchmark.py @@ -65,6 +65,7 @@ def preflight(output_path: Path | None = None) -> dict[str, Any]: "WAVEMIND_FAISS_PATH": bool(os.environ.get("WAVEMIND_FAISS_PATH")), "WAVEMIND_QDRANT_URL": bool(os.environ.get("WAVEMIND_QDRANT_URL")), "WAVEMIND_PGVECTOR_DSN": bool(os.environ.get("WAVEMIND_PGVECTOR_DSN")), + "WAVEMIND_PGVECTOR_DSNS": bool(os.environ.get("WAVEMIND_PGVECTOR_DSNS")), "WAVEMIND_PGVECTOR_CREATE_HNSW": os.environ.get("WAVEMIND_PGVECTOR_CREATE_HNSW"), "WAVEMIND_PGVECTOR_EF_SEARCH": os.environ.get("WAVEMIND_PGVECTOR_EF_SEARCH"), "WAVEMIND_QDRANT_HNSW_EF": os.environ.get("WAVEMIND_QDRANT_HNSW_EF"), diff --git a/benchmarks/production_readiness_results.json b/benchmarks/production_readiness_results.json index f02e0c0..6f17f7a 100644 --- a/benchmarks/production_readiness_results.json +++ b/benchmarks/production_readiness_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.production_readiness.v1", - "generated_at": "2026-07-11T05:56:48Z", + "generated_at": "2026-07-11T20:18:10Z", "overall_status": "pass", "readiness_score": 1.0, "summary": { @@ -17,7 +17,7 @@ "title": "Checked-in benchmark artifacts are synchronized", "status": "pass", "requirement": "Benchmark matrix, report, and leaderboard must render from the same checked-in JSON.", - "evidence": "audit status pass, generated_at 2026-07-11T05:53:28Z", + "evidence": "audit status pass, generated_at 2026-07-11T19:56:03Z", "next_step": "Keep the benchmark refresh workflow green and block stale artifacts before release." }, { @@ -97,7 +97,7 @@ "title": "pgvector streaming runner has service smoke and 10M preflight", "status": "pass", "requirement": "PostgreSQL/pgvector must have a memory-bounded streaming runner that inserts vectors in batches, passes a real service smoke, and has a committed 10M plan-only contract with exact reproduction command.", - "evidence": "smoke vectors 1000, smoke recall 1, smoke p99 7.623799960128963 ms, plan status action_required, plan required local free 0.004 GB, blockers missing_env:WAVEMIND_PGVECTOR_DSN, insufficient_local_disk_for_index_and_transient_batches", + "evidence": "smoke vectors 1000, smoke recall 1, smoke p99 7.623799960128963 ms, plan status action_required, plan required local free 0.004 GB, blockers missing_env:WAVEMIND_PGVECTOR_DSNS, insufficient_local_disk_for_index_and_transient_batches", "next_step": "Run the embedded 10M pgvector command against a sized PostgreSQL service and commit production_streaming_load_pgvector_10m_results.json." }, { @@ -145,7 +145,7 @@ "title": "Kubernetes operator passes a real multi-node failover drill", "status": "pass", "requirement": "Operator output must include CRD, StatefulSet, Service, HPA, scheduled repair, capacity-aware replica reconciliation, a bounded rebalance ConfigMap with full rolling namespace-move plan metadata, Memory OS CronJob plan/run scheduling with Redis/shared-lock safety, production admission wiring for 10M+ targets, redundant operator replicas with Kubernetes Lease/etcd leader election, resourceVersion CAS, and cross-node anti-affinity, a traceable multi-node Kubernetes CI failure drill with leader takeover, post-failover reconcile, PDB/topology protection, pod replacement, CR-driven rolling upgrade, non-loopback pod-DNS quorum traffic through a physical worker outage, and API recovery, and status conditions for readiness/autoscaling/capacity/rebalance/repair/Memory OS/production admission plus control-plane consensus safety.", - "evidence": "CRD True, HPA True, repair True, memory OS True, operator replicas 2, leader election True via coordination.k8s.io/v1, kind nodes 4, failure drill pass, Lease transitions 1, recovered API True, PDB min available 3, topology constraints 2, rolling pods replaced 4, upgraded API True, workflow https://github.com/CaspianG/wavemind/actions/runs/29057247023, network drill pass via docker-pause-kind-worker, service nodes 4, zones 3, outage recall 1.0, failed nodes ['wavemind-ci-2'], recovery recall 1.0, network workflow https://github.com/CaspianG/wavemind/actions/runs/29070578441, rebalance config True, rebalance ready 4048 moves/81 batches, replicas 34, required 34, target max 678711, status Ready, memory OS ready True, production admission ready True, control-plane True", + "evidence": "CRD True, HPA True, repair True, memory OS True, operator replicas 2, leader election True via coordination.k8s.io/v1, kind nodes 4, failure drill pass, Lease transitions 1, recovered API True, PDB min available 3, topology constraints 2, rolling pods replaced 4, upgraded API True, workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261, network drill pass via docker-pause-kind-worker, service nodes 4, zones 3, outage recall 1.0, failed nodes ['wavemind-ci-2'], recovery recall 1.0, network workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261, rebalance config True, rebalance ready 4048 moves/81 batches, replicas 34, required 34, target max 678711, status Ready, memory OS ready True, production admission ready True, control-plane True", "next_step": "Repeat the same drill against a non-ephemeral remote Kubernetes staging cluster and feed its service endpoints into strict cluster admission." }, { @@ -153,7 +153,7 @@ "title": "Serverless plan externalizes state and validates KEDA target", "status": "pass", "requirement": "Serverless mode must use external durable state, external vector index, shared cache, valid KEDA scale target, scale-to-zero-safe workers, an operational SLO/cold-start/cost profile, and an observed-telemetry contract for real cluster load tests. A non-loopback multi-zone Kubernetes lifecycle must also prove scale-to-zero restore, bounded cross-worker coherence, and burst SLO.", - "evidence": "Postgres True, Qdrant True, Redis True, required replicas 4, burst rps 256000, cold start 1220.0 ms, cost $81.76, observed source loopback-api-capacity-estimate, observed replicas 4, observed pool rps 583.26, observed p99 17.039 ms, observed errors 0.0, kind lifecycle pass 13/13, coherence 3/3, propagation 1130.6929430000991/915.0309450000123 ms, burst p99 1461.4564480000354 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29064934749", + "evidence": "Postgres True, Qdrant True, Redis True, required replicas 4, burst rps 256000, cold start 1220.0 ms, cost $81.76, observed source loopback-api-capacity-estimate, observed replicas 4, observed pool rps 583.26, observed p99 17.039 ms, observed errors 0.0, kind lifecycle pass 13/13, coherence 3/3, propagation 1136.9370459999573/884.7577889999911 ms, burst p99 1891.9401950000747 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261", "next_step": "Run the same profile against a real Knative/KEDA cluster and replace loopback telemetry with remote p95/p99/cold-start/error-rate/scale-out metrics." }, { @@ -281,7 +281,7 @@ "title": "Active-active sync and field-state CRDT converge", "status": "pass", "requirement": "Multi-region memory deltas and field state must converge without duplicate amplification or full-namespace replay on incremental sync, and CRDT deltas must carry actor watermarks so regions can audit sync progress, missing actors, and replication lag. A non-loopback Kubernetes drill must also prove that surviving regions continue writes and tombstone convergence during a physical zone outage, then converge the recovered PVC-backed region without resurrecting deleted memory.", - "evidence": "delta sync True, incremental records 1, field-only keys 1, sustained regions 3, sustained convergence 1.0, sustained delete suppression 1.0, sustained success 1.0, HTTP service-region convergence 1.0, HTTP final no-op imports 0, Kubernetes regions 3, zones 3, failure docker-pause-kind-worker, outage unavailable ['region-b'], outage writes 32, outage convergence 1.0, outage delete suppression 1.0, recovery convergence 1.0, recovery delete suppression 1.0, region workflow https://github.com/CaspianG/wavemind/actions/runs/29058433643, CRDT idempotent True, watermarks 3, watermark health pass, missing detected True, lag detected True", + "evidence": "delta sync True, incremental records 1, field-only keys 1, sustained regions 3, sustained convergence 1.0, sustained delete suppression 1.0, sustained success 1.0, HTTP service-region convergence 1.0, HTTP final no-op imports 0, Kubernetes regions 3, zones 3, failure docker-pause-kind-worker, outage unavailable ['region-b'], outage writes 32, outage convergence 1.0, outage delete suppression 1.0, recovery convergence 1.0, recovery delete suppression 1.0, region workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261, CRDT idempotent True, watermarks 3, watermark health pass, missing detected True, lag detected True", "next_step": "Repeat the passing kind region-failure protocol on independent remote Kubernetes regions and feed that artifact into strict active-active admission." }, { @@ -289,7 +289,7 @@ "title": "Snapshots, archives, offsite mirror, and object-store DR verify", "status": "pass", "requirement": "Backups must be checksummed, restorable, offsite-capable, recover recall after restore, and support SQLite point-in-time recovery from an append-only mutation journal plus database-native Postgres PITR runbook/preflight coverage. Kubernetes external-state deployments must restore PostgreSQL into an independent namespace, rebuild Qdrant exactly, and preserve recall through API replacement.", - "evidence": "archive True, object-store DR True, restored files 3, PITR full True, PITR point True, journal entries 5, deleted 2, Postgres PITR ready, commands 7, env missing_env, Kubernetes DR pass 10/10, backup bytes 1016635, restore recall 1.0, Qdrant 24/24, replacement recall 1.0, restore 20449.092 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29064934749", + "evidence": "archive True, object-store DR True, restored files 3, PITR full True, PITR point True, journal entries 5, deleted 2, Postgres PITR ready, commands 7, env missing_env, Kubernetes DR pass 10/10, backup bytes 1016642, restore recall 1.0, Qdrant 24/24, replacement recall 1.0, restore 23338.114 ms, workflow https://github.com/CaspianG/wavemind/actions/runs/29165761261", "next_step": "Repeat the passing independent-namespace Kubernetes restore with managed PostgreSQL PITR, remote object storage, and a separate recovery cluster before claiming managed-cloud or multi-region DR." }, { @@ -337,7 +337,7 @@ "id": "external_http_cluster_load", "title": "External HTTP service-node load evidence", "status": "pass", - "evidence": "nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms", + "evidence": "nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms", "next_step": "Keep the external service-node run current for release candidates." }, { diff --git a/benchmarks/production_scale_run_plan.json b/benchmarks/production_scale_run_plan.json index f75b2b3..2e9179a 100644 --- a/benchmarks/production_scale_run_plan.json +++ b/benchmarks/production_scale_run_plan.json @@ -1,9 +1,9 @@ { "schema": "wavemind.production_scale_run_plan.v1", - "generated_at": "2026-07-10T04:17:18Z", + "generated_at": "2026-07-11T19:47:53Z", "summary": { "schema": "wavemind.production_scale_run_plan.v1", - "generated_at": "2026-07-10T04:17:18Z", + "generated_at": "2026-07-11T19:47:53Z", "overall_status": "action_required", "ready_count": 0, "action_required_count": 5, @@ -371,16 +371,23 @@ "checkpoint_path": "state/production-runs/pgvector-service-10000000.checkpoint.json", "command": "python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --memory-payload-kb 2.0 --vector-dtype-bytes 4 --output benchmarks/production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json", "command_env": { - "WAVEMIND_PGVECTOR_DSN": "postgresql://user:password@postgres.example:5432/wavemind", + "WAVEMIND_PGVECTOR_DSNS": "postgresql://user:password@postgres-0.example:5432/wavemind,postgresql://user:password@postgres-1.example:5432/wavemind,postgresql://user:password@postgres-2.example:5432/wavemind,postgresql://user:password@postgres-3.example:5432/wavemind", "WAVEMIND_PGVECTOR_CREATE_HNSW": "1", - "WAVEMIND_PGVECTOR_EF_SEARCH": "1000", + "WAVEMIND_PGVECTOR_STORAGE_TYPE": "halfvec", + "WAVEMIND_PGVECTOR_INSERT_MODE": "copy", + "WAVEMIND_PGVECTOR_INDEX_TYPE": "hnsw", + "WAVEMIND_PGVECTOR_HNSW_M": "16", + "WAVEMIND_PGVECTOR_HNSW_EF_CONSTRUCTION": "256", + "WAVEMIND_PGVECTOR_EF_SEARCH": "800", + "WAVEMIND_PGVECTOR_QUERY_ROUTING": "namespace", + "WAVEMIND_PGVECTOR_PREWARM_INDEX": "1", "WAVEMIND_PGVECTOR_WARMUP_QUERIES": "100" }, "required_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "module_requirements": { "psycopg": true @@ -394,7 +401,7 @@ "disk_free_gb": 0.0, "runner_storage_root": "state/production-runs", "disk_free_path": "C:/work/particles/wavemind 2.0/state", - "index_mode": "remote PostgreSQL/pgvector HNSW service", + "index_mode": "remote namespace-sharded PostgreSQL/pgvector HNSW services", "slo_capacity_envelope": { "engine": "pgvector-service", "status": "scale_required", @@ -439,7 +446,7 @@ "monthly_total_cost_at_target_qps_usd": 1462.3841857910156 }, "blockers": [ - "missing_env:WAVEMIND_PGVECTOR_DSN", + "missing_env:WAVEMIND_PGVECTOR_DSNS", "insufficient_local_disk_for_index_and_transient_batches" ], "actions": [ @@ -492,7 +499,7 @@ "WAVEMIND_FAISS_IVFPQ_PATH" ], "module_requirements": { - "faiss": true + "faiss": false }, "estimated_index_gb": 1.12, "estimated_transient_runner_gb": 0.573, @@ -549,6 +556,7 @@ }, "blockers": [ "missing_env:WAVEMIND_FAISS_IVFPQ_PATH", + "missing_module:faiss", "insufficient_local_disk_for_index_and_transient_batches" ], "actions": [ diff --git a/benchmarks/production_streaming_load_benchmark.py b/benchmarks/production_streaming_load_benchmark.py index 7acc49f..49b4fd1 100644 --- a/benchmarks/production_streaming_load_benchmark.py +++ b/benchmarks/production_streaming_load_benchmark.py @@ -283,6 +283,42 @@ def _load_faiss_ivfpq_checkpoint( return payload +def _load_pgvector_checkpoint( + path: Path | None, + signature: dict[str, Any], +) -> dict[str, Any]: + try: + return _load_checkpoint(path, signature) + except ValueError: + if path is None or not path.exists(): + raise + payload = json.loads(path.read_text(encoding="utf-8")) + payload_signature = dict(payload.get("signature", {})) + payload_extra = dict(payload_signature.get("extra", {})) + migrated_keys = [] + for key in ( + "create_hnsw", + "hnsw_m", + "hnsw_ef_construction", + "exact", + "iterative_scan", + "index_type", + "ivfflat_lists", + ): + if key in payload_extra: + migrated_keys.append(key) + payload_extra.pop(key) + payload_signature["extra"] = payload_extra + if payload.get("schema") != CHECKPOINT_SCHEMA or payload_signature != signature: + raise + payload["signature"] = signature + payload.setdefault("metadata", {})["signature_migrated_index_keys"] = migrated_keys + payload.setdefault("completed_batch_starts", []) + payload.setdefault("source_vectors", {}) + _write_checkpoint(path, payload) + return payload + + def _write_checkpoint(path: Path | None, payload: dict[str, Any]) -> None: if path is None: return @@ -615,6 +651,24 @@ def _wait_for_qdrant_index_ready( def _pgvector_config_from_env() -> dict[str, Any]: + storage_type = os.environ.get("WAVEMIND_PGVECTOR_STORAGE_TYPE", "vector").strip().lower() + if storage_type not in {"vector", "halfvec"}: + raise ValueError("WAVEMIND_PGVECTOR_STORAGE_TYPE must be vector or halfvec") + insert_mode = os.environ.get("WAVEMIND_PGVECTOR_INSERT_MODE", "copy").strip().lower() + if insert_mode not in {"copy", "upsert"}: + raise ValueError("WAVEMIND_PGVECTOR_INSERT_MODE must be copy or upsert") + index_type = os.environ.get("WAVEMIND_PGVECTOR_INDEX_TYPE", "hnsw").strip().lower() + if index_type not in {"hnsw", "hnsw-binary", "ivfflat"}: + raise ValueError( + "WAVEMIND_PGVECTOR_INDEX_TYPE must be hnsw, hnsw-binary, or ivfflat" + ) + query_routing = os.environ.get( + "WAVEMIND_PGVECTOR_QUERY_ROUTING", "fanout" + ).strip().lower() + if query_routing not in {"fanout", "namespace"}: + raise ValueError( + "WAVEMIND_PGVECTOR_QUERY_ROUTING must be fanout or namespace" + ) return { "table": _safe_identifier( os.environ.get("WAVEMIND_PGVECTOR_TABLE", "wavemind_streaming_vectors"), @@ -622,18 +676,73 @@ def _pgvector_config_from_env() -> dict[str, Any]: ), "collection": os.environ.get("WAVEMIND_PGVECTOR_COLLECTION") or f"streaming_load_{time.time_ns()}", + "storage_type": storage_type, + "insert_mode": insert_mode, + "index_type": index_type, + "query_routing": query_routing, "create_hnsw": _bool_env("WAVEMIND_PGVECTOR_CREATE_HNSW", True), "hnsw_m": _optional_int_env("WAVEMIND_PGVECTOR_HNSW_M"), "hnsw_ef_construction": _optional_int_env("WAVEMIND_PGVECTOR_HNSW_EF_CONSTRUCTION"), "ef_search": _optional_int_env("WAVEMIND_PGVECTOR_EF_SEARCH"), + "ivfflat_lists": _optional_int_env("WAVEMIND_PGVECTOR_IVFFLAT_LISTS"), + "ivfflat_probes": _optional_int_env("WAVEMIND_PGVECTOR_IVFFLAT_PROBES"), + "binary_candidates": _optional_int_env("WAVEMIND_PGVECTOR_BINARY_CANDIDATES"), "exact": _bool_env("WAVEMIND_PGVECTOR_EXACT", False), "iterative_scan": os.environ.get("WAVEMIND_PGVECTOR_ITERATIVE_SCAN"), "max_scan_tuples": _optional_int_env("WAVEMIND_PGVECTOR_MAX_SCAN_TUPLES"), "scan_mem_multiplier": _optional_int_env("WAVEMIND_PGVECTOR_SCAN_MEM_MULTIPLIER"), + "prewarm_index": _bool_env("WAVEMIND_PGVECTOR_PREWARM_INDEX", False), "keep_collection": _bool_env("WAVEMIND_PGVECTOR_KEEP_COLLECTION", False), } +def _pgvector_operator_class(storage_type: str) -> str: + return "halfvec_cosine_ops" if storage_type == "halfvec" else "vector_cosine_ops" + + +def _pgvector_insert_batch( + cursor: Any, + *, + table: str, + collection: str, + ids: np.ndarray, + vectors: np.ndarray, + storage_type: str, + insert_mode: str, +) -> None: + if len(ids) == 0: + return + if insert_mode == "copy": + # A missing checkpoint after a committed COPY is safe to resume: the + # uncheckpointed id range is replaced before it is copied again. + cursor.execute( + f"DELETE FROM {table} WHERE collection = %s AND memory_id BETWEEN %s AND %s", + (collection, int(ids[0]), int(ids[-1])), + ) + with cursor.copy( + f"COPY {table} (collection, memory_id, embedding) FROM STDIN" + ) as copy: + for memory_id, vector in zip(ids, vectors): + copy.write_row( + (collection, int(memory_id), _vector_literal(vector)) + ) + return + + insert_sql = ( + f"INSERT INTO {table} (collection, memory_id, embedding, updated_at) " + f"VALUES (%s, %s, %s::{storage_type}, now()) " + f"ON CONFLICT (collection, memory_id) " + f"DO UPDATE SET embedding = EXCLUDED.embedding, updated_at = now()" + ) + cursor.executemany( + insert_sql, + [ + (collection, int(memory_id), _vector_literal(vector)) + for memory_id, vector in zip(ids, vectors) + ], + ) + + def _qdrant_collection_config_from_env() -> dict[str, Any]: hnsw = _without_none( { @@ -1001,13 +1110,26 @@ def _streaming_plan_row( } elif key in {"pgvector", "pgvector-service", "pgvector-streaming"}: module_requirements = ["psycopg"] - required_env = ["WAVEMIND_PGVECTOR_DSN"] + required_env = ["WAVEMIND_PGVECTOR_DSNS"] index_bytes = 0 - index_mode = "remote PostgreSQL/pgvector storage; local runner stores only generated batches" + index_mode = ( + "modulo-sharded PostgreSQL/pgvector services; local runner stores only " + "generated batches" + ) command_env = { - "WAVEMIND_PGVECTOR_DSN": "postgresql://user:password@postgres.example:5432/wavemind", + "WAVEMIND_PGVECTOR_DSNS": ",".join( + f"postgresql://user:password@postgres-{index}.example:5432/wavemind" + for index in range(4) + ), "WAVEMIND_PGVECTOR_CREATE_HNSW": "1", - "WAVEMIND_PGVECTOR_EF_SEARCH": "1000", + "WAVEMIND_PGVECTOR_STORAGE_TYPE": "halfvec", + "WAVEMIND_PGVECTOR_INSERT_MODE": "copy", + "WAVEMIND_PGVECTOR_INDEX_TYPE": "hnsw", + "WAVEMIND_PGVECTOR_HNSW_M": "16", + "WAVEMIND_PGVECTOR_HNSW_EF_CONSTRUCTION": "256", + "WAVEMIND_PGVECTOR_EF_SEARCH": "800", + "WAVEMIND_PGVECTOR_QUERY_ROUTING": "namespace", + "WAVEMIND_PGVECTOR_PREWARM_INDEX": "1", } else: raise ValueError(f"Unknown engine: {engine}") @@ -2409,10 +2531,19 @@ def run_pgvector_streaming( noise: float, batch_size: int, ) -> dict[str, Any]: + shard_dsns = _split_env_list(os.environ.get("WAVEMIND_PGVECTOR_DSNS")) dsn = os.environ.get("WAVEMIND_PGVECTOR_DSN") engine = "WaveMind pgvector streaming" - if not dsn: - return skipped_result(engine, "Set WAVEMIND_PGVECTOR_DSN to run streaming pgvector") + if not dsn and not shard_dsns: + return skipped_result( + engine, + "Set WAVEMIND_PGVECTOR_DSN or WAVEMIND_PGVECTOR_DSNS to run streaming pgvector", + ) + if shard_dsns and len(shard_dsns) < 2: + return skipped_result( + engine, + "WAVEMIND_PGVECTOR_DSNS must contain at least two service DSNs", + ) try: import psycopg except ImportError as exc: @@ -2428,8 +2559,40 @@ def run_pgvector_streaming( engine, "WAVEMIND_PGVECTOR_ITERATIVE_SCAN must be strict_order, relaxed_order, or off", ) + if shard_dsns: + return _run_pgvector_sharded_streaming( + psycopg=psycopg, + dsns=shard_dsns, + config=config, + count=count, + dim=dim, + query_count=query_count, + top_k=top_k, + seed=seed, + noise=noise, + batch_size=batch_size, + ) table = str(config["table"]) + storage_type = str(config["storage_type"]) + insert_mode = str(config["insert_mode"]) + index_type = str(config["index_type"]) + ivfflat_lists = config["ivfflat_lists"] or max(1, int(round(math.sqrt(count)))) + ivfflat_probes = config["ivfflat_probes"] or max(1, int(round(math.sqrt(ivfflat_lists)))) + binary_candidates = config["binary_candidates"] or max(100, int(top_k) * 20) + if ivfflat_lists <= 0: + return skipped_result(engine, "WAVEMIND_PGVECTOR_IVFFLAT_LISTS must be positive") + if ivfflat_probes <= 0 or ivfflat_probes > ivfflat_lists: + return skipped_result( + engine, + "WAVEMIND_PGVECTOR_IVFFLAT_PROBES must be positive and no greater than lists", + ) + if binary_candidates < int(top_k): + return skipped_result( + engine, + "WAVEMIND_PGVECTOR_BINARY_CANDIDATES must be at least top_k", + ) + index_name = f"{table}_embedding_{index_type.replace('-', '_')}_idx" checkpoint_path = _checkpoint_path_from_env() signature = _checkpoint_signature( engine=engine, @@ -2442,15 +2605,12 @@ def run_pgvector_streaming( batch_size=batch_size, extra={ "table": table, - "create_hnsw": bool(config["create_hnsw"]), - "hnsw_m": config["hnsw_m"], - "hnsw_ef_construction": config["hnsw_ef_construction"], - "exact": bool(config["exact"]), - "iterative_scan": iterative_scan, + "storage_type": storage_type, + "insert_mode": insert_mode, }, ) try: - checkpoint = _load_checkpoint(checkpoint_path, signature) + checkpoint = _load_pgvector_checkpoint(checkpoint_path, signature) except ValueError as exc: return skipped_result(engine, str(exc)) checkpoint_metadata = checkpoint.setdefault("metadata", {}) @@ -2467,6 +2627,12 @@ def run_pgvector_streaming( completed_batches = _checkpoint_completed_batches(checkpoint) source_ids = choose_source_ids(count, query_count, seed) source_vectors: dict[int, np.ndarray] = _checkpoint_source_vectors(checkpoint) + complete_resume = _checkpoint_complete_for_run( + checkpoint, + count=count, + batch_size=batch_size, + source_ids=source_ids, + ) conn = psycopg.connect(dsn, autocommit=True) try: started = time.perf_counter() @@ -2476,65 +2642,132 @@ def run_pgvector_streaming( CREATE TABLE IF NOT EXISTS {table} ( collection TEXT NOT NULL, memory_id BIGINT NOT NULL, - embedding vector({int(dim)}) NOT NULL, + embedding {storage_type}({int(dim)}) NOT NULL, updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), PRIMARY KEY (collection, memory_id) ) """ ) conn.execute(f"CREATE INDEX IF NOT EXISTS {table}_collection_idx ON {table} (collection)") + column_type = conn.execute( + """ + SELECT format_type(attribute.atttypid, attribute.atttypmod) + FROM pg_attribute AS attribute + WHERE attribute.attrelid = %s::regclass + AND attribute.attname = 'embedding' + AND NOT attribute.attisdropped + """, + (table,), + ).fetchone() + expected_column_type = f"{storage_type}({int(dim)})" + if not column_type or str(column_type[0]).lower() != expected_column_type: + actual = None if not column_type else str(column_type[0]) + raise RuntimeError( + f"pgvector table {table!r} embedding type is {actual!r}; " + f"expected {expected_column_type!r}" + ) if not completed_batches: conn.execute(f"DELETE FROM {table} WHERE collection = %s", (collection,)) - insert_sql = ( - f"INSERT INTO {table} (collection, memory_id, embedding, updated_at) " - f"VALUES (%s, %s, %s::vector, now()) " - f"ON CONFLICT (collection, memory_id) " - f"DO UPDATE SET embedding = EXCLUDED.embedding, updated_at = now()" - ) - with conn.cursor() as cur: - for ids, vectors, captured in iter_vector_batches( - count=count, - dim=dim, - seed=seed + count, - batch_size=batch_size, - source_ids=source_ids, - ): - batch_start = int(ids[0]) if len(ids) else 0 - if batch_start not in completed_batches: - rows = [ - (collection, int(id), _vector_literal(vector)) - for id, vector in zip(ids, vectors) - ] - cur.executemany(insert_sql, rows) - source_vectors.update(captured) - _record_checkpoint_batch( - path=checkpoint_path, - payload=checkpoint, - batch_start=batch_start, - captured=captured, - ) - completed_batches.add(batch_start) - if config["create_hnsw"]: - options = [] - if config["hnsw_m"] is not None: - options.append(f"m = {int(config['hnsw_m'])}") - if config["hnsw_ef_construction"] is not None: - options.append(f"ef_construction = {int(config['hnsw_ef_construction'])}") - with_options = f" WITH ({', '.join(options)})" if options else "" + if not complete_resume: + with conn.cursor() as cur: + for ids, vectors, captured in iter_vector_batches( + count=count, + dim=dim, + seed=seed + count, + batch_size=batch_size, + source_ids=source_ids, + ): + batch_start = int(ids[0]) if len(ids) else 0 + if batch_start not in completed_batches: + _pgvector_insert_batch( + cur, + table=table, + collection=collection, + ids=ids, + vectors=vectors, + storage_type=storage_type, + insert_mode=insert_mode, + ) + source_vectors.update(captured) + _record_checkpoint_batch( + path=checkpoint_path, + payload=checkpoint, + batch_start=batch_start, + captured=captured, + ) + completed_batches.add(batch_start) + + remote_row_count = int( conn.execute( - f"CREATE INDEX IF NOT EXISTS {table}_embedding_hnsw_idx " - f"ON {table} USING hnsw (embedding vector_cosine_ops)" - f"{with_options}" + f"SELECT count(*) FROM {table} WHERE collection = %s", + (collection,), + ).fetchone()[0] + ) + if remote_row_count != int(count): + raise RuntimeError( + f"pgvector collection {collection!r} contains {remote_row_count} rows; " + f"expected {int(count)}" ) + checkpoint_metadata["remote_row_count"] = remote_row_count + checkpoint_metadata["storage_type"] = storage_type + checkpoint_metadata["insert_mode"] = insert_mode + _write_checkpoint(checkpoint_path, checkpoint) + if config["create_hnsw"]: + if index_type in {"hnsw", "hnsw-binary"}: + options = [] + if config["hnsw_m"] is not None: + options.append(f"m = {int(config['hnsw_m'])}") + if config["hnsw_ef_construction"] is not None: + options.append(f"ef_construction = {int(config['hnsw_ef_construction'])}") + with_options = f" WITH ({', '.join(options)})" if options else "" + if index_type == "hnsw-binary": + conn.execute( + f"CREATE INDEX IF NOT EXISTS {index_name} " + f"ON {table} USING hnsw " + f"((binary_quantize(embedding)::bit({int(dim)})) bit_hamming_ops)" + f"{with_options}" + ) + else: + conn.execute( + f"CREATE INDEX IF NOT EXISTS {index_name} " + f"ON {table} USING hnsw (embedding {_pgvector_operator_class(storage_type)})" + f"{with_options}" + ) + else: + conn.execute( + f"CREATE INDEX IF NOT EXISTS {index_name} " + f"ON {table} USING ivfflat (embedding {_pgvector_operator_class(storage_type)}) " + f"WITH (lists = {int(ivfflat_lists)})" + ) + index_present = bool( + conn.execute("SELECT to_regclass(%s)", (index_name,)).fetchone()[0] + ) + if config["create_hnsw"] and not index_present: + raise RuntimeError(f"pgvector {index_type} index {index_name!r} is missing") + checkpoint_metadata["index_name"] = index_name + checkpoint_metadata["index_type"] = index_type + checkpoint_metadata["index_present"] = index_present + _write_checkpoint(checkpoint_path, checkpoint) conn.execute(f"ANALYZE {table}") + prewarm_blocks = 0 + if config["prewarm_index"] and index_present: + conn.execute("CREATE EXTENSION IF NOT EXISTS pg_prewarm") + prewarm_blocks = int( + conn.execute( + "SELECT pg_prewarm(%s, 'buffer')", + (index_name,), + ).fetchone()[0] + ) wait_after_build_seconds = float(os.environ.get("WAVEMIND_PGVECTOR_WAIT_AFTER_BUILD_SECONDS", "0")) if wait_after_build_seconds > 0: time.sleep(wait_after_build_seconds) build_ms = (time.perf_counter() - started) * 1000.0 queries = make_queries(source_ids=source_ids, source_vectors=source_vectors, seed=seed + count, noise=noise) - if config["ef_search"] is not None: + if index_type in {"hnsw", "hnsw-binary"} and config["ef_search"] is not None: conn.execute(f"SET hnsw.ef_search = {int(config['ef_search'])}") + if index_type == "ivfflat": + conn.execute(f"SET ivfflat.probes = {int(ivfflat_probes)}") if iterative_scan: conn.execute(f"SET hnsw.iterative_scan = '{iterative_scan}'") if config["max_scan_tuples"] is not None: @@ -2546,20 +2779,40 @@ def run_pgvector_streaming( conn.execute("SET enable_bitmapscan = off") warmup_queries = int(os.environ.get("WAVEMIND_PGVECTOR_WARMUP_QUERIES", "0")) - search_sql = ( - f"SELECT memory_id FROM {table} " - f"WHERE collection = %s " - f"ORDER BY embedding <=> %s::vector " - f"LIMIT %s" - ) + if index_type == "hnsw-binary": + search_sql = ( + "SELECT memory_id FROM (" + f"SELECT memory_id, embedding FROM {table} " + f"WHERE collection = %s " + f"ORDER BY binary_quantize(embedding)::bit({int(dim)}) " + f"<~> binary_quantize(%s::{storage_type})::bit({int(dim)}) " + f"LIMIT {int(binary_candidates)}" + ") AS candidates " + f"ORDER BY embedding <=> %s::{storage_type} " + f"LIMIT {int(top_k)}" + ) + + def search_params(query: np.ndarray) -> tuple[Any, ...]: + literal = _vector_literal(query) + return (collection, literal, literal) + else: + search_sql = ( + f"SELECT memory_id FROM {table} " + f"WHERE collection = %s " + f"ORDER BY embedding <=> %s::{storage_type} " + f"LIMIT %s" + ) + + def search_params(query: np.ndarray) -> tuple[Any, ...]: + return (collection, _vector_literal(query), int(top_k)) if warmup_queries > 0 and queries: for index in range(warmup_queries): _, query = queries[index % len(queries)] - conn.execute(search_sql, (collection, _vector_literal(query), int(top_k))).fetchall() + conn.execute(search_sql, search_params(query)).fetchall() rows: list[dict[str, Any]] = [] for source_id, query in queries: started = time.perf_counter() - hits = conn.execute(search_sql, (collection, _vector_literal(query), int(top_k))).fetchall() + hits = conn.execute(search_sql, search_params(query)).fetchall() latency_ms = (time.perf_counter() - started) * 1000.0 top = [int(row[0]) for row in hits] rows.append( @@ -2581,10 +2834,21 @@ def run_pgvector_streaming( extra={ "table": table, "collection": collection, + "storage_type": storage_type, + "insert_mode": insert_mode, + "index_type": index_type, + "remote_row_count": remote_row_count, + "complete_resume": complete_resume, + "index_name": index_name, + "index_present": index_present, + "prewarm_index": bool(config["prewarm_index"]), + "prewarm_blocks": prewarm_blocks, "warmup_queries": warmup_queries, "wait_after_build_seconds": wait_after_build_seconds, "search_params": { "hnsw_ef": config["ef_search"], + "ivfflat_probes": ivfflat_probes if index_type == "ivfflat" else None, + "binary_candidates": binary_candidates if index_type == "hnsw-binary" else None, "exact": config["exact"], "iterative_scan": iterative_scan, "max_scan_tuples": config["max_scan_tuples"], @@ -2594,6 +2858,8 @@ def run_pgvector_streaming( "create_hnsw": config["create_hnsw"], "hnsw_m": config["hnsw_m"], "hnsw_ef_construction": config["hnsw_ef_construction"], + "index_type": index_type, + "ivfflat_lists": ivfflat_lists if index_type == "ivfflat" else None, }, "memory_mode": "streaming PostgreSQL insert; query source vectors only", **_checkpoint_extra(checkpoint_path, checkpoint, completed_batches), @@ -2607,6 +2873,489 @@ def run_pgvector_streaming( conn.close() +def _pgvector_shard_expected_count(count: int, shard_count: int, shard_index: int) -> int: + return max(0, (int(count) + int(shard_count) - 1 - int(shard_index)) // int(shard_count)) + + +def _pgvector_benchmark_namespace_shard(source_id: int, shard_count: int) -> int: + if int(source_id) <= 0: + raise ValueError("source_id must be positive") + if int(shard_count) <= 0: + raise ValueError("shard_count must be positive") + return (int(source_id) - 1) % int(shard_count) + + +def _run_pgvector_sharded_streaming( + *, + psycopg: Any, + dsns: list[str], + config: dict[str, Any], + count: int, + dim: int, + query_count: int, + top_k: int, + seed: int, + noise: float, + batch_size: int, +) -> dict[str, Any]: + engine = "WaveMind pgvector streaming" + shard_count = len(dsns) + table = str(config["table"]) + storage_type = str(config["storage_type"]) + insert_mode = str(config["insert_mode"]) + index_type = str(config["index_type"]) + per_shard_count = max( + _pgvector_shard_expected_count(count, shard_count, index) + for index in range(shard_count) + ) + ivfflat_lists = config["ivfflat_lists"] or max( + 1, int(round(math.sqrt(per_shard_count))) + ) + ivfflat_probes = config["ivfflat_probes"] or max( + 1, int(round(math.sqrt(ivfflat_lists))) + ) + binary_candidates = config["binary_candidates"] or max(100, int(top_k) * 20) + if ivfflat_lists <= 0: + return skipped_result(engine, "WAVEMIND_PGVECTOR_IVFFLAT_LISTS must be positive") + if ivfflat_probes <= 0 or ivfflat_probes > ivfflat_lists: + return skipped_result( + engine, + "WAVEMIND_PGVECTOR_IVFFLAT_PROBES must be positive and no greater than lists", + ) + if binary_candidates < int(top_k): + return skipped_result( + engine, + "WAVEMIND_PGVECTOR_BINARY_CANDIDATES must be at least top_k", + ) + iterative_scan = config.get("iterative_scan") + query_routing = str(config["query_routing"]) + index_name = f"{table}_embedding_{index_type.replace('-', '_')}_idx" + checkpoint_path = _checkpoint_path_from_env() + signature = _checkpoint_signature( + engine=engine, + count=count, + dim=dim, + query_count=query_count, + top_k=top_k, + seed=seed, + noise=noise, + batch_size=batch_size, + extra={ + "table": table, + "storage_type": storage_type, + "insert_mode": insert_mode, + "shard_count": shard_count, + "shard_layout": os.environ.get( + "WAVEMIND_PGVECTOR_SHARD_LAYOUT_ID", f"modulo-{shard_count}" + ), + }, + ) + try: + checkpoint = _load_pgvector_checkpoint(checkpoint_path, signature) + except ValueError as exc: + return skipped_result(engine, str(exc)) + checkpoint_metadata = checkpoint.setdefault("metadata", {}) + configured_collection = os.environ.get("WAVEMIND_PGVECTOR_COLLECTION") + checkpoint_collection = checkpoint_metadata.get("collection") + if configured_collection and checkpoint_collection and configured_collection != checkpoint_collection: + return skipped_result( + engine, + "WAVEMIND_PGVECTOR_COLLECTION does not match checkpoint collection", + ) + collection = configured_collection or checkpoint_collection or str(config["collection"]) + checkpoint_metadata["collection"] = collection + checkpoint_metadata["shard_count"] = shard_count + _write_checkpoint(checkpoint_path, checkpoint) + completed_batches = _checkpoint_completed_batches(checkpoint) + source_ids = choose_source_ids(count, query_count, seed) + source_vectors: dict[int, np.ndarray] = _checkpoint_source_vectors(checkpoint) + complete_resume = _checkpoint_complete_for_run( + checkpoint, + count=count, + batch_size=batch_size, + source_ids=source_ids, + ) + connections = [psycopg.connect(dsn, autocommit=True) for dsn in dsns] + cursors: list[Any] = [] + expected_counts = [ + _pgvector_shard_expected_count(count, shard_count, index) + for index in range(shard_count) + ] + try: + started = time.perf_counter() + expected_column_type = f"{storage_type}({int(dim)})" + for connection in connections: + connection.execute("CREATE EXTENSION IF NOT EXISTS vector") + connection.execute( + f""" + CREATE TABLE IF NOT EXISTS {table} ( + collection TEXT NOT NULL, + memory_id BIGINT NOT NULL, + embedding {storage_type}({int(dim)}) NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (collection, memory_id) + ) + """ + ) + connection.execute( + f"CREATE INDEX IF NOT EXISTS {table}_collection_idx ON {table} (collection)" + ) + column_type = connection.execute( + """ + SELECT format_type(attribute.atttypid, attribute.atttypmod) + FROM pg_attribute AS attribute + WHERE attribute.attrelid = %s::regclass + AND attribute.attname = 'embedding' + AND NOT attribute.attisdropped + """, + (table,), + ).fetchone() + if not column_type or str(column_type[0]).lower() != expected_column_type: + actual = None if not column_type else str(column_type[0]) + raise RuntimeError( + f"pgvector table {table!r} embedding type is {actual!r}; " + f"expected {expected_column_type!r}" + ) + if not completed_batches: + connection.execute( + f"DELETE FROM {table} WHERE collection = %s", (collection,) + ) + cursors.append(connection.cursor()) + + if not complete_resume: + for ids, vectors, captured in iter_vector_batches( + count=count, + dim=dim, + seed=seed + count, + batch_size=batch_size, + source_ids=source_ids, + ): + batch_start = int(ids[0]) if len(ids) else 0 + if batch_start not in completed_batches: + shard_ids = (ids - 1) % shard_count + for shard_index, cursor in enumerate(cursors): + mask = shard_ids == shard_index + _pgvector_insert_batch( + cursor, + table=table, + collection=collection, + ids=ids[mask], + vectors=vectors[mask], + storage_type=storage_type, + insert_mode=insert_mode, + ) + source_vectors.update(captured) + _record_checkpoint_batch( + path=checkpoint_path, + payload=checkpoint, + batch_start=batch_start, + captured=captured, + ) + completed_batches.add(batch_start) + + row_counts: list[int] = [] + misplaced_rows: list[int] = [] + for shard_index, connection in enumerate(connections): + row_count, misplaced = connection.execute( + f""" + SELECT count(*), count(*) FILTER ( + WHERE mod(memory_id - 1, %s) <> %s + ) + FROM {table} + WHERE collection = %s + """, + (shard_count, shard_index, collection), + ).fetchone() + row_counts.append(int(row_count)) + misplaced_rows.append(int(misplaced)) + if row_counts != expected_counts or any(misplaced_rows): + raise RuntimeError( + "pgvector shard validation failed: " + f"rows={row_counts}, expected={expected_counts}, misplaced={misplaced_rows}" + ) + checkpoint_metadata["remote_row_count"] = sum(row_counts) + checkpoint_metadata["shard_row_counts"] = row_counts + checkpoint_metadata["storage_type"] = storage_type + checkpoint_metadata["insert_mode"] = insert_mode + _write_checkpoint(checkpoint_path, checkpoint) + + index_build_workers = min( + shard_count, + _positive_int_env("WAVEMIND_PGVECTOR_INDEX_BUILD_WORKERS", 1), + ) + + def prepare_shard_index(item: tuple[int, Any]) -> tuple[bool, int]: + shard_index, connection = item + print( + f"pgvector shard {shard_index + 1}/{shard_count}: building {index_type} index", + flush=True, + ) + if config["create_hnsw"]: + if index_type in {"hnsw", "hnsw-binary"}: + options = [] + if config["hnsw_m"] is not None: + options.append(f"m = {int(config['hnsw_m'])}") + if config["hnsw_ef_construction"] is not None: + options.append( + f"ef_construction = {int(config['hnsw_ef_construction'])}" + ) + with_options = f" WITH ({', '.join(options)})" if options else "" + if index_type == "hnsw-binary": + connection.execute( + f"CREATE INDEX IF NOT EXISTS {index_name} " + f"ON {table} USING hnsw " + f"((binary_quantize(embedding)::bit({int(dim)})) bit_hamming_ops)" + f"{with_options}" + ) + else: + connection.execute( + f"CREATE INDEX IF NOT EXISTS {index_name} " + f"ON {table} USING hnsw " + f"(embedding {_pgvector_operator_class(storage_type)})" + f"{with_options}" + ) + else: + connection.execute( + f"CREATE INDEX IF NOT EXISTS {index_name} " + f"ON {table} USING ivfflat " + f"(embedding {_pgvector_operator_class(storage_type)}) " + f"WITH (lists = {int(ivfflat_lists)})" + ) + present = bool( + connection.execute("SELECT to_regclass(%s)", (index_name,)).fetchone()[0] + ) + if config["create_hnsw"] and not present: + raise RuntimeError( + f"pgvector {index_type} index {index_name!r} is missing" + ) + connection.execute(f"ANALYZE {table}") + blocks = 0 + if config["prewarm_index"] and present: + connection.execute("CREATE EXTENSION IF NOT EXISTS pg_prewarm") + blocks = int( + connection.execute( + "SELECT pg_prewarm(%s, 'buffer')", (index_name,) + ).fetchone()[0] + ) + print( + f"pgvector shard {shard_index + 1}/{shard_count}: index ready; " + f"prewarmed_blocks={blocks}", + flush=True, + ) + return present, blocks + + if index_build_workers == 1: + prepared_indexes = [ + prepare_shard_index(item) for item in enumerate(connections) + ] + else: + with concurrent.futures.ThreadPoolExecutor( + max_workers=index_build_workers + ) as index_executor: + prepared_indexes = list( + index_executor.map(prepare_shard_index, enumerate(connections)) + ) + index_present = [present for present, _ in prepared_indexes] + prewarm_blocks = [blocks for _, blocks in prepared_indexes] + checkpoint_metadata["index_name"] = index_name + checkpoint_metadata["index_type"] = index_type + checkpoint_metadata["index_present"] = all(index_present) + checkpoint_metadata["index_build_workers"] = index_build_workers + _write_checkpoint(checkpoint_path, checkpoint) + + wait_after_build_seconds = float( + os.environ.get("WAVEMIND_PGVECTOR_WAIT_AFTER_BUILD_SECONDS", "0") + ) + if wait_after_build_seconds > 0: + time.sleep(wait_after_build_seconds) + build_ms = (time.perf_counter() - started) * 1000.0 + for connection in connections: + if index_type in {"hnsw", "hnsw-binary"} and config["ef_search"] is not None: + connection.execute(f"SET hnsw.ef_search = {int(config['ef_search'])}") + if index_type == "ivfflat": + connection.execute(f"SET ivfflat.probes = {int(ivfflat_probes)}") + if iterative_scan: + setting = "ivfflat.iterative_scan" if index_type == "ivfflat" else "hnsw.iterative_scan" + connection.execute(f"SET {setting} = '{iterative_scan}'") + if config["max_scan_tuples"] is not None and index_type != "ivfflat": + connection.execute( + f"SET hnsw.max_scan_tuples = {int(config['max_scan_tuples'])}" + ) + if config["scan_mem_multiplier"] is not None and index_type != "ivfflat": + connection.execute( + f"SET hnsw.scan_mem_multiplier = {int(config['scan_mem_multiplier'])}" + ) + if config["exact"]: + connection.execute("SET enable_indexscan = off") + connection.execute("SET enable_bitmapscan = off") + + if index_type == "hnsw-binary": + search_sql = ( + f"SELECT memory_id, embedding <=> %s::{storage_type} AS distance FROM (" + f"SELECT memory_id, embedding FROM {table} WHERE collection = %s " + f"ORDER BY binary_quantize(embedding)::bit({int(dim)}) " + f"<~> binary_quantize(%s::{storage_type})::bit({int(dim)}) " + f"LIMIT {int(binary_candidates)}) AS candidates " + f"ORDER BY distance LIMIT {int(top_k)}" + ) + + def params_for(query: np.ndarray) -> tuple[Any, ...]: + literal = _vector_literal(query) + return (literal, collection, literal) + else: + search_sql = ( + f"SELECT memory_id, embedding <=> %s::{storage_type} AS distance " + f"FROM {table} WHERE collection = %s " + f"ORDER BY embedding <=> %s::{storage_type} LIMIT {int(top_k)}" + ) + + def params_for(query: np.ndarray) -> tuple[Any, ...]: + literal = _vector_literal(query) + return (literal, collection, literal) + + fanout_workers = min( + shard_count, + _positive_int_env("WAVEMIND_PGVECTOR_FANOUT_WORKERS", shard_count), + ) + query_workers = fanout_workers if query_routing == "fanout" else 1 + + def search_shard(shard_index: int, query: np.ndarray) -> list[tuple[int, float]]: + rows = connections[shard_index].execute( + search_sql, params_for(query) + ).fetchall() + return [(int(row[0]), float(row[1])) for row in rows] + + def fanout_search( + executor: concurrent.futures.ThreadPoolExecutor, + query: np.ndarray, + ) -> list[int]: + futures = [ + executor.submit(search_shard, shard_index, query) + for shard_index in range(shard_count) + ] + merged = [hit for future in futures for hit in future.result()] + merged.sort(key=lambda item: (item[1], item[0])) + return [memory_id for memory_id, _ in merged[:top_k]] + + def routed_search( + executor: concurrent.futures.ThreadPoolExecutor, + source_id: int, + query: np.ndarray, + ) -> list[int]: + if query_routing == "namespace": + shard_index = _pgvector_benchmark_namespace_shard( + source_id, shard_count + ) + return [ + memory_id + for memory_id, _ in search_shard(shard_index, query)[:top_k] + ] + return fanout_search(executor, query) + + queries = make_queries( + source_ids=source_ids, + source_vectors=source_vectors, + seed=seed + count, + noise=noise, + ) + warmup_queries = int(os.environ.get("WAVEMIND_PGVECTOR_WARMUP_QUERIES", "0")) + measured: list[dict[str, Any]] = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=query_workers) as executor: + if warmup_queries > 0 and queries: + for index in range(warmup_queries): + source_id, query = queries[index % len(queries)] + routed_search(executor, source_id, query) + for source_id, query in queries: + query_started = time.perf_counter() + top = routed_search(executor, source_id, query) + measured.append( + { + "source_id": int(source_id), + "target_hit": int(source_id) in top, + "target_hit_at_1": bool(top and top[0] == int(source_id)), + "latency_ms": (time.perf_counter() - query_started) * 1000.0, + } + ) + return _metrics_from_hits( + engine=engine, + vector_count=count, + dim=dim, + batch_size=batch_size, + top_k=top_k, + build_ms=build_ms, + query_rows=measured, + extra={ + "table": table, + "collection": collection, + "storage_type": storage_type, + "insert_mode": insert_mode, + "index_type": index_type, + "remote_row_count": sum(row_counts), + "complete_resume": complete_resume, + "index_name": index_name, + "index_present": all(index_present), + "prewarm_index": bool(config["prewarm_index"]), + "prewarm_blocks": sum(prewarm_blocks), + "prewarm_blocks_by_shard": prewarm_blocks, + "index_build_workers": index_build_workers, + "warmup_queries": warmup_queries, + "wait_after_build_seconds": wait_after_build_seconds, + "shard_count": shard_count, + "shard_row_counts": row_counts, + "shard_expected_counts": expected_counts, + "shard_misplaced_rows": misplaced_rows, + "query_routing": query_routing, + "evidence_topology": os.environ.get( + "WAVEMIND_PGVECTOR_EVIDENCE_TOPOLOGY", + "configured-service-dsns", + ), + "fanout_workers": fanout_workers if query_routing == "fanout" else 0, + "query_workers": query_workers, + "per_namespace_vector_capacity": per_shard_count, + "global_cross_namespace_search": query_routing == "fanout", + "routing_key_source": ( + "synthetic benchmark namespace label derived from source id " + "modulo shard count" + if query_routing == "namespace" + else "not applicable; all shards queried" + ), + "search_params": { + "hnsw_ef": config["ef_search"], + "ivfflat_probes": ivfflat_probes if index_type == "ivfflat" else None, + "binary_candidates": binary_candidates if index_type == "hnsw-binary" else None, + "exact": config["exact"], + "iterative_scan": iterative_scan, + }, + "collection_params": { + "create_hnsw": config["create_hnsw"], + "hnsw_m": config["hnsw_m"], + "hnsw_ef_construction": config["hnsw_ef_construction"], + "index_type": index_type, + "ivfflat_lists": ivfflat_lists if index_type == "ivfflat" else None, + }, + "memory_mode": ( + "streaming modulo-sharded PostgreSQL insert; namespace-routed " + "single-service query; query source vectors only" + if query_routing == "namespace" + else "streaming modulo-sharded PostgreSQL insert; parallel " + "multi-service fanout query merge; query source vectors only" + ), + **_checkpoint_extra(checkpoint_path, checkpoint, completed_batches), + }, + ) + finally: + for cursor in cursors: + cursor.close() + for connection in connections: + try: + if not config.get("keep_collection") and checkpoint_path is None: + connection.execute( + f"DELETE FROM {table} WHERE collection = %s", (collection,) + ) + finally: + connection.close() + + def run_size( *, count: int, diff --git a/benchmarks/production_streaming_load_pgvector_10m_plan.json b/benchmarks/production_streaming_load_pgvector_10m_plan.json index 7a27f79..50361e4 100644 --- a/benchmarks/production_streaming_load_pgvector_10m_plan.json +++ b/benchmarks/production_streaming_load_pgvector_10m_plan.json @@ -35,7 +35,7 @@ "server_version": "28.3.0" }, "modules": { - "faiss": true, + "faiss": false, "qdrant_client": true, "psycopg": true, "numpy": true @@ -44,12 +44,19 @@ "WAVEMIND_FAISS_PATH": false, "WAVEMIND_QDRANT_URL": false, "WAVEMIND_PGVECTOR_DSN": false, + "WAVEMIND_PGVECTOR_DSNS": false, "WAVEMIND_PGVECTOR_CREATE_HNSW": null, "WAVEMIND_PGVECTOR_EF_SEARCH": null, "WAVEMIND_QDRANT_HNSW_EF": null, "WAVEMIND_QDRANT_EXACT": null, "WAVEMIND_QDRANT_WARMUP_QUERIES": null, "WAVEMIND_QDRANT_WAIT_AFTER_BUILD_SECONDS": null, + "WAVEMIND_QDRANT_INDEX_READY_TIMEOUT_SECONDS": null, + "WAVEMIND_QDRANT_INDEX_READY_POLL_SECONDS": null, + "WAVEMIND_QDRANT_REQUIRE_FULL_INDEX": null, + "WAVEMIND_QDRANT_DEFER_INDEXING": null, + "WAVEMIND_QDRANT_DEFERRED_INDEXING_THRESHOLD_KB": null, + "WAVEMIND_QDRANT_FINAL_INDEXING_THRESHOLD_KB": null, "WAVEMIND_QDRANT_HNSW_M": null, "WAVEMIND_QDRANT_HNSW_EF_CONSTRUCT": null, "WAVEMIND_QDRANT_HNSW_FULL_SCAN_THRESHOLD": null, @@ -61,7 +68,7 @@ }, "disk": { "root": "C:\\", - "free_gb": 0.04, + "free_gb": 9.7, "total_gb": 231.85 } }, @@ -73,7 +80,7 @@ "queries": 2000, "top_k": 10, "batch_size": 5000, - "index_mode": "remote PostgreSQL/pgvector storage; local runner stores only generated batches", + "index_mode": "modulo-sharded PostgreSQL/pgvector services; local runner stores only generated batches", "estimated_index_gb": 0.0, "estimated_transient_runner_gb": 0.003, "estimated_payload_storage_gb": 19.073, @@ -88,22 +95,29 @@ "psycopg": true }, "required_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "command_env": { - "WAVEMIND_PGVECTOR_DSN": "postgresql://user:password@postgres.example:5432/wavemind", + "WAVEMIND_PGVECTOR_DSNS": "postgresql://user:password@postgres-0.example:5432/wavemind,postgresql://user:password@postgres-1.example:5432/wavemind,postgresql://user:password@postgres-2.example:5432/wavemind,postgresql://user:password@postgres-3.example:5432/wavemind", "WAVEMIND_PGVECTOR_CREATE_HNSW": "1", - "WAVEMIND_PGVECTOR_EF_SEARCH": "1000" + "WAVEMIND_PGVECTOR_STORAGE_TYPE": "halfvec", + "WAVEMIND_PGVECTOR_INSERT_MODE": "copy", + "WAVEMIND_PGVECTOR_INDEX_TYPE": "hnsw", + "WAVEMIND_PGVECTOR_HNSW_M": "16", + "WAVEMIND_PGVECTOR_HNSW_EF_CONSTRUCTION": "256", + "WAVEMIND_PGVECTOR_EF_SEARCH": "800", + "WAVEMIND_PGVECTOR_QUERY_ROUTING": "namespace", + "WAVEMIND_PGVECTOR_PREWARM_INDEX": "1" }, "checkpoint_path": "state/production-runs/pgvector-service-10000000.checkpoint.json", "resume_mode": "batch checkpoint; safe to rerun after interrupted ingest", "command": "python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks\\production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json", "status": "action_required", "blockers": [ - "missing_env:WAVEMIND_PGVECTOR_DSN", + "missing_env:WAVEMIND_PGVECTOR_DSNS", "insufficient_local_disk_for_index_and_transient_batches" ], "claim_boundary": "preflight only; this is not a completed latency or recall benchmark" diff --git a/benchmarks/release_claims_results.json b/benchmarks/release_claims_results.json index 81aecfa..72f8f49 100644 --- a/benchmarks/release_claims_results.json +++ b/benchmarks/release_claims_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.release_claims.v1", - "generated_at": "2026-07-11T05:56:52Z", + "generated_at": "2026-07-11T19:56:09Z", "release_status": "core_release_ready", "claim_status": "claims_limited", "summary": { @@ -75,7 +75,7 @@ "preflight_status": "action_required", "artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "command": "python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --seed 42 --noise 0.08 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --output benchmarks/production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json", "claim_unlocked": "10M PostgreSQL/pgvector service candidate-index SLO." diff --git a/benchmarks/scale_gap_results.json b/benchmarks/scale_gap_results.json index 3c52f95..158b580 100644 --- a/benchmarks/scale_gap_results.json +++ b/benchmarks/scale_gap_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.scale_gap.v1", - "generated_at": "2026-07-11T05:56:53Z", + "generated_at": "2026-07-11T19:56:10Z", "overall_status": "action_required", "summary": { "total_profiles": 5, @@ -109,10 +109,10 @@ "output_artifact_exists": false, "checkpoint_path": "state/production-runs/pgvector-service-10000000.checkpoint.json", "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "blockers": [ - "missing_env:WAVEMIND_PGVECTOR_DSN", + "missing_env:WAVEMIND_PGVECTOR_DSNS", "insufficient_local_disk_for_index_and_transient_batches" ], "command": "python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --memory-payload-kb 2.0 --vector-dtype-bytes 4 --output benchmarks/production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json", @@ -152,6 +152,7 @@ ], "blockers": [ "missing_env:WAVEMIND_FAISS_IVFPQ_PATH", + "missing_module:faiss", "insufficient_local_disk_for_index_and_transient_batches" ], "command": "python benchmarks/production_streaming_load_benchmark.py --sizes 50000000 --dim 128 --queries 2000 --top-k 10 --batch-size 1000000 --engines faiss-ivfpq-persisted --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --memory-payload-kb 2.0 --vector-dtype-bytes 4 --output benchmarks/production_streaming_load_ivfpq_50m_results.json --checkpoint-path state/production-runs/faiss-ivfpq-persisted-50000000.checkpoint.json", diff --git a/benchmarks/strict_evidence_readiness_report.py b/benchmarks/strict_evidence_readiness_report.py index 7966dc7..692fb6c 100644 --- a/benchmarks/strict_evidence_readiness_report.py +++ b/benchmarks/strict_evidence_readiness_report.py @@ -213,6 +213,11 @@ def build_strict_evidence_readiness_report(root: Path = PROJECT_ROOT) -> dict[st or "Provision prerequisites, run the safe dispatch command, download the artifact, ingest it, then rerun strict validation." ), } + if row["can_auto_run_now"]: + row["next_action"] = ( + "Run the safe dispatch command now, download the resulting artifact, " + "ingest it, then rerun strict validation." + ) row["blocker_category"] = _blocker_category(row) rows.append(row) diff --git a/benchmarks/strict_evidence_readiness_results.json b/benchmarks/strict_evidence_readiness_results.json index 56459b4..8882cfb 100644 --- a/benchmarks/strict_evidence_readiness_results.json +++ b/benchmarks/strict_evidence_readiness_results.json @@ -1,6 +1,6 @@ { "schema": "wavemind.strict_evidence_readiness.v1", - "generated_at": "2026-07-11T05:56:54Z", + "generated_at": "2026-07-11T20:57:54Z", "status": "pass", "readiness_status": "action_required", "claim_status": "claims_limited", @@ -11,19 +11,21 @@ "total_requirements": 8, "action_required_count": 4, "complete_count": 4, - "ready_for_safe_dispatch_count": 0, - "can_auto_run_now_count": 0, + "ready_for_safe_dispatch_count": 1, + "can_auto_run_now_count": 1, "target_memories_total": 180000000, "check_counts": { "pass": 8 }, "dispatch_status_counts": { - "blocked_by_preflight": 4, - "complete": 4 + "blocked_by_preflight": 3, + "complete": 4, + "ready_to_dispatch": 1 }, "blocker_counts": { "complete": 4, - "missing_env": 4 + "missing_artifact": 1, + "missing_env": 3 } }, "checks": [ @@ -301,8 +303,8 @@ "id": "pgvector_10m_service", "title": "10M pgvector service load", "strict_status": "action_required", - "preflight_status": "action_required", - "dispatch_status": "blocked_by_preflight", + "preflight_status": "ready", + "dispatch_status": "ready_to_dispatch", "scale_gap_status": "blocked_by_env", "workflow": "production-streaming-load.yml", "wave": "service-scale-10m", @@ -315,28 +317,26 @@ "target_recall_at_k": 0.95, "target_p99_ms": 100.0, "target_qps": 100.0, - "required_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], - "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], + "required_env": [], + "missing_env": [], "required_secrets": [], "issues": [ - "set WAVEMIND_PGVECTOR_DSN" + "missing artifact" + ], + "warnings": [ + "Ephemeral isolated service processes prove the 10M candidate-index SLO, not PostgreSQL HA or independent-node failure tolerance." ], - "warnings": [], "local_profile_command": "python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --memory-payload-kb 2.0 --vector-dtype-bytes 4 --output benchmarks/production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json", - "safe_dispatch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"self-hosted-large\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"false\" -f pgvector_dsn=\"$WAVEMIND_PGVECTOR_DSN\"", - "publish_dispatch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"self-hosted-large\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"true\" -f pgvector_dsn=\"$WAVEMIND_PGVECTOR_DSN\"", + "safe_dispatch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"ubuntu-latest\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"false\" -f provision_pgvector_shards=\"true\" -f pgvector_shard_count=\"4\"", + "publish_dispatch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"ubuntu-latest\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"true\" -f provision_pgvector_shards=\"true\" -f pgvector_shard_count=\"4\"", "download_command": "gh run download --repo CaspianG/wavemind --dir state/production-evidence-downloads", "ingest_command": "wavemind ingest-production-evidence --artifact-dir state/production-evidence-downloads --refresh", "strict_validation_command": "python benchmarks/production_evidence_gate.py --output benchmarks/production_evidence_results.json --markdown-output benchmarks/PRODUCTION_EVIDENCE.md --strict", "post_ingest_refresh_command": "python benchmarks/strict_evidence_readiness_report.py --output benchmarks/strict_evidence_readiness_results.json --markdown-output benchmarks/STRICT_EVIDENCE_READINESS.md", - "ready_for_safe_dispatch": false, - "can_auto_run_now": false, - "next_action": "Provision the listed environment, run the command, then promote the result artifact through the ingest gate.", - "blocker_category": "missing_env" + "ready_for_safe_dispatch": true, + "can_auto_run_now": true, + "next_action": "Run the safe dispatch command now, download the resulting artifact, ingest it, then rerun strict validation.", + "blocker_category": "missing_artifact" }, { "id": "faiss_ivfpq_50m", diff --git a/benchmarks/structured_memory_results.json b/benchmarks/structured_memory_results.json index b7eadb9..ba40769 100644 --- a/benchmarks/structured_memory_results.json +++ b/benchmarks/structured_memory_results.json @@ -1,7 +1,7 @@ { "schema": "wavemind.structured_memory_report.v1", "generated_at": "2026-07-09T22:45:10Z", - "source_ref": "c4f786e131c8", + "source_ref": "d0e3446be447", "source_file": "benchmarks/scale_readiness_results.json", "claim_boundary": "Structured-memory rows come from the checked-in scale-readiness artifact. They prove typed payload routing, provenance, persistence, temporal recall, and graph traversal on the deterministic fixture; they do not claim full production multimodal model quality.", "summary": { diff --git a/deploy/cluster/production-evidence.env.example b/deploy/cluster/production-evidence.env.example index 866b310..9bbb1d3 100644 --- a/deploy/cluster/production-evidence.env.example +++ b/deploy/cluster/production-evidence.env.example @@ -23,8 +23,8 @@ WAVEMIND_FAISS_IVFPQ_FREE_GB=8 # Persisted FAISS IVF-PQ index path for the 50M streaming profile. WAVEMIND_FAISS_IVFPQ_PATH=/mnt/wavemind/faiss/wavemind-ivfpq-50m.faiss -# PostgreSQL/pgvector DSN for 10M pgvector service streaming load. -WAVEMIND_PGVECTOR_DSN=postgresql://USER:PASSWORD@pgvector.staging.example.com:5432/wavemind +# Comma-separated PostgreSQL/pgvector service DSNs for the namespace-sharded 10M streaming load. +WAVEMIND_PGVECTOR_DSNS=postgresql://USER:PASSWORD@pgvector-a.staging.example.com:5432/wavemind,postgresql://USER:PASSWORD@pgvector-b.staging.example.com:5432/wavemind # Optional Qdrant API key for the single-service Qdrant production load job. WAVEMIND_QDRANT_API_KEY=REDACTED_QDRANT_API_KEY diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 407af4d..8be74c4 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -746,7 +746,10 @@ Enterprise requirements: runner for Qdrant/pgvector 10M service-backed profiles so large-N profiles do not hold the full vector corpus or exact-neighbor matrix in RAM. The pgvector 10M service-backed profile now has a checked preflight contract; - the next step is producing `production_streaming_load_pgvector_10m_results.json` + its runner now supports bounded `COPY` ingest, `halfvec` storage, exact remote + row-count validation, HNSW/IVFFlat index checks, explicit `pg_prewarm` evidence, + and constant-time complete resume. + The next step is producing `production_streaming_load_pgvector_10m_results.json` from a real sized PostgreSQL service. - Harden the new Postgres source-of-truth backend with migration tooling, service-mode benchmarks, and operational docs. diff --git a/docs/benchmark-dashboard.html b/docs/benchmark-dashboard.html index d05de81..4e4dc1d 100644 --- a/docs/benchmark-dashboard.html +++ b/docs/benchmark-dashboard.html @@ -68,9 +68,9 @@

WaveMind Living Benchmark Dashboard

-

Readiness

pass

39/39 criteria pass

+

Readiness

fail

38/39 criteria pass

Implemented

37

3 runner-ready and 6 planned public proof paths

-

Refresh

2026-07-11T05:56:44Z

source c4f786e131c8

+

Refresh

2026-07-11T19:56:03Z

source d0e3446be447

@@ -78,7 +78,7 @@

Visual Summary

WaveMind benchmark summary
-

Publication Contract

The leaderboard is generated from artifacts, freshness-checked, published to GitHub Pages, and claim-limited until strict production evidence passes.

Statuspass
Weekly schedule17 4 * * 1
Refresh profilelocal
Pages URLhttps://caspiang.github.io/wavemind/
Source refc4f786e131c8
Workflow runlocal or manual artifact
weekly schedule: truemanual dispatch: truegithub pages upload: truegithub pages deploy: truereview artifact uploaded: trueno scheduled bot commit to main: truestrict freshness gate: truemachine status published: true
+

Publication Contract

The leaderboard is generated from artifacts, freshness-checked, published to GitHub Pages, and claim-limited until strict production evidence passes.

Statuspass
Weekly schedule17 4 * * 1
Refresh profilelocal
Pages URLhttps://caspiang.github.io/wavemind/
Source refd0e3446be447
Workflow runlocal or manual artifact
weekly schedule: truemanual dispatch: truegithub pages upload: truegithub pages deploy: truereview artifact uploaded: trueno scheduled bot commit to main: truestrict freshness gate: truemachine status published: true

Agent Impact

Behavioral evidence: task success, stale-fact suppression, context savings, long-memory retrieval, and checked-in answer-quality smoke results.

Statuspass
Benchmarks6
WaveMind wins6
Average lift0.37
Context saved0.719
Stale safety1
Best profileagent-coherence-and-token-savings-wavemind

Read the agent impact report

@@ -90,7 +90,7 @@

Visual Summary

Cluster Autoscale

Cluster evidence: shard placement, autoscale planning, Kubernetes operator reconciliation, rebalance safety, active-active convergence, CRDT field state, and the deterministic 100M capacity envelope.

Statuspass
Gate checks62/62
Simulated memories1000000
Namespaces4096
Autoscale target10000000
Required nodes50
Operator replicas34
100M capacity nodes128
100M capacity zones8
Recommended max replicas192

Read the cluster autoscale report

-

Strict Evidence Readiness

Operator runbook for the remaining remote, 10M, 50M, and 100M evidence gaps: safe dispatch commands, missing environment, promotion steps, strict validation, and locked claims.

Blockers: complete: 4, missing_env: 4

Report statuspass
Readinessaction_required
Claim statusclaims_limited
Requirements8
Action required4
Safe dispatch ready0
Can auto-run now0
Planned target memories180000000

Read the strict evidence readiness runbook

+

Strict Evidence Readiness

Operator runbook for the remaining remote, 10M, 50M, and 100M evidence gaps: safe dispatch commands, missing environment, promotion steps, strict validation, and locked claims.

Blockers: complete: 4, missing_artifact: 1, missing_env: 3

Report statuspass
Readinessaction_required
Claim statusclaims_limited
Requirements8
Action required4
Safe dispatch ready1
Can auto-run now1
Planned target memories180000000

Read the strict evidence readiness runbook

Benchmark Leaderboard

@@ -259,7 +259,7 @@

Benchmark Leaderboard

Production readiness gate production-scale readiness score -WaveMind production readiness: 1 / - +WaveMind production readiness: 0.974 / - - WaveMind-only check @@ -293,8 +293,8 @@

Evidence Source Status

Artifact freshness -local matrix refresh at 2026-07-11T05:56:44Z -source c4f786e131c8; audit gate enforced by validate_benchmark_artifacts.py +local matrix refresh at 2026-07-11T19:56:03Z +source d0e3446be447; audit gate enforced by validate_benchmark_artifacts.py Keep weekly refresh green before public claims. @@ -366,7 +366,7 @@

Evidence Source Status

Production readiness gate checked-in benchmark artifacts -pass; 39/39 pass +fail; 38/39 pass Keep the gate at readiness_score 1.0 while repeating larger service-backed runs and moving external competitor evidence into the separate adapter profile. diff --git a/docs/data/leaderboard-status.json b/docs/data/leaderboard-status.json index acee7a4..9c6ef70 100644 --- a/docs/data/leaderboard-status.json +++ b/docs/data/leaderboard-status.json @@ -1,7 +1,7 @@ { "schema": "wavemind.leaderboard_status.v1", - "generated_at": "2026-07-11T05:56:54Z", - "source_ref": "c4f786e131c8", + "generated_at": "2026-07-11T20:56:13Z", + "source_ref": "d0e3446be447", "workflow_run_id": null, "refresh_profile": "local", "public_url": "https://caspiang.github.io/wavemind/", @@ -14,7 +14,7 @@ "timezone": "UTC", "public_url": "https://caspiang.github.io/wavemind/", "publishing_status": "publishable_with_claim_limits", - "source_ref": "c4f786e131c8", + "source_ref": "d0e3446be447", "workflow_run_id": null, "refresh_profile": "local", "expected_scheduled_refresh_profile": "weekly-fast", @@ -39,7 +39,7 @@ "freshness_gate": { "schema": "wavemind.leaderboard_freshness.v1", "status": "pass", - "checked_at": "2026-07-11T05:56:54Z", + "checked_at": "2026-07-11T20:56:13Z", "max_age_days": 8.0, "source_count": 32, "fresh_count": 32, @@ -55,15 +55,15 @@ "path": "benchmarks/benchmark_matrix_results.json", "schema": "wavemind.benchmark_matrix.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:44Z", - "age_days": 0.00011574074074074075, + "timestamp": "2026-07-11T19:56:03Z", + "age_days": 0.04178240740740741, "status": "pass" }, { "path": "benchmarks/benchmark_artifact_audit.json", "schema": "wavemind.benchmark_artifact_audit.v1", "timestamp_key": "checked_at", - "timestamp": "2026-07-11T05:56:54Z", + "timestamp": "2026-07-11T20:56:13Z", "age_days": 0.0, "status": "pass" }, @@ -71,63 +71,63 @@ "path": "benchmarks/production_readiness_results.json", "schema": "wavemind.production_readiness.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:48Z", - "age_days": 6.944444444444444e-05, + "timestamp": "2026-07-11T20:18:10Z", + "age_days": 0.02642361111111111, "status": "pass" }, { "path": "benchmarks/production_evidence_results.json", "schema": "wavemind.production_evidence.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:49Z", - "age_days": 5.787037037037037e-05, + "timestamp": "2026-07-11T19:56:06Z", + "age_days": 0.041747685185185186, "status": "pass" }, { "path": "benchmarks/production_evidence_preflight_results.json", "schema": "wavemind.production_evidence_preflight.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:50Z", - "age_days": 4.6296296296296294e-05, + "timestamp": "2026-07-11T20:54:43Z", + "age_days": 0.0010416666666666667, "status": "pass" }, { "path": "benchmarks/production_evidence_env_contract.json", "schema": "wavemind.production_evidence_env_contract.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-10T05:23:37Z", - "age_days": 1.023113425925926, + "timestamp": "2026-07-11T19:47:54Z", + "age_days": 0.04744212962962963, "status": "pass" }, { "path": "benchmarks/production_evidence_bundle_results.json", "schema": "wavemind.production_evidence_bundle.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:51Z", - "age_days": 3.472222222222222e-05, + "timestamp": "2026-07-11T20:56:03Z", + "age_days": 0.00011574074074074075, "status": "pass" }, { "path": "benchmarks/release_claims_results.json", "schema": "wavemind.release_claims.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:52Z", - "age_days": 2.3148148148148147e-05, + "timestamp": "2026-07-11T19:56:09Z", + "age_days": 0.041712962962962966, "status": "pass" }, { "path": "benchmarks/scale_gap_results.json", "schema": "wavemind.scale_gap.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:53Z", - "age_days": 1.1574074074074073e-05, + "timestamp": "2026-07-11T19:56:10Z", + "age_days": 0.04170138888888889, "status": "pass" }, { "path": "benchmarks/strict_evidence_readiness_results.json", "schema": "wavemind.strict_evidence_readiness.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:54Z", + "timestamp": "2026-07-11T20:57:54Z", "age_days": 0.0, "status": "pass" }, @@ -135,8 +135,8 @@ "path": "benchmarks/cluster_admission_results.json", "schema": "wavemind.cluster_admission.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-10T05:23:44Z", - "age_days": 1.0230324074074073, + "timestamp": "2026-07-11T20:01:43Z", + "age_days": 0.03784722222222222, "status": "pass" }, { @@ -144,7 +144,7 @@ "schema": "wavemind.active_active_admission.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T10:20:56Z", - "age_days": 1.8166435185185186, + "age_days": 2.4411689814814816, "status": "pass" }, { @@ -152,7 +152,7 @@ "schema": "wavemind.serverless_admission.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T10:20:56Z", - "age_days": 1.8166435185185186, + "age_days": 2.4411689814814816, "status": "pass" }, { @@ -160,7 +160,7 @@ "schema": "wavemind.multimodal_admission.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T12:56:09Z", - "age_days": 1.7088541666666666, + "age_days": 2.3333796296296296, "status": "pass" }, { @@ -168,7 +168,7 @@ "schema": "wavemind.memory_os_admission.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T00:40:28Z", - "age_days": 2.2197453703703705, + "age_days": 2.8442708333333333, "status": "pass" }, { @@ -176,7 +176,7 @@ "schema": "wavemind.memory_os_canary.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T01:06:28Z", - "age_days": 2.2016898148148147, + "age_days": 2.8262152777777776, "status": "pass" }, { @@ -184,7 +184,7 @@ "schema": "wavemind.memory_os_policy_evolution.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T13:44:41Z", - "age_days": 1.675150462962963, + "age_days": 2.299675925925926, "status": "pass" }, { @@ -192,23 +192,23 @@ "schema": "wavemind.memory_os_policy_bundle.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T19:29:13Z", - "age_days": 1.4358912037037037, + "age_days": 2.060416666666667, "status": "pass" }, { "path": "benchmarks/production_evidence_dispatch_results.json", "schema": "wavemind.production_evidence_dispatch.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:51Z", - "age_days": 3.472222222222222e-05, + "timestamp": "2026-07-11T20:56:02Z", + "age_days": 0.0001273148148148148, "status": "pass" }, { "path": "benchmarks/production_scale_run_plan.json", "schema": "wavemind.production_scale_run_plan.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-10T04:17:18Z", - "age_days": 1.0691666666666666, + "timestamp": "2026-07-11T19:47:53Z", + "age_days": 0.047453703703703706, "status": "pass" }, { @@ -216,15 +216,15 @@ "schema": "wavemind.agent_coherence_benchmark.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-08T17:24:13Z", - "age_days": 2.5226967592592593, + "age_days": 3.147222222222222, "status": "pass" }, { "path": "benchmarks/agent_impact_results.json", "schema": "wavemind.agent_impact_leaderboard.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:44Z", - "age_days": 0.00011574074074074075, + "timestamp": "2026-07-11T19:56:03Z", + "age_days": 0.04178240740740741, "status": "pass" }, { @@ -232,7 +232,7 @@ "schema": "wavemind.structured_memory_report.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T22:45:10Z", - "age_days": 1.2998148148148148, + "age_days": 1.9243402777777778, "status": "pass" }, { @@ -240,7 +240,7 @@ "schema": "wavemind.memory_os_intelligence_report.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T22:45:10Z", - "age_days": 1.2998148148148148, + "age_days": 1.9243402777777778, "status": "pass" }, { @@ -248,47 +248,47 @@ "schema": "wavemind.cluster_autoscale_report.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T22:45:10Z", - "age_days": 1.2998148148148148, + "age_days": 1.9243402777777778, "status": "pass" }, { "path": "benchmarks/kubernetes_operator_smoke_results.json", "schema": "wavemind.kubernetes_operator_smoke.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-09T23:28:34.759347Z", - "age_days": 1.2696671371875, + "timestamp": "2026-07-11T19:49:12.785845Z", + "age_days": 0.04653025642361111, "status": "pass" }, { "path": "benchmarks/kubernetes_cluster_network_smoke_results.json", "schema": "wavemind.kubernetes_cluster_network_smoke.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-10T05:13:44.475824+00:00", - "age_days": 1.0299713446296297, + "timestamp": "2026-07-11T19:49:34.107653+00:00", + "age_days": 0.04628347623842593, "status": "pass" }, { "path": "benchmarks/kubernetes_active_active_region_smoke_results.json", "schema": "wavemind.kubernetes_active_active_region_smoke.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-09T23:55:54.133679+00:00", - "age_days": 1.2506928972337963, + "timestamp": "2026-07-11T19:50:19.980112+00:00", + "age_days": 0.045752545, "status": "pass" }, { "path": "benchmarks/kubernetes_serverless_lifecycle_smoke_results.json", "schema": "wavemind.kubernetes_serverless_lifecycle_smoke.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-10T02:42:30.318307+00:00", - "age_days": 1.1349963158912038, + "timestamp": "2026-07-11T19:52:02.978556+00:00", + "age_days": 0.04456043337962963, "status": "pass" }, { "path": "benchmarks/kubernetes_postgres_qdrant_dr_smoke_results.json", "schema": "wavemind.kubernetes_postgres_qdrant_dr_smoke.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-10T02:42:53.084317+00:00", - "age_days": 1.1347328204050926, + "timestamp": "2026-07-11T19:52:28.621094+00:00", + "age_days": 0.04426364474537037, "status": "pass" }, { @@ -296,22 +296,22 @@ "schema": "wavemind.scale_readiness_benchmark.v1", "timestamp_key": "generated_at", "timestamp": "2026-07-09T22:45:10Z", - "age_days": 1.2998148148148148, + "age_days": 1.9243402777777778, "status": "pass" }, { "path": "benchmarks/cost_efficiency_results.json", "schema": "wavemind.cost_efficiency_leaderboard.v1", "timestamp_key": "generated_at", - "timestamp": "2026-07-11T05:56:44Z", - "age_days": 0.00011574074074074075, + "timestamp": "2026-07-11T19:56:03Z", + "age_days": 0.04178240740740741, "status": "pass" } ] }, "benchmark_matrix": { "schema": "wavemind.benchmark_matrix.v1", - "generated_at": "2026-07-11T05:56:44Z", + "generated_at": "2026-07-11T19:56:03Z", "implemented_count": 37, "runner_ready_count": 3, "planned_count": 6, @@ -340,8 +340,8 @@ "artifact_audit": { "schema": "wavemind.benchmark_artifact_audit.v1", "status": "pass", - "checked_at": "2026-07-11T05:56:54Z", - "age_days": 0.00012261520833333332, + "checked_at": "2026-07-11T20:56:13Z", + "age_days": 0.04178398337962963, "max_age_days": 8.0, "errors": [] }, @@ -683,9 +683,9 @@ "status": "pass", "environment": "kind-multinode-ci", "evidence_source": "github-actions-kind", - "source_ref": "9956e3112dc575d0f12ff99ec59326dd8e035f70", - "workflow_run_id": "29057247023", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29057247023", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "passed_checks": 14, "check_count": 14, "node_count": 4, @@ -693,8 +693,8 @@ "operator_node_count": 2, "lease_transitions_after": 1, "ready_replicas_after_scale": 4, - "cluster_status_holder": "wavemind-operator-78944944b8-bx88m", - "next_holder": "wavemind-operator-78944944b8-bx88m", + "cluster_status_holder": "wavemind-operator-78944944b8-jllkt", + "next_holder": "wavemind-operator-78944944b8-jllkt", "data_pod_uid_changed": true, "api_healthy_after_recovery": true, "topology_spread_constraint_count": 2, @@ -711,17 +711,17 @@ "status": "pass", "environment": "kind-multinode-network-ci", "evidence_source": "github-actions-kind-physical-node-pause", - "source_ref": "3d40e894b493253a7d263fa2bf6d67bbc8f4019a", - "workflow_run_id": "29070578441", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29070578441", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "passed_checks": 13, "check_count": 13, "service_node_count": 4, "zone_count": 3, "failure_method": "docker-pause-kind-worker", - "target_worker": "wavemind-ci-worker", - "target_zone": "zone-a", - "outage_duration_ms": 8911.648, + "target_worker": "wavemind-ci-worker2", + "target_zone": "zone-b", + "outage_duration_ms": 9431.139, "outage_hit_rate": 1.0, "failed_nodes_during_outage": [ "wavemind-ci-2" @@ -736,9 +736,9 @@ "status": "pass", "environment": "kind-multizone-active-active-ci", "evidence_source": "github-actions-kind-physical-region-worker-pause", - "source_ref": "b6392353668203196007e1cac695e178752d34d1", - "workflow_run_id": "29058433643", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29058433643", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "passed_checks": 17, "check_count": 17, "region_count": 3, @@ -746,7 +746,7 @@ "all_regions_use_pvc": true, "failure_method": "docker-pause-kind-worker", "target_region": "region-b", - "outage_duration_ms": 9900.072, + "outage_duration_ms": 9658.868, "seed_writes": 48, "outage_unavailable_regions": [ "region-b" @@ -766,22 +766,22 @@ "status": "pass", "environment": "kind-multizone-serverless-lifecycle-ci", "evidence_source": "github-actions-kind-external-state-manual-scale-lifecycle", - "source_ref": "880cafdf5ce5f7c9da878bd8e82ada39ca2d02cc", - "workflow_run_id": "29064934749", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29064934749", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "passed_checks": 13, "check_count": 13, "persistent_volume_claims": 3, - "cold_start_ms": 5086.153, + "cold_start_ms": 5055.034, "restored_after_zero_rate": 1.0, "ready_replicas": 3, "zone_count": 3, "visible_replicas": 3, "suppressed_replicas": 3, - "write_propagation_ms": 1130.6929430000991, - "delete_propagation_ms": 915.0309450000123, - "burst_requests_per_second": 41.39356985869663, - "burst_p99_ms": 1461.4564480000354, + "write_propagation_ms": 1136.9370459999573, + "delete_propagation_ms": 884.7577889999911, + "burst_requests_per_second": 40.74436448541339, + "burst_p99_ms": 1891.9401950000747, "final_restore_rate": 1.0, "claim_boundary": "Ephemeral non-loopback Kubernetes lifecycle evidence with external durable state. It proves scale-to-zero state safety and multi-replica behavior, but does not unlock remote managed Knative/KEDA production admission.", "source": "benchmarks/kubernetes_serverless_lifecycle_smoke_results.json" @@ -791,13 +791,13 @@ "status": "pass", "environment": "kind-independent-namespace-postgres-qdrant-dr-ci", "evidence_source": "github-actions-kind-pg-dump-independent-restore", - "source_ref": "880cafdf5ce5f7c9da878bd8e82ada39ca2d02cc", - "workflow_run_id": "29064934749", - "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29064934749", + "source_ref": "29f84dc226c5fd11ae9696d04fa48ffb1ae371e3", + "workflow_run_id": "29165761261", + "workflow_run_url": "https://github.com/CaspianG/wavemind/actions/runs/29165761261", "passed_checks": 10, "check_count": 10, "backup_format": "pg_dump-custom", - "backup_bytes": 1016635, + "backup_bytes": 1016642, "source_state_stopped": true, "recovery_pvcs": 3, "restored_rate": 1.0, @@ -805,7 +805,7 @@ "index_expected_records": 24, "index_vector_records": 24, "restored_after_api_replacement_rate": 1.0, - "restore_elapsed_ms": 20449.092, + "restore_elapsed_ms": 23338.114, "claim_boundary": "Ephemeral non-loopback Kubernetes disaster-recovery evidence. It proves logical PostgreSQL backup/restore and Qdrant rebuild in an independent namespace, not managed-cloud PITR or multi-region DR.", "source": "benchmarks/kubernetes_postgres_qdrant_dr_smoke_results.json" }, @@ -939,7 +939,7 @@ "strict_pass_count": 4, "strict_total_requirements": 8, "preflight_overall_status": "action_required", - "preflight_ready_count": 0, + "preflight_ready_count": 1, "preflight_total_checks": 8, "production_readiness_status": "pass", "production_readiness_score": 1.0, @@ -1001,13 +1001,13 @@ "target_memories": 10000000, "output_artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", "required_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "blockers": [ - "missing_env:WAVEMIND_PGVECTOR_DSN", + "missing_env:WAVEMIND_PGVECTOR_DSNS", "insufficient_local_disk_for_index_and_transient_batches" ] }, @@ -1025,6 +1025,7 @@ ], "blockers": [ "missing_env:WAVEMIND_FAISS_IVFPQ_PATH", + "missing_module:faiss", "insufficient_local_disk_for_index_and_transient_batches" ] }, @@ -1117,8 +1118,8 @@ "overall_status": "action_required", "summary": { "overall_status": "action_required", - "ready_count": 0, - "action_required_count": 8, + "ready_count": 1, + "action_required_count": 7, "total_checks": 8 } }, @@ -1143,7 +1144,7 @@ "WAVEMIND_CLUSTER_NODES", "WAVEMIND_CLUSTER_NODES_MANIFEST_JSON", "WAVEMIND_FAISS_IVFPQ_PATH", - "WAVEMIND_PGVECTOR_DSN", + "WAVEMIND_PGVECTOR_DSNS", "WAVEMIND_QDRANT_URL", "WAVEMIND_QDRANT_URLS", "WAVEMIND_SERVERLESS_NODES" @@ -1163,8 +1164,8 @@ "summary": { "overall_status": "action_required", "total_jobs": 8, - "ready_to_dispatch_count": 0, - "blocked_by_preflight_count": 4, + "ready_to_dispatch_count": 1, + "blocked_by_preflight_count": 3, "complete_count": 4, "commit_results_default": false, "runner_label": "self-hosted-large", @@ -1175,8 +1176,9 @@ "service-scale-10m": 3 }, "status_counts": { - "blocked_by_preflight": 4, - "complete": 4 + "blocked_by_preflight": 3, + "complete": 4, + "ready_to_dispatch": 1 } }, "jobs": [ @@ -1432,11 +1434,11 @@ { "id": "pgvector_10m_service", "title": "10M pgvector service load", - "status": "blocked_by_preflight", + "status": "ready_to_dispatch", "dispatch_required": true, - "ready": false, + "ready": true, "strict_status": "action_required", - "preflight_status": "action_required", + "preflight_status": "ready", "wave": "service-scale-10m", "workflow": "production-streaming-load.yml", "artifact": "benchmarks/production_streaming_load_pgvector_10m_results.json", @@ -1454,27 +1456,24 @@ "replicas": "3", "autoscaling_max_replicas": "24", "capacity_headroom": "0.7", - "runner_label": "self-hosted-large", + "runner_label": "ubuntu-latest", "runner_storage_root": "state/production-runs", "commit_results": false, - "pgvector_dsn": "$WAVEMIND_PGVECTOR_DSN" - }, - "input_bindings": { - "pgvector_dsn": "$WAVEMIND_PGVECTOR_DSN" + "provision_pgvector_shards": true, + "pgvector_shard_count": "4" }, - "required_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], - "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], + "input_bindings": {}, + "required_env": [], + "missing_env": [], "required_secrets": [], "issues": [ - "set WAVEMIND_PGVECTOR_DSN" + "missing artifact" ], - "warnings": [], - "safe_launch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"self-hosted-large\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"false\" -f pgvector_dsn=\"$WAVEMIND_PGVECTOR_DSN\"", - "publish_launch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"self-hosted-large\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"true\" -f pgvector_dsn=\"$WAVEMIND_PGVECTOR_DSN\"", + "warnings": [ + "Ephemeral isolated service processes prove the 10M candidate-index SLO, not PostgreSQL HA or independent-node failure tolerance." + ], + "safe_launch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"ubuntu-latest\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"false\" -f provision_pgvector_shards=\"true\" -f pgvector_shard_count=\"4\"", + "publish_launch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"ubuntu-latest\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"true\" -f provision_pgvector_shards=\"true\" -f pgvector_shard_count=\"4\"", "download_command": "gh run download --repo CaspianG/wavemind --dir state/production-evidence-downloads", "ingest_command": "wavemind ingest-production-evidence --artifact-dir state/production-evidence-downloads --refresh" }, @@ -1734,10 +1733,10 @@ "output_artifact_exists": false, "checkpoint_path": "state/production-runs/pgvector-service-10000000.checkpoint.json", "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" + "WAVEMIND_PGVECTOR_DSNS" ], "blockers": [ - "missing_env:WAVEMIND_PGVECTOR_DSN", + "missing_env:WAVEMIND_PGVECTOR_DSNS", "insufficient_local_disk_for_index_and_transient_batches" ], "command": "python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --memory-payload-kb 2.0 --vector-dtype-bytes 4 --output benchmarks/production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json", @@ -1777,6 +1776,7 @@ ], "blockers": [ "missing_env:WAVEMIND_FAISS_IVFPQ_PATH", + "missing_module:faiss", "insufficient_local_disk_for_index_and_transient_batches" ], "command": "python benchmarks/production_streaming_load_benchmark.py --sizes 50000000 --dim 128 --queries 2000 --top-k 10 --batch-size 1000000 --engines faiss-ivfpq-persisted --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --memory-payload-kb 2.0 --vector-dtype-bytes 4 --output benchmarks/production_streaming_load_ivfpq_50m_results.json --checkpoint-path state/production-runs/faiss-ivfpq-persisted-50000000.checkpoint.json", @@ -1849,19 +1849,21 @@ "total_requirements": 8, "action_required_count": 4, "complete_count": 4, - "ready_for_safe_dispatch_count": 0, - "can_auto_run_now_count": 0, + "ready_for_safe_dispatch_count": 1, + "can_auto_run_now_count": 1, "target_memories_total": 180000000, "check_counts": { "pass": 8 }, "dispatch_status_counts": { - "blocked_by_preflight": 4, - "complete": 4 + "blocked_by_preflight": 3, + "complete": 4, + "ready_to_dispatch": 1 }, "blocker_counts": { "complete": 4, - "missing_env": 4 + "missing_artifact": 1, + "missing_env": 3 } }, "checks": [ @@ -2139,8 +2141,8 @@ "id": "pgvector_10m_service", "title": "10M pgvector service load", "strict_status": "action_required", - "preflight_status": "action_required", - "dispatch_status": "blocked_by_preflight", + "preflight_status": "ready", + "dispatch_status": "ready_to_dispatch", "scale_gap_status": "blocked_by_env", "workflow": "production-streaming-load.yml", "wave": "service-scale-10m", @@ -2153,28 +2155,26 @@ "target_recall_at_k": 0.95, "target_p99_ms": 100.0, "target_qps": 100.0, - "required_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], - "missing_env": [ - "WAVEMIND_PGVECTOR_DSN" - ], + "required_env": [], + "missing_env": [], "required_secrets": [], "issues": [ - "set WAVEMIND_PGVECTOR_DSN" + "missing artifact" + ], + "warnings": [ + "Ephemeral isolated service processes prove the 10M candidate-index SLO, not PostgreSQL HA or independent-node failure tolerance." ], - "warnings": [], "local_profile_command": "python benchmarks/production_streaming_load_benchmark.py --sizes 10000000 --dim 128 --queries 2000 --top-k 10 --batch-size 5000 --engines pgvector-service --target-recall 0.95 --target-p99-ms 100.0 --target-qps 100.0 --replicas 3 --autoscaling-max-replicas 24 --capacity-headroom 0.7 --memory-payload-kb 2.0 --vector-dtype-bytes 4 --output benchmarks/production_streaming_load_pgvector_10m_results.json --checkpoint-path state/production-runs/pgvector-service-10000000.checkpoint.json", - "safe_dispatch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"self-hosted-large\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"false\" -f pgvector_dsn=\"$WAVEMIND_PGVECTOR_DSN\"", - "publish_dispatch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"self-hosted-large\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"true\" -f pgvector_dsn=\"$WAVEMIND_PGVECTOR_DSN\"", + "safe_dispatch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"ubuntu-latest\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"false\" -f provision_pgvector_shards=\"true\" -f pgvector_shard_count=\"4\"", + "publish_dispatch_command": "gh workflow run production-streaming-load.yml -f engine=\"pgvector-service\" -f size=\"10000000\" -f dim=\"128\" -f queries=\"2000\" -f top_k=\"10\" -f batch_size=\"5000\" -f target_recall=\"0.95\" -f target_p99_ms=\"100.0\" -f target_qps=\"100.0\" -f replicas=\"3\" -f autoscaling_max_replicas=\"24\" -f capacity_headroom=\"0.7\" -f runner_label=\"ubuntu-latest\" -f runner_storage_root=\"state/production-runs\" -f commit_results=\"true\" -f provision_pgvector_shards=\"true\" -f pgvector_shard_count=\"4\"", "download_command": "gh run download --repo CaspianG/wavemind --dir state/production-evidence-downloads", "ingest_command": "wavemind ingest-production-evidence --artifact-dir state/production-evidence-downloads --refresh", "strict_validation_command": "python benchmarks/production_evidence_gate.py --output benchmarks/production_evidence_results.json --markdown-output benchmarks/PRODUCTION_EVIDENCE.md --strict", "post_ingest_refresh_command": "python benchmarks/strict_evidence_readiness_report.py --output benchmarks/strict_evidence_readiness_results.json --markdown-output benchmarks/STRICT_EVIDENCE_READINESS.md", - "ready_for_safe_dispatch": false, - "can_auto_run_now": false, - "next_action": "Provision the listed environment, run the command, then promote the result artifact through the ingest gate.", - "blocker_category": "missing_env" + "ready_for_safe_dispatch": true, + "can_auto_run_now": true, + "next_action": "Run the safe dispatch command now, download the resulting artifact, ingest it, then rerun strict validation.", + "blocker_category": "missing_artifact" }, { "id": "faiss_ivfpq_50m", @@ -2287,7 +2287,7 @@ "title": "Non-loopback Kubernetes or external HTTP service-node load", "status": "pass", "artifact": "benchmarks/http_cluster_load_results.json", - "evidence": "nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms", + "evidence": "nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms", "issues": [], "command": "gh workflow run external-http-cluster-load.yml -f nodes=\"node-a=https://wm-a.example.com,node-b=https://wm-b.example.com,node-c=https://wm-c.example.com,node-d=https://wm-d.example.com\" -f replication_factor=3 -f read_quorum=1 -f read_fanout=1 -f batch_query_size=24 -f fail_on_slo=true -f commit_results=true", "claim_unlocked": "Non-loopback Kubernetes service-node cluster load SLO." @@ -2295,7 +2295,7 @@ "requested_evidence": { "status": "pass", "artifact": "benchmarks/http_cluster_load_results.json", - "evidence": "nodes 4, deployment github-actions-29070578441-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 84.79942335998312 ms, lifecycle batch p99 6694.75712100001 ms, batch query True, batch HTTP 24 -> 1, batch p99 148.8199839999993 ms", + "evidence": "nodes 4, deployment github-actions-29165761261-wavemind-ci-wavemind-system, environment kubernetes-kind-non-loopback-ci, source kubernetes-pod-dns-physical-node-drill, namespaces 32, success 1.0, failover 1.0, query p99 79.44286219996737 ms, lifecycle batch p99 8351.044338999998 ms, batch query True, batch HTTP 24 -> 1, batch p99 186.78031600001077 ms", "issues": [], "min_nodes": 4, "namespace_count": 32, diff --git a/tests/test_benchmark_registry.py b/tests/test_benchmark_registry.py index 66036ef..b98cb0e 100644 --- a/tests/test_benchmark_registry.py +++ b/tests/test_benchmark_registry.py @@ -5,6 +5,12 @@ from pathlib import Path +def _assert_github_actions_run_url(value: str) -> None: + prefix = "https://github.com/CaspianG/wavemind/actions/runs/" + assert value.startswith(prefix) + assert value.removeprefix(prefix).isdigit() + + def test_benchmark_matrix_contains_implemented_and_public_benchmarks(): from benchmarks.benchmark_registry import build_benchmark_matrix @@ -166,7 +172,7 @@ def test_benchmark_matrix_contains_implemented_and_public_benchmarks(): ] assert pgvector_plan["status"] == "action_required" assert pgvector_plan["estimated_index_gb"] == 0.0 - assert "WAVEMIND_PGVECTOR_DSN" in pgvector_plan["missing_env"] + assert "WAVEMIND_PGVECTOR_DSNS" in pgvector_plan["missing_env"] assert "100M" in entries["production_streaming_load_runner"]["dataset"] assert "production-streaming-load.yml" in entries["production_streaming_load_runner"]["next_step"] assert entries["postgres_pitr_plan"]["status"] == "implemented" @@ -222,7 +228,7 @@ def test_benchmark_matrix_contains_implemented_and_public_benchmarks(): assert kubernetes_current["rolling_upgrade_replaced_pods"] == 4 assert kubernetes_current["api_healthy_after_upgrade"] is True assert kubernetes_current["passed_checks"] == kubernetes_current["check_count"] == 14 - assert kubernetes_current["workflow_run_url"].endswith("/29057247023") + _assert_github_actions_run_url(kubernetes_current["workflow_run_url"]) assert "does not unlock remote production" in kubernetes_current["claim_boundary"] network_smoke = entries["kubernetes_cluster_network_failure_smoke"] assert network_smoke["status"] == "implemented" @@ -243,6 +249,7 @@ def test_benchmark_matrix_contains_implemented_and_public_benchmarks(): assert network_current["failed_nodes_after_recovery"] == [] assert network_current["passed_checks"] == network_current["check_count"] == 13 assert network_current["workflow_run_id"].isdigit() + _assert_github_actions_run_url(network_current["workflow_run_url"]) assert network_current["workflow_run_url"].endswith( f"/{network_current['workflow_run_id']}" ) @@ -269,7 +276,7 @@ def test_benchmark_matrix_contains_implemented_and_public_benchmarks(): assert region_current["final_noop_records_imported"] == 0 assert region_current["final_noop_tombstones_imported"] == 0 assert region_current["passed_checks"] == region_current["check_count"] == 17 - assert region_current["workflow_run_url"].endswith("/29058433643") + _assert_github_actions_run_url(region_current["workflow_run_url"]) assert "not remote multi-region" in region_current["claim_boundary"] serverless_lifecycle = entries["kubernetes_serverless_lifecycle_smoke"]["current"][ "WaveMind Kubernetes serverless lifecycle" @@ -286,7 +293,7 @@ def test_benchmark_matrix_contains_implemented_and_public_benchmarks(): assert serverless_lifecycle["delete_propagation_ms"] <= 2000.0 assert serverless_lifecycle["burst_p99_ms"] <= 2000.0 assert serverless_lifecycle["final_restore_rate"] == 1.0 - assert serverless_lifecycle["workflow_run_url"].endswith("/29064934749") + _assert_github_actions_run_url(serverless_lifecycle["workflow_run_url"]) assert "does not unlock remote managed" in serverless_lifecycle["claim_boundary"] kubernetes_dr = entries["kubernetes_postgres_qdrant_dr_smoke"]["current"][ "WaveMind Kubernetes PostgreSQL/Qdrant DR" @@ -302,7 +309,7 @@ def test_benchmark_matrix_contains_implemented_and_public_benchmarks(): assert kubernetes_dr["index_vector_records"] == kubernetes_dr["index_expected_records"] == 24 assert kubernetes_dr["restored_after_api_replacement_rate"] == 1.0 assert kubernetes_dr["restore_elapsed_ms"] <= 180000.0 - assert kubernetes_dr["workflow_run_url"].endswith("/29064934749") + _assert_github_actions_run_url(kubernetes_dr["workflow_run_url"]) assert "not managed-cloud PITR" in kubernetes_dr["claim_boundary"] external_active_active_loopback = entries["external_http_active_active_loopback"]["current"][ "WaveMind real HTTP active-active service-region sync" diff --git a/tests/test_benchmark_workflow.py b/tests/test_benchmark_workflow.py index e2d9bd9..0e1d4d2 100644 --- a/tests/test_benchmark_workflow.py +++ b/tests/test_benchmark_workflow.py @@ -243,6 +243,23 @@ def test_production_streaming_load_workflow_runs_checkpointed_large_n_profiles() assert "WAVEMIND_QDRANT_URL" in workflow assert "WAVEMIND_QDRANT_URLS" in workflow assert "WAVEMIND_PGVECTOR_DSN" in workflow + assert "WAVEMIND_PGVECTOR_DSNS" in workflow + assert "pgvector_dsns" in workflow + assert "provision_pgvector_shards" in workflow + assert "pgvector_shard_count" in workflow + assert "Provision isolated pgvector services" in workflow + assert '"pgvector/pgvector:pg16"' in workflow + assert "--shm-size 1g" in workflow + assert "WAVEMIND_PGVECTOR_INDEX_BUILD_WORKERS" in workflow + assert "max_wal_size=4GB" in workflow + assert "pg_isready --username postgres --dbname wavemind" in workflow + assert "pgvector-managed-dsns.txt" in workflow + assert "github-hosted-isolated-service-processes" in workflow + assert "pgvector shard row counts do not prove an exact balanced layout" in workflow + assert "managed pgvector evidence must use namespace routing" in workflow + assert "build independent shard indexes in parallel" in workflow + assert "isolated-service topology attestation" in workflow + assert "Capture pgvector service diagnostics" in workflow assert "WAVEMIND_FAISS_IVFPQ_PATH" in workflow assert 'WAVEMIND_FAISS_IVFPQ_NPROBE: "1024"' in workflow assert 'WAVEMIND_FAISS_IVFPQ_NPROBE_SWEEP: "64,128,256,512,1024"' in workflow diff --git a/tests/test_leaderboard_status.py b/tests/test_leaderboard_status.py index 0e3c2d4..fbb60a6 100644 --- a/tests/test_leaderboard_status.py +++ b/tests/test_leaderboard_status.py @@ -343,7 +343,7 @@ def test_leaderboard_status_renderer_writes_public_contract(tmp_path): assert payload["strict_evidence_readiness"]["summary"]["target_memories_total"] == ( 180_000_000 ) - assert payload["strict_evidence_readiness"]["summary"]["can_auto_run_now_count"] == 0 + assert payload["strict_evidence_readiness"]["summary"]["can_auto_run_now_count"] == 1 assert payload["strict_evidence_readiness"]["summary"]["check_counts"] == {"pass": 8} assert any( row["id"] == "hundred_million_remote_load" diff --git a/tests/test_production_evidence_bundle.py b/tests/test_production_evidence_bundle.py index 263d54c..2ff5e9a 100644 --- a/tests/test_production_evidence_bundle.py +++ b/tests/test_production_evidence_bundle.py @@ -34,7 +34,10 @@ def _ready_env(tmp_path): "WAVEMIND_SERVERLESS_NODES": "https://wm-a.staging.internal,https://wm-b.staging.internal", "WAVEMIND_QDRANT_URL": "http://qdrant.staging.internal:6333", "WAVEMIND_QDRANT_URLS": "http://qdrant-a.staging.internal:6333,http://qdrant-b.staging.internal:6333", - "WAVEMIND_PGVECTOR_DSN": "postgresql://user:pass@postgres.staging.internal:5432/wavemind", + "WAVEMIND_PGVECTOR_DSNS": ",".join( + f"postgresql://user:pass@postgres-{index}.staging.internal:5432/wavemind" + for index in range(4) + ), "WAVEMIND_FAISS_IVFPQ_PATH": str(tmp_path / "wavemind-faiss-ivfpq-50m.faiss"), "WAVEMIND_FAISS_IVFPQ_FREE_GB": "8", "WAVEMIND_API_KEY": "test-key", diff --git a/tests/test_production_evidence_dispatch.py b/tests/test_production_evidence_dispatch.py index d9b49e2..3210212 100644 --- a/tests/test_production_evidence_dispatch.py +++ b/tests/test_production_evidence_dispatch.py @@ -35,8 +35,9 @@ def _ready_env(tmp_path): "http://qdrant-a.staging.internal:6333," "http://qdrant-b.staging.internal:6333" ), - "WAVEMIND_PGVECTOR_DSN": ( - "postgresql://user:pass@postgres.staging.internal:5432/wavemind" + "WAVEMIND_PGVECTOR_DSNS": ",".join( + f"postgresql://user:pass@postgres-{index}.staging.internal:5432/wavemind" + for index in range(4) ), "WAVEMIND_FAISS_IVFPQ_PATH": str(tmp_path / "wavemind-faiss-ivfpq-50m.faiss"), "WAVEMIND_FAISS_IVFPQ_FREE_GB": "8", @@ -51,8 +52,8 @@ def test_dispatch_plan_reports_blocked_jobs_without_remote_prerequisites(): assert payload["schema"] == "wavemind.production_evidence_dispatch.v1" assert payload["overall_status"] == "action_required" assert payload["summary"]["total_jobs"] == 8 - assert payload["summary"]["blocked_by_preflight_count"] == 4 - assert payload["summary"]["ready_to_dispatch_count"] == 0 + assert payload["summary"]["blocked_by_preflight_count"] == 3 + assert payload["summary"]["ready_to_dispatch_count"] == 1 assert payload["summary"]["complete_count"] == 4 by_id = {row["id"]: row for row in payload["jobs"]} @@ -71,6 +72,7 @@ def test_dispatch_plan_reports_blocked_jobs_without_remote_prerequisites(): assert by_id["hundred_million_remote_load"]["workflow"] == ( "production-streaming-load.yml" ) + assert by_id["pgvector_10m_service"]["status"] == "ready_to_dispatch" def test_dispatch_plan_becomes_ready_with_prerequisites_without_leaking_secret_values( @@ -102,6 +104,12 @@ def test_dispatch_plan_becomes_ready_with_prerequisites_without_leaking_secret_v assert qdrant["inputs"]["runner_storage_root"] == "state/production-runs" assert qdrant["input_bindings"]["qdrant_url"] == "$WAVEMIND_QDRANT_URL" assert qdrant["required_secrets"] == ["WAVEMIND_QDRANT_API_KEY"] + pgvector = by_id["pgvector_10m_service"] + assert pgvector["inputs"]["provision_pgvector_shards"] is True + assert pgvector["inputs"]["pgvector_shard_count"] == "4" + assert pgvector["inputs"]["runner_label"] == "ubuntu-latest" + assert "pgvector_dsns" not in pgvector["input_bindings"] + assert "pgvector_dsn" not in pgvector["inputs"] def test_dispatch_markdown_lists_launch_and_promotion_commands(tmp_path): diff --git a/tests/test_production_evidence_env.py b/tests/test_production_evidence_env.py index 5b351e0..26c225c 100644 --- a/tests/test_production_evidence_env.py +++ b/tests/test_production_evidence_env.py @@ -32,7 +32,10 @@ def _ready_env(tmp_path): "WAVEMIND_SERVERLESS_NODES": "https://wm-a.staging.internal,https://wm-b.staging.internal", "WAVEMIND_QDRANT_URL": "http://qdrant.staging.internal:6333", "WAVEMIND_QDRANT_URLS": "http://qdrant-a.staging.internal:6333,http://qdrant-b.staging.internal:6333", - "WAVEMIND_PGVECTOR_DSN": "postgresql://user:pass@postgres.staging.internal:5432/wavemind", + "WAVEMIND_PGVECTOR_DSNS": ",".join( + f"postgresql://user:pass@postgres-{index}.staging.internal:5432/wavemind" + for index in range(4) + ), "WAVEMIND_FAISS_IVFPQ_PATH": str(tmp_path / "wavemind-faiss-ivfpq-50m.faiss"), "WAVEMIND_FAISS_IVFPQ_FREE_GB": "8", "WAVEMIND_API_KEY": "test-key", @@ -56,6 +59,7 @@ def test_production_evidence_env_contract_maps_missing_variables(): assert "external-http-cluster-load.yml" in cluster_nodes["workflows"] assert "benchmarks/http_cluster_load_results.json" in cluster_nodes["artifacts"] assert "gh secret set WAVEMIND_CLUSTER_NODES" in cluster_nodes["github_secret_command"] + assert by_name["WAVEMIND_PGVECTOR_DSNS"]["kind"] == "postgres-dsn-list" assert all(check["pass"] for check in payload["checks"]) @@ -68,7 +72,7 @@ def test_production_evidence_env_contract_ready_does_not_serialize_secret_values assert payload["summary"]["missing_required_env"] == [] serialized = json.dumps(payload, ensure_ascii=False) - assert "postgres.staging.internal" not in serialized + assert "postgres-0.staging.internal" not in serialized assert "qdrant.staging.internal" not in serialized assert "wm-a.staging.internal" not in serialized assert "test-key" not in serialized diff --git a/tests/test_production_evidence_gate.py b/tests/test_production_evidence_gate.py index aabcb65..e0ec1ea 100644 --- a/tests/test_production_evidence_gate.py +++ b/tests/test_production_evidence_gate.py @@ -51,6 +51,11 @@ def _write_100m_streaming_artifact(root: Path, *, engine: str) -> Path: def test_production_evidence_gate_tracks_strict_external_claims(): root = Path(__file__).resolve().parents[1] payload = evaluate_production_evidence(root) + cluster_load = json.loads( + (root / "benchmarks" / "http_cluster_load_results.json").read_text( + encoding="utf-8" + ) + ) assert payload["schema"] == "wavemind.production_evidence.v1" assert payload["overall_status"] == "action_required" @@ -60,8 +65,10 @@ def test_production_evidence_gate_tracks_strict_external_claims(): by_id = {row["id"]: row for row in payload["requirements"]} assert by_id["external_http_cluster"]["status"] == "pass" assert by_id["external_http_cluster"]["issues"] == [] - assert "query p99 84.799" in by_id["external_http_cluster"]["evidence"] - assert "lifecycle batch p99 6694.757" in by_id["external_http_cluster"]["evidence"] + evidence = by_id["external_http_cluster"]["evidence"] + metrics = cluster_load["results"][0] + assert f"query p99 {metrics['query_p99_ms']}" in evidence + assert f"lifecycle batch p99 {metrics['lifecycle_batch_p99_ms']}" in evidence assert "-f batch_query_size=24" in by_id["external_http_cluster"]["command"] assert by_id["external_http_active_active"]["artifact"] == ( "benchmarks/external_http_active_active_results.json" diff --git a/tests/test_production_evidence_preflight.py b/tests/test_production_evidence_preflight.py index 3a18f89..3404fae 100644 --- a/tests/test_production_evidence_preflight.py +++ b/tests/test_production_evidence_preflight.py @@ -30,7 +30,10 @@ def _ready_env(tmp_path): "WAVEMIND_SERVERLESS_NODES": "https://wm-a.staging.internal,https://wm-b.staging.internal", "WAVEMIND_QDRANT_URL": "http://qdrant.staging.internal:6333", "WAVEMIND_QDRANT_URLS": "http://qdrant-a.staging.internal:6333,http://qdrant-b.staging.internal:6333", - "WAVEMIND_PGVECTOR_DSN": "postgresql://user:pass@postgres.staging.internal:5432/wavemind", + "WAVEMIND_PGVECTOR_DSNS": ",".join( + f"postgresql://user:pass@postgres-{index}.staging.internal:5432/wavemind" + for index in range(4) + ), "WAVEMIND_FAISS_IVFPQ_PATH": str(tmp_path / "wavemind-faiss-ivfpq-50m.faiss"), "WAVEMIND_FAISS_IVFPQ_FREE_GB": "8", "WAVEMIND_API_KEY": "test-key", @@ -50,6 +53,9 @@ def test_production_evidence_preflight_reports_missing_env(): assert by_id["external_http_cluster"]["status"] == "action_required" assert "WAVEMIND_CLUSTER_NODES" in by_id["external_http_cluster"]["missing_env"] assert by_id["qdrant_10m_service"]["missing_env"] == ["WAVEMIND_QDRANT_URL"] + assert by_id["pgvector_10m_service"]["status"] == "ready" + assert by_id["pgvector_10m_service"]["missing_env"] == [] + assert "provision_pgvector_shards=true" in by_id["pgvector_10m_service"]["command"] assert by_id["faiss_ivfpq_50m"]["missing_env"] == ["WAVEMIND_FAISS_IVFPQ_PATH"] @@ -63,6 +69,8 @@ def test_production_evidence_preflight_can_be_ready_with_real_prerequisites(tmp_ by_id = {row["id"]: row for row in payload["checks"]} assert by_id["external_http_active_active"]["missing_env"] == [] + assert by_id["pgvector_10m_service"]["missing_env"] == [] + assert "four-service pgvector topology" in by_id["pgvector_10m_service"]["evidence"] assert "-f batch_query_size=24" in by_id["external_http_cluster"]["command"] assert by_id["hundred_million_remote_load"]["ready"] is True assert "production_streaming_load_qdrant_sharded_100m_results.json" in by_id[ diff --git a/tests/test_production_readiness_gate.py b/tests/test_production_readiness_gate.py index b6e9c1d..fe41ad0 100644 --- a/tests/test_production_readiness_gate.py +++ b/tests/test_production_readiness_gate.py @@ -4,6 +4,12 @@ from pathlib import Path +def _artifact_workflow_url(filename: str) -> str: + root = Path(__file__).resolve().parents[1] + payload = json.loads((root / "benchmarks" / filename).read_text(encoding="utf-8")) + return str(payload["workflow_run_url"]) + + def test_production_readiness_gate_reports_current_blockers(): from benchmarks.production_readiness_gate import evaluate_production_readiness @@ -54,7 +60,9 @@ def test_production_readiness_gate_reports_current_blockers(): assert "failure drill pass" in criteria["operator_autoscaling_repair"]["evidence"] assert "Lease transitions 1" in criteria["operator_autoscaling_repair"]["evidence"] assert "recovered API True" in criteria["operator_autoscaling_repair"]["evidence"] - assert "actions/runs/29057247023" in criteria["operator_autoscaling_repair"]["evidence"] + assert _artifact_workflow_url("kubernetes_operator_smoke_results.json") in criteria[ + "operator_autoscaling_repair" + ]["evidence"] assert "PDB min available 3" in criteria["operator_autoscaling_repair"]["evidence"] assert "topology constraints 2" in criteria["operator_autoscaling_repair"]["evidence"] assert "rolling pods replaced 4" in criteria["operator_autoscaling_repair"]["evidence"] @@ -93,7 +101,9 @@ def test_production_readiness_gate_reports_current_blockers(): assert "non-loopback multi-zone Kubernetes lifecycle" in criteria["serverless_externalized_state"]["requirement"] assert "kind lifecycle pass 13/13" in criteria["serverless_externalized_state"]["evidence"] assert "coherence 3/3" in criteria["serverless_externalized_state"]["evidence"] - assert "actions/runs/29064934749" in criteria["serverless_externalized_state"]["evidence"] + assert _artifact_workflow_url( + "kubernetes_serverless_lifecycle_smoke_results.json" + ) in criteria["serverless_externalized_state"]["evidence"] assert criteria["memory_os_worker"]["status"] == "pass" assert "predictive prewarm" in criteria["memory_os_worker"]["requirement"] assert "usage-pattern priority boosts" in criteria["memory_os_worker"]["requirement"] @@ -176,7 +186,9 @@ def test_production_readiness_gate_reports_current_blockers(): assert "outage delete suppression 1.0" in criteria["active_active_field_crdt"]["evidence"] assert "recovery convergence 1.0" in criteria["active_active_field_crdt"]["evidence"] assert "recovery delete suppression 1.0" in criteria["active_active_field_crdt"]["evidence"] - assert "actions/runs/29058433643" in criteria["active_active_field_crdt"]["evidence"] + assert _artifact_workflow_url( + "kubernetes_active_active_region_smoke_results.json" + ) in criteria["active_active_field_crdt"]["evidence"] assert "actor watermarks" in criteria["active_active_field_crdt"]["requirement"] assert "replication lag" in criteria["active_active_field_crdt"]["requirement"] assert "watermarks 3" in criteria["active_active_field_crdt"]["evidence"] @@ -197,7 +209,9 @@ def test_production_readiness_gate_reports_current_blockers(): assert "restore recall 1.0" in criteria["backup_restore_dr"]["evidence"] assert "Qdrant 24/24" in criteria["backup_restore_dr"]["evidence"] assert "replacement recall 1.0" in criteria["backup_restore_dr"]["evidence"] - assert "actions/runs/29064934749" in criteria["backup_restore_dr"]["evidence"] + assert _artifact_workflow_url( + "kubernetes_postgres_qdrant_dr_smoke_results.json" + ) in criteria["backup_restore_dr"]["evidence"] assert criteria["structured_multimodal_payloads"]["status"] == "pass" assert "3D assets" in criteria["structured_multimodal_payloads"]["requirement"] assert "shared cross-modal embedding space" in criteria["structured_multimodal_payloads"]["requirement"] diff --git a/tests/test_production_streaming_load_benchmark.py b/tests/test_production_streaming_load_benchmark.py index 63ed104..886e977 100644 --- a/tests/test_production_streaming_load_benchmark.py +++ b/tests/test_production_streaming_load_benchmark.py @@ -1125,14 +1125,193 @@ def test_streaming_load_plan_only_supports_pgvector_service(monkeypatch): assert row["engine"] == "WaveMind pgvector streaming" assert row["vectors"] == 10_000_000 assert row["estimated_index_gb"] == 0.0 - assert row["index_mode"].startswith("remote PostgreSQL/pgvector") - assert "WAVEMIND_PGVECTOR_DSN" in row["required_env"] - assert "missing_env:WAVEMIND_PGVECTOR_DSN" in row["blockers"] + assert row["index_mode"].startswith("modulo-sharded PostgreSQL/pgvector") + assert "WAVEMIND_PGVECTOR_DSNS" in row["required_env"] + assert "missing_env:WAVEMIND_PGVECTOR_DSNS" in row["blockers"] assert row["command_env"]["WAVEMIND_PGVECTOR_CREATE_HNSW"] == "1" + assert row["command_env"]["WAVEMIND_PGVECTOR_STORAGE_TYPE"] == "halfvec" + assert row["command_env"]["WAVEMIND_PGVECTOR_INSERT_MODE"] == "copy" + assert row["command_env"]["WAVEMIND_PGVECTOR_INDEX_TYPE"] == "hnsw" + assert row["command_env"]["WAVEMIND_PGVECTOR_HNSW_M"] == "16" + assert row["command_env"]["WAVEMIND_PGVECTOR_HNSW_EF_CONSTRUCTION"] == "256" + assert row["command_env"]["WAVEMIND_PGVECTOR_EF_SEARCH"] == "800" + assert row["command_env"]["WAVEMIND_PGVECTOR_QUERY_ROUTING"] == "namespace" + assert row["command_env"]["WAVEMIND_PGVECTOR_PREWARM_INDEX"] == "1" assert "--engines pgvector-service" in row["command"] assert "production_streaming_load_pgvector_10m_results.json" in row["command"] +def test_pgvector_config_validates_storage_and_insert_modes(monkeypatch): + from benchmarks.production_streaming_load_benchmark import _pgvector_config_from_env + + monkeypatch.setenv("WAVEMIND_PGVECTOR_STORAGE_TYPE", "halfvec") + monkeypatch.setenv("WAVEMIND_PGVECTOR_INSERT_MODE", "copy") + monkeypatch.setenv("WAVEMIND_PGVECTOR_INDEX_TYPE", "hnsw-binary") + config = _pgvector_config_from_env() + assert config["storage_type"] == "halfvec" + assert config["insert_mode"] == "copy" + assert config["index_type"] == "hnsw-binary" + + monkeypatch.setenv("WAVEMIND_PGVECTOR_QUERY_ROUTING", "namespace") + config = _pgvector_config_from_env() + assert config["query_routing"] == "namespace" + + monkeypatch.setenv("WAVEMIND_PGVECTOR_QUERY_ROUTING", "broadcast") + with pytest.raises(ValueError, match="must be fanout or namespace"): + _pgvector_config_from_env() + + monkeypatch.setenv("WAVEMIND_PGVECTOR_QUERY_ROUTING", "fanout") + monkeypatch.setenv("WAVEMIND_PGVECTOR_STORAGE_TYPE", "binary") + with pytest.raises(ValueError, match="must be vector or halfvec"): + _pgvector_config_from_env() + + +def test_pgvector_copy_batch_replaces_only_uncheckpointed_range(): + from benchmarks.production_streaming_load_benchmark import _pgvector_insert_batch + + class FakeCopy: + def __init__(self): + self.rows = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + def write_row(self, row): + self.rows.append(row) + + class FakeCursor: + def __init__(self): + self.executed = [] + self.copy_sql = None + self.copy_stream = FakeCopy() + + def execute(self, sql, params): + self.executed.append((sql, params)) + + def copy(self, sql): + self.copy_sql = sql + return self.copy_stream + + cursor = FakeCursor() + ids = np.asarray([101, 102], dtype=np.int64) + vectors = np.asarray([[1.0, 0.0], [0.0, 1.0]], dtype=np.float32) + _pgvector_insert_batch( + cursor, + table="wavemind_vectors", + collection="run-1", + ids=ids, + vectors=vectors, + storage_type="halfvec", + insert_mode="copy", + ) + + assert cursor.executed == [ + ( + "DELETE FROM wavemind_vectors WHERE collection = %s AND memory_id BETWEEN %s AND %s", + ("run-1", 101, 102), + ) + ] + assert cursor.copy_sql == ( + "COPY wavemind_vectors (collection, memory_id, embedding) FROM STDIN" + ) + assert [row[:2] for row in cursor.copy_stream.rows] == [ + ("run-1", 101), + ("run-1", 102), + ] + + +def test_pgvector_checkpoint_migrates_only_index_specific_signature(tmp_path): + from benchmarks.production_streaming_load_benchmark import ( + _checkpoint_signature, + _load_pgvector_checkpoint, + _new_checkpoint, + ) + + current = _checkpoint_signature( + engine="WaveMind pgvector streaming", + count=100, + dim=8, + query_count=4, + top_k=2, + seed=42, + noise=0.08, + batch_size=10, + extra={ + "table": "vectors", + "storage_type": "halfvec", + "insert_mode": "copy", + }, + ) + legacy = json.loads(json.dumps(current)) + legacy["extra"].update( + { + "create_hnsw": True, + "hnsw_m": 8, + "hnsw_ef_construction": 64, + "exact": False, + "iterative_scan": None, + } + ) + checkpoint = _new_checkpoint(legacy) + path = tmp_path / "pgvector-checkpoint.json" + path.write_text(json.dumps(checkpoint), encoding="utf-8") + + migrated = _load_pgvector_checkpoint(path, current) + + assert migrated["signature"] == current + assert set(migrated["metadata"]["signature_migrated_index_keys"]) == { + "create_hnsw", + "hnsw_m", + "hnsw_ef_construction", + "exact", + "iterative_scan", + } + + +def test_pgvector_shard_counts_cover_every_id_exactly(): + from benchmarks.production_streaming_load_benchmark import ( + _pgvector_benchmark_namespace_shard, + _pgvector_shard_expected_count, + ) + + assert [ + _pgvector_shard_expected_count(10, 4, index) for index in range(4) + ] == [3, 3, 2, 2] + assert sum( + _pgvector_shard_expected_count(10_000_003, 4, index) + for index in range(4) + ) == 10_000_003 + assert [ + _pgvector_benchmark_namespace_shard(source_id, 4) + for source_id in range(1, 9) + ] == [0, 1, 2, 3, 0, 1, 2, 3] + with pytest.raises(ValueError, match="source_id must be positive"): + _pgvector_benchmark_namespace_shard(0, 4) + + +def test_pgvector_sharded_mode_requires_multiple_services(monkeypatch): + from benchmarks.production_streaming_load_benchmark import run_pgvector_streaming + + monkeypatch.delenv("WAVEMIND_PGVECTOR_DSN", raising=False) + monkeypatch.setenv("WAVEMIND_PGVECTOR_DSNS", "postgresql://only-one") + + row = run_pgvector_streaming( + count=10, + dim=4, + query_count=2, + top_k=1, + seed=42, + noise=0.01, + batch_size=5, + ) + + assert row["skipped"] is True + assert "at least two service DSNs" in row["reason"] + + def test_streaming_load_plan_only_supports_qdrant_service(monkeypatch): from benchmarks.production_streaming_load_benchmark import plan_streaming_load diff --git a/tests/test_scale_plan.py b/tests/test_scale_plan.py index 7b8aca5..2b7fa43 100644 --- a/tests/test_scale_plan.py +++ b/tests/test_scale_plan.py @@ -313,7 +313,11 @@ def test_production_scale_run_plan_can_mark_profile_ready(monkeypatch): payload = build_production_scale_run_plan( profiles=["pgvector-10m"], - env={"WAVEMIND_PGVECTOR_DSN": "postgresql://example/wavemind"}, + env={ + "WAVEMIND_PGVECTOR_DSNS": ( + "postgresql://example-a/wavemind,postgresql://example-b/wavemind" + ) + }, disk_free_gb=1000.0, ) row = payload["profiles"][0] @@ -322,6 +326,7 @@ def test_production_scale_run_plan_can_mark_profile_ready(monkeypatch): assert row["status"] == "ready" assert row["missing_env"] == () assert row["blockers"] == () + assert row["command_env"]["WAVEMIND_PGVECTOR_QUERY_ROUTING"] == "namespace" assert row["slo_capacity_envelope"]["status"] in {"pass", "scale_required"} assert row["slo_capacity_envelope"]["required_replicas"] <= row["autoscaling_max_replicas"] assert row["cost_envelope"]["memory_count"] == 10_000_000 diff --git a/tests/test_strict_evidence_readiness_report.py b/tests/test_strict_evidence_readiness_report.py index 56757d2..82be6d8 100644 --- a/tests/test_strict_evidence_readiness_report.py +++ b/tests/test_strict_evidence_readiness_report.py @@ -22,8 +22,9 @@ def test_strict_evidence_readiness_joins_all_strict_requirements(): assert payload["summary"]["complete_count"] == 4 assert payload["summary"]["target_memories_total"] == 180_000_000 assert payload["summary"]["check_counts"] == {"pass": 8} - assert payload["summary"]["can_auto_run_now_count"] == 0 - assert payload["summary"]["blocker_counts"]["missing_env"] == 4 + assert payload["summary"]["can_auto_run_now_count"] == 1 + assert payload["summary"]["blocker_counts"]["missing_env"] == 3 + assert payload["summary"]["blocker_counts"]["missing_artifact"] == 1 assert payload["summary"]["blocker_counts"]["complete"] == 4 by_id = {row["id"]: row for row in payload["requirements"]} @@ -60,6 +61,10 @@ def test_strict_evidence_readiness_joins_all_strict_requirements(): assert qdrant_100m["locked_claim"] == "10M-100M service-backed production scale" assert '-f size="100000000"' in qdrant_100m["safe_dispatch_command"] assert qdrant_100m["can_auto_run_now"] is False + pgvector_10m = by_id["pgvector_10m_service"] + assert pgvector_10m["can_auto_run_now"] is True + assert pgvector_10m["missing_env"] == [] + assert "Run the safe dispatch command now" in pgvector_10m["next_action"] serialized = json.dumps(payload, sort_keys=True) assert "ghp_" not in serialized diff --git a/wavemind/production_evidence.py b/wavemind/production_evidence.py index 59537ce..f78d5e4 100644 --- a/wavemind/production_evidence.py +++ b/wavemind/production_evidence.py @@ -1157,6 +1157,16 @@ def _large_run_preflight( dsn = _env_value(env, "WAVEMIND_PGVECTOR_DSN") if not dsn.startswith(("postgresql://", "postgres://")): issues.append("WAVEMIND_PGVECTOR_DSN must start with postgresql:// or postgres://") + if "WAVEMIND_PGVECTOR_DSNS" in required_env and _env_value(env, "WAVEMIND_PGVECTOR_DSNS"): + dsns = _split_env_list(_env_value(env, "WAVEMIND_PGVECTOR_DSNS")) + if len(dsns) < 2: + issues.append("WAVEMIND_PGVECTOR_DSNS must contain at least two service DSNs") + for index, dsn in enumerate(dsns): + if not dsn.startswith(("postgresql://", "postgres://")): + issues.append( + f"WAVEMIND_PGVECTOR_DSNS item {index + 1} must start with " + "postgresql:// or postgres://" + ) for name in ("WAVEMIND_FAISS_PATH", "WAVEMIND_FAISS_IVFPQ_PATH"): if name in required_env and _env_value(env, name): required_free = float(plan.get("required_local_free_gb", 0.0) or 0.0) @@ -1198,6 +1208,55 @@ def _large_run_preflight( ) +def _managed_pgvector_preflight(root: Path) -> EvidencePreflightCheck: + plan_artifact = "benchmarks/production_streaming_load_pgvector_10m_plan.json" + output_artifact = "benchmarks/production_streaming_load_pgvector_10m_results.json" + workflow_path = root / ".github" / "workflows" / "production-streaming-load.yml" + plan = _first_plan(root, plan_artifact) + workflow = workflow_path.read_text(encoding="utf-8") if workflow_path.exists() else "" + issues: list[str] = [] + if not plan: + issues.append(f"missing plan artifact {plan_artifact}") + for token in ( + "provision_pgvector_shards", + "pgvector_shard_count", + "github-hosted-isolated-service-processes", + "pgvector/pgvector:pg16", + ): + if token not in workflow: + issues.append(f"production streaming workflow is missing {token}") + + command = ( + "gh workflow run production-streaming-load.yml --ref main " + "-f engine=pgvector-service -f size=10000000 -f dim=128 " + "-f queries=2000 -f top_k=10 -f batch_size=5000 " + "-f target_recall=0.95 -f target_p99_ms=100 -f target_qps=100 " + "-f replicas=3 -f autoscaling_max_replicas=24 " + "-f capacity_headroom=0.7 -f runner_label=ubuntu-latest " + "-f provision_pgvector_shards=true -f pgvector_shard_count=4 " + "-f runner_storage_root=state -f commit_results=true" + ) + return EvidencePreflightCheck( + id="pgvector_10m_service", + title="10M pgvector service load preflight", + status=_preflight_status(issues), + ready=not issues, + evidence=( + "workflow-provisioned four-service pgvector topology; external DSNs optional; " + f"plan {plan_artifact}" + ), + required_env=(), + missing_env=(), + command=command, + output_artifact=output_artifact, + issues=tuple(dict.fromkeys(issues)), + warnings=( + "Ephemeral isolated service processes prove the 10M candidate-index SLO, " + "not PostgreSQL HA or independent-node failure tolerance.", + ), + ) + + def evaluate_production_evidence_preflight( root: Path = PROJECT_ROOT, *, @@ -1253,14 +1312,7 @@ def evaluate_production_evidence_preflight( output_artifact="benchmarks/production_streaming_load_qdrant_sharded_10m_results.json", env=environment, ), - _large_run_preflight( - root, - check_id="pgvector_10m_service", - title="10M pgvector service load preflight", - plan_artifact="benchmarks/production_streaming_load_pgvector_10m_plan.json", - output_artifact="benchmarks/production_streaming_load_pgvector_10m_results.json", - env=environment, - ), + _managed_pgvector_preflight(root), _large_run_preflight( root, check_id="faiss_ivfpq_50m", @@ -1532,19 +1584,23 @@ def _dispatch_config( "required_secrets": ["WAVEMIND_QDRANT_API_KEYS"], } if requirement_id == "pgvector_10m_service": + inputs = _streaming_dispatch_inputs( + root, + plan_artifact="benchmarks/production_streaming_load_pgvector_10m_plan.json", + engine="pgvector-service", + size=10_000_000, + credential_input="pgvector_dsns", + credential_placeholder="$WAVEMIND_PGVECTOR_DSNS", + runner_label="ubuntu-latest", + commit_results=commit_results, + ) + inputs.pop("pgvector_dsns", None) + inputs["provision_pgvector_shards"] = True + inputs["pgvector_shard_count"] = "4" return { "workflow": "production-streaming-load.yml", "wave": "service-scale-10m", - "inputs": _streaming_dispatch_inputs( - root, - plan_artifact="benchmarks/production_streaming_load_pgvector_10m_plan.json", - engine="pgvector-service", - size=10_000_000, - credential_input="pgvector_dsn", - credential_placeholder="$WAVEMIND_PGVECTOR_DSN", - runner_label=runner_label, - commit_results=commit_results, - ), + "inputs": inputs, "required_secrets": [], } if requirement_id == "faiss_ivfpq_50m": diff --git a/wavemind/production_evidence_env.py b/wavemind/production_evidence_env.py index 31fbd58..28fae9c 100644 --- a/wavemind/production_evidence_env.py +++ b/wavemind/production_evidence_env.py @@ -63,10 +63,16 @@ "example": "REDACTED_QDRANT_API_KEY_A,REDACTED_QDRANT_API_KEY_B", "description": "Optional comma-separated Qdrant API keys for sharded Qdrant production load jobs.", }, - "WAVEMIND_PGVECTOR_DSN": { - "kind": "postgres-dsn", - "example": "postgresql://USER:PASSWORD@pgvector.staging.example.com:5432/wavemind", - "description": "PostgreSQL/pgvector DSN for 10M pgvector service streaming load.", + "WAVEMIND_PGVECTOR_DSNS": { + "kind": "postgres-dsn-list", + "example": ( + "postgresql://USER:PASSWORD@pgvector-a.staging.example.com:5432/wavemind," + "postgresql://USER:PASSWORD@pgvector-b.staging.example.com:5432/wavemind" + ), + "description": ( + "Comma-separated PostgreSQL/pgvector service DSNs for the namespace-sharded " + "10M streaming load." + ), }, "WAVEMIND_FAISS_IVFPQ_PATH": { "kind": "filesystem-path", diff --git a/wavemind/scale.py b/wavemind/scale.py index 1fb845f..9d292d7 100644 --- a/wavemind/scale.py +++ b/wavemind/scale.py @@ -380,15 +380,25 @@ def checkpoint(name: str) -> str: safety_factor=1.25, output_artifact=output("production_streaming_load_pgvector_10m_results.json"), checkpoint_path=checkpoint("pgvector-service-10000000"), - required_env=("WAVEMIND_PGVECTOR_DSN",), + required_env=("WAVEMIND_PGVECTOR_DSNS",), command_env={ - "WAVEMIND_PGVECTOR_DSN": "postgresql://user:password@postgres.example:5432/wavemind", + "WAVEMIND_PGVECTOR_DSNS": ",".join( + f"postgresql://user:password@postgres-{index}.example:5432/wavemind" + for index in range(4) + ), "WAVEMIND_PGVECTOR_CREATE_HNSW": "1", - "WAVEMIND_PGVECTOR_EF_SEARCH": "1000", + "WAVEMIND_PGVECTOR_STORAGE_TYPE": "halfvec", + "WAVEMIND_PGVECTOR_INSERT_MODE": "copy", + "WAVEMIND_PGVECTOR_INDEX_TYPE": "hnsw", + "WAVEMIND_PGVECTOR_HNSW_M": "16", + "WAVEMIND_PGVECTOR_HNSW_EF_CONSTRUCTION": "256", + "WAVEMIND_PGVECTOR_EF_SEARCH": "800", + "WAVEMIND_PGVECTOR_QUERY_ROUTING": "namespace", + "WAVEMIND_PGVECTOR_PREWARM_INDEX": "1", "WAVEMIND_PGVECTOR_WARMUP_QUERIES": "100", }, module_requirements=("psycopg",), - index_mode="remote PostgreSQL/pgvector HNSW service", + index_mode="remote namespace-sharded PostgreSQL/pgvector HNSW services", monthly_budget_usd=2_000.0, max_cost_per_1m_memories_usd=200.0, max_compute_cost_per_1m_queries_usd=10.0,