Merge develop→main: gap waves 3–6 + GT-540 (evidence seam live, enforcer parity, RAG read+write)

.harness/scripts/ci/14-rag-index-sync.mjs

@@ -21,6 +21,10 @@ import { readFileSync, writeFileSync, existsSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { createRagAdapter } from './rag-port.mjs';
 import { syncIndex, chunkIds } from './rag-sync.mjs';
+// Side-effect import: registers the durable `pgvector` adapter (GT-538 / ADR-0112)
+// so EVOLITH_RAG_PROVIDER=pgvector resolves instead of failing closed. The port
+// itself stays vendor-neutral; the vendor is wired only here, at the CI seam.
+import './rag-pgvector.mjs';
 
 const RAG_SYNC_ENABLED = process.env.EVOLITH_RAG_SYNC === 'true';
 

.harness/scripts/ci/rag-embed-integration.test.mjs

@@ -0,0 +1,121 @@
+/**
+ * GT-539 — Integration of the real embedding model into the durable pgvector
+ * adapter and the write-side sync. STUBBED DB client + STUBBED sidecar fetch;
+ * the live sidecar (running Qwen3) is deploy-gated and NOT exercised here.
+ *
+ * Covers: configured ⇒ real-model path (dim 1024 + model id in corpus_version);
+ * unconfigured ⇒ hashEmbed offline default; wrong-dim sidecar ⇒ fail closed.
+ */
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { syncIndex } from './rag-sync.mjs';
+import { pgvectorAdapter, RAG_EMBEDDING_DIM, HASH_EMBED_MODEL_ID } from './rag-pgvector.mjs';
+import { RagPortError } from './rag-port.mjs';
+
+const DOC = `# Title\n\nintro\n\n## Alpha\n\nalpha body\n\n## Beta\n\nbeta body\n`;
+const URL = 'http://localhost:8085/embed';
+const vec = (n, f = 0.1) => Array.from({ length: n }, () => f);
+
+function dbStub() {
+  const calls = [];
+  return {
+    calls,
+    async query(text, params) {
+      calls.push({ text, params });
+      return { rowCount: Array.isArray(params?.[0]) ? params[0].length : 1 };
+    },
+  };
+}
+
+/** A sidecar fetch stub returning `dim`-length vectors for every input. */
+function sidecarStub(dim = RAG_EMBEDDING_DIM) {
+  const calls = [];
+  const fn = async (url, opts) => {
+    const { input } = JSON.parse(opts.body);
+    calls.push({ url, input });
+    return { ok: true, status: 200, async json() { return { data: input.map(() => ({ embedding: vec(dim) })) }; } };
+  };
+  fn.calls = calls;
+  return fn;
+}
+
+/** Run a block with EVOLITH_RAG_EMBED_URL unset (isolated offline default). */
+async function withoutSidecarEnv(fn) {
+  const saved = process.env.EVOLITH_RAG_EMBED_URL;
+  delete process.env.EVOLITH_RAG_EMBED_URL;
+  try {
+    return await fn();
+  } finally {
+    if (saved !== undefined) process.env.EVOLITH_RAG_EMBED_URL = saved;
+  }
+}
+
+test('unconfigured adapter uses the hashEmbed offline default (model id = hash-sha256@1024)', async () => {
+  await withoutSidecarEnv(async () => {
+    const a = pgvectorAdapter({ client: dbStub() });
+    assert.equal(a.embeddingModelId, HASH_EMBED_MODEL_ID);
+    const [v] = await a.embed(['hello']);
+    assert.equal(v.length, RAG_EMBEDDING_DIM);
+  });
+});
+
+test('configured adapter uses the real model path (sidecar called, dim 1024)', async () => {
+  const sidecar = sidecarStub();
+  const a = pgvectorAdapter({ client: dbStub(), url: URL, fetch: sidecar });
+  assert.equal(a.embeddingModelId, 'qwen3-embedding-0.6b');
+  const out = await a.embed(['x', 'y']);
+  assert.equal(out.length, 2);
+  assert.equal(out[0].length, RAG_EMBEDDING_DIM);
+  assert.ok(sidecar.calls.length >= 1, 'the sidecar must be called on the real-model path');
+});
+
+test('configured sync records the model id in each chunk corpus_version (ADR-0090 §3)', async () => {
+  const db = dbStub();
+  const a = pgvectorAdapter({ client: db, url: URL, fetch: sidecarStub() });
+  const receipt = await syncIndex({
+    adapter: a,
+    changed: [{ sourceFile: 'reference/x.md', content: DOC }],
+    corpusVersion: 'gitsha123',
+  });
+  assert.equal(receipt.corpusVersion, 'gitsha123+qwen3-embedding-0.6b');
+
+  const upsertCalls = db.calls.filter((c) => /INSERT INTO rag_chunks/.test(c.text));
+  assert.ok(upsertCalls.length >= 2, 'expected multiple chunks upserted');
+  for (const c of upsertCalls) {
+    // param[8] is the corpus_version column; param[9] is the 1024-dim vector literal.
+    assert.equal(c.params[8], 'gitsha123+qwen3-embedding-0.6b');
+    assert.match(c.params[9], /^\[/);
+  }
+});
+
+test('offline sync still declares the (hash) model id in corpus_version', async () => {
+  await withoutSidecarEnv(async () => {
+    const a = pgvectorAdapter({ client: dbStub() });
+    const receipt = await syncIndex({
+      adapter: a,
+      changed: [{ sourceFile: 'reference/x.md', content: DOC }],
+      corpusVersion: 'gitsha123',
+    });
+    assert.equal(receipt.corpusVersion, `gitsha123+${HASH_EMBED_MODEL_ID}`);
+  });
+});
+
+test('configured adapter fails closed on a wrong-dimension sidecar response', async () => {
+  const a = pgvectorAdapter({ client: dbStub(), url: URL, fetch: sidecarStub(768) });
+  await assert.rejects(() => a.embed(['x']), RagPortError);
+});
+
+test('an injected embedder with the wrong dim is rejected at construction (fail closed)', () => {
+  assert.throws(
+    () => pgvectorAdapter({ client: dbStub(), embedder: { modelId: 'x', dim: 512, embed: async () => [] } }),
+    RagPortError,
+  );
+});
+
+test('adapter embed() re-checks vector dimension even from an injected embedder', async () => {
+  const a = pgvectorAdapter({
+    client: dbStub(),
+    embedder: { modelId: 'liar', dim: RAG_EMBEDDING_DIM, embed: async (t) => t.map(() => vec(999)) },
+  });
+  await assert.rejects(() => a.embed(['x']), RagPortError);
+});

.harness/scripts/ci/rag-embed-qwen3.mjs

@@ -0,0 +1,130 @@
+/**
+ * GT-539 / ADR-0112 §1,§2,§4,§5 — Real embedding model behind the RAG port.
+ *
+ * A model-agnostic embedder that computes vectors by calling a LOCAL inference
+ * SIDECAR over HTTP. Per ADR-0112 §4 the Node process never runs the model
+ * in-process — the sidecar is the platform boundary and embeddings are computed
+ * on-perimeter, so there is NO corpus egress. The default model is
+ * `Qwen3-Embedding-0.6B` at dimension 1024 (ADR-0112 §1/§2), but the endpoint,
+ * model id and dimension are all env/config-driven so switching model is a
+ * re-embed, not a code change (the port stays model-agnostic — ADR-0090 §3).
+ *
+ * The network client is an INJECTED seam (`config.fetch`) so unit tests never
+ * touch the network. This module NEVER imports a network library at load time.
+ *
+ * Fails closed on: a missing endpoint, a transport error, a non-OK response, an
+ * unparseable body, a wrong vector count, or ANY vector whose length != the
+ * expected dimension. Dimension consistency is load-bearing — vectors of a
+ * different dimension are not cosine-comparable with the store (ADR-0112 §2/§5).
+ */
+
+import { RagPortError } from './rag-port.mjs';
+
+/** ADR-0112 §1 — default self-hosted OSS model (Apache-2.0), lean footprint. */
+export const DEFAULT_EMBED_MODEL = 'qwen3-embedding-0.6b';
+
+/** ADR-0112 §2 — Qwen3 Matryoshka maximum; the pgvector column is vector(1024). */
+export const DEFAULT_EMBED_DIM = 1024;
+
+/** Resolve the configured sidecar URL (injected config wins over env). */
+function resolveUrl(config = {}) {
+  return config.url || config.embedUrl || process.env.EVOLITH_RAG_EMBED_URL || null;
+}
+
+/**
+ * True when an on-perimeter inference sidecar is configured (injected config or
+ * env). When false, callers keep the offline `hashEmbed` default — the sidecar
+ * is opt-in so dry-run/tests never require it.
+ */
+export function isQwen3Configured(config = {}) {
+  return Boolean(resolveUrl(config));
+}
+
+/** Accept OpenAI-style {data:[{embedding}]}, TEI-style [[...]], or {embeddings}. */
+function extractVectors(body) {
+  const pick = (e) => (Array.isArray(e) ? e : e && e.embedding);
+  if (Array.isArray(body)) return body.map(pick);
+  if (body && Array.isArray(body.data)) return body.data.map(pick);
+  if (body && Array.isArray(body.embeddings)) return body.embeddings.map(pick);
+  return null;
+}
+
+/**
+ * Build a Qwen3 sidecar embedder.
+ *
+ * @param {object}   [config]
+ * @param {string}   [config.url]    sidecar endpoint (else EVOLITH_RAG_EMBED_URL)
+ * @param {string}   [config.model]  model id (else EVOLITH_RAG_EMBED_MODEL / default)
+ * @param {number}   [config.dim]    expected output dimension (default 1024)
+ * @param {Function} [config.fetch]  injected fetch seam (else global fetch)
+ * @returns {{ modelId: string, dim: number, embed(texts: string[]): Promise<number[][]> }}
+ */
+export function makeQwen3Embedder(config = {}) {
+  const url = resolveUrl(config);
+  if (!url) {
+    throw new RagPortError(
+      'Qwen3 embedder requires a sidecar URL (config.url or EVOLITH_RAG_EMBED_URL)',
+    );
+  }
+  const modelId = config.model || process.env.EVOLITH_RAG_EMBED_MODEL || DEFAULT_EMBED_MODEL;
+  const dim = Number(config.dim || DEFAULT_EMBED_DIM);
+  // Injected client seam — tests pass a stub; real runs use global fetch.
+  const doFetch = config.fetch || globalThis.fetch;
+  if (typeof doFetch !== 'function') {
+    throw new RagPortError(
+      'Qwen3 embedder requires a fetch implementation (config.fetch or a global fetch)',
+    );
+  }
+
+  return {
+    modelId,
+    dim,
+    async embed(texts) {
+      if (!Array.isArray(texts)) throw new RagPortError('embed() expects an array of texts');
+      if (texts.length === 0) return [];
+
+      let res;
+      try {
+        res = await doFetch(url, {
+          method: 'POST',
+          headers: { 'content-type': 'application/json' },
+          body: JSON.stringify({ model: modelId, input: texts }),
+        });
+      } catch (err) {
+        // Fail closed — never silently fall back to a non-semantic embedding.
+        throw new RagPortError(`Qwen3 sidecar request failed (${url}): ${err.message}`);
+      }
+
+      if (!res || res.ok === false) {
+        const status = res && res.status != null ? res.status : 'no-response';
+        throw new RagPortError(`Qwen3 sidecar returned a non-OK response (status ${status})`);
+      }
+
+      let body;
+      try {
+        body = await res.json();
+      } catch (err) {
+        throw new RagPortError(`Qwen3 sidecar returned an unparseable body: ${err.message}`);
+      }
+
+      const vectors = extractVectors(body);
+      if (!Array.isArray(vectors) || vectors.length !== texts.length) {
+        throw new RagPortError(
+          `Qwen3 sidecar returned ${Array.isArray(vectors) ? vectors.length : typeof vectors} ` +
+            `vector(s) for ${texts.length} input(s)`,
+        );
+      }
+      for (let i = 0; i < vectors.length; i++) {
+        const v = vectors[i];
+        if (!Array.isArray(v) || v.length !== dim) {
+          // Fail closed — a mismatched dimension is not cosine-comparable.
+          throw new RagPortError(
+            `Qwen3 sidecar vector ${i} has dimension ` +
+              `${Array.isArray(v) ? v.length : typeof v}, expected ${dim} (ADR-0112 §2/§5)`,
+          );
+        }
+      }
+      return vectors;
+    },
+  };
+}

.harness/scripts/ci/rag-embed-qwen3.test.mjs

@@ -0,0 +1,122 @@
+/**
+ * GT-539 — Unit tests for the Qwen3 sidecar embedder (rag-embed-qwen3.mjs).
+ *
+ * Every case uses a STUBBED fetch — the network is never touched. The live
+ * sidecar (an actually-running Qwen3 model) is deploy-gated and NOT exercised
+ * here (ADR-0112 §4).
+ */
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import {
+  makeQwen3Embedder,
+  isQwen3Configured,
+  DEFAULT_EMBED_MODEL,
+  DEFAULT_EMBED_DIM,
+} from './rag-embed-qwen3.mjs';
+import { RagPortError } from './rag-port.mjs';
+
+const URL = 'http://localhost:8085/embed';
+const vec = (n, fill = 0.1) => Array.from({ length: n }, () => fill);
+
+/** Build an injectable fetch stub that records calls and shapes its response. */
+function fetchStub({ vectors = [], ok = true, status = 200, shape = 'openai', throwErr = null, badJson = false } = {}) {
+  const calls = [];
+  const fn = async (url, opts) => {
+    calls.push({ url, opts, body: opts && JSON.parse(opts.body) });
+    if (throwErr) throw new Error(throwErr);
+    return {
+      ok,
+      status,
+      async json() {
+        if (badJson) throw new Error('invalid json');
+        if (shape === 'openai') return { data: vectors.map((v) => ({ embedding: v })) };
+        if (shape === 'tei') return vectors; // bare array of vectors
+        if (shape === 'embeddings') return { embeddings: vectors };
+        return {};
+      },
+    };
+  };
+  fn.calls = calls;
+  return fn;
+}
+
+test('isQwen3Configured reflects config.url', () => {
+  assert.equal(isQwen3Configured({}), false);
+  assert.equal(isQwen3Configured({ url: URL }), true);
+});
+
+test('defaults: model qwen3-embedding-0.6b, dim 1024 (ADR-0112 §1/§2)', () => {
+  const e = makeQwen3Embedder({ url: URL, fetch: fetchStub() });
+  assert.equal(e.modelId, DEFAULT_EMBED_MODEL);
+  assert.equal(e.modelId, 'qwen3-embedding-0.6b');
+  assert.equal(e.dim, 1024);
+  assert.equal(DEFAULT_EMBED_DIM, 1024);
+});
+
+test('embed posts {model,input} and returns dim-1024 vectors (openai shape)', async () => {
+  const stub = fetchStub({ vectors: [vec(1024), vec(1024, 0.2)] });
+  const e = makeQwen3Embedder({ url: URL, fetch: stub });
+  const out = await e.embed(['a', 'b']);
+  assert.equal(out.length, 2);
+  assert.equal(out[0].length, 1024);
+  assert.equal(stub.calls.length, 1);
+  assert.equal(stub.calls[0].url, URL);
+  assert.equal(stub.calls[0].opts.method, 'POST');
+  assert.deepEqual(stub.calls[0].body, { model: 'qwen3-embedding-0.6b', input: ['a', 'b'] });
+});
+
+test('embed accepts TEI bare-array and {embeddings} response shapes', async () => {
+  const e1 = makeQwen3Embedder({ url: URL, fetch: fetchStub({ vectors: [vec(1024)], shape: 'tei' }) });
+  assert.equal((await e1.embed(['x']))[0].length, 1024);
+  const e2 = makeQwen3Embedder({ url: URL, fetch: fetchStub({ vectors: [vec(1024)], shape: 'embeddings' }) });
+  assert.equal((await e2.embed(['x']))[0].length, 1024);
+});
+
+test('empty input short-circuits without a network call (no egress)', async () => {
+  const stub = fetchStub();
+  const e = makeQwen3Embedder({ url: URL, fetch: stub });
+  assert.deepEqual(await e.embed([]), []);
+  assert.equal(stub.calls.length, 0);
+});
+
+test('custom model + Matryoshka dim are honoured', async () => {
+  const e = makeQwen3Embedder({ url: URL, model: 'qwen3-embedding-4b', dim: 512, fetch: fetchStub({ vectors: [vec(512)] }) });
+  assert.equal(e.modelId, 'qwen3-embedding-4b');
+  assert.equal(e.dim, 512);
+  assert.equal((await e.embed(['x']))[0].length, 512);
+});
+
+test('wrong-dimension response fails closed (768 != 1024)', async () => {
+  const e = makeQwen3Embedder({ url: URL, fetch: fetchStub({ vectors: [vec(768)] }) });
+  await assert.rejects(() => e.embed(['x']), RagPortError);
+});
+
+test('vector-count mismatch fails closed', async () => {
+  const e = makeQwen3Embedder({ url: URL, fetch: fetchStub({ vectors: [vec(1024)] }) });
+  await assert.rejects(() => e.embed(['x', 'y']), RagPortError);
+});
+
+test('non-OK sidecar response fails closed', async () => {
+  const e = makeQwen3Embedder({ url: URL, fetch: fetchStub({ ok: false, status: 503 }) });
+  await assert.rejects(() => e.embed(['x']), RagPortError);
+});
+
+test('transport error fails closed (never silently degrades)', async () => {
+  const e = makeQwen3Embedder({ url: URL, fetch: fetchStub({ throwErr: 'ECONNREFUSED' }) });
+  await assert.rejects(() => e.embed(['x']), RagPortError);
+});
+
+test('unparseable body fails closed', async () => {
+  const e = makeQwen3Embedder({ url: URL, fetch: fetchStub({ badJson: true }) });
+  await assert.rejects(() => e.embed(['x']), RagPortError);
+});
+
+test('missing url throws (fail closed at construction)', () => {
+  const saved = process.env.EVOLITH_RAG_EMBED_URL;
+  delete process.env.EVOLITH_RAG_EMBED_URL;
+  try {
+    assert.throws(() => makeQwen3Embedder({ fetch: fetchStub() }), RagPortError);
+  } finally {
+    if (saved !== undefined) process.env.EVOLITH_RAG_EMBED_URL = saved;
+  }
+});