feat: persist declared concepts, and stop losing short follow-ups in recall

dist/dsh.js

@@ -9,10 +9,11 @@ import { randomUUID } from "node:crypto";
 import { dshMemorySource, isDshMemorySource } from "./src/format/dsh-source.js";
 import { DshExtractionUnavailableError, resolveDshExtractionReasoning, } from "./src/engine/dsh-extraction-route.js";
 import { openDb } from "./src/store/db.js";
-import { allActiveNodes, getRecentTurnMemoriesBySession, getStats, getVectorStats, getNextUnextractedTurn, getUnextractedTurn, getExtractionStats, getPendingSessionIds, getExtractionCompletedTurn, getNodeSources, markMessagesExtracted, markExtractionTurnCompleted, quarantineMessages, recordExtractionFailure, requeueQuarantined, saveMessageOnce, upsertNode, upsertTurnMemory, replaceNavigationTriples, } from "./src/store/store.js";
+import { allActiveNodes, getRecentTurnMemoriesBySession, getStats, getVectorStats, getNextUnextractedTurn, getUnextractedTurn, getExtractionStats, getPendingSessionIds, getExtractionCompletedTurn, getNodeSources, markMessagesExtracted, markExtractionTurnCompleted, quarantineMessages, recordExtractionFailure, requeueQuarantined, saveMessageOnce, upsertNode, upsertTurnMemory, replaceNavigationTriples, applyConceptExtraction, } from "./src/store/store.js";
 import { Extractor } from "./src/extractor/extract.js";
 import { GRAPH_EXTRACTION_TOOL, GRAPH_EXTRACTION_TOOL_NAME, } from "./src/extractor/contract.js";
 import { Recaller } from "./src/recaller/recall.js";
+import { createHttpReranker } from "./src/recaller/rerank.js";
 import { assembleContext } from "./src/format/assemble.js";
 import { replaceDshArchivedPrefix, selectDshRollingCompactionRange, } from "./src/format/dsh-compaction.js";
 import { replaceDshCompletedTurnTrace, projectDshCompletedTurnMemory, selectDshCompletedTurnTraceRange, } from "./src/format/dsh-turn-projection.js";
@@ -25,6 +26,13 @@ import { messageRetentionPolicyRevision, normalizeMessageRetentionPolicy, runMes
 export const name = "graph-memory-dsh";
 export const inject = ["tools", "llm", "systemPrompt", "agentLoop", "agents", "sessions", "credentials", "tokenMeter"];
 const HOST = "dsh";
+/**
+ * Retrieval pool multiplier applied before the visibility filter. Five slots
+ * per rendered slot is enough to backfill a block that a same-session answer
+ * would otherwise empty, while costing nothing measurable: the vector path is
+ * already a full scan and the lexical path is a single indexed LIKE.
+ */
+const RECALL_OVER_FETCH = 5;
 function sessionKey(id) {
     return `${HOST}:${String(id)}`;
 }
@@ -111,12 +119,20 @@ export function apply(ctx, input = {}) {
         compactTurnCount: maintenanceInterval,
         recallMaxNodes,
         semanticScoreThreshold: input.semanticScoreThreshold ?? DEFAULT_CONFIG.semanticScoreThreshold,
+        rerankTopN: input.rerankTopN ?? 6,
         embedding,
     };
     const extractionEnabled = input.extractionEnabled ?? true;
     const recallEnabled = input.recallEnabled ?? true;
     const db = openDb(config.dbPath, { busyTimeoutMs: input.dbBusyTimeoutMs });
     const recaller = new Recaller(db, config);
+    if (input.rerankEndpoint) {
+        recaller.setRerankFn(createHttpReranker({
+            endpoint: input.rerankEndpoint,
+            model: input.rerankModel,
+            timeoutMs: input.rerankTimeoutMs ?? 1500,
+        }));
+    }
     const latestRoute = new Map();
     const extractChain = new Map();
     const turnCounts = new Map();
@@ -281,6 +297,13 @@ export function apply(ctx, input = {}) {
             })),
         });
         replaceNavigationTriples(db, turnMemory, result.triples);
+        // The concept layer rides on the same extraction payload. Writing it here
+        // is what keeps the durable graph current: without this step the host only
+        // ever stored navigation, so every concept had to be recorded by hand.
+        const concept = applyConceptExtraction(db, sid, turnMemory.sources, result.entities ?? [], result.relations ?? []);
+        if (concept.nodes || concept.edges) {
+            ctx.logger.info(`[graph-memory] DSH stored ${concept.nodes} concept nodes and ${concept.edges} concept edges`);
+        }
         // The navigation graph becomes queryable only after its atomic SPO write.
         // This runs inside the existing background extraction worker, never in the
         // foreground turn, and makes the newest completed memory available to PPR.
@@ -539,7 +562,13 @@ export function apply(ctx, input = {}) {
             // is still in flight. Historical recall must wait for that shared probe;
             // otherwise the very first cross-session question can miss all vectors.
             await embeddingReady;
-            const recalled = await recaller.recall(query);
+            // Retrieval runs on a wider pool than the rendered width. The filters
+            // below drop memory already visible verbatim in this session's fresh
+            // window, and a pool sized exactly to the render width could be emptied
+            // entirely by that filter — a short follow-up whose top hits all came
+            // from the live session then injected nothing at all. Over-fetching lets
+            // the block backfill from cross-session or archived evidence instead.
+            const recalled = await recaller.recall(query, { overFetch: RECALL_OVER_FETCH });
             signal?.throwIfAborted?.();
             const key = String(id);
             const currentSession = sessionKey(id);
@@ -555,8 +584,8 @@ export function apply(ctx, input = {}) {
                     && isDshMemorySource(event?.data?.source)
                     && event?.surfaceOp?.op === "replace";
             });
-            const recalledNodes = filterDshRecallNodes(recalled.nodes, getNodeSources(db, recalled.nodes.map(node => node.id)), currentSession, visibleMessageIds, hasArchivedHistory);
-            const recalledMemories = filterDshRecallMemories(recalled.turnMemories, currentSession, visibleMessageIds);
+            const recalledNodes = filterDshRecallNodes(recalled.nodes, getNodeSources(db, recalled.nodes.map(node => node.id)), currentSession, visibleMessageIds, hasArchivedHistory).slice(0, config.recallMaxNodes);
+            const recalledMemories = filterDshRecallMemories(recalled.turnMemories, currentSession, visibleMessageIds).slice(0, config.recallMaxNodes);
             if (!recalledNodes.length && !recalledMemories.length)
                 return decision;
             const recalledIds = new Set(recalledNodes.map(node => node.id));

dist/src/consolidate/cluster.js

@@ -0,0 +1,156 @@
+/**
+ * Clustering primitives used by memory consolidation.
+ *
+ * Adapted from slow-stack/mneme (MIT License) — `dsh-mneme/src/dream/clustering.js`,
+ * https://github.com/slow-stack/mneme. The algorithms are reproduced unchanged;
+ * only types and the grouping key are parameterized so they fit this store's
+ * turn-memory shape. Original copyright (c) slow-stack/mneme contributors.
+ *
+ * Kept as a standalone module: nothing here touches the database, so it can be
+ * exercised and tested without a store, and the consolidation layer can be
+ * dropped without leaving a mark on recall or extraction.
+ */
+/**
+ * Cosine similarity of two equal-length vectors. Returns 0 for empty input,
+ * mismatched lengths, or a zero-norm vector (numerically stable rather than
+ * NaN, so a degenerate embedding cannot poison a whole run).
+ */
+export function cosineSimilarity(a, b) {
+    if (!a || !b || a.length === 0 || a.length !== b.length)
+        return 0;
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    for (let i = 0; i < a.length; i += 1) {
+        dot += a[i] * b[i];
+        na += a[i] * a[i];
+        nb += b[i] * b[i];
+    }
+    if (na === 0 || nb === 0)
+        return 0;
+    return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+/**
+ * K-Means over numeric vectors, returning one index array per cluster.
+ *
+ * k-means++ seeding, and empty clusters are reseeded mid-iteration so a bad
+ * draw cannot collapse the result to fewer clusters than requested.
+ */
+export function kMeans(vectors, k, opts = {}) {
+    const n = vectors.length;
+    if (n === 0)
+        return [];
+    const { maxIter = 100, tol = 1e-4, random = Math.random } = opts;
+    if (k <= 0 || k >= n)
+        return [vectors.map((_, index) => index)];
+    const dim = vectors[0].length;
+    const randomIndex = () => Math.floor(random() * n);
+    const squaredDist = (v, c) => {
+        let sum = 0;
+        for (let d = 0; d < dim; d += 1)
+            sum += (v[d] - c[d]) * (v[d] - c[d]);
+        return sum;
+    };
+    // First centroid uniform, later ones sampled proportional to squared distance.
+    const centroids = [Float32Array.from(vectors[randomIndex()])];
+    while (centroids.length < k) {
+        const dist = new Array(n);
+        let sum = 0;
+        for (let i = 0; i < n; i += 1) {
+            let best = Infinity;
+            for (const centroid of centroids)
+                best = Math.min(best, squaredDist(vectors[i], centroid));
+            dist[i] = best;
+            sum += best;
+        }
+        if (sum === 0) {
+            centroids.push(Float32Array.from(vectors[randomIndex()]));
+        }
+        else {
+            let r = random() * sum;
+            let pick = 0;
+            for (let i = 0; i < n; i += 1) {
+                r -= dist[i];
+                if (r <= 0) {
+                    pick = i;
+                    break;
+                }
+            }
+            centroids.push(Float32Array.from(vectors[pick]));
+        }
+    }
+    const assignment = new Array(n).fill(0);
+    for (let iter = 0; iter < maxIter; iter += 1) {
+        let changed = false;
+        for (let i = 0; i < n; i += 1) {
+            let best = 0;
+            let bestD = Infinity;
+            for (let j = 0; j < k; j += 1) {
+                const d = squaredDist(vectors[i], centroids[j]);
+                if (d < bestD) {
+                    bestD = d;
+                    best = j;
+                }
+            }
+            if (assignment[i] !== best) {
+                assignment[i] = best;
+                changed = true;
+            }
+        }
+        if (!changed)
+            break;
+        const sums = Array.from({ length: k }, () => new Float32Array(dim));
+        const counts = new Array(k).fill(0);
+        for (let i = 0; i < n; i += 1) {
+            const cluster = assignment[i];
+            counts[cluster] += 1;
+            for (let d = 0; d < dim; d += 1)
+                sums[cluster][d] += vectors[i][d];
+        }
+        let maxMove = 0;
+        for (let j = 0; j < k; j += 1) {
+            if (counts[j] === 0) {
+                centroids[j] = Float32Array.from(vectors[randomIndex()]);
+                maxMove = Infinity;
+                continue;
+            }
+            for (let d = 0; d < dim; d += 1) {
+                const mean = sums[j][d] / counts[j];
+                const move = mean - centroids[j][d];
+                if (move * move > maxMove)
+                    maxMove = move * move;
+                centroids[j][d] = mean;
+            }
+        }
+        if (Math.sqrt(maxMove) < tol)
+            break;
+    }
+    const clusters = Array.from({ length: k }, () => []);
+    for (let i = 0; i < n; i += 1)
+        clusters[assignment[i]].push(i);
+    return clusters.filter(cluster => cluster.length > 0);
+}
+/** Cluster items by their vectors, preserving the original object references. */
+export function clusterByVector(items, vectors, k) {
+    return kMeans(vectors, k).map(cluster => cluster.map(index => items[index]));
+}
+/**
+ * Pairs that are highly similar yet might contradict each other.
+ *
+ * Similarity is only a candidate signal: `0.9` means "read these two together",
+ * never "these two disagree". Only items sharing the same group key are paired,
+ * because a merge or conflict between unrelated kinds of memory is meaningless.
+ */
+export function findPotentialConflicts(items, vectors, keyOf, threshold = 0.85) {
+    const pairs = [];
+    for (let i = 0; i < items.length; i += 1) {
+        for (let j = i + 1; j < items.length; j += 1) {
+            if (keyOf(items[i]) !== keyOf(items[j]))
+                continue;
+            const similarity = cosineSimilarity(vectors[i], vectors[j]);
+            if (similarity > threshold)
+                pairs.push({ a: items[i], b: items[j], similarity });
+        }
+    }
+    return pairs;
+}

dist/src/extractor/contract.js

@@ -9,6 +9,30 @@ const NavigationTripleSchema = Type.Object({
     predicate: Type.String({ minLength: 1, description: "Short relation phrase." }),
     object: Type.String({ minLength: 1, description: "Concrete searchable object phrase." }),
 }, { additionalProperties: false });
+const ConceptEntitySchema = Type.Object({
+    name: Type.String({
+        minLength: 1,
+        maxLength: 24,
+        description: "Reusable noun phrase naming one concept; never a sentence, a status word, or a bare quantity.",
+    }),
+    type: Type.String({
+        enum: ["EVENT", "SKILL", "TASK"],
+        description: "Concept kind: EVENT happened, SKILL reusable capability, TASK goal or pending work.",
+    }),
+    description: Type.String({
+        minLength: 1,
+        description: "One sentence stating what this concept is, using only evidence from this turn.",
+    }),
+}, { additionalProperties: false });
+const ConceptRelationSchema = Type.Object({
+    from: Type.String({ minLength: 1, description: "Subject name; must equal one entities[].name value." }),
+    to: Type.String({ minLength: 1, description: "Object name; must equal one entities[].name value." }),
+    type: Type.String({
+        enum: ["RELATES", "SUPERSEDES", "USED_SKILL", "SOLVED_BY", "REQUIRES", "PATCHES", "CONFLICTS_WITH"],
+        description: "Declared relation kind between the two concepts.",
+    }),
+    instruction: Type.String({ minLength: 1, description: "Short natural-language statement of the relation." }),
+}, { additionalProperties: false });
 /**
  * Provider-facing extraction contract.
  *
@@ -27,11 +51,17 @@ export const GRAPH_EXTRACTION_SCHEMA = Type.Object({
     triples: Type.Array(NavigationTripleSchema, {
         description: "Required SPO relations derived only from summary; use an empty array when none are explicit.",
     }),
+    entities: Type.Optional(Type.Array(ConceptEntitySchema, {
+        description: "Optional durable concept list for this turn; omit or use [] when the turn yields no reusable concept.",
+    })),
+    relations: Type.Optional(Type.Array(ConceptRelationSchema, {
+        description: "Optional typed links between entries of entities; both endpoints must be declared concepts.",
+    })),
 }, { additionalProperties: false });
 export const GRAPH_EXTRACTION_TOOL_NAME = "submit_result";
 export const GRAPH_EXTRACTION_TOOL = Object.freeze({
     name: GRAPH_EXTRACTION_TOOL_NAME,
-    description: "Submit exactly the three required fields summary, outcome, and triples. Derive zero or more subject-predicate-object triples only from the summary; triples must be [] when no relation is explicit. Emit no text.",
+    description: "Submit exactly the three required fields summary, outcome, and triples. Derive zero or more subject-predicate-object triples only from the summary; triples must be [] when no relation is explicit. Optionally add entities and relations when the turn names reusable concepts. Emit no text.",
     parameters: GRAPH_EXTRACTION_SCHEMA,
 });
 /** Fail closed before normalization or persistence when the contract is incomplete. */

dist/src/extractor/extract.js

@@ -41,9 +41,11 @@ const EXTRACT_SYS = `【任务】
 1. summary 用一句简短、自包含的话写清本轮对象、结论和最终回答明确报告的完成情况；不复述过程，不添加输入未表达的信息。
 2. outcome 按最终回答选择 completed、partial、failed、informational、unknown 之一。
 3. triples 只从 summary 拆分。subject 和 object 使用具体可检索短语，predicate 使用简短自然语言；没有明确关系时使用空数组，不补充、不猜测。
+4. entities 可选，用于记录本轮出现的、以后还会被问到的概念。每个概念给三样：name 是 2~16 字的名词短语（禁止整句，禁止“全部正常 / 已完成 / 72 个”这类状态与数量）；type 取 EVENT（发生过的事）、SKILL（可复用的工具、技能、脚本、服务、配置）、TASK（目标、待办、缺口）；description 用一句话说明它是什么，只用本轮信息。本轮没有可复用概念时省略该字段。
+5. relations 可选，只在确有明确关系时输出：from 与 to 必须是 entities 里出现过的 name，type 取 RELATES、SUPERSEDES、USED_SKILL、SOLVED_BY、REQUIRES、PATCHES、CONFLICTS_WITH 之一，instruction 用一句中文说明关系。没有关系时省略该字段。
 
 【输出合同】
-只调用 submit_result 一次。参数对象必须且只能包含 summary、outcome、triples 三个顶层字段，三个字段都不能省略。不要输出解释或正文。
+只调用 submit_result 一次。参数对象必须包含 summary、outcome、triples 三个顶层字段，三个字段都不能省略；可选包含 entities 与 relations。不要输出解释或正文。
 
 示例一：
 输入：用户要求把周会改到周四；最终回答确认日程已更新。
@@ -57,7 +59,11 @@ const EXTRACT_SYS = `【任务】
 输入只确认稍后继续讨论，没有新结论。
 输出：{"summary":"本轮确认稍后继续讨论，未产生新结论。","outcome":"informational","triples":[]}
 
-调用前自检：参数是否恰好包含三个顶层字段；outcome 是否属于枚举；没有关系时 triples 是否仍明确写为 []。`;
+示例四（带概念）：
+输入：助手把抽取路由钉到本地 27B，避免打到线上账号。
+输出：{"summary":"抽取路由已改为本地 27B，不再使用线上账号。","outcome":"completed","triples":[{"subject":"抽取路由","predicate":"改为","object":"本地 27B"}],"entities":[{"name":"抽取路由","type":"SKILL","description":"抽取请求使用的模型路由配置"},{"name":"本地 27B","type":"SKILL","description":"本机 llama-swap 上的主模型"},{"name":"线上账号","type":"EVENT","description":"此前误用线上 API 的问题"}],"relations":[{"from":"抽取路由","to":"本地 27B","type":"REQUIRES","instruction":"路由改为依赖本地模型"},{"from":"抽取路由","to":"线上账号","type":"SOLVED_BY","instruction":"改路由后不再走线上账号"}]}
+
+调用前自检：是否包含 summary、outcome、triples 三个必需字段；outcome 是否属于枚举；没有关系时 triples 是否仍明确写为 []；若输出 entities 与 relations，relation 两端是否都能在 entities 的 name 中找到，且 type 是否在枚举内。`;
 // ─── 提取 User Prompt ───────────────────────────────────────────
 const EXTRACT_USER = (msgs, priorTurns) => `<Previous Turn Summaries>
 ${priorTurns.length ? JSON.stringify(priorTurns.map(memory => memory.summary)) : "（无）"}
@@ -94,6 +100,11 @@ export class Extractor {
             return {
                 turn: { summary: p.summary, outcome: p.outcome },
                 triples: p.triples.map(triple => ({ ...triple })),
+                // Optional blocks are copied verbatim and default to empty. The contract
+                // decides whether they are well formed; a host that does not persist
+                // concepts simply ignores them, so the field stays additive.
+                entities: (p.entities ?? []).map(entity => ({ ...entity })),
+                relations: (p.relations ?? []).map(relation => ({ ...relation })),
             };
         }
         catch (err) {