diff --git a/.changeset/page-citation-assets-metadata.md b/.changeset/page-citation-assets-metadata.md new file mode 100644 index 0000000..d088ce6 --- /dev/null +++ b/.changeset/page-citation-assets-metadata.md @@ -0,0 +1,6 @@ +--- +"@ontos-ai/knowhere-sdk": major +"@ontos-ai/knowhere-mcp": major +--- + +Preserve page citation asset descriptors only in chunk metadata, remove the deprecated SDK-side page citation asset generation options, and split local-cache/server-safe result handling into explicit `knowledge.parseToLocalCache(...)`, `knowledge.importJobResult(...)`, and `knowledge.loadJobResult(...)` methods. diff --git a/.changeset/parsed-storage-redesign.md b/.changeset/parsed-storage-redesign.md new file mode 100644 index 0000000..348b686 --- /dev/null +++ b/.changeset/parsed-storage-redesign.md @@ -0,0 +1,11 @@ +--- +"@ontos-ai/knowhere-sdk": minor +"@ontos-ai/knowhere-mcp": minor +--- + +Move parsed document reads to a storage-first SDK model with remote chunk fallback. + +Adds parsed snapshot storage types, `knowledge.withParsedStorage(...)`, +`knowledge.syncParsedDocument(...)`, paged `readChunks(...)` display params, +durable asset URL hardening, bounded remote grep/outline fallback, and explicit +disk parsed storage for MCP cache-directory usage. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d72fd52 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.pdf binary diff --git a/.gitignore b/.gitignore index 6be33cb..ecc2c5a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Dependencies node_modules/ yarn.lock +.repos/effect # Build output dist/ diff --git a/2026-07-04-222626-local-command-caveatcaveat-the-messages-below.txt b/2026-07-04-222626-local-command-caveatcaveat-the-messages-below.txt new file mode 100644 index 0000000..02f8500 --- /dev/null +++ b/2026-07-04-222626-local-command-caveatcaveat-the-messages-below.txt @@ -0,0 +1,1837 @@ +╭─── Claude Code v2.1.201 ─────────────────────────────────────────────────────╮ +│ │ Tips for getting started │ +│ Welcome back! │ Run /init to create a CLAUDE.m… │ +│ │ ─────────────────────────────── │ +│ ▐▛███▜▌ │ What's new │ +│ ▝▜█████▛▘ │ Claude Sonnet 5 sessions no lo… │ +│ ▘▘ ▝▝ │ Changed `AskUserQuestion` dial… │ +│ │ Changed the "default" permissi… │ +│ Opus 4.8 · API Usage Billing │ /release-notes for more │ +│ ~/github.com/ontosAI/knowhere-node-sdk │ │ +╰──────────────────────────────────────────────────────────────────────────────╯ + + +❯ /clear + +❯ read the new design @docs/new-design.md, and lets plan on the notebookside + /home/suguan/github.com/ontosAI/knowhere-notebook + ⎿  Read docs/new-design.md (119 lines) + + Thought for 25s (ctrl+o to expand) + +● I've read the design doc. Now let me explore the notebook codebase to + understand the current implementation before planning. Let me launch parallel + Explore agents. + +● 3 background agents launched (↓ to manage) + ├ Explore parse/upload flow & source status + ├ Explore chunk read/display/chat & assets + └ Explore Blob, scheduler & SDK config + +● The three Explore agents are running in parallel. While they investigate, let + me also look at the SDK side to understand what's already built, since the + design references specific SDK methods. Let me read the current SDK state to + understand the contract the notebook will consume. + +● Bash(git log --oneline -15 && echo "---STRUCTURE---" && find src -type f -name + "*.ts" 2>/dev/null | head -60) + ⎿  2d006ae Add parsed storage backed knowledge reads + b5b4339 Clarify local knowledge result imports + cbd8753 Add parsed result snapshot storage + … +61 lines (ctrl+o to expand) + + Thought for 8s (ctrl+o to expand) + +● The SDK side is already substantially built. Let me read the key contract + files that the notebook will consume — the storage adapter interface, the + knowledge client public API, and relevant types. + + Thought for 19s, read 3 files (ctrl+o to expand) + +● Now let me read the storage types — this is the critical interface the + notebook must implement with Vercel Blob. + + Thought for 10s, read 1 file (ctrl+o to expand) + +● I now have a complete picture of the SDK contract. Let me read the disk + storage implementation as a reference model for the Vercel Blob adapter the + notebook will implement. + + Thought for 16s, read 1 file (ctrl+o to expand) + +● I've fully mapped the SDK contract. The key surfaces the notebook must + integrate with: + + - ParsedDocumentStorage interface (manifest, chunk pages, assets, sync + progress) — notebook implements a Vercel Blob version + - ParsedDocumentSyncScheduler — notebook implements an Upstash/Vercel-safe + version + - knowledge.withParsedStorage({ storage, scheduler, limits }) — config entry + point + - readChunks({ assetUrlPolicy: 'durable' }), grepChunks, getDocumentOutline — + reads + - syncParsedDocument — for parse-flow "sync before complete"; importJobResult + replaces cacheJobResult + + Now I'm waiting on the three Explore agents scanning the notebook repo + (Blob/scheduler/SDK config; chunk read/display/chat/assets; parse + flow/status/source_parse_results). I'll continue planning once they report + back. + +✻ Sautéed for 2m 3s + +● Agent "Explore Blob, scheduler & SDK config" finished · 4m 54s + +● First agent is back with a thorough map. Two more agents still running (chunk + read/display/chat/assets, and parse flow/status). Waiting for those before I + design the plan. + + Key takeaway from this report: the notebook currently uses the older + KnowhereAssetStorageOptions adapter + loadJobResult and reimplements + progress/batching/deadline logic itself. The redesign migrates it to the newer + ParsedDocumentStorage interface via withParsedStorage. The existing Vercel + Blob helpers, Upstash workflow self-continuation pattern, and + source-parse-result-repository are the seams to build on. + +✻ Baked for 9s + +● Agent "Explore chunk read/display/chat & assets" finished · 6m 15s + +● Second agent back. One more running (parse flow / source status / + source_parse_results schema / migrations). Critical new facts from this + report: + + - Notebook already has a rich chunk-read path with a Blob snapshot and asset + mirroring — but it reimplements the manifest/chunk-page/asset logic itself + (chunks/server.ts, remote-parsed-snapshot-sync.ts, + parse-result-storage-adapter.ts), all built on the old loadJobResult + + KnowhereAssetStorageOptions adapter, not readChunks/ParsedDocumentStorage. + - Chat uses Knowhere agentic retrieval.query — no + grepChunks/getDocumentOutline today. + - Asset hardening (copy-to-blob) already exists in two places + (media-asset-hardening.ts for chat, chunks/server.ts mirroring for the + viewer). + + Waiting on the parse-flow/status/schema agent before finalizing the plan. + +✻ Cooked for 11s + +● Agent "Explore parse/upload flow & source status" finished · 6m 30s + + Thought for 1m 26s (ctrl+o to expand) + +● All three Explore agents are back. I now have a complete two-sided picture: + + SDK side (already built, this branch): ParsedDocumentStorage interface, + withParsedStorage({storage, scheduler, limits}), + readChunks({assetUrlPolicy:'durable'}), grepChunks, getDocumentOutline, + syncParsedDocument, importJobResult, resumable background sync with progress + cursor + scheduler + deadline limits, and synchronous asset hardening. + + Notebook side (uses the OLD path): + - Reads via client.documents.listChunks + a hand-rolled Blob + snapshot/chunk-page cache + asset mirroring (chunks/server.ts, + remote-parsed-snapshot-sync.ts, parse-result-storage-adapter.ts, + parsed-result-assets.ts), all on loadJobResult + the old + KnowhereAssetStorageOptions adapter — not the new ParsedDocumentStorage. + - Parse lifecycle: uploading|parsing|ready|failed (text column, atomic + requiredStatus guards) driven by an Upstash workflow with a + deterministic-workflowRunId continuation pattern. Marks ready after the + snapshot is written. + - source_parse_results has resultBlobUrl, snapshotManifestUrl/Key, assetUrls — + no revision_key, sync_status, or failure_stage. Migrations in drizzle/, next + is 0011. + - Chat uses agentic retrieval.query (no grep/outline). Asset hardening + (copy-to-Blob) already exists twice: media-asset-hardening.ts (chat) and the + viewer mirror in chunks/server.ts. + + This is a substantial migration. A few forks materially change the plan, so + let me confirm scope before I design it. + +● User answered Claude's questions: + ⎿  · How aggressively should the notebook migrate to the SDK's new + ParsedDocumentStorage + readChunks path? → Full rip-and-replace + (Recommended) + · Beyond chunk read/display + parse lifecycle, should this effort also + unify chat citation asset hardening onto the SDK's durable policy? → + Include chat hardening (Recommended) + · How should the non-blocking background sync (for read-misses on existing + remote docs with no Blob snapshot) run on Vercel? → QStash-enqueued + resumable sync (Recommended) + + Thought for 9s (ctrl+o to expand) + +● All three recommended. Let me read the critical notebook files directly to + ground the plan before writing it — the current storage adapter, the reconcile + workflow, the chunk server path, the schema, and the SDK client wiring. + + Thought for 25s, read 3 files (ctrl+o to expand) + +● Now let me read the remaining critical files: the chunk read path, route + orchestration, schema, repository, and chat wiring. + + Thought for 22s, read 2 files (ctrl+o to expand) +● This is a large, well-understood migration. Let me read the last few grounding + files — schema, the source service (snapshot orchestration), and the workflow + runtime — then dispatch a Plan agent to synthesize the design. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && sed -n '60,200p' + src/infrastructure/db/schema.ts) + ⎿   * of a Knowhere parse + index job; the actual chunks / file bytes liv + e + * upstream in Knowhere, not here. + … +138 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 24s, read 2 files (ctrl+o to expand) + +● I have a thorough picture of both sides now. Let me dispatch a Plan agent to + synthesize the detailed implementation design, giving it the full context I've + gathered. It can read the chat-hardening files itself. + +● Plan(Design notebook migration plan) + ⎿  Done (43 tool uses · 257.7k tokens · 7m 36s) + (ctrl+o to expand) + +● The Plan agent produced a thorough, well-grounded design. I have everything I + need. Let me write the final plan file, then present it for approval. + +● Updated plan + ⎿  /plan to preview + +● User approved Claude's plan + ⎿  Plan saved to: ~/.claude/plans/read-the-new-design-precious-tower.md · + /plan to edit + Notebook Migration to SDK Parsed-Document Storage + + Context + + docs/new-design.md moves parsed chunk read/sync behavior into the Knowhere + SDK so Notebook, CLI, and MCP share one model. The SDK side is already + built on the current branch (ParsedDocumentStorage interface, + withParsedStorage, readChunks/grepChunks/getDocumentOutline, resumable + syncParsedDocument, synchronous assetUrlPolicy:'durable' hardening). This + plan covers the Notebook side + (/home/suguan/github.com/ontosAI/knowhere-notebook), which today still uses + the old path: loadJobResult + the KnowhereAssetStorageOptions adapter, + plus hand-rolled Blob snapshot readers, a chunk-page cache, and asset + mirroring. + + The intended outcome: Notebook provides one Vercel-Blob + ParsedDocumentStorage adapter + an Upstash/QStash-safe scheduler, routes + all chunk reads/display/chat asset hardening through the SDK, and marks + sources ready only after parsed storage is synced. Existing remote docs + remain readable immediately (SDK falls back to remote, backfills Blob in + the background). + + Decisions taken with the user: + 1. Full rip-and-replace of the hand-rolled snapshot machinery. + 2. Include chat: unify citation/media asset hardening onto the SDK durable + policy; keep agentic retrieval.query. + 3. QStash-enqueued resumable background sync (reuse the + deterministic-workflowRunId continuation pattern); parse flow syncs before + marking ready. + + SDK contract being consumed (no SDK changes) + + - knowledge.withParsedStorage({ storage, scheduler?, limits? }) → new + configured Knowledge (unwrapped copy). + - ParsedDocumentStorage (src/types/storage.ts:142): + readManifest/writeManifest, readChunkPage/writeChunkPage, + writeAsset/getAssetUrl, readSyncProgress/writeSyncProgress. Reference impl: + src/storage/disk-parsed-document-storage.ts. Storage is only ever called + with {documentId, revisionKey} — never workspace/source. + - Freshness: manifest is fresh iff manifest.revisionKey === revisionKey, + where revisionKey = jobResultId ?? jobId. Pass revisionKey into reads to + hit storage without a remote probe. + - readChunks(params) — paged (page/pageSize/assetUrlPolicy) cannot combine + with sectionPath/startChunk/endChunk/chunkId. Never returns zero for a + ready remote doc (falls back to remote listChunks, returns bounded page, + fire-and-forget scheduler.schedule to backfill). + - assetUrlPolicy:'durable' + configured storage → SDK synchronously hardens + primary assetUrl and metadata.pageAssets[].assetUrl via + writeAsset/getAssetUrl; on failure omits the URL (never leaks a presigned + Knowhere URL). + - syncParsedDocument({documentId, revisionKey}) — resumable via + sync-progress; bounded by limits; returns completed:false at page/deadline + bounds (re-invoke to continue); commits manifest only after all pages + present. + + Implementation + + 1. Vercel-Blob ParsedDocumentStorage adapter — + src/domains/sources/parsed-document-blob-storage.ts (new) + + class BlobParsedDocumentStorage implements ParsedDocumentStorage, + constructed workspace-scoped ({ workspaceId, blobStore? }) with an + injectable blobStore seam (default wraps @vercel/blob get/put/head/del), + mirroring the seam pattern in + chunks/server.ts/parse-result-storage-adapter.ts. + + Key layout (documentId-scoped, workspace-rooted — resolves the scoping + mismatch since one client serves many docs and the adapter only receives + documentId): + workspaces/{workspaceId}/parsed-documents/{documentId}/{revisionKey}/manife + st/current.json + workspaces/{workspaceId}/parsed-documents/{documentId}/{revisionKey}/chunks + /page-{N}.json + workspaces/{workspaceId}/parsed-documents/{documentId}/{revisionKey}/assets + /{normalizedSourcePath} + workspaces/{workspaceId}/parsed-documents/{documentId}/{revisionKey}/sync-p + rogress.json + - reads → blobStore.get + JSON parse, null on BlobNotFoundError/404 (reuse + readBlobText). readChunkPage ignores chunkType (SDK filters after read). + - JSON writes → put(..., {access:"public", allowOverwrite:true, + contentType:"application/json; charset=utf-8"}). + - writeAsset → put(Buffer.from(body), {access:"public", + allowOverwrite:true, contentType, multipart:true}) → {sourcePath, url: + blob.url}. getAssetUrl → head(key).url or null. + - Sanitize documentId/revisionKey/sourcePath with a traversal guard (mirror + DiskParsedDocumentStorage.normalizeRelativeStoragePath: reject .., + absolute, empty segments). Keying under {revisionKey} gives the freshness + contract for free. Chat's existing isNotebookOwnedAssetUrl already treats + *.blob.vercel-storage.com as owned, so these durable URLs pass through + un-re-hardened. + + 2. Scheduler + sync route + + - src/domains/sources/parsed-document-sync-scheduler.ts (new): + createQStashSyncScheduler({workspaceId, sourceId, documentId, revisionKey, + apiKey}). schedule(task) ignores the closure (can't serialize the client) + and triggers a QStash continuation, reusing background-reconcile.ts's + trigger shape: Client.trigger({ url: + ${NOTEBOOK_PUBLIC_URL}/api/sources/parsed-sync, body, workflowRunId: + ${documentId}-sync-${revisionKey}-${segmentIndex}, retries: 3 }). Add a + process-level cooldown guard keyed by documentId|revisionKey (like + lastTriggeredAtBySourceId) so read-path misses don't storm QStash. + - src/app/api/sources/parsed-sync/route.ts + + src/domains/sources/parsed-sync-route-workflow.ts (new), modeled on + source-reconcile-route-workflow.ts: rebuild a parsed-storage knowledge, + loop context.run("sync-N", () => knowledge.syncParsedDocument({documentId, + revisionKey})) until completed:true or budget (~45s / N segments); on + incomplete trigger continuation segmentIndex+1; on complete for the parse + flow mark ready (§5); failureFunction writes sync_status='failed' + + failure_stage='storage_sync' (does not fail an already-ready source — it + still serves via remote fallback). Partial-snapshot safety is SDK-side. + + 3. Client wiring — src/integrations/knowhere.ts + + Add makeKnowhereClientWithParsedStorage(apiKey, { workspaceId, scheduler?, + limits? }) → { client, knowledge }. Build the wrapped client via existing + makeKnowhereClient (keep for retrieval/jobs/documents), then knowledge = + client.knowledge.withParsedStorage({ storage: new + BlobParsedDocumentStorage({workspaceId}), scheduler, limits }). Invoking + withParsedStorage through the logging Proxy gives correct this and returns + a functional unwrapped Knowledge (its internal listChunks won't be logged — + acceptable). Vercel-safe limits: chunkPageSize:200, remotePageSize:100, + maxPagesPerSync:10, maxAssetsPerSync:20, syncDeadlineMs:8000, + outlineMaxPages:50, grepMaxPages:50. + + Scoping note: route-chunks reads one source at a time → build per-source + knowledge with a document-bound QStash scheduler. Chat's multi-doc asset + loop → build one workspace knowledge with a no-op/coalescing scheduler (its + reads use synchronous 'durable' hardening; full background sync isn't its + job). + + 4. Read path rewrite + + - Delete src/domains/chunks/server.ts (snapshot readers, + warmChunkPageCache, mirror machinery, chunk-page cache, scheduleWarm). + Reduce src/domains/chunks/index.ts to view mappers: keep toParsedChunkView, + resolveChunkConnectionTargets, getChunkPageParams, + ChunkPage/ChunkPageParams; add toParsedChunkViewFromReadChunk(chunk: + KnowledgeReadChunk, sourceTitle, documentId) (map chunkId/type/content/read + ableContent/sectionPath/filePath/assetUrl/metadata/pageNums/connections). + No assetUrlsByFilePath re-mapping — durable hardening already rewrote asset + URLs. + - src/domains/sources/route-chunks.ts for a ready workspace source: + - Paged display → knowledge.readChunks({documentId, revisionKey, page, + pageSize, assetUrlPolicy:'durable'}); pagination from + {page,pageSize,totalChunks,totalPages}. + - Load-all → loop readChunks({documentId, revisionKey, page:N, + pageSize:200, assetUrlPolicy:'durable'}) until page >= totalPages, then + resolveChunkConnectionTargets. + - Tree/structure (if any caller needs it) → + knowledge.getDocumentOutline({documentId, revisionKey}). + - Pass revisionKey from source_parse_results.revision_key (fallback + source.knowhereJobId); if unknown, omit → SDK probes remote once, returns + immediately, schedules background sync. + - New gating: source.status !== 'ready' → 202 "still preparing"; ready → + always serve. Delete ensureParsedSnapshotForRead/isCompleteSnapshot and the + snapshot/snapshotStore/onRevisionKey plumbing. + - Demo kind unchanged (HTTP demoApi.fetchChunkPage). Remote-library kind: + keep localizeRemoteDocument to get documentId, then readChunks; delete the + syncRemoteParsedSnapshot call (first read triggers background sync). + - Update counts.ts to use getDocumentOutline(...).totalChunks (or + readChunks(...).totalChunks) instead of readParsedResultSnapshotManifest. + + 5. Parse flow — source-reconcile-route-workflow.ts + + Replace the load-job-result-snapshot step (loadJobResult({storageAdapter})) + with a resumable sync loop: after ready-to-prepare {jobId, documentId}, + persist revision_key (= jobResultId ?? jobId, probe + listChunks(documentId,{page:1,pageSize:1}) if needed) + + sync_status='running'; build parsed-storage knowledge (§3); loop + context.run("sync-N", () => knowledge.syncParsedDocument({documentId, + revisionKey})). If completed:false at budget → trigger parsed-sync + continuation and return with source still parsing. On completed:true → + markSourceReadyAfterReconciliation + sync_status='completed'. On sync + failure after successful parse → markFailed with + failure_stage='storage_sync'. Use documentId-based (resumable, bounded) + sync, not jobId-based (one-shot zip load), to respect serverless deadlines. + markReadyEffect already hard-guards requiredStatus:'parsing', so a late + continuation can't double-ready. + + Retry (retry.ts / route-retry.ts): dispatch on failure_stage. 'parse'/null + → existing reparse from originalBlobUrl. 'storage_sync' → new + resumeParsedSync that re-enqueues the continuation from stored + documentId+revision_key (no reparse), resets sync_status='running'. + + 6. DB migration 0011 (new only; never edit existing) + + src/infrastructure/db/schema.ts: + - sourceParseResults: add revisionKey: text("revision_key"), syncStatus: + text("sync_status") (pending|running|completed|failed), syncError: + text("sync_error"). Resumable cursor stays in Blob sync-progress.json + (SDK-managed) — no cursor columns. Keep legacy + resultBlobUrl/snapshotManifest* for back-compat. + - sources: add failureStage: text("failure_stage") (nullable; + parse|storage_sync). + - pnpm db:generate → drizzle/0011_*.sql. + - Repos: source-parse-result-repository.ts (persist/read new cols + + updateSyncStatusEffect), workflow-runtime.ts facade (updateSyncStatus, + setFailureStage), source-row-repository.ts (markFailed accepts + failureStage; markReady clears it). + - Back-compat: old rows have revision_key = NULL → reads omit the fast-path + key, probe remote once, serve remote, schedule background sync that + populates the new layout. Old blob objects orphaned (later cleanup sweep, + out of scope). + + 7. Chat unification — route-answer.ts + media-asset-hardening.ts + + Keep agentic retrieval.query, citation selection, page-citation-assets.ts. + Delete copyAssetToBlob, resolveAssetFetchRequest, parseDemoAssetRoute, + getChatAssetBlobPathname, the chat-assets/ fetch-and-put path, and + HardeningContext.blobStore/fetchAsset. Keep isNotebookOwnedAssetUrl, + resolveParsedAssetUrl, enrichRetrievalResultsWithAssetUrls. + New loadSourceAssetUrls: build workspace parsed-storage knowledge once; per + source readChunks({documentId, revisionKey, page:N, pageSize:200, + assetUrlPolicy:'durable'}) (loop pages), build assetUrlsByFilePath from + chunk.filePath → chunk.assetUrl and metadata.pageAssets[].artifactRef → + assetUrl. This is the single hardening path (durable policy writes any + missing asset and returns durable URLs). hardenMediaAssetUrls shrinks to: + owned → keep; else resolve against the durable map; else keep raw (no + copy-to-blob). + + Critical files + + - New: src/domains/sources/parsed-document-blob-storage.ts, + parsed-document-sync-scheduler.ts, parsed-sync-route-workflow.ts, + src/app/api/sources/parsed-sync/route.ts, drizzle/0011_*.sql. + - Change: src/integrations/knowhere.ts, + src/domains/sources/route-chunks.ts, source-reconcile-route-workflow.ts, + retry.ts/route-retry.ts, src/infrastructure/db/schema.ts, + source-parse-result-repository.ts, workflow-runtime.ts, + source-row-repository.ts, src/domains/chunks/index.ts, counts.ts, + src/domains/chat/route-answer.ts, media-asset-hardening.ts. + - Delete: src/domains/chunks/server.ts, remote-parsed-snapshot-sync.ts, and + the snapshot-reader half of parse-result-storage-adapter.ts; strip + snapshot plumbing from service.ts, route-dependencies.ts, route-types.ts. + + Ordered task list + + 1. DB migration + repos — schema cols, pnpm db:generate → 0011, extend + repositories/facade (no behavior change). + 2. Blob storage adapter + tests (key layout, seam, traversal guard, + freshness). + 3. Client wiring — makeKnowhereClientWithParsedStorage + limits; verify + Proxy interaction. + 4. Scheduler + sync route — deterministic runIds + cooldown; continuation + loop. + 5. Parse flow — syncParsedDocument loop + continuation + ready-gating + + failure_stage. + 6. Read path — collapse route-chunks onto readChunks/getDocumentOutline; + add mapper; update counts.ts; delete chunks/server.ts. + 7. Chat unification — new loadSourceAssetUrls; shrink + media-asset-hardening.ts; update route-answer.ts. + 8. Retry — failure_stage dispatch (resume sync vs reparse). + 9. Delete legacy — parse-result-storage-adapter.ts snapshot readers, + remote-parsed-snapshot-sync.ts, chunks/server.ts; strip plumbing. + 10. Tests + verify. + + Risks + + - Logging Proxy vs withParsedStorage (unwrapped result; internal listChunks + unlogged — acceptable). + - Per-request storage scoping: scheduler only gets a closure → pre-bind + per-document (route-chunks) or no-op scheduler (chat asset reads). + - Serverless deadlines → documentId-based resumable sync + conservative + limits + QStash continuation. + - Demo (HTTP-only) and remote-library (localizeRemoteDocument first) kinds. + - Back-compat: old source_parse_results rows self-heal via remote fallback + + background re-sync; old blobs orphaned. + - Asset volume: synchronous durable hardening on a visible page is bounded + by page size but adds latency on image-heavy pages. + + Verification + + - Unit/integration (Vitest): storage adapter round-trip + guards; + scheduler/sync-route continuation + failure staging; route-chunks + paged/load-all/202/ready + demo + remote-library; reconcile parse flow + (syncParsedDocument, continuation, ready-only-on-complete, storage_sync + failure); repo persistence of revision_key/sync_status; retry failure_stage + dispatch; chat owned-URL passthrough + reference resolution + no + copy-to-blob. + - pnpm typecheck, pnpm test, pnpm db:generate (confirm only 0011 added). + - Smoke (local Knowhere + Notebook): upload a multi-page PDF with + images/tables → wait until ready → inspect Blob manifest/pages/assets under + parsed-documents/... → open chunks panel (durable Blob asset URLs render) + → chat with a citation (image/page citation renders durable Blob URL, never + a presigned Knowhere URL). Also verify an existing remote doc with no Blob + snapshot returns chunks immediately from remote and backfills Blob in the + background. + + Thought for 22s (ctrl+o to expand) + +● Plan approved. Let me clean up the stale task list (those are from a prior + SDK/CLI effort) and set up tasks for this notebook migration. First let me + check the notebook repo's git state, since all work happens there. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && git branch + --show-current && echo "---STATUS---" && git status --short && echo + "---SCRIPTS---" && node -…) + ⎿  feature/wangbinqi/notebook-page-citation-assets + ---STATUS--- + ---SCRIPTS--- + … +20 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 15s (ctrl+o to expand) + +● Clean tree, on a feature branch. Let me clear the stale tasks (prior SDK/CLI + work) and set up this migration's task list, then create a working branch on + the notebook repo. + + I'll replace the stale tasks with the plan's ordered steps. + Thought for 18s (ctrl+o to expand) + +● Let me set up the key dependencies to reflect the plan's order, then create a + working branch on the notebook repo. + +● Task graph is set. Now the branch. The notebook is on + feature/wangbinqi/notebook-page-citation-assets; per the git guidelines I + should branch for this new work rather than commit onto that one. Let me + create a branch in the correct xxx/wangbinqi/xxx format from the current + branch (which has the page-citation work this builds on). + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && git checkout -b + feature/wangbinqi/notebook-parsed-document-storage && git branch + --show-current) + ⎿  Switched to a new branch 'feature/wangbinqi/notebook-parsed-document-s + torage' + feature/wangbinqi/notebook-parsed-document-storage + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Branch created. Now Task #7 (DB migration + repos) and Task #8 (Blob storage + adapter) are both unblocked and independent — I'll start with them. Let me + begin with Task #8 (the Blob adapter), since it's the foundation the client + wiring and everything downstream depends on, and read the SDK's + DiskParsedDocumentStorage traversal guard details plus the notebook's existing + blob-seam conventions I need to mirror. + + Let me start Task #8. I already have the SDK's DiskParsedDocumentStorage and + parse-result-storage-adapter.ts in context. Let me check how @vercel/blob's + get/head are typed in the installed version and how the existing seam handles + get return shape. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && find + node_modules/@vercel/blob/dist -name "*.d.ts" | head && echo "---" && grep + -rn "export declare func…) + ⎿  node_modules/@vercel/blob/dist/index.d.ts + node_modules/@vercel/blob/dist/create-folder-vlS2Pu_G.d.ts + node_modules/@vercel/blob/dist/client.d.ts + … +16 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && sed -n '1,120p' + node_modules/@vercel/blob/dist/create-folder-vlS2Pu_G.d.ts | grep -n + "GetBlobResult\|Hea…) + ⎿  1:import { Readable } from 'stream'; + 39: * Defines the content type of the blob. By default, this value + is inferred from the pathname. Sent as the 'content-type' header when + … +5 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -n + "GetBlobResult\|HeadBlobResult\|GetCommandOptions\|interface .*Blob\|type + GetBlob\|type HeadBlob…) + ⎿  4:interface BlobCommandOptions { + 21:interface CommonCreateBlobOptions extends BlobCommandOptions { + 112:interface PutBlobResult { + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk +● The GetBlobResult/HeadBlobResult types live elsewhere. The existing + parse-result-storage-adapter.ts already models get returning {statusCode, + stream} and head returning {url, contentType, size}, so I have the shapes I + need from the code I've already read. Let me find where these types are + declared to get head's exact return. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -rn + "GetBlobResult\b\|HeadBlobResult\b" node_modules/@vercel/blob/dist/*.d.ts + | grep -v "export {"…) + ⎿  node_modules/@vercel/blob/dist/index.d.ts:76:interface HeadBlobResult + { + node_modules/@vercel/blob/dist/index.d.ts:121:declare function head(ur + … +13 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 12s, read 1 file (ctrl+o to expand) + +● I have all the types I need. Let me confirm the SDK's public type exports so I + can import the ParsedDocumentStorage interface and snapshot types by name. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -n + "ParsedDocumentStorage\|KnowhereParsedSnapshotManifest\|KnowhereParsedSnap + shotChunkPage\|ParsedD…) + ⎿  410:interface KnowhereParsedSnapshotChunkPageReference { + 417:interface KnowhereParsedSnapshotManifest { + 426: readonly chunkPages: readonly KnowhereParsedSnapshotChunkPageR + … +2 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 13s (ctrl+o to expand) + +● The dist typings are stale — they don't export the new + ParsedDocumentStorage/ParsedDocumentSyncProgress types the notebook needs to + import. Since the SDK is pnpm-linked via dist/, I need to confirm whether the + SDK dist is built from the current source. Let me check. + + Thought for 13s, listed 1 directory (ctrl+o to expand) + +● The SDK repo's own dist/index.d.ts has withParsedStorage/ParsedDocumentStorage + (22 hits), but the notebook's node_modules copy is stale (only 4 hits, + missing the new types). pnpm file: links can be copies. Let me check the link + type and whether the notebook's copy needs a rebuild/reinstall. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && ls -la + node_modules/@ontos-ai/ && echo "---is it symlink?---" && readlink -f + node_modules/@ontos-ai/know…) + ⎿  total 48 + drwxr-xr-x 2 suguan suguan 4096 Jul 3 23:40 . + drwxr-xr-x 68 suguan suguan 36864 Jul 4 15:36 .. + … +9 lines (ctrl+o to expand) + + Thought for 21s (ctrl+o to expand) + +● The notebook's pnpm copy of the SDK is a stale snapshot (0 hits for + withParsedStorage), even though the SDK repo's own dist/ is current. pnpm + copied the package at install time. I need to rebuild the SDK dist (to be + safe) and refresh the notebook's copy so the new ParsedDocumentStorage types + are importable. Let me check the SDK build script and rebuild. + +● Bash(node -e "const p=require('./package.json'); console.log('version', + p.version); console.log(JSON.stringify(p.scripts,null,2))") + ⎿  version 2.0.0 + { + "build": "tsup src/index.ts --format cjs,esm --dts --clean", + … +23 lines (ctrl+o to expand) + +● Let me rebuild the SDK dist and then refresh the notebook's pnpm copy so the + new types are available. First rebuild the SDK. + +● Bash(pnpm run build 2>&1 | tail -20 && echo "---verify types present---" && + grep -c "withParsedStorage" dist/index.d.ts) + ⎿  > @ontos-ai/knowhere-sdk@2.0.0 build /home/suguan/github.com/ontosAI/k + nowhere-node-sdk + > tsup src/index.ts --format cjs,esm --dts --clean + … +18 lines (ctrl+o to expand) + + Thought for 15s, searched for 1 pattern (ctrl+o to expand) + +● All the new types are now exported from the freshly built dist: + ParsedDocumentStorage, ParsedDocumentSyncProgress, + ParsedDocumentStorageConfig, ParsedDocumentStorageLimits, + KnowhereParsedSnapshotManifest, KnowhereParsedSnapshotChunkPage, etc. Now + refresh the notebook's pnpm copy so it picks up the rebuilt dist. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm install 2>&1 + | tail -15 && echo "---verify notebook copy now current---" && grep -c + "withParsedStor…) + ⎿  │ esbuild@0.27.7, msgpackr-extract@3.0.3, msw@2.14.3. + │ + │ Run "pnpm approve-builds" to pick which dependencies should be all + … +18 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -c + "interface ParsedDocumentStorage\b\|ParsedDocumentSyncProgress\|ParsedDocu + mentStorageConfig\|Par…) + ⎿  12 + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● The notebook now sees the new types. Now I'll write the Blob adapter (Task + #8). All work happens in the notebook repo. Let me create + parsed-document-blob-storage.ts. + +● Write(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/parsed-docume + nt-blob-storage.ts) + ⎿  Wrote 291 lines to ../knowhere-notebook/src/domains/sources/parsed-doc + ument-blob-storage.ts + 2 import "server-only" + 3 import { del, get, head, put, BlobNotFoundError } from "@vercel/ + blob" + 4 import type { + 5 KnowhereParsedSnapshotChunkPage, + 6 KnowhereParsedSnapshotManifest, + 7 ParsedDocumentStorage, + 8 ParsedDocumentStorageAsset, + 9 ParsedDocumentStorageAssetParams, + 10 ParsedDocumentStorageChunkPageParams, + … +314 lines (ctrl+o to expand) + +● Now let me check an existing test in the sources domain to match the test + conventions before writing the adapter's test. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && ls + src/domains/sources/*.test.ts 2>/dev/null | head && echo "---sample test + header---" && sed -n '1,40p'…) + ⎿  src/domains/sources/background-reconcile.test.ts + src/domains/sources/blob-upload.test.ts + src/domains/sources/counts.test.ts + … +48 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Now I'll write the adapter test with an in-memory blob store fake. + +● Write(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/parsed-docume + nt-blob-storage.test.ts) + ⎿  Wrote 219 lines to ../knowhere-notebook/src/domains/sources/parsed-doc + ument-blob-storage.test.ts + 1 import { describe, expect, it } from "vitest" + 2 import type { + 3 KnowhereParsedSnapshotChunkPage, + 4 KnowhereParsedSnapshotManifest, + 5 ParsedDocumentSyncProgress, + 6 } from "@ontos-ai/knowhere-sdk" + 7 + 8 import { + 9 BlobParsedDocumentStorage, + 10 type ParsedDocumentBlobStore, + … +229 lines (ctrl+o to expand) + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm vitest run + src/domains/sources/parsed-document-blob-storage.test.ts 2>&1 | tail -30) + ⎿   RUN v4.1.5 /home/suguan/github.com/ontosAI/knowhere-notebook + … +5 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● All 7 tests pass. Let me typecheck the new file, then mark Task #8 done and + move to Task #9 (client wiring). + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -20) + ⎿  > knowhere-notebook@0.1.0 typecheck /home/suguan/github.com/ontosAI/kn + owhere-notebook + > tsc --noEmit + … +9 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk +● A type-only issue in the test fake's new Response(body) when body is a Buffer. + Let me fix by normalizing to a Uint8Array in the fake. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/parsed-docum + ent-blob-storage.test.ts) + ⎿  Added 5 lines, removed 1 line + 27 get: async (pathname) => { + 28 const object = objects.get(pathname) + 29 if (!object) return null + 30 + const body = + 31 + typeof object.body === "string" + 32 + ? object.body + 33 + : new Uint8Array(object.body) + 34 return { + 35 statusCode: 200, + 32 - stream: new Response(object.body).body as ReadableStrea + -m, + 36 + stream: new Response(body).body as ReadableStream, + 37 } + 38 }, + 39 put: async (pathname, body, options) => { + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -8 && echo "===TEST===" && pnpm vitest run + src/domains/sources/parsed-documen…) + ⎿  > knowhere-notebook@0.1.0 typecheck /home/suguan/github.com/ontosAI/kn + owhere-notebook + > tsc --noEmit + … +8 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Typecheck clean, 7/7 tests pass. Task #8 done. Now Task #9 (client wiring) and + Task #7 (DB migration + repos) are both unblocked. Let me mark #8 complete + and start #9 (client wiring), which unblocks the most downstream work. + + Before wiring, I need to understand the SDK's + KnowledgeParsedStorageOptions/ParsedDocumentStorageConfig shape and how + withParsedStorage behaves through the logging Proxy. I have the SDK source. + Let me check route-dependencies.ts and request-context.ts to see how clients + are currently built and where a parsed-storage-configured knowledge should be + surfaced. + + Thought for 18s, read 1 file (ctrl+o to expand) + +● The Proxy interaction is clear from the SDK source: + client.knowledge.withParsedStorage(cfg) resolves through the namespace proxy + via Reflect.apply(fn, realKnowledge, [cfg]), and withParsedStorage returns new + Knowledge(this.client, ...) where this.client is the real unwrapped Knowhere + — so it works correctly. Let me check route-types.ts for the + SourceRouteKnowhereClient shape and how client.knowledge is typed/used. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -n + "SourceRouteKnowhereClient\|knowledge\|documents\|ChunkKnowhereClient" + src/domains/sources/route…) + ⎿  2: ChunkKnowhereClient, + 28:type SourceRouteKnowhereClient = UploadKnowhereClient & + 29: ChunkKnowhereClient & { + … +9 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 15s (ctrl+o to expand) + +● Now I'll add the client wiring helper. It returns { client, knowledge } where + knowledge is configured with the Blob storage + scheduler + limits. The + scheduler is injected by the caller (route-chunks binds per-document, chat + uses a no-op). + +● Update(~/github.com/ontosAI/knowhere-notebook/src/integrations/knowhere.ts) + ⎿  Added 53 lines + 1 import Knowhere from "@ontos-ai/knowhere-sdk" + 2 +import type { + 3 + Knowledge, + 4 + ParsedDocumentStorageLimits, + 5 + ParsedDocumentSyncScheduler, + 6 +} from "@ontos-ai/knowhere-sdk" + 7 import { logger } from "@/lib/logger" + 8 +import { BlobParsedDocumentStorage } from "@/domains/sources/pa + +rsed-document-blob-storage" + 9 + 10 /** + 11 + * Vercel-safe defaults for parsed-document reads and backgroun + +d sync. Each read + 12 + * miss and each background sync step is bounded by page count + +and a deadline so + 13 + * a single serverless invocation stays well under the platform + + ceiling; the + 14 + * SDK returns `completed:false` and the caller re-enqueues to + +continue. + 15 + */ + 16 +const defaultParsedStorageLimits: ParsedDocumentStorageLimits = + + { + 17 + chunkPageSize: 200, + 18 + remotePageSize: 100, + 19 + maxPagesPerSync: 10, + 20 + maxAssetsPerSync: 20, + 21 + syncDeadlineMs: 8000, + 22 + grepMaxPages: 50, + 23 + grepDeadlineMs: 8000, + 24 + outlineMaxPages: 50, + 25 + outlineDeadlineMs: 8000, + 26 +} + 27 + + 28 +type ParsedStorageOptions = { + 29 + readonly workspaceId: string + 30 + readonly scheduler?: ParsedDocumentSyncScheduler + 31 + readonly limits?: ParsedDocumentStorageLimits + 32 +} + 33 + + 34 +/** + 35 * Create a Knowhere client with the given API key. + 36 * Use for per-request clients created from Dashboard-issued JW + Ts. + 37 */ + ... + 44 return wrapKnowhereClient(client) + 45 } + 46 + 47 +/** + 48 + * Create a Knowhere client plus a `Knowledge` configured with + +a Vercel-Blob + 49 + * `ParsedDocumentStorage`. Reads through `knowledge` serve fro + +m Blob first and + 50 + * fall back to Knowhere remote transparently; the `scheduler` + +(when provided) + 51 + * backfills Blob in the background. Use `client` for retrieval + +/documents/jobs. + 52 + * + 53 + * `withParsedStorage` is invoked through the logging Proxy so + +`this` binds to + 54 + * the real Knowledge; the returned Knowledge wraps the unwrapp + +ed inner client + 55 + * (its internal `documents.listChunks` calls are not logged, w + +hich is fine). + 56 + */ + 57 +export function makeKnowhereClientWithParsedStorage( + 58 + apiKey: string, + 59 + options: ParsedStorageOptions, + 60 +): { readonly client: Knowhere; readonly knowledge: Knowledge } + + { + 61 + const client = makeKnowhereClient(apiKey) + 62 + const knowledge = client.knowledge.withParsedStorage({ + 63 + storage: new BlobParsedDocumentStorage({ workspaceId: optio + +ns.workspaceId }), + 64 + scheduler: options.scheduler, + 65 + limits: options.limits ?? defaultParsedStorageLimits, + 66 + }) + 67 + return { client, knowledge } + 68 +} + 69 + + 70 function wrapKnowhereClient(client: Knowhere): Knowhere { + 71 return new Proxy(client, { + 72 get(target, prop, receiver) { + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -15) + ⎿  > knowhere-notebook@0.1.0 typecheck /home/suguan/github.com/ontosAI/kn + owhere-notebook + > tsc --noEmit + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Typecheck clean. Task #9 done. Now Task #7 (DB migration + repos) is unblocked + and it, plus #10/#12, become available. Task #7 is a prerequisite for the + parse flow, read path, scheduler, and retry — let me do it next. + + Let me read the parse-result repository and source-row repository to see + exactly what to extend. + + Read 1 file (ctrl+o to expand) + +● Now let me read the source-row-repository's markFailedEffect and + markReadyEffect to add failureStage, and the updateInWorkspaceWithDb + signature. + + Read 1 file (ctrl+o to expand) + +● Now let me look at the SourceRowRepository type / interface signatures near + the top and SourceUpdate type. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && sed -n '1,120p' + src/domains/sources/source-row-repository.ts) + ⎿  import "server-only" + + import { and, desc, eq, isNull, sql } from "drizzle-orm" + … +117 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Now I have everything for Task #7. Let me make the schema changes. First, the + two schema edits: add failureStage to sources and + revisionKey/syncStatus/syncError to sourceParseResults. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/infrastructure/db/schema.ts) + ⎿  File must be read first + +● I read that region via sed, not the Read tool. Let me Read the two schema + regions properly, then edit. + + Read 1 file (ctrl+o to expand) + +● Update(~/github.com/ontosAI/knowhere-notebook/src/infrastructure/db/schema.ts) + ⎿  Added 1 line + 95 sizeBytes: bigint("size_bytes", { mode: "number" }).notNul + l(), + 96 status: text("status").notNull(), + 97 failureReason: text("failure_reason"), + 98 + failureStage: text("failure_stage"), + 99 knowhereJobId: text("knowhere_job_id"), + 100 knowhereDocumentId: text("knowhere_document_id"), + 101 stagedBlobPathname: text("staged_blob_pathname"), + +● Now update the doc comment for the sources table to document failure_stage, + and the source_parse_results table. Let me read those regions. + + Read 1 file (ctrl+o to expand) +● Update(~/github.com/ontosAI/knowhere-notebook/src/infrastructure/db/schema.ts) + ⎿  Added 3 lines + 177 resultBlobUrl: text("result_blob_url").notNull(), + 178 snapshotManifestUrl: text("snapshot_manifest_url"), + 179 snapshotManifestKey: text("snapshot_manifest_key"), + 180 + revisionKey: text("revision_key"), + 181 + syncStatus: text("sync_status"), + 182 + syncError: text("sync_error"), + 183 assetUrls: jsonb("asset_urls") + 184 .$type>>() + 185 .notNull(), + +● Now update the two doc comments to reflect the new columns. Let me read the + sources comment block and the source_parse_results comment block. + + Read 1 file (ctrl+o to expand) + +● Update(~/github.com/ontosAI/knowhere-notebook/src/infrastructure/db/schema.ts) + ⎿  Added 12 lines, removed 2 lines + 163 * Notebook-owned parse-result artifact index for one source. + 164 * + 165 * Blob is the Notebook-owned read model for parsed chunks aft + er source + 166 - * reconciliation completes. This row stores the current parse + -d snapshot + 167 - * manifest and the file-path-to-public-URL map for parsed med + -ia artifacts. + 166 + * reconciliation completes. The parsed snapshot itself (manif + +est, chunk pages, + 167 + * assets, and resumable sync progress) lives in Vercel Blob u + +nder + 168 + * `workspaces/{ws}/parsed-documents/{documentId}/{revisionKey + +}/...`, managed by + 169 + * the SDK `ParsedDocumentStorage`. This row records: + 170 + * - `revision_key` — current parsed revision (jobResultId + +?? jobId); the + 171 + * storage fast-path key passed into SDK + + reads + 172 + * - `sync_status` — pending | running | completed | faile + +d for the + 173 + * background/parse-time storage sync + 174 + * - `sync_error` — last storage-sync error detail when s + +ync_status=failed + 175 + * - `result_blob_url` / `snapshot_manifest_*` — legacy colu + +mns retained for + 176 + * rows written by the pre-migration man + +ifest format + 177 + * - `asset_urls` — legacy file-path-to-public-URL map fo + +r older rows + 178 */ + 179 export const sourceParseResults = pgTable( + 180 "source_parse_results", + +● Now update the sources doc comment to mention failure_stage. + + Read 1 file (ctrl+o to expand) + +● Update(~/github.com/ontosAI/knowhere-notebook/src/infrastructure/db/schema.ts) + ⎿  Added 2 lines + 66 * - `size_bytes` — original upload size (for display + quo + ta) + 67 * - `status` — lifecycle: uploading | parsing | ready + | failed + 68 * - `failure_reason` — human-readable error text when status + =failed + 69 + * - `failure_stage` — which stage failed: parse | storage_s + +ync; drives + 70 + * whether a retry reparses or only resu + +mes storage sync + 71 * - `knowhere_job_id` — set once the parse job is creat + ed + 72 * - `knowhere_document_id` — set when parsing completes; use + d to import + 73 * parsed snapshots and to exclude + a source from a + +● Now generate the migration. Let me run pnpm db:generate. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm db:generate + 2>&1 | tail -20 && echo "---new files---" && git status --short drizzle/) + ⎿  > knowhere-notebook@0.1.0 db:generate /home/suguan/github.com/ontosAI/ + knowhere-notebook + > drizzle-kit generate + … +17 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Migration 0011_brainy_the_leader.sql generated (only 0011, no edits to + existing). Let me verify its contents are exactly the four ADD COLUMNs. + + Read 1 file (ctrl+o to expand) + +● Clean — exactly the four additive columns, all nullable. Now extend the + repositories. First the parse-result repository: add + revisionKey/syncStatus/syncError to the save input and progress reads, and an + updateSyncStatusEffect. Let me update source-parse-result-repository.ts. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 21 lines + 14 readonly resultBlobUrl: string + 15 readonly snapshotManifestUrl?: string + 16 readonly snapshotManifestKey?: string + 17 + readonly revisionKey?: string + 18 + readonly syncStatus?: SourceParseSyncStatus + 19 + readonly syncError?: string | null + 20 readonly assetUrlsByFilePath: Readonly + > + 21 } + 22 + 23 +export type SourceParseSyncStatus = + 24 + | "pending" + 25 + | "running" + 26 + | "completed" + 27 + | "failed" + 28 + + 29 type SourceParseResultProgress = { + 30 readonly resultBlobUrl: string + 31 readonly snapshotManifestUrl?: string | null + 32 readonly snapshotManifestKey?: string | null + 33 + readonly revisionKey?: string | null + 34 + readonly syncStatus?: string | null + 35 + readonly syncError?: string | null + 36 readonly assetUrlsByFilePath: Readonly + > + 37 } + 38 + ... + 40 readonly resultBlobUrl: string + 41 readonly snapshotManifestUrl?: string | null + 42 readonly snapshotManifestKey?: string | null + 43 + readonly revisionKey?: string | null + 44 + readonly syncStatus?: string | null + 45 + readonly syncError?: string | null + 46 readonly assetUrlsByFilePath: Readonly + > + 47 } + 48 + 49 +type UpdateSyncStatusInput = { + 50 + readonly revisionKey?: string + 51 + readonly syncStatus: SourceParseSyncStatus + 52 + readonly syncError?: string | null + 53 +} + 54 + + 55 type SourceParseResultRepository = { + 56 readonly saveParseResultEffect: ( + 57 workspaceId: string, + +● Now add updateSyncStatusEffect to the repository interface and implementation, + and thread the new columns through save/merge/read. Let me update the + interface type. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 5 lines + 75 workspaceId: string, + 76 sourceId: string, + 77 ) => Effect.Effect>, never, D + bClient> + 78 + readonly updateSyncStatusEffect: ( + 79 + workspaceId: string, + 80 + sourceId: string, + 81 + input: UpdateSyncStatusInput, + 82 + ) => Effect.Effect + 83 } + 84 + 85 export function buildAtomicAssetUrlsMergeSql( + +● Now thread the new columns through saveParseResultEffect and + mergeParseAssetUrlsEffect insert+update sets. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 6 lines + 105 resultBlobUrl: input.resultBlobUrl, + 106 snapshotManifestUrl: input.snapshotManifestUrl, + 107 snapshotManifestKey: input.snapshotManifestKey, + 108 + revisionKey: input.revisionKey, + 109 + syncStatus: input.syncStatus, + 110 + syncError: input.syncError, + 111 assetUrls: input.assetUrlsByFilePath, + 112 }) + 113 .onConflictDoUpdate({ + ... + 116 resultBlobUrl: input.resultBlobUrl, + 117 snapshotManifestUrl: input.snapshotManifestUrl, + 118 snapshotManifestKey: input.snapshotManifestKey, + 119 + revisionKey: input.revisionKey, + 120 + syncStatus: input.syncStatus, + 121 + syncError: input.syncError, + 122 assetUrls: input.assetUrlsByFilePath, + 123 updatedAt: sql`now()`, + 124 }, + +● Now the merge effect: it also should persist revisionKey/syncStatus when + provided. Let me update its insert and update sets. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 6 lines + 146 resultBlobUrl: input.resultBlobUrl, + 147 snapshotManifestUrl: input.snapshotManifestUrl, + 148 snapshotManifestKey: input.snapshotManifestKey, + 149 + revisionKey: input.revisionKey, + 150 + syncStatus: input.syncStatus, + 151 + syncError: input.syncError, + 152 assetUrls: input.assetUrlsByFilePath, + 153 }) + 154 .onConflictDoUpdate({ + ... + 157 resultBlobUrl: input.resultBlobUrl, + 158 snapshotManifestUrl: input.snapshotManifestUrl, + 159 snapshotManifestKey: input.snapshotManifestKey, + 160 + revisionKey: input.revisionKey, + 161 + syncStatus: input.syncStatus, + 162 + syncError: input.syncError, + 163 assetUrls: buildAtomicAssetUrlsMergeSql( + 164 input.assetUrlsByFilePath, + 165 ), + +● Now update getParseResultProgressEffect to select and return the new columns, + and add updateSyncStatusEffect, plus register it in the export. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 33 lines + 187 resultBlobUrl: sourceParseResults.resultBlobUrl, + 188 snapshotManifestUrl: sourceParseResults.snapshotMa + nifestUrl, + 189 snapshotManifestKey: sourceParseResults.snapshotMa + nifestKey, + 190 + revisionKey: sourceParseResults.revisionKey, + 191 + syncStatus: sourceParseResults.syncStatus, + 192 + syncError: sourceParseResults.syncError, + 193 assetUrls: sourceParseResults.assetUrls, + 194 }) + 195 .from(sourceParseResults) + ... + 203 resultBlobUrl: progress.resultBlobUrl, + 204 snapshotManifestUrl: progress.snapshotManifestUrl, + 205 snapshotManifestKey: progress.snapshotManifestKey, + 206 + revisionKey: progress.revisionKey, + 207 + syncStatus: progress.syncStatus, + 208 + syncError: progress.syncError, + 209 assetUrlsByFilePath: progress.assetUrls, + 210 } + 211 }) + 212 + 213 +const updateSyncStatusEffect: SourceParseResultRepository["upd + +ateSyncStatusEffect"] = + 214 + (workspaceId: string, sourceId: string, input: UpdateSyncSta + +tusInput) => + 215 + Effect.gen(function* () { + 216 + const db = yield* DbClient + 217 + const source = yield* Effect.promise(() => + 218 + sourceRowRepository.findInWorkspaceWithDb(db, workspac + +eId, sourceId), + 219 + ) + 220 + if (!source) return null + 221 + + 222 + const [result] = yield* Effect.promise(() => + 223 + db + 224 + .update(sourceParseResults) + 225 + .set({ + 226 + ...(input.revisionKey !== undefined + 227 + ? { revisionKey: input.revisionKey } + 228 + : {}), + 229 + syncStatus: input.syncStatus, + 230 + syncError: input.syncError ?? null, + 231 + updatedAt: sql`now()`, + 232 + }) + 233 + .where(eq(sourceParseResults.sourceId, sourceId)) + 234 + .returning(), + 235 + ) + 236 + + 237 + return result ?? null + 238 + }) + 239 + + 240 const getParseAssetUrlsEffect: SourceParseResultRepository["ge + tParseAssetUrlsEffect"] = + 241 (workspaceId: string, sourceId: string) => + 242 Effect.gen(function* () { + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 1 line + 270 getParseResultProgressEffect, + 271 getParseSnapshotMetadataEffect, + 272 getParseAssetUrlsEffect, + 273 + updateSyncStatusEffect, + 274 } + +● Now I need to check how repository.ts re-exports these (workflow-runtime + imports sourceRepository.saveParseResultEffect etc). Let me check + repository.ts. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -n + "ParseResult\|updateSyncStatus\|updateRevisionKey\|sourceParseResultReposi + tory\|\.\.\.\|export c…) + ⎿  4:import { sourceParseResultRepository } from "./source-parse-result-r + epository" + 17: readonly updateRevisionKeyEffect: typeof sourceRowRepository.upda + … +15 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Let me read repository.ts fully to add the new effects + (updateSyncStatusEffect, and markFailedEffect/markReadyEffect will need + failureStage support) and the interface. + + Read 1 file (ctrl+o to expand) + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/repository.t + s) + ⎿  Added 1 line + 23 readonly getParseResultProgressEffect: typeof sourceParseResu + ltRepository.getParseResultProgressEffect + 24 readonly getParseSnapshotMetadataEffect: typeof sourceParseRe + sultRepository.getParseSnapshotMetadataEffect + 25 readonly getParseAssetUrlsEffect: typeof sourceParseResultRep + ository.getParseAssetUrlsEffect + 26 + readonly updateSyncStatusEffect: typeof sourceParseResultRepo + +sitory.updateSyncStatusEffect + 27 } + 28 + 29 export const sourceRepository: SourceRepository = { + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/repository.t + s) ⎿  Added 1 line + 50 getParseSnapshotMetadataEffect: + 51 sourceParseResultRepository.getParseSnapshotMetadataEffect, + 52 getParseAssetUrlsEffect: sourceParseResultRepository.getParse + AssetUrlsEffect, + 53 + updateSyncStatusEffect: sourceParseResultRepository.updateSyn + +cStatusEffect, + 54 } + +● Now add failureStage support to markFailedEffect and clear it in + markReadyEffect in source-row-repository.ts. I need to add failureStage to the + SourceUpdate type and thread an optional param through markFailedEffect. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-row-r + epository.ts) + ⎿  Added 1 line + 26 | "sizeBytes" + 27 | "status" + 28 | "failureReason" + 29 + | "failureStage" + 30 | "knowhereJobId" + 31 | "knowhereDocumentId" + 32 | "stagedBlobPathname" + +● Now update the markFailedEffect signature in the interface and impl to accept + an optional failureStage, and clear failureStage in markReadyEffect and + markParsingEffect. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-row-r + epository.ts) + ⎿  Added 1 line + 84 sourceId: string, + 85 reason: string, + 86 requiredStatus?: string, + 87 + failureStage?: string, + 88 ) => Effect.Effect + 89 readonly clearStagedBlobEffect: ( + 90 workspaceId: string, + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-row-r + epository.ts) + ⎿  Added 2 lines + 195 knowhereJobId: jobId, + 196 knowhereDocumentId: documentId, + 197 failureReason: null, + 198 + failureStage: null, + 199 }, requiredStatus) + 200 + 201 const markReadyEffect: SourceRowRepository["markReadyEffect"] + = ( + ... + 207 status: "ready", + 208 knowhereDocumentId: documentId, + 209 failureReason: null, + 210 + failureStage: null, + 211 }, "parsing") + 212 + 213 const updateRevisionKeyEffect: SourceRowRepository["updateRevi + sionKeyEffect"] = ( + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-row-r + epository.ts) + ⎿  Added 2 lines + 224 sourceId: string, + 225 reason: string, + 226 requiredStatus?: string, + 227 + failureStage?: string, + 228 ) => + 229 updateInWorkspaceEffect(workspaceId, sourceId, { + 230 status: "failed", + 231 failureReason: reason, + 232 + failureStage: failureStage ?? null, + 233 }, requiredStatus) + 234 + 235 const clearStagedBlobEffect: SourceRowRepository["clearStagedB + lobEffect"] = ( + +● Now update the workflow-runtime.ts facade: markFailed should accept + failureStage, and add updateSyncStatus. Let me update the type and impls. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 1 line + 38 sourceId: string, + 39 reason: string, + 40 requiredStatus?: string, + 41 + failureStage?: string, + 42 ) => Promise + 43 } + 44 + +● Now add updateSyncStatus to the SourceWorkflowRuntime type and implement it. + Let me add to the type block after saveParseResult/mergeParseAssetUrls. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 5 lines + 106 sourceId: string, + 107 input: SaveSourceParseResultInput, + 108 ) => Promise + 109 + readonly updateSyncStatus: ( + 110 + workspaceId: string, + 111 + sourceId: string, + 112 + input: UpdateSyncStatusInput, + 113 + ) => Promise + 114 readonly softDelete: ( + 115 workspaceId: string, + 116 sourceId: string, + +● Now add the UpdateSyncStatusInput type alias near the other Parameters<...> + aliases, update markFailed impl to pass failureStage, add updateSyncStatus + impl, and register it in the export object. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 4 lines + 17 typeof sourceRepository.saveParseResultEffect + 18 >[2] + 19 + 20 +type UpdateSyncStatusInput = Parameters< + 21 + typeof sourceRepository.updateSyncStatusEffect + 22 +>[2] + 23 + + 24 type UpsertMaterializedDemoSourceInput = Parameters< + 25 typeof sourceRepository.upsertMaterializedDemoSourceEffect + 26 >[1] + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 8 lines, removed 1 line + 210 sourceId: string, + 211 reason: string, + 212 requiredStatus?: string, + 213 + failureStage?: string, + 214 ) => + 215 databaseRuntime.runPromise( + 215 - sourceRepository.markFailedEffect(workspaceId, sourceId, r + -eason, requiredStatus), + 216 + sourceRepository.markFailedEffect( + 217 + workspaceId, + 218 + sourceId, + 219 + reason, + 220 + requiredStatus, + 221 + failureStage, + 222 + ), + 223 ) + 224 + 225 const clearStagedBlob: SourceWorkflowRuntime["clearStagedBlob" + ] = ( + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 9 lines + 262 sourceRepository.mergeParseAssetUrlsEffect(workspaceId, so + urceId, input), + 263 ) + 264 + 265 +const updateSyncStatus: SourceWorkflowRuntime["updateSyncStatu + +s"] = ( + 266 + workspaceId: string, + 267 + sourceId: string, + 268 + input: UpdateSyncStatusInput, + 269 +) => + 270 + databaseRuntime.runPromise( + 271 + sourceRepository.updateSyncStatusEffect(workspaceId, sourc + +eId, input), + 272 + ) + 273 + + 274 const getParseResultProgress: SourceWorkflowRuntime["getParseR + esultProgress"] = + 275 (workspaceId: string, sourceId: string) => + 276 databaseRuntime.runPromise( + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 1 line + 342 mergeParseAssetUrls, + 343 saveParseResult, + 344 softDelete, + 345 + updateSyncStatus, + 346 upsertMaterializedDemoSource, + 347 } + +● Now typecheck to catch anything I missed in Task #7. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -20) + ⎿   Type 'undefined' is not assignable to type 'string | null'. + src/domains/sources/route-service.test.ts(18,7): error TS2741: Propert + y 'failureStage' is missing in type '{ id: string; workspaceId: string + … +61 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● The new non-null failureStage column on the Source type breaks test fixtures + that build full Source objects. Test fixtures likely use a shared builder. Let + me find how test Source objects are constructed — there's probably a factory + to update in one place. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -rln + "failureReason: null\|failureReason:" src --include=*.ts | grep -i + "test\|fixture\|factory\|mo…) + ⎿  (eval):1: no matches found: --include=*.ts + ---any shared source factory?--- + (eval):1: no matches found: --include=*.ts + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -rln + "buildSource\|makeSource\|createSource\|sourceFixture\|baseSource\|fakeSou + rce\|SourceFactory"…) + ⎿  src/app/api/sources/route.ts + src/app/api/sources/[sourceId]/chunks/route.ts + src/app/api/demo-sources/materialize/route.test.ts + … +53 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 21s (ctrl+o to expand) + +● The type is legitimately correct (real column), so the fixtures need updating. + A few already show failureStage? optional — meaning they spread from a shared + base. Let me find that base fixture and see the pattern. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== + source-reconcile-workflow.test.ts 1-40 ===" && sed -n '1,45p' + src/domains/sources/source-recon…) + ⎿  === source-reconcile-workflow.test.ts 1-40 === + import { describe, expect, it, vi } from "vitest" + import type { JobResult } from "@ontos-ai/knowhere-sdk" + … +43 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● These are per-file inline Source fixtures, each listing all columns. I need to + add failureStage: null to each. Rather than edit each manually, let me find + every occurrence of the fixture shape (the line failureReason: null, + immediately followed by knowhereJobId) in test/source files and insert + failureStage: null, after it. Let me first identify exactly which files + construct full Source objects (the failing set plus others), by searching for + the adjacent-line pattern. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -rlzP + "failureReason:[^\n]*\n\s*knowhereJobId:" src | tr '\0' '\n') ⎿  src/app/api/sources/[sourceId]/chunks/route.test.ts + src/app/api/sources/[sourceId]/route.test.ts + src/app/api/sources/route.test.ts + … +23 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Two of these are non-test source files (demo-source-repository.ts, + source-row-repository.ts) — those are DB inserts where the column is optional + (nullable, no default needed), so I should NOT add failureStage there. The + rest are test fixtures constructing full Source objects. Let me verify the two + source files don't need it (insert values, not full Source literals), then do + a targeted insert across the test files only. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== + demo-source-repository.ts ===" && grep -nP "failureReason:" + src/domains/sources/demo-source-re…) + ⎿  === demo-source-repository.ts === + 100: failureReason: null, + 113: failureReason: null, + … +7 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 30s (ctrl+o to expand) + +● These are full Source/NewSource literals (the two non-test source files are DB + insert/update values where the nullable column needs nothing). The Source + type now requires failureStage, so every full-source test fixture needs + failureStage: null. That's ~24 test files with the identical failureReason: … + ⏎ knowhereJobId: shape — a scripted insert is the right tool here rather than + dozens of manual edits. I'll insert failureStage: null, (matching indentation) + after each failureReason: line that's immediately followed by knowhereJobId:, + across test files only. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && files=$(grep + -rlzP "failureReason:[^\n]*\n\s*knowhereJobId:" src | tr '\0' '\n' | grep + "\.test\.ts$") &&…) + ⎿  src/app/api/sources/[sourceId]/chunks/route.test.ts + src/app/api/sources/[sourceId]/route.test.ts + src/app/api/sources/route.test.ts + … +48 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 27s (ctrl+o to expand) + +● The verify shows the insertion did NOT actually apply (line 26-27 unchanged), + and the bracketed [sourceId] paths errored due to shell globbing. Let me redo + this correctly — test on one file first to confirm the perl regex works. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && perl -0pi -e + 's/(failureReason:[^\n]*,\n)([ \t]*)(knowhereJobId:)/$1${2}failureStage: + null,\n$2$3/g' src…) + ⎿  26: failureReason: null, + 27- failureStage: null, + 28- knowhereJobId: "job_1", + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● The regex works when run per-file (the earlier for loop failed because the + file list from grep -rlz retained NUL/globbing issues). Now apply to the + remaining files using a null-safe loop. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -rlzP + "failureReason:[^\n]*\n\s*knowhereJobId:" src | while IFS= read -r -d '' + f; do case "$f" in *…) + ⎿  DONE + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● source-reconcile-workflow.test.ts was already processed once — the loop would + double-insert. Let me check it didn't get a duplicate, then typecheck. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== dup + check (any file with 2 consecutive failureStage lines) ===" && grep -rzoP + "failureStage: n…) + ⎿  === dup check (any file with 2 consecutive failureStage lines) === + === reconcile-workflow fixture === + 26: failureReason: null, + … +64 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Progress but several remain. Two distinct cases: + 1. workflow-runtime.test.ts, route-service.test.ts — full Source literals + where knowhereJobId comes BEFORE failureReason (different field order), so my + regex missed them. + 2. retry.test.ts:118, upload.test.ts:18, view.test.ts:7, + initial-state.test.ts:642 — these show failureStage?: undefined meaning they + spread a base fixture that has failureStage optional, OR use Partial overrides + producing undefined. Let me look at each failing spot. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== + workflow-runtime.test.ts 20-45 ===" && sed -n '20,45p' + src/domains/sources/workflow-runtime.te…) + ⎿  === workflow-runtime.test.ts 20-45 === + repository.markSourceFailed("workspace_1", "source_1", "failed") + , + … +54 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● These two have a conditional failureReason: status === ... ? ... : null, + (workflow-runtime) — the [^\n]*,\n matched but the ternary contains no issue; + actually route-service.test.ts line 25 has plain failureReason: null, then + knowhereJobId: — my regex should have caught it. It seems these files weren't + in my processed list or the earlier run's file list differed. Let me just add + failureStage: null, after the failureReason line in these two specific files. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && for f in + src/domains/sources/workflow-runtime.test.ts + src/domains/sources/route-service.test.ts; do perl…) + ⎿  === workflow-runtime === + 33: failureReason: status === "failed" ? "failed" : null, + 34- failureStage: null, + … +4 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Now the other four (retry.test.ts:118, upload.test.ts:18, view.test.ts:7, + initial-state.test.ts:642) show failureStage?: undefined, meaning they spread + a base object that itself now has failureStage but the override object passes + undefined, OR they build via a helper with Partial. Let me look at these. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== + retry.test.ts 100-125 ===" && sed -n '100,125p' + src/domains/sources/retry.test.ts && echo "===…) + ⎿  === retry.test.ts 100-125 === + } + … +55 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● These makeSource helpers use ...overrides — the failureStage?: undefined in + the error is because the fixture object still lacks failureStage before the + spread. My regex requires failureReason:...,\nknowhereJobId:. In + view.test.ts the pattern matches (line 14→15), so why did it fail? The earlier + bulk loop reprocessed the already-done file and possibly the file list from + grep -rlz didn't include these because of how I piped. Let me just re-run the + per-file perl on the four remaining files directly. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && for f in + src/domains/sources/retry.test.ts src/domains/sources/upload.test.ts + src/domains/sources/view.t…) + ⎿  verify: + -- retry -- + 2 + … +5 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -20) + ⎿   Types of property 'failureStage' are incompatible. + Type 'string | null | undefined' is not assignable to type 'string + | null'. + … +47 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Fewer errors. counts.test.ts and reconcile.test.ts still need failureStage (my + regex likely missed them — perhaps failureReason uses a different value with + a trailing comment or the following line isn't knowhereJobId). And now a + SourceParseResult fixture in remote-parsed-snapshot-sync.test.ts:302 needs + revisionKey/syncStatus/syncError. Let me inspect these three spots. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== + counts.test.ts 5-15 ===" && sed -n '5,15p' + src/domains/sources/counts.test.ts && echo "=== rec…) + ⎿  === counts.test.ts 5-15 === + + import type { Source } from "@/infrastructure/db/schema" + … +45 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● counts.test.ts and reconcile.test.ts have the standard shape, so my regex + should have caught them — they must not have been in the grep list when I ran + the loop (glob issue). Let me apply per-file, and add the three + SourceParseResult columns to the parse-result fixture. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && for f in + src/domains/sources/counts.test.ts src/domains/sources/reconcile.test.ts; + do perl -0pi -e 's/(f…) + ⎿  src/domains/sources/reconcile.test.ts:1 + src/domains/sources/counts.test.ts:1 + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Now the SourceParseResult fixture. Let me add revisionKey/syncStatus/syncError + to it after snapshotManifestKey. + + Read 1 file (ctrl+o to expand) +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/remote-parse + d-snapshot-sync.test.ts) + ⎿  Added 3 lines + 308 "https://blob.example/workspaces/workspace_1/sources/000 + 00000-0000-0000-0000-000000000009/parsed-result/manifest/curre + nt.json", + 309 snapshotManifestKey: + 310 "workspaces/workspace_1/sources/00000000-0000-0000-0000- + 000000000009/parsed-result/manifest/current.json", + 311 + revisionKey: null, + 312 + syncStatus: null, + 313 + syncError: null, + 314 assetUrls: {}, + 315 createdAt: new Date("2026-05-10T00:00:00.000Z"), + 316 updatedAt: new Date("2026-05-10T00:00:00.000Z"), + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -20) + ⎿   Types of property 'failureStage' are incompatible. + Type 'string | null | undefined' is not assignable to type 'string + | null'. + … +46 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Still a handful: chat/service.test.ts, chunks/index.test.ts, + chunks/server.test.ts, and remote-parsed-snapshot-sync.test.ts:277. My bulk + loop clearly didn't touch several files consistently. Let me just check which + test files with a full Source literal still lack failureStage and fix them all + in one reliable pass. The safest approach: for every .test.ts under src that + has the failureReason:...\n...knowhereJobId: pattern but NOT already + failureStage, apply the perl. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && changed=""; while + IFS= read -r -d '' f; do if ! grep -qzP + "failureReason:[^\n]*,\n\s*failureStage:" "$f"…) + ⎿  patched: + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● The while-loop input got mangled. Let me take a simpler, deterministic path: + iterate a plain explicit list of the four remaining failing files. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && for f in + "src/domains/chat/service.test.ts" "src/domains/chunks/index.test.ts" + "src/domains/chunks/serve…) + ⎿  src/domains/chat/service.test.ts -> 1 + src/domains/chunks/index.test.ts -> 1 + src/domains/chunks/server.test.ts -> 1 + src/domains/sources/remote-parsed-snapshot-sync.test.ts -> 1 + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -20) + ⎿   Types of property 'failureStage' are incompatible. + Type 'string | null | undefined' is not assignable to type 'string + | null'. + … +46 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● More chat test files. The bulk loop is clearly unreliable in this shell. Let + me get the definitive list of ALL remaining offenders from tsc directly and + fix exactly those. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | grep -oP "^src/[^(]+\.test\.ts" | sort -u) + ⎿  src/app/api/demo-sources/materialize/route.test.ts + src/app/api/sources/route.test.ts + src/domains/chat/index.test.ts + … +4 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Seven files. Let me apply the perl to each explicitly (avoiding the loop-input + issue), then check whether any have a different field ordering. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && perl -0pi -e + 's/(failureReason:[^\n]*,\n)([ \t]*)(knowhereJobId:)/$1${2}failureStage: + null,\n$2$3/g' "sr…) + ⎿  src/app/api/demo-sources/materialize/route.test.ts -> 1 + src/app/api/sources/route.test.ts -> 1 + src/domains/chat/index.test.ts -> 1 + … +4 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● All patched. Some of these files (e.g. route.test.ts had failureStage reported + missing at multiple lines) may have more than one Source literal — the count + of 1 suggests only one matched. Let me typecheck to see if any remain. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -25) + ⎿  > knowhere-notebook@0.1.0 typecheck /home/suguan/github.com/ontosAI/kn + owhere-notebook + > tsc --noEmit + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Please run /login · API Error: 403 预扣费额度失败, 用户剩余额度: + ¥0.155160, 需要预扣费额度: ¥0.495542 (request id: + 202607041425149789930498268d9d69rP7bjXn) + +✻ Crunched for 1h 3m 4s \ No newline at end of file diff --git a/README.md b/README.md index 0b09f67..9dbbb1c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ npm install @ontos-ai/knowhere-sdk **Requirements:** -- Node.js >= 20.19.0 +- Node.js >= 22.13.0 - npm >= 10.0.0 - TypeScript >= 5.0 (optional, for type checking) @@ -145,6 +145,53 @@ const result = await client.parse({ }); ``` +### Page Citation Assets + +Page citation assets are generated by Knowhere during page-memory parsing. The +SDK does not render PDF pages locally; it preserves the page image descriptors +returned by the server on page chunks and retrieval metadata. + +```typescript +import Knowhere from '@ontos-ai/knowhere-sdk'; + +const client = new Knowhere({ apiKey: process.env.KNOWHERE_API_KEY }); + +const result = await client.parse({ + file: './manual.pdf', +}); + +for (const chunk of result.pageChunks) { + console.log(chunk.metadata.pageAssets); +} +``` + +`metadata.pageAssets` entries point at concrete page images stored with the Knowhere +result. `artifactRef` is the durable result artifact path; `assetUrl` is an +optional server-generated access URL. + +```typescript +type PageCitationAsset = { + pageNum: number; + artifactRef: string; + assetUrl?: string; + contentType: 'image/png' | 'image/jpeg'; + width?: number; + height?: number; + source: 'knowhere-rendered-page-citation-source'; +}; +``` + +Document chunk APIs can include signed page image URLs when requested: + +```typescript +const chunks = await client.documents.listChunks('doc_123', { + chunkType: 'page', + includeAssetUrls: true, +}); + +console.log(chunks.chunks[0]?.metadata.pageAssets?.[0]?.assetUrl); +``` + ### Low-Level API For granular control over the job lifecycle: @@ -299,12 +346,16 @@ console.log(archived.status); ### Local Knowledge Tools -The SDK can also keep parsed results in a local cache and run exact inspection -tools over that cached copy. This is the implementation used by the separate -`@ontos-ai/knowhere-mcp` package. +The SDK can also run exact inspection tools over parsed results. Local import +helpers still write under the SDK cache directory by design, while published +`documentId` reads use configured parsed storage first and fall back to +Knowhere's remote chunk API without importing the result ZIP into local disk. +Server workflows can configure parsed storage with +`client.knowledge.withParsedStorage(...)` so Notebook, MCP, and CLI surfaces +share the same snapshot model. ```typescript -const parsed = await client.knowledge.parse({ +const parsed = await client.knowledge.parseToLocalCache({ file: './manual.pdf', localDocumentId: 'manual', }); @@ -334,27 +385,49 @@ console.log(grep.matches); console.log(serverSearch.references); ``` -Local grep and reads use the cached parse result, not server-side chunk scans. -Search uses the Knowhere API retrieval query; local document IDs only help map -returned server document IDs back to local cache IDs when available. -If a search result only has a published `documentId`, or an async parse flow only -has a completed `jobId`, read-oriented helpers can accept that remote identifier -directly and will sync the result into the local cache before reading. For a -`documentId`, the SDK resolves the document's current published `jobId` and then -downloads the parser result ZIP through the same `cacheJobResult(...)` path: +Local grep and reads use the cached parse result. Published `documentId` grep +streams `documents.listChunks(...)` page by page when parsed storage is missing +or stale. Search uses the Knowhere API retrieval query; local document IDs only +help map returned server document IDs back to local cache IDs when available. +When `knowledge.parseToLocalCache(...)`, `knowledge.importJobResult(...)`, or +`knowledge.loadJobResult(...)` runs with configured parsed storage, the SDK +writes chunk pages, durable assets, sync progress, and a committed +`manifest/current.json` before returning. Partial snapshots are ignored until +the manifest is committed. ```typescript -const remoteRead = await client.knowledge.readChunks({ +const knowledge = client.knowledge.withParsedStorage({ + storage: myParsedDocumentStorage, + scheduler: myBackgroundScheduler, + limits: { remotePageSize: 100, maxPagesPerSync: 10 }, +}); +``` + +If a search result only has a published `documentId`, read-oriented helpers can +accept that remote identifier directly. `readChunks` supports display paging; +`assetUrlPolicy: 'durable'` hardens only returned assets through configured +storage and omits asset URLs if hardening fails: + +```typescript +const remoteRead = await knowledge.readChunks({ documentId: 'doc_123', - sectionPath: 'Overview', - limit: 5, + page: 1, + pageSize: 20, + chunkType: 'page', + assetUrlPolicy: 'durable', }); -const remoteOutline = await client.knowledge.getDocumentOutline({ +const remoteOutline = await knowledge.getDocumentOutline({ documentId: 'doc_123', }); -const jobRead = await client.knowledge.readChunks({ +const grep = await knowledge.grepChunks({ + documentId: 'doc_123', + pattern: 'warranty', + maxResults: 10, +}); + +const jobRead = await knowledge.readChunks({ jobId: jobResult.jobId, localDocumentId: 'manual', limit: 5, @@ -383,9 +456,13 @@ if (status.job.isDone && status.cache.document) { When the job was started through `client.knowledge.startParse(...)`, `getJobStatus(...)` automatically caches the completed result locally the first -time it observes `status.job.isDone`. Use `cacheJobResult(...)` only to recover a -completed job that was not started through the local knowledge helper, or to -retry a cache step explicitly. +time it observes `status.job.isDone`. Use `importJobResult(...)` to recover a +completed job into the local cache when it was not started through the local +knowledge helper, or to retry a local import step explicitly. Use +`loadJobResult(...)` for server workflows that should load a completed result +without creating SDK local-disk cache state. Use `syncParsedDocument(...)` to +explicitly resume or retry parsed-storage sync for an existing `documentId`, +`jobId`, or local parsed result. Follow-up queries can exclude documents or sections for one request: diff --git a/docs/new-design.md b/docs/new-design.md new file mode 100644 index 0000000..63bbf1e --- /dev/null +++ b/docs/new-design.md @@ -0,0 +1,118 @@ + # SDK Parsed Storage Redesign + + ## Summary + Move parsed chunk read/sync behavior into the SDK so Notebook, CLI, and MCP share one model. Callers provide a parsed-document storage adapter; SDK reads storage first, falls back to Knowhere remote storage + transparently, and schedules non-blocking storage sync on read misses. Parse flow is different: after parsing succeeds, SDK must sync to provided storage before the caller treats the document as complete/ + ready. + + ## Public SDK Shape + - Keep `knowledge.readChunks`, `knowledge.grepChunks`, and `knowledge.getDocumentOutline` as the only public read methods; do not add public `readChunkPage`. + - Extend `readChunks` with paged display params: + - `page`, `pageSize`, `chunkType`, `assetUrlPolicy?: "none" | "durable"`. + - Paged mode must not combine with scan-heavy filters like `sectionPath`, `startChunk`, `endChunk`, or `chunkId`. + - Add storage configuration on the SDK/knowledge client, e.g. `knowledge.withParsedStorage({ storage, scheduler, limits })`. + - Replace new usage of `cacheJobResult` with clearer methods: + - `importJobResult` means explicit local import. + - `syncParsedDocument` means write a parsed snapshot/assets to the configured storage. + - Keep `cacheJobResult` only as deprecated compatibility alias. + + ## SDK Implementation + - Define a `ParsedDocumentStorage` adapter for parsed snapshots, not just assets: + - read/write manifest + - read/write chunk pages + - read/write durable assets + - get asset URL + - read/write sync progress metadata + - Use `revisionKey = jobResultId ?? jobId` for freshness. Storage is valid only when its manifest revision matches the current remote revision. + - Read flow: + - resolve one document reference: `documentId`, `localDocumentId`, or `jobId` + - try configured storage first + - if missing/stale, read remote via `documents.listChunks` + - return the requested bounded result immediately + - schedule background sync into configured storage + - Parse flow: + - parse completes on Knowhere + - SDK writes chunks/assets/manifest to configured storage + - only then returns completion to the caller + - Background sync must be resumable and Vercel-safe: + - cursor/progress includes `documentId`, `revisionKey`, next chunk page, next asset index + - bounded by page count, asset count, and deadline per step + - final manifest is written only after all required pages/assets are present + - partial snapshots are ignored by reads until manifest commit + + ## Grep And Outline + - `grepChunks` supports only one specific document, never broad workspace/search scope. + - Broad search continues to use Knowhere `retrieval.query`. + - Remote grep streams `documents.listChunks(..., includeAssetUrls: false)` page by page, applies literal/regex matching SDK-side, and stops at `maxResults`, deadline, max pages, or end of document. + - Grep response should include `truncated` and a continuation cursor when it stops early. + - `getDocumentOutline` reads full outline from storage when available. On remote fallback, build a lightweight section outline from chunk `sectionPath` within configured page/deadline limits and mark it + truncated if incomplete. + + ## Notebook Changes + - Notebook uses SDK directly for chunk read/display/chat tooling; do not route this through MCP. + - Notebook provides a Vercel Blob `ParsedDocumentStorage` adapter and an Upstash/Vercel-safe scheduler. + - New Notebook parse/upload flow remains hidden behind `parsing`: + - Knowhere parse success plus Blob sync success -> mark source `ready` + - Knowhere parse failure -> mark source `failed` + - Blob sync failure after parse success -> mark source `failed` with `failureStage: "storage_sync"` + - Retry behavior: + - parse failure retries parse/upload + - `storage_sync` failure resumes sync from existing `documentId/jobId/revisionKey`, no reparse + - Existing remote Knowhere documents are readable immediately: + - SDK reads remote if Blob snapshot is missing/stale + - Blob sync is non-blocking + - missing Blob data must not produce zero chunks or non-ready behavior + - User-visible asset URLs must be durable Notebook Blob URLs: + - `readChunks` display calls use `assetUrlPolicy: "durable"` + - only visible/cited assets are hardened synchronously + - grep uses no asset URLs + - if hardening fails, return text and omit the asset URL rather than exposing a presigned Knowhere URL + + ## CLI And MCP + - CLI/MCP configure disk `ParsedDocumentStorage` explicitly. + - CLI removes duplicate `documentId -> listChunks -> importJobResult` logic and delegates reads to SDK. + - MCP continues delegating to SDK `readChunks`, `grepChunks`, and `getDocumentOutline`; update descriptions/tests for storage-first remote-fallback behavior. + - Remote reads must not silently write local disk unless disk storage was explicitly configured. + + ## Database And Compatibility + - Add a new Notebook migration only; do not edit existing migrations. + - Store snapshot/cache metadata in `source_parse_results`, including at least: + - `revision_key` + - `sync_status` or equivalent + - `failure_stage` / sync failure detail where appropriate + - progress cursor for resumable sync if not stored elsewhere + - Keep existing snapshot manifest shape compatible where possible, but treat committed manifest as the only readable snapshot marker. + + ## Test Plan + - SDK: + - storage hit reads without remote calls + - stale/missing storage falls back to remote + - remote reads schedule non-blocking sync + - parse writes configured storage before completion + - paged `readChunks` behavior and invalid param combinations + - single-document bounded remote grep with truncation cursor + - remote outline fallback and storage outline path + - no implicit disk writes without configured disk storage + - Notebook: + - parse remains `parsing` until Blob sync completes + - parse success plus Blob sync failure marks `failed` with `storage_sync` + - retry of `storage_sync` resumes sync without reparsing + - existing remote document with no Blob snapshot returns chunks from remote + - missing/stale Blob never returns zero chunks for ready remote docs + - displayed/chat citation assets are Blob URLs or omitted, never Knowhere presigned URLs + - CLI/MCP: + - remote document read/grep/outline works through SDK + - disk storage is explicit + - duplicate remote-to-local import path is removed + - Verification: + - run targeted SDK storage/read tests + - run Notebook parse/sync/chunk/chat tests + - run CLI read/inspect tests and MCP tests + - run feasible typecheck/lint/test suites per repo + - smoke test local Knowhere + Notebook: parse PDF, wait until ready, inspect Blob manifest/pages/assets, open chunks, chat with citation, verify durable Blob URLs + + ## Assumptions + - `documents.listChunks` is the authoritative remote parsed read API for published parsed v2 documents. + - `jobResultId ?? jobId` is sufficient as the revision key. + - Full job-result ZIP loading is allowed for explicit import/sync workflows, but not for Notebook request-time reads. + - Notebook request-time work is bounded to the requested page/window plus visible/cited asset hardening. diff --git a/package.json b/package.json index 936d54f..9492eac 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "LICENSE" ], "engines": { - "node": ">=20.19.0", + "node": ">=22.13.0", "npm": ">=10.0.0", "pnpm": ">=9.0.0" }, diff --git a/packages/mcp/README.md b/packages/mcp/README.md index d0e434a..d71e7ed 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -19,9 +19,11 @@ npx -y @ontos-ai/knowhere-mcp login npx -y @ontos-ai/knowhere-mcp ``` -The server uses stdio transport and stores expanded Knowhere result files under -the SDK local knowledge cache by default. `knowhere-mcp login` opens the -Knowhere dashboard in your browser and stores a local MCP login at +The server uses stdio transport. Parse tools store expanded Knowhere result +files under the SDK local knowledge cache; published `documentId` reads use the +SDK's remote chunk fallback unless the host explicitly configures a cache +directory for parsed-storage snapshots. `knowhere-mcp login` opens the Knowhere +dashboard in your browser and stores a local MCP login at `~/.knowhere-node-sdk/mcp/auth.json`. During login, the dashboard asks for a Permission: @@ -196,18 +198,23 @@ When logged in with Read only permission, the MCP server exposes only passed to outline/read/grep tools. - `knowhere_delete_document`: archive, or soft-delete, a published Knowhere document through the Knowhere API. -- `knowhere_get_document_outline`: inspect a parsed document outline by passing - `localDocumentId`, published `documentId`, or completed `jobId`. -- `knowhere_read_chunks`: read exact chunks from a parsed document by passing - `localDocumentId`, published `documentId`, or completed `jobId`. -- `knowhere_grep_chunks`: run literal or regex grep over parsed document chunks - by passing `localDocumentId`, published `documentId`, or completed `jobId`. +- `knowhere_get_document_outline`: inspect one parsed document outline by + passing `localDocumentId`, published `documentId`, or completed `jobId`. +- `knowhere_read_chunks`: read exact chunks from one parsed document by passing + `localDocumentId`, published `documentId`, or completed `jobId`. Use + `page`/`pageSize` for display reads; `assetUrlPolicy: "durable"` returns only + storage-hardened asset URLs. +- `knowhere_grep_chunks`: run literal or regex grep over one parsed document by + passing `localDocumentId`, published `documentId`, or completed `jobId`. + Broad workspace search belongs to `knowhere_search`. - `knowhere_search`: search published documents through the Knowhere API retrieval query. -Read-related tool responses include `document.resultDirectoryPath`; expanded +Local-cache tool responses include `document.resultDirectoryPath`; expanded chunks are stored at `/chunks.json` for direct filesystem -reads when needed. +reads when a local cache entry exists. Remote fallback responses use +`remote:` or `parsed-storage:` markers instead of +claiming a local expanded result directory. ## Package Boundary diff --git a/packages/mcp/src/__tests__/mcp.test.ts b/packages/mcp/src/__tests__/mcp.test.ts index 497ce3b..4a39656 100644 --- a/packages/mcp/src/__tests__/mcp.test.ts +++ b/packages/mcp/src/__tests__/mcp.test.ts @@ -30,8 +30,8 @@ describe('knowhere MCP wrapper', () => { expect(statusTool?.description).toContain('5s, 10s, 20s, 40s, 80s'); expect(statusTool?.description).toContain('Large PDFs or OCR-heavy files can take 10+ minutes'); expect(listTool?.description).toContain('remote API'); - expect(readTool?.description).toContain('resultDirectoryPath'); - expect(readTool?.description).toContain('chunks.json'); + expect(readTool?.description).toContain('configured parsed storage first'); + expect(readTool?.description).toContain('assetUrlPolicy is durable'); await client.close(); await server.close(); }); @@ -107,7 +107,7 @@ describe('knowhere MCP wrapper', () => { }, }); - expect(knowhereClient.knowledge.parse).toHaveBeenCalledWith({ + expect(knowhereClient.knowledge.parseToLocalCache).toHaveBeenCalledWith({ file: './report.md', fileName: undefined, namespace: undefined, @@ -160,16 +160,21 @@ describe('knowhere MCP wrapper', () => { name: 'knowhere_read_chunks', arguments: { documentId: 'doc_remote', - sectionPath: 'Overview', + revisionKey: 'jres_remote', + page: 1, + pageSize: 3, chunkType: 'page', limit: 3, + assetUrlPolicy: 'none', }, }); await client.callTool({ name: 'knowhere_grep_chunks', arguments: { documentId: 'doc_remote', + revisionKey: 'jres_remote', pattern: 'revenue', + continuationCursor: 'cursor-1', chunkType: 'page', }, }); @@ -179,13 +184,18 @@ describe('knowhere MCP wrapper', () => { }); expect(knowhereClient.knowledge.readChunks).toHaveBeenCalledWith({ documentId: 'doc_remote', - sectionPath: 'Overview', + revisionKey: 'jres_remote', + page: 1, + pageSize: 3, chunkType: 'page', limit: 3, + assetUrlPolicy: 'none', }); expect(knowhereClient.knowledge.grepChunks).toHaveBeenCalledWith({ documentId: 'doc_remote', + revisionKey: 'jres_remote', pattern: 'revenue', + continuationCursor: 'cursor-1', chunkType: 'page', }); await client.close(); @@ -318,6 +328,7 @@ describe('knowhere MCP wrapper', () => { expect(knowhereClient.knowledge.withCacheDirectory).toHaveBeenCalledWith( '/tmp/knowhere-mcp-cache', ); + expect(knowhereClient.knowledge.withParsedStorage).toHaveBeenCalledOnce(); expect(knowhereClient.knowledge.recoverPendingAsyncParseJobs).toHaveBeenCalledOnce(); expect(knowhereClient.knowledge.readChunks).toHaveBeenCalledWith({ localDocumentId: 'local-report', @@ -414,7 +425,7 @@ function createClient(): Knowhere & { knowledge: KnowledgeWithMocks; } { const knowledge: KnowledgeWithMocks = { - parse: vi.fn().mockResolvedValue({ + parseToLocalCache: vi.fn().mockResolvedValue({ document: { localDocumentId: 'local-report' }, result: { jobId: 'job-1' }, }), @@ -430,7 +441,7 @@ function createClient(): Knowhere & { document: { localDocumentId: 'local-report' }, }, }), - cacheJobResult: vi.fn().mockResolvedValue({ + importJobResult: vi.fn().mockResolvedValue({ document: { localDocumentId: 'local-report' }, }), recoverPendingAsyncParseJobs: vi.fn().mockResolvedValue({ @@ -448,8 +459,10 @@ function createClient(): Knowhere & { grepChunks: vi.fn().mockResolvedValue({ matches: [] }), search: vi.fn().mockResolvedValue({ results: [] }), withCacheDirectory: vi.fn(), + withParsedStorage: vi.fn(), }; knowledge.withCacheDirectory.mockReturnValue(knowledge as unknown as Knowledge); + knowledge.withParsedStorage.mockReturnValue(knowledge as unknown as Knowledge); const archiveDocument = vi.fn().mockResolvedValue({ documentId: 'doc-1', status: 'archived', @@ -482,10 +495,10 @@ function createClient(): Knowhere & { type KnowledgeWithMocks = Pick< Knowledge, - | 'parse' + | 'parseToLocalCache' | 'startParse' | 'getJobStatus' - | 'cacheJobResult' + | 'importJobResult' | 'recoverPendingAsyncParseJobs' | 'listDocuments' | 'getDocumentOutline' @@ -493,11 +506,12 @@ type KnowledgeWithMocks = Pick< | 'grepChunks' | 'search' | 'withCacheDirectory' + | 'withParsedStorage' > & { - parse: Mock; + parseToLocalCache: Mock; startParse: Mock; getJobStatus: Mock; - cacheJobResult: Mock; + importJobResult: Mock; recoverPendingAsyncParseJobs: Mock; listDocuments: Mock; getDocumentOutline: Mock; @@ -505,4 +519,5 @@ type KnowledgeWithMocks = Pick< grepChunks: Mock; search: Mock; withCacheDirectory: Mock; + withParsedStorage: Mock; }; diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 5a4bf32..da0e5e1 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -1,6 +1,9 @@ +import path from 'path'; + import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; import { + DiskParsedDocumentStorage, Knowhere, VERSION, ValidationError, @@ -51,10 +54,7 @@ export async function createKnowhereMcpServer( authTokenProvider: options?.authTokenProvider, baseURL: options?.baseURL, }); - const knowledge: Knowledge = - options?.cacheDirectory === undefined - ? client.knowledge - : client.knowledge.withCacheDirectory(options.cacheDirectory); + const knowledge: Knowledge = createKnowledgeClient(client, options?.cacheDirectory); if (options?.recoverPendingJobsOnStart !== false) { await knowledge.recoverPendingAsyncParseJobs(); } @@ -82,7 +82,7 @@ export async function createKnowhereMcpServer( }, async (input) => createToolResult( - await knowledge.parse({ + await knowledge.parseToLocalCache({ url: input.url, namespace: input.namespace, localDocumentId: input.localDocumentId, @@ -109,7 +109,7 @@ export async function createKnowhereMcpServer( }, async (input) => createToolResult( - await knowledge.parse({ + await knowledge.parseToLocalCache({ file: input.file, fileName: input.fileName, namespace: input.namespace, @@ -227,11 +227,12 @@ export async function createKnowhereMcpServer( 'knowhere_get_document_outline', { description: - 'Return the outline for a parsed document. Pass localDocumentId, published documentId, or completed jobId. The response document includes resultDirectoryPath; expanded chunks are stored in chunks.json under that directory.', + 'Return the outline for one parsed document. Pass localDocumentId, published documentId, or completed jobId. The SDK reads configured parsed storage first, falls back to remote document chunks for documentId reads, and schedules bounded sync only when parsed storage is explicitly configured.', inputSchema: { localDocumentId: z.string().optional(), documentId: z.string().optional(), jobId: z.string().optional(), + revisionKey: z.string().optional(), }, outputSchema: objectOutputSchema, }, @@ -242,17 +243,21 @@ export async function createKnowhereMcpServer( 'knowhere_read_chunks', { description: - 'Read exact chunks from a parsed document. Pass localDocumentId, published documentId, or completed jobId. The response document includes resultDirectoryPath; expanded chunks are stored in chunks.json under that directory.', + 'Read exact chunks from one parsed document. Pass localDocumentId, published documentId, or completed jobId. page/pageSize are for display reads and cannot be combined with sectionPath/startChunk/endChunk/chunkId. The SDK reads configured parsed storage first, falls back to remote document chunks for documentId reads, and only returns durable asset URLs when assetUrlPolicy is durable and storage hardening succeeds.', inputSchema: { localDocumentId: z.string().optional(), documentId: z.string().optional(), jobId: z.string().optional(), + revisionKey: z.string().optional(), + page: z.number().int().positive().optional(), + pageSize: z.number().int().positive().optional(), sectionPath: z.string().optional(), startChunk: z.number().int().positive().optional(), endChunk: z.number().int().positive().optional(), chunkId: z.string().optional(), chunkType: z.enum(['text', 'image', 'table', 'page']).optional(), limit: z.number().int().positive().optional(), + assetUrlPolicy: z.enum(['none', 'durable']).optional(), }, outputSchema: objectOutputSchema, }, @@ -263,12 +268,14 @@ export async function createKnowhereMcpServer( 'knowhere_grep_chunks', { description: - 'Run grep-style literal or regex matching against parsed document chunks. Pass localDocumentId, published documentId, or completed jobId. The response document includes resultDirectoryPath; expanded chunks are stored in chunks.json under that directory.', + 'Run grep-style literal or regex matching against one parsed document. Pass localDocumentId, published documentId, or completed jobId. Broad workspace search belongs to knowhere_search; documentId grep streams remote chunks without asset URLs when parsed storage is missing or stale.', inputSchema: { localDocumentId: z.string().optional(), documentId: z.string().optional(), jobId: z.string().optional(), + revisionKey: z.string().optional(), pattern: z.string(), + continuationCursor: z.string().optional(), isRegex: z.boolean().optional(), isCaseSensitive: z.boolean().optional(), maxResults: z.number().int().positive().optional(), @@ -307,6 +314,16 @@ export async function runKnowhereMcpServer(options?: KnowhereMcpServerOptions): await server.connect(transport); } +function createKnowledgeClient(client: Knowhere, cacheDirectory: string | undefined): Knowledge { + if (cacheDirectory === undefined) { + return client.knowledge; + } + + return client.knowledge.withCacheDirectory(cacheDirectory).withParsedStorage({ + storage: new DiskParsedDocumentStorage(path.join(cacheDirectory, 'parsed-documents')), + }); +} + function createToolResult(result: ToolResult): { content: { type: 'text'; text: string }[]; structuredContent: { result: ToolResult }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 14c47ba..bc78944 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,7 +44,7 @@ importers: version: 3.8.4 tsup: specifier: ^8.0.0 - version: 8.5.1(postcss@8.5.15)(typescript@5.9.3) + version: 8.5.1(postcss@8.5.15)(typescript@5.9.3)(yaml@2.9.0) typescript: specifier: ^5.3.0 version: 5.9.3 @@ -53,7 +53,7 @@ importers: version: 8.62.0(eslint@10.5.0)(typescript@5.9.3) vitest: specifier: ^4.0.18 - version: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)) + version: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)) packages/mcp: dependencies: @@ -72,13 +72,13 @@ importers: version: 25.9.4 tsup: specifier: ^8.0.0 - version: 8.5.1(postcss@8.5.15)(typescript@5.9.3) + version: 8.5.1(postcss@8.5.15)(typescript@5.9.3)(yaml@2.9.0) typescript: specifier: ^5.3.0 version: 5.9.3 vitest: specifier: ^4.0.18 - version: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)) + version: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)) packages: @@ -2130,6 +2130,11 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -2802,7 +2807,7 @@ snapshots: obug: 2.1.3 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)) + vitest: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)) '@vitest/expect@4.1.9': dependencies: @@ -2813,13 +2818,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.9(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7))': + '@vitest/mocker@4.1.9(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.9 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.1.0(@types/node@25.9.4)(esbuild@0.27.7) + vite: 8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0) '@vitest/pretty-format@4.1.9': dependencies: @@ -3700,11 +3705,12 @@ snapshots: mlly: 1.8.2 pathe: 2.0.3 - postcss-load-config@6.0.1(postcss@8.5.15): + postcss-load-config@6.0.1(postcss@8.5.15)(yaml@2.9.0): dependencies: lilconfig: 3.1.3 optionalDependencies: postcss: 8.5.15 + yaml: 2.9.0 postcss@8.5.15: dependencies: @@ -3995,7 +4001,7 @@ snapshots: tslib@2.8.1: optional: true - tsup@8.5.1(postcss@8.5.15)(typescript@5.9.3): + tsup@8.5.1(postcss@8.5.15)(typescript@5.9.3)(yaml@2.9.0): dependencies: bundle-require: 5.1.0(esbuild@0.27.7) cac: 6.7.14 @@ -4006,7 +4012,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.5.15) + postcss-load-config: 6.0.1(postcss@8.5.15)(yaml@2.9.0) resolve-from: 5.0.0 rollup: 4.62.2 source-map: 0.7.6 @@ -4062,7 +4068,7 @@ snapshots: vary@1.1.2: {} - vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7): + vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -4073,11 +4079,12 @@ snapshots: '@types/node': 25.9.4 esbuild: 0.27.7 fsevents: 2.3.3 + yaml: 2.9.0 - vitest@4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)): + vitest@4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.9 - '@vitest/mocker': 4.1.9(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)) + '@vitest/mocker': 4.1.9(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.9 '@vitest/runner': 4.1.9 '@vitest/snapshot': 4.1.9 @@ -4094,7 +4101,7 @@ snapshots: tinyexec: 1.2.4 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vite: 8.1.0(@types/node@25.9.4)(esbuild@0.27.7) + vite: 8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 25.9.4 @@ -4122,6 +4129,9 @@ snapshots: wrappy@1.0.2: {} + yaml@2.9.0: + optional: true + yocto-queue@0.1.0: {} zod-to-json-schema@3.25.2(zod@3.25.76): diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index 0883637..fa78008 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -5,7 +5,13 @@ import { Knowhere } from '../client.js'; import { ValidationError } from '../errors/base.js'; import { Agent as HttpAgent } from 'http'; import { Agent as HttpsAgent } from 'https'; -import type { ParseResult, TextChunk } from '../types/result.js'; +import JSZip from 'jszip'; +import type { ParseResult, PageChunk, TextChunk } from '../types/result.js'; +import type { + KnowhereAssetStorageAdapter, + KnowhereAssetStorageObject, + KnowhereAssetStorageWriteResult, +} from '../types/storage.js'; describe('Knowhere Client', () => { // Store original env @@ -202,6 +208,94 @@ describe('Knowhere Client', () => { expect(result.documentId).toBe('doc-123'); }); + it('should store parse result assets when a storage adapter is provided', async () => { + const pageChunk: PageChunk = { + chunkId: 'page-1', + type: 'page', + content: 'Page one summary', + contentSource: 'summary', + path: 'report.pdf/Page 1', + metadata: { + pageNums: [1], + pageAssets: [ + { + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + contentType: 'image/png', + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + }; + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => + Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }), + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + + vi.spyOn(client.jobs, 'load').mockResolvedValueOnce({ + manifest: { + version: '2.0', + jobId: 'job-123', + sourceFileName: 'report.pdf', + statistics: { + totalChunks: 1, + textChunks: 0, + imageChunks: 0, + tableChunks: 0, + pageChunks: 1, + }, + }, + chunks: [pageChunk], + textChunks: [], + imageChunks: [], + tableChunks: [], + pageChunks: [pageChunk], + rawZip: await createZipBuffer({ + 'page_citation_assets/page-1.png': 'page-one-image', + }), + jobId: 'job-123', + statistics: { + totalChunks: 1, + textChunks: 0, + imageChunks: 0, + tableChunks: 0, + pageChunks: 1, + }, + getChunk: vi.fn(), + save: vi.fn(), + }); + + const result = await client.parse({ + url: 'https://example.com/doc.pdf', + storageAdapter: { + adapter, + keyPrefix: 'workspaces/workspace-1/sources/source-1/parsed-result', + }, + }); + + expect(writeObject).toHaveBeenCalledWith( + expect.objectContaining({ + key: 'workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + contentType: 'image/png', + }), + ); + expect(result.pageChunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + expect(result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(result.documentId).toBe('doc-123'); + }); + it('should parse document from file path', async () => { const result = await client.parse({ file: './test.pdf', @@ -492,3 +586,11 @@ describe('Knowhere Client', () => { }); }); }); + +async function createZipBuffer(files: Readonly>): Promise { + const zip = new JSZip(); + for (const [filePath, content] of Object.entries(files)) { + zip.file(filePath, Buffer.from(content)); + } + return Buffer.from(await zip.generateAsync({ type: 'arraybuffer' })); +} diff --git a/src/client.ts b/src/client.ts index 7b21b0e..00ae169 100644 --- a/src/client.ts +++ b/src/client.ts @@ -13,6 +13,7 @@ import { Knowledge } from './knowledge/index.js'; import { DEFAULT_BASE_URL, ENV } from './constants.js'; import { ValidationError } from './errors/index.js'; import { enrichParseResult } from './lib/utils.js'; +import { storeParseResultAssets } from './storage/asset-storage.js'; function inferFileName(file: ParseParams['file'], explicitFileName?: string): string | undefined { if (explicitFileName) { @@ -145,7 +146,13 @@ export class Knowhere { verifyChecksum: params.verifyChecksum, }); - return enrichParseResult(result, jobResult); + const enrichedResult = enrichParseResult(result, { + namespace: jobResult.namespace, + documentId: jobResult.documentId ?? params.documentId, + }); + + const storedResult = await storeParseResultAssets(enrichedResult, params.storageAdapter); + return storedResult.result; } /** diff --git a/src/index.ts b/src/index.ts index ea39b30..99cbda8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,7 @@ export type { DocumentChunkPagination, DocumentChunkResponse, DocumentChunkType, + DocumentPageCitationSource, DocumentListPagination, DocumentListParams, DocumentListResponse, @@ -49,6 +50,32 @@ export type { ParseParams, UploadProgress, PollProgress, + KnowhereAssetStorageAdapter, + KnowhereAssetStorageBody, + KnowhereAssetStorageHead, + KnowhereAssetStorageObject, + KnowhereAssetStorageOptions, + KnowhereAssetStorageResult, + KnowhereAssetStorageWriteResult, + KnowhereParsedSnapshot, + KnowhereParsedSnapshotChunk, + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotChunkPageReference, + KnowhereParsedSnapshotManifest, + ParsedDocumentAssetUrlPolicy, + ParsedDocumentStorage, + ParsedDocumentStorageAsset, + ParsedDocumentStorageAssetParams, + ParsedDocumentStorageChunkPageParams, + ParsedDocumentStorageConfig, + ParsedDocumentStorageDocument, + ParsedDocumentStorageLimits, + ParsedDocumentStorageManifestParams, + ParsedDocumentSyncProgress, + ParsedDocumentSyncScheduler, + PageCitationAsset, + PageCitationAssetContentType, + PageCitationAssetSource, // Result types ParseResult, Manifest, @@ -67,11 +94,15 @@ export type { KnowledgeCacheDocumentParams, KnowledgeCacheJobResultParams, KnowledgeDocumentReference, + KnowledgeImportJobResultParams, + KnowledgeJobResultResponse, + KnowledgeLoadJobResultParams, KnowledgeParseParams, LocalKnowledgeDocument, LocalKnowledgeParseResponse, KnowledgeOutline, KnowledgeSection, + KnowledgeParsedStorageOptions, KnowledgeReadParams, KnowledgeReadChunk, KnowledgeReadResponse, @@ -83,8 +114,14 @@ export type { KnowledgeSearchResponse, KnowledgeSearchResult, KnowledgeStartupRecoveryResponse, + KnowledgeSyncParsedDocumentParams, + KnowledgeSyncParsedDocumentResponse, } from './types/index.js'; +export { storeParseResultAssets } from './storage/asset-storage.js'; +export { DiskParsedDocumentStorage } from './storage/disk-parsed-document-storage.js'; +export { syncParseResultToParsedDocumentStorage } from './storage/parsed-document-storage.js'; + // Errors export { // Base errors diff --git a/src/knowledge/__tests__/knowledge.test.ts b/src/knowledge/__tests__/knowledge.test.ts index c589116..f928699 100644 --- a/src/knowledge/__tests__/knowledge.test.ts +++ b/src/knowledge/__tests__/knowledge.test.ts @@ -1,13 +1,24 @@ import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import path from 'path'; +import JSZip from 'jszip'; import { describe, it, expect, vi, afterEach } from 'vitest'; import { Knowledge } from '../knowledge.js'; import type { Knowhere } from '../../client.js'; import type { + Chunk, + DocumentChunk, + DocumentChunkListParams, DocumentChunkListResponse, + KnowhereAssetStorageAdapter, + KnowhereAssetStorageObject, + KnowhereAssetStorageWriteResult, + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, ParseResult, + ParsedDocumentStorage, + ParsedDocumentSyncProgress, TextChunk, TableChunk, } from '../../types/index.js'; @@ -17,6 +28,7 @@ describe('Knowledge', () => { afterEach(async () => { vi.restoreAllMocks(); + vi.unstubAllGlobals(); await Promise.all( tempDirectories.map((directory) => rm(directory, { recursive: true, force: true })), ); @@ -29,7 +41,7 @@ describe('Knowledge', () => { const { client, parse } = createClient(parseResult); const knowledge = new Knowledge(client, { cacheDirectory }); - const response = await knowledge.parse({ + const response = await knowledge.parseToLocalCache({ url: 'https://example.com/report.md', namespace: 'support-center', localDocumentId: 'local-report', @@ -59,6 +71,24 @@ describe('Knowledge', () => { await expectFileMissing(path.join(cacheDirectory, 'documents', 'local-report', 'result.zip')); }); + it('should keep the deprecated parse alias for compatibility', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = createParseResult(); + const { client, parse } = createClient(parseResult); + const knowledge = new Knowledge(client, { cacheDirectory }); + + const response = await knowledge.parse({ + url: 'https://example.com/report.md', + localDocumentId: 'local-report', + }); + + expect(parse).toHaveBeenCalledWith({ + url: 'https://example.com/report.md', + localDocumentId: 'local-report', + }); + expect(response.document.localDocumentId).toBe('local-report'); + }); + it('should reject local document ids that resolve outside the cache', async () => { const cacheDirectory = await createTempDirectory(); const siblingDirectory = path.join(path.dirname(cacheDirectory), 'knowhere-victim'); @@ -71,7 +101,7 @@ describe('Knowledge', () => { await writeFile(path.join(siblingDirectory, 'sentinel.txt'), 'keep'); await expect( - knowledge.parse({ + knowledge.parseToLocalCache({ url: 'https://example.com/report.md', localDocumentId: `../../${path.basename(siblingDirectory)}`, }), @@ -138,12 +168,12 @@ describe('Knowledge', () => { expect(documents).toHaveLength(1); }); - it('should still allow manual completed job result caching', async () => { + it('should import a completed job result into the local cache', async () => { const cacheDirectory = await createTempDirectory(); const { client, jobsLoad } = createClient(createParseResult()); const knowledge = new Knowledge(client, { cacheDirectory }); - const cached = await knowledge.cacheJobResult({ + const cached = await knowledge.importJobResult({ jobId: 'job-1', localDocumentId: 'local-report', verifyChecksum: false, @@ -153,6 +183,20 @@ describe('Knowledge', () => { expect(cached.document.localDocumentId).toBe('local-report'); }); + it('should keep the deprecated job result cache alias for compatibility', async () => { + const cacheDirectory = await createTempDirectory(); + const { client, jobsLoad } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }); + + const cached = await knowledge.cacheJobResult({ + jobId: 'job-1', + localDocumentId: 'local-report', + }); + + expect(jobsLoad).toHaveBeenCalledWith('job-1', { verifyChecksum: undefined }); + expect(cached.document.localDocumentId).toBe('local-report'); + }); + it('should recover and cache pending async parse jobs on startup', async () => { const cacheDirectory = await createTempDirectory(); const { client, jobsGet, jobsLoad } = createClient(createParseResult()); @@ -203,7 +247,7 @@ describe('Knowledge', () => { expect(read.nextChunk).toBeUndefined(); }); - it('should sync a remote document id into the local cache before reads', async () => { + it('should read a remote document id without importing into the local cache', async () => { const cacheDirectory = await createTempDirectory(); const { client, documentsListChunks, jobsLoad } = createClient(createParseResult()); const knowledge = new Knowledge(client, { cacheDirectory }); @@ -223,15 +267,17 @@ describe('Knowledge', () => { expect(documentsListChunks).toHaveBeenCalledWith('doc_remote', { page: 1, - pageSize: 1, + pageSize: 100, + chunkType: undefined, + includeAssetUrls: false, }); - expect(jobsLoad).toHaveBeenCalledWith('job_remote', { verifyChecksum: undefined }); + expect(jobsLoad).not.toHaveBeenCalled(); expect(read.document).toMatchObject({ localDocumentId: 'doc_remote', documentId: 'doc_remote', namespace: 'support-center', jobId: 'job_remote', - sourceFileName: 'job_remote.md', + sourceFileName: 'doc_remote', chunkCount: 3, }); expect(read.chunks.map((chunk) => chunk.chunkId)).toEqual(['chunk-intro']); @@ -240,9 +286,140 @@ describe('Knowledge', () => { chunkId: 'chunk-intro', sectionPath: 'Intro', }); - expect(documents).toHaveLength(1); - await expectFileExists(path.join(cacheDirectory, 'documents', 'doc_remote', 'manifest.json')); - await expectFileExists(path.join(cacheDirectory, 'documents', 'doc_remote', 'chunks.json')); + expect(documents).toHaveLength(0); + await expectFileMissing(path.join(cacheDirectory, 'documents', 'doc_remote', 'manifest.json')); + await expectFileMissing(path.join(cacheDirectory, 'documents', 'doc_remote', 'chunks.json')); + }); + + it('should read configured parsed storage for an explicit revision without remote calls', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = createParseResult(); + const storage = createInMemoryParsedStorage(); + storage.seedSnapshot({ + documentId: 'doc_remote', + revisionKey: 'jres_remote', + result: parseResult, + }); + const { client, documentsListChunks } = createClient(parseResult); + const knowledge = new Knowledge(client, { cacheDirectory }).withParsedStorage({ storage }); + + const read = await knowledge.readChunks({ + documentId: 'doc_remote', + revisionKey: 'jres_remote', + page: 1, + pageSize: 2, + }); + + expect(documentsListChunks).not.toHaveBeenCalled(); + expect(read.document.resultDirectoryPath).toBe('parsed-storage:doc_remote'); + expect(read.chunks.map((chunk) => chunk.chunkId)).toEqual(['chunk-intro', 'chunk-table']); + expect(read.totalChunks).toBe(3); + }); + + it('should fall back to remote chunks and schedule parsed storage sync on storage misses', async () => { + const cacheDirectory = await createTempDirectory(); + const storage = createInMemoryParsedStorage(); + const scheduledTasks: Array<() => Promise> = []; + const scheduler = { + schedule: vi.fn((task: () => Promise): void => { + scheduledTasks.push(task); + }), + }; + const { client, documentsListChunks, jobsLoad } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }).withParsedStorage({ + storage, + scheduler, + limits: { + remotePageSize: 2, + maxPagesPerSync: 10, + }, + }); + + const read = await knowledge.readChunks({ + documentId: 'doc_remote', + page: 1, + pageSize: 2, + }); + + expect(read.chunks.map((chunk) => chunk.chunkId)).toEqual(['chunk-intro', 'chunk-table']); + expect(jobsLoad).not.toHaveBeenCalled(); + expect(scheduler.schedule).toHaveBeenCalledOnce(); + expect(storage.manifestCount()).toBe(0); + + await scheduledTasks[0]?.(); + + expect(documentsListChunks).toHaveBeenCalledWith('doc_remote', { + page: 1, + pageSize: 2, + includeAssetUrls: true, + }); + expect(storage.manifestCount()).toBe(1); + expect(storage.getManifest('doc_remote', 'jres_remote')?.totalChunks).toBe(3); + expect(storage.getProgress('doc_remote', 'jres_remote')?.status).toBe('completed'); + }); + + it('should sync configured parsed storage before parse returns', async () => { + const cacheDirectory = await createTempDirectory(); + const storage = createInMemoryParsedStorage(); + const { client } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }).withParsedStorage({ storage }); + + const parsed = await knowledge.parseToLocalCache({ + url: 'https://example.com/report.md', + localDocumentId: 'local-report', + }); + + expect(parsed.document.localDocumentId).toBe('local-report'); + expect(storage.getManifest('doc-1', 'job-1')).toMatchObject({ + documentId: 'doc-1', + revisionKey: 'job-1', + totalChunks: 3, + }); + expect(storage.getPage('doc-1', 'job-1', 1)?.chunks).toHaveLength(3); + }); + + it('should reject paged read params combined with scan-heavy filters', async () => { + const cacheDirectory = await createTempDirectory(); + const { client, documentsListChunks } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }); + + await expect( + knowledge.readChunks({ + documentId: 'doc_remote', + page: 1, + sectionPath: 'Intro', + }), + ).rejects.toThrow(/cannot be combined/); + expect(documentsListChunks).not.toHaveBeenCalled(); + }); + + it('should grep a remote document with truncation cursor without asset URLs', async () => { + const cacheDirectory = await createTempDirectory(); + const { client, documentsListChunks } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }); + + const grep = await knowledge.grepChunks({ + documentId: 'doc_remote', + pattern: 'revenue', + maxResults: 1, + }); + + expect(grep.matches).toHaveLength(1); + expect(grep.truncated).toBe(true); + expect(grep.continuationCursor).toEqual(expect.any(String)); + const resumed = await knowledge.grepChunks({ + documentId: 'doc_remote', + pattern: 'revenue', + maxResults: 1, + continuationCursor: grep.continuationCursor, + }); + expect(resumed.matches[0]?.chunkId).toBe('chunk-table'); + expect(documentsListChunks).toHaveBeenCalledWith('doc_remote', { + page: 1, + pageSize: 100, + chunkType: undefined, + includeAssetUrls: false, + }); }); it('should fail clearly when a remote document id has no published job id', async () => { @@ -269,7 +446,7 @@ describe('Knowledge', () => { documentId: 'doc_missing_job', limit: 5, }), - ).rejects.toThrow('Cannot sync server document doc_missing_job'); + ).rejects.toThrow('Cannot read server document doc_missing_job'); }); it('should sync a completed job id into the local cache before reads', async () => { @@ -355,7 +532,7 @@ describe('Knowledge', () => { const cacheDirectory = await createTempDirectory(); const { client, retrievalQuery } = createClient(createParseResult()); const knowledge = new Knowledge(client, { cacheDirectory }); - await knowledge.parse({ + await knowledge.parseToLocalCache({ url: 'https://example.com/report.md', localDocumentId: 'local-report', }); @@ -427,13 +604,199 @@ describe('Knowledge', () => { expect(savedTable).toBe('
Revenue
'); }); + it('should persist page assets from parse results into local read chunks', async () => { + const knowledge = await createKnowledgeWithCachedResult(createPageParseResultWithAssets()); + + const read = await knowledge.readChunks({ + localDocumentId: 'local-report', + chunkType: 'page', + limit: 1, + }); + + expect(read.chunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + width: 120, + height: 240, + }), + ]); + expect(read.chunks[0]).not.toHaveProperty('pageAssets'); + }); + + it('should import server-provided page assets without SDK-side rendering', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = await createPageParseResultWithRawPageAsset(); + const { client, jobsLoad, documentsGetPageCitationSource } = createClient(parseResult); + const knowledge = new Knowledge(client, { cacheDirectory }); + + const cached = await knowledge.importJobResult({ + jobId: 'job-1', + localDocumentId: 'local-report', + }); + const read = await knowledge.readChunks({ + localDocumentId: 'local-report', + chunkType: 'page', + limit: 1, + }); + + expect(jobsLoad).toHaveBeenCalledWith('job-1', { verifyChecksum: undefined }); + expect(documentsGetPageCitationSource).not.toHaveBeenCalled(); + expect(cached.result.pageChunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + }), + ]); + expect(cached.result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(read.chunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + width: 120, + height: 240, + }), + ]); + expect(read.chunks[0]).not.toHaveProperty('pageAssets'); + }); + + it('should load job result assets through a storage adapter without saving locally', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = await createPageParseResultWithRawPageAsset(); + const { client, jobsLoad } = createClient(parseResult); + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => + Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }), + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + const knowledge = new Knowledge(client, { cacheDirectory }); + + const loaded = await knowledge.loadJobResult({ + jobId: 'job-1', + storageAdapter: { + adapter, + keyPrefix: 'workspaces/workspace-1/sources/source-1/parsed-result', + }, + }); + const documents = await knowledge.listDocuments(); + + expect(jobsLoad).toHaveBeenCalledWith('job-1', { verifyChecksum: undefined }); + expect(documents).toEqual([]); + expect(loaded.assetUrlsByFilePath).toEqual({ + 'page_citation_assets/page-1.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }); + expect(loaded.parsedSnapshot).toMatchObject({ + manifestKey: 'workspaces/workspace-1/sources/source-1/parsed-result/manifest/current.json', + manifestUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/manifest/current.json', + manifest: { + kind: 'knowhere-parsed-result-snapshot', + jobId: 'job-1', + documentId: 'doc-1', + totalChunks: 1, + chunkPages: [ + { + page: 1, + key: 'workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + url: 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + }, + ], + }, + }); + expect(loaded.result.pageChunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + expect(loaded.result.pageChunks[0]).not.toHaveProperty('pageAssets'); + await expectFileMissing(path.join(cacheDirectory, 'documents')); + }); + + it('should import job result assets through a storage adapter and preserve metadata shape', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = await createPageParseResultWithRawPageAsset(); + const { client } = createClient(parseResult); + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => + Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }), + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + const knowledge = new Knowledge(client, { cacheDirectory }); + + const cached = await knowledge.importJobResult({ + jobId: 'job-1', + localDocumentId: 'local-report', + storageAdapter: { + adapter, + keyPrefix: 'workspaces/workspace-1/sources/source-1/parsed-result', + }, + }); + const read = await knowledge.readChunks({ + localDocumentId: 'local-report', + chunkType: 'page', + limit: 1, + }); + + expect(cached.assetUrlsByFilePath).toEqual({ + 'page_citation_assets/page-1.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }); + expect(cached.parsedSnapshot).toMatchObject({ + manifestKey: 'workspaces/workspace-1/sources/source-1/parsed-result/manifest/current.json', + manifestUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/manifest/current.json', + manifest: { + kind: 'knowhere-parsed-result-snapshot', + jobId: 'job-1', + documentId: 'doc-1', + totalChunks: 1, + chunkPages: [ + { + page: 1, + key: 'workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + url: 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + }, + ], + }, + }); + expect(cached.result.pageChunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + expect(cached.result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(read.chunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + expect(read.chunks[0]).not.toHaveProperty('pageAssets'); + }); + async function createKnowledgeWithCachedResult( parseResult = createParseResult(), ): Promise { const cacheDirectory = await createTempDirectory(); const { client } = createClient(parseResult); const knowledge = new Knowledge(client, { cacheDirectory }); - await knowledge.parse({ + await knowledge.parseToLocalCache({ url: 'https://example.com/report.md', localDocumentId: 'local-report', }); @@ -454,6 +817,7 @@ function createClient(parseResult: ParseResult): { jobsGet: ReturnType; jobsLoad: ReturnType; documentsListChunks: ReturnType; + documentsGetPageCitationSource: ReturnType; retrievalQuery: ReturnType; } { const parse = vi.fn().mockResolvedValue(parseResult); @@ -491,22 +855,43 @@ function createClient(parseResult: ParseResult): { }, }), ); - const documentsListChunks = vi.fn().mockImplementation( - (_documentId: string): Promise => - Promise.resolve({ - documentId: 'doc_remote', - namespace: 'support-center', - jobId: 'job_remote', - jobResultId: 'jres_remote', - chunks: [], - pagination: { - page: 1, - pageSize: 1, - total: 0, - totalPages: 1, - }, - }), - ); + const documentsListChunks = vi + .fn() + .mockImplementation( + ( + documentId: string, + params?: DocumentChunkListParams, + ): Promise => { + const page = params?.page ?? 1; + const pageSize = params?.pageSize ?? 50; + const chunks = toRemoteDocumentChunks(parseResult, documentId, params); + const pageChunks = chunks.slice((page - 1) * pageSize, page * pageSize); + return Promise.resolve({ + documentId, + namespace: 'support-center', + jobId: documentId === 'doc_remote' ? 'job_remote' : parseResult.jobId, + jobResultId: documentId === 'doc_remote' ? 'jres_remote' : `jres_${parseResult.jobId}`, + chunks: pageChunks, + pagination: { + page, + pageSize, + total: chunks.length, + totalPages: Math.max(1, Math.ceil(chunks.length / pageSize)), + }, + }); + }, + ); + const documentsGetPageCitationSource = vi.fn().mockResolvedValue({ + documentId: 'doc-1', + namespace: 'support-center', + jobId: 'job-1', + jobResultId: 'jres-1', + variant: 'normalized_pdf', + fileName: 'report.pdf', + contentType: 'application/pdf', + url: 'https://assets.example/source.pdf', + expiresAt: new Date('2026-01-01T00:00:00.000Z'), + }); const retrievalQuery = vi.fn().mockResolvedValue({ namespace: 'support-center', query: 'margin', @@ -545,6 +930,7 @@ function createClient(parseResult: ParseResult): { }, documents: { listChunks: documentsListChunks, + getPageCitationSource: documentsGetPageCitationSource, }, retrieval: { query: retrievalQuery, @@ -555,6 +941,7 @@ function createClient(parseResult: ParseResult): { jobsGet, jobsLoad, documentsListChunks, + documentsGetPageCitationSource, retrievalQuery, }; } @@ -567,6 +954,204 @@ async function expectFileMissing(filePath: string): Promise { await expect(access(filePath)).rejects.toMatchObject({ code: 'ENOENT' }); } +function toRemoteDocumentChunks( + result: ParseResult, + documentId: string, + params?: DocumentChunkListParams, +): DocumentChunk[] { + return result.chunks + .map((chunk, index): DocumentChunk => { + const sectionPath = chunk.path.replace(`${result.manifest.sourceFileName}/`, ''); + const filePath = + chunk.type === 'image' || chunk.type === 'table' + ? chunk.filePath + : typeof chunk.metadata.filePath === 'string' + ? chunk.metadata.filePath + : undefined; + return { + id: `${documentId}-${chunk.chunkId}`, + chunkId: chunk.chunkId, + chunkType: chunk.type, + contentSource: chunk.contentSource, + content: chunk.content, + sectionPath, + sourceChunkPath: chunk.path, + filePath, + sortOrder: index, + metadata: chunk.metadata, + assetUrl: + params?.includeAssetUrls && (chunk.type === 'image' || chunk.type === 'table') + ? chunk.assetUrl + : undefined, + }; + }) + .filter((chunk) => !params?.chunkType || chunk.chunkType === params.chunkType); +} + +function createInMemoryParsedStorage(): ParsedDocumentStorage & { + seedSnapshot(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly result: ParseResult; + }): void; + getManifest(documentId: string, revisionKey: string): KnowhereParsedSnapshotManifest | undefined; + getPage( + documentId: string, + revisionKey: string, + page: number, + ): KnowhereParsedSnapshotChunkPage | undefined; + getProgress(documentId: string, revisionKey: string): ParsedDocumentSyncProgress | undefined; + manifestCount(): number; +} { + const manifests = new Map(); + const pages = new Map(); + const assetUrls = new Map(); + const progressByRevision = new Map(); + + function getRevisionKey(documentId: string, revisionKey: string): string { + return `${documentId}:${revisionKey}`; + } + + function getPageKey(documentId: string, revisionKey: string, page: number): string { + return `${getRevisionKey(documentId, revisionKey)}:${page}`; + } + + function seedSnapshot(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly result: ParseResult; + }): void { + const page = createSnapshotPage(params); + const manifest = createSnapshotManifest(params); + pages.set(getPageKey(params.documentId, params.revisionKey, 1), page); + manifests.set(getRevisionKey(params.documentId, params.revisionKey), manifest); + } + + return { + seedSnapshot, + getManifest: (documentId: string, revisionKey: string) => + manifests.get(getRevisionKey(documentId, revisionKey)), + getPage: (documentId: string, revisionKey: string, page: number) => + pages.get(getPageKey(documentId, revisionKey, page)), + getProgress: (documentId: string, revisionKey: string) => + progressByRevision.get(getRevisionKey(documentId, revisionKey)), + manifestCount: () => manifests.size, + readManifest: (params): Promise => + Promise.resolve(manifests.get(getRevisionKey(params.documentId, params.revisionKey)) ?? null), + writeManifest: (params): Promise => { + manifests.set(getRevisionKey(params.documentId, params.revisionKey), params.manifest); + return Promise.resolve(); + }, + readChunkPage: (params): Promise => + Promise.resolve( + pages.get(getPageKey(params.documentId, params.revisionKey, params.page)) ?? null, + ), + writeChunkPage: (params): Promise => { + pages.set(getPageKey(params.documentId, params.revisionKey, params.page.page), params.page); + return Promise.resolve(); + }, + writeAsset: (params): Promise<{ readonly sourcePath: string; readonly url?: string }> => { + const key = `${getRevisionKey(params.documentId, params.revisionKey)}:${params.sourcePath}`; + const url = `https://blob.example/${params.documentId}/${params.revisionKey}/${params.sourcePath}`; + assetUrls.set(key, url); + return Promise.resolve({ + sourcePath: params.sourcePath, + url, + }); + }, + getAssetUrl: (params): Promise => + Promise.resolve( + assetUrls.get( + `${getRevisionKey(params.documentId, params.revisionKey)}:${params.sourcePath}`, + ) ?? null, + ), + readSyncProgress: (params): Promise => + Promise.resolve( + progressByRevision.get(getRevisionKey(params.documentId, params.revisionKey)) ?? null, + ), + writeSyncProgress: (params): Promise => { + progressByRevision.set(getRevisionKey(params.documentId, params.revisionKey), params); + return Promise.resolve(); + }, + }; +} + +function createSnapshotManifest(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly result: ParseResult; +}): KnowhereParsedSnapshotManifest { + return { + version: 1, + kind: 'knowhere-parsed-result-snapshot', + jobId: params.result.jobId, + revisionKey: params.revisionKey, + documentId: params.documentId, + namespace: params.result.namespace, + sourceFileName: params.result.manifest.sourceFileName, + totalChunks: params.result.chunks.length, + typeCounts: countFixtureChunkTypes(params.result), + chunkPageSize: params.result.chunks.length, + chunkPages: [ + { + page: 1, + pageSize: params.result.chunks.length, + chunkCount: params.result.chunks.length, + key: 'parsed/chunks/page-1.json', + }, + ], + assetUrlsByFilePath: {}, + createdAt: '2026-01-01T00:00:00.000Z', + }; +} + +function createSnapshotPage(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly result: ParseResult; +}): KnowhereParsedSnapshotChunkPage { + return { + version: 1, + jobId: params.result.jobId, + revisionKey: params.revisionKey, + documentId: params.documentId, + namespace: params.result.namespace, + sourceFileName: params.result.manifest.sourceFileName, + page: 1, + pageSize: params.result.chunks.length, + total: params.result.chunks.length, + totalPages: 1, + chunks: params.result.chunks.map((chunk, index) => ({ + id: chunk.chunkId, + chunkId: chunk.chunkId, + chunkType: chunk.type, + contentSource: chunk.contentSource, + content: chunk.content, + sectionPath: chunk.path.replace(`${params.result.manifest.sourceFileName}/`, ''), + sourceChunkPath: chunk.path, + filePath: + chunk.type === 'image' || chunk.type === 'table' + ? chunk.filePath + : typeof chunk.metadata.filePath === 'string' + ? chunk.metadata.filePath + : undefined, + sortOrder: index + 1, + metadata: chunk.metadata, + assetUrl: chunk.type === 'image' || chunk.type === 'table' ? chunk.assetUrl : undefined, + })), + }; +} + +function countFixtureChunkTypes(result: ParseResult): Readonly> { + return result.chunks.reduce>( + (counts, chunk) => { + counts[chunk.type] += 1; + return counts; + }, + { text: 0, image: 0, table: 0, page: 0 }, + ); +} + function createParseResult(): ParseResult { const chunks: [TextChunk, TableChunk, TextChunk] = [ { @@ -675,3 +1260,90 @@ function createParseResultWithFullPaths(): ParseResult { }; return result; } + +function createPageParseResult(chunks: Chunk[] = [createPageChunk()]): ParseResult { + const pageChunks = chunks.filter( + (chunk): chunk is Extract => chunk.type === 'page', + ); + + return { + manifest: { + version: '2.0', + jobId: 'job-1', + sourceFileName: 'report.pdf', + statistics: { + totalChunks: chunks.length, + textChunks: chunks.filter((chunk) => chunk.type === 'text').length, + imageChunks: chunks.filter((chunk) => chunk.type === 'image').length, + tableChunks: chunks.filter((chunk) => chunk.type === 'table').length, + pageChunks: pageChunks.length, + }, + }, + chunks, + rawZip: Buffer.from('not-used-in-tests'), + namespace: 'support-center', + documentId: 'doc-1', + textChunks: chunks.filter((chunk): chunk is TextChunk => chunk.type === 'text'), + imageChunks: [], + tableChunks: [], + pageChunks, + jobId: 'job-1', + statistics: { + totalChunks: chunks.length, + textChunks: chunks.filter((chunk) => chunk.type === 'text').length, + imageChunks: chunks.filter((chunk) => chunk.type === 'image').length, + tableChunks: chunks.filter((chunk) => chunk.type === 'table').length, + pageChunks: pageChunks.length, + }, + getChunk: (chunkId: string) => chunks.find((chunk) => chunk.chunkId === chunkId), + save: vi.fn(), + }; +} + +function createPageChunk(): Chunk { + return { + chunkId: 'page-1', + type: 'page', + content: 'Page one summary.', + contentSource: 'summary', + path: 'report.pdf/Page 1', + metadata: { summary: 'Page one summary.', pageNums: [1] }, + }; +} + +function createPageParseResultWithAssets(): ParseResult { + const chunks: Chunk[] = [ + { + chunkId: 'page-1', + type: 'page', + content: 'Page one summary.', + contentSource: 'summary', + path: 'report.pdf/Page 1', + metadata: { + summary: 'Page one summary.', + pageNums: [1], + pageAssets: [ + { + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: 'https://assets.example/page-1.png', + contentType: 'image/png', + width: 120, + height: 240, + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + }, + ]; + + return createPageParseResult(chunks); +} + +async function createPageParseResultWithRawPageAsset(): Promise { + const result = createPageParseResultWithAssets(); + const zip = new JSZip(); + zip.file('page_citation_assets/page-1.png', Buffer.from('page-one-image')); + result.rawZip = Buffer.from(await zip.generateAsync({ type: 'arraybuffer' })); + return result; +} diff --git a/src/knowledge/index.ts b/src/knowledge/index.ts index 3a5c5c5..4ed7014 100644 --- a/src/knowledge/index.ts +++ b/src/knowledge/index.ts @@ -10,8 +10,12 @@ export type { KnowledgeGrepMatch, KnowledgeGrepParams, KnowledgeGrepResponse, + KnowledgeImportJobResultParams, + KnowledgeJobResultResponse, + KnowledgeLoadJobResultParams, KnowledgeOutline, KnowledgeParseParams, + KnowledgeParsedStorageOptions, KnowledgeReadChunk, KnowledgeReadParams, KnowledgeReadResponse, @@ -21,6 +25,8 @@ export type { KnowledgeSearchResult, KnowledgeSection, KnowledgeStartupRecoveryResponse, + KnowledgeSyncParsedDocumentParams, + KnowledgeSyncParsedDocumentResponse, LocalKnowledgeDocument, LocalKnowledgeParseResponse, } from './types.js'; diff --git a/src/knowledge/knowledge.ts b/src/knowledge/knowledge.ts index edecc3b..1993437 100644 --- a/src/knowledge/knowledge.ts +++ b/src/knowledge/knowledge.ts @@ -3,12 +3,15 @@ import { createHash } from 'crypto'; import type { Knowhere } from '../client.js'; import type { Chunk, + DocumentChunk, DocNavSection, DocumentChunkListResponse, ParseResult, RetrievalResult, } from '../types/index.js'; import { ValidationError } from '../errors/index.js'; +import { storeParseResultAssets } from '../storage/asset-storage.js'; +import { syncParseResultToParsedDocumentStorage } from '../storage/parsed-document-storage.js'; import { LocalKnowledgeStore } from './local-store.js'; import type { IndexedKnowledgeChunk, @@ -22,8 +25,12 @@ import type { KnowledgeGrepMatch, KnowledgeGrepParams, KnowledgeGrepResponse, + KnowledgeImportJobResultParams, + KnowledgeJobResultResponse, + KnowledgeLoadJobResultParams, KnowledgeOutline, KnowledgeParseParams, + KnowledgeParsedStorageOptions, KnowledgeReadChunk, KnowledgeReadParams, KnowledgeReadResponse, @@ -33,38 +40,93 @@ import type { KnowledgeSearchResult, KnowledgeSection, KnowledgeStartupRecoveryResponse, + KnowledgeSyncParsedDocumentParams, + KnowledgeSyncParsedDocumentResponse, KnowledgeChunkType, LocalKnowledgeDocument, LocalKnowledgeParseResponse, } from './types.js'; +import type { + KnowhereParsedSnapshotChunk, + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, + ParsedDocumentAssetUrlPolicy, + ParsedDocumentStorageConfig, + ParsedDocumentStorageLimits, +} from '../types/storage.js'; const DEFAULT_READ_LIMIT = 12; const MAX_READ_LIMIT = 40; +const DEFAULT_PAGE_SIZE = 20; +const MAX_PAGE_SIZE = 200; const DEFAULT_GREP_LIMIT = 20; const MAX_GREP_LIMIT = 50; const DEFAULT_CONTEXT_CHARS = 80; +const DEFAULT_REMOTE_SCAN_PAGE_SIZE = 100; +const DEFAULT_MAX_SYNC_PAGES = 10; +const DEFAULT_MAX_SYNC_ASSETS = 20; +const DEFAULT_SYNC_DEADLINE_MS = 8000; +const DEFAULT_GREP_MAX_PAGES = 50; +const DEFAULT_GREP_DEADLINE_MS = 8000; +const DEFAULT_OUTLINE_MAX_PAGES = 50; +const DEFAULT_OUTLINE_DEADLINE_MS = 8000; const SAFE_LOCAL_DOCUMENT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; const REMOTE_DOCUMENT_ID_PATTERN = /^doc[_-]/; export class Knowledge { private readonly client: Knowhere; private readonly store: LocalKnowledgeStore; - - constructor(client: Knowhere, options?: { cacheDirectory?: string }) { + private readonly cacheDirectory: string | undefined; + private readonly parsedStorageConfig: ParsedDocumentStorageConfig | undefined; + + constructor( + client: Knowhere, + options?: { + cacheDirectory?: string; + parsedStorage?: ParsedDocumentStorageConfig; + }, + ) { this.client = client; + this.cacheDirectory = options?.cacheDirectory; + this.parsedStorageConfig = options?.parsedStorage; this.store = new LocalKnowledgeStore(options?.cacheDirectory); } withCacheDirectory(cacheDirectory: string): Knowledge { - return new Knowledge(this.client, { cacheDirectory }); + return new Knowledge(this.client, { + cacheDirectory, + parsedStorage: this.parsedStorageConfig, + }); } - async parse(params: KnowledgeParseParams): Promise { - const result = await this.client.parse(params); + withParsedStorage(parsedStorage: KnowledgeParsedStorageOptions): Knowledge { + return new Knowledge(this.client, { + cacheDirectory: this.cacheDirectory, + parsedStorage, + }); + } + + async parseToLocalCache(params: KnowledgeParseParams): Promise { + const loadedResult = params.storageAdapter + ? await this.parseWithoutClientStorageAdapter(params) + : await this.client.parse(params); + const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter); + const syncedStorage = await this.syncParseResultToConfiguredStorage(storedAssets.result); + const result = syncedStorage.result; const document = await this.store.saveResult(result, { localDocumentId: params.localDocumentId, }); - return { document, result }; + return { + document, + result, + assetUrlsByFilePath: storedAssets.assetUrlsByFilePath, + parsedSnapshot: storedAssets.snapshot, + }; + } + + /** @deprecated Use parseToLocalCache for local cache imports or the top-level client.parse for server-safe parsing. */ + async parse(params: KnowledgeParseParams): Promise { + return this.parseToLocalCache(params); } async startParse(params: KnowledgeAsyncParseParams): Promise { @@ -99,16 +161,121 @@ export class Knowledge { }; } - async cacheJobResult( - params: KnowledgeCacheJobResultParams, - ): Promise { - const result = await this.client.jobs.load(params.jobId, { + async loadJobResult(params: KnowledgeLoadJobResultParams): Promise { + const loadedResult = await this.client.jobs.load(params.jobId, { verifyChecksum: params.verifyChecksum, }); + const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter); + const syncedStorage = await this.syncParseResultToConfiguredStorage(storedAssets.result); + const result = syncedStorage.result; + return { + result, + assetUrlsByFilePath: storedAssets.assetUrlsByFilePath, + parsedSnapshot: syncedStorage.snapshot ?? storedAssets.snapshot, + }; + } + + async importJobResult( + params: KnowledgeImportJobResultParams, + ): Promise { + const stored = await this.loadJobResult(params); + const result = stored.result; const document = await this.store.saveResult(result, { localDocumentId: params.localDocumentId, }); - return { document, result }; + return { + document, + result, + assetUrlsByFilePath: stored.assetUrlsByFilePath, + parsedSnapshot: stored.parsedSnapshot, + }; + } + + /** @deprecated Use importJobResult for local cache imports or loadJobResult for server-safe loads. */ + async cacheJobResult( + params: KnowledgeCacheJobResultParams, + ): Promise { + return this.importJobResult(params); + } + + async syncParsedDocument( + params: KnowledgeSyncParsedDocumentParams, + ): Promise { + const normalized = normalizeDocumentReference(params); + const parsedStorageConfig = this.requireParsedStorageConfig(); + + if (normalized.documentId) { + return this.syncRemoteDocumentToParsedStorage({ + documentId: normalized.documentId, + revisionKey: params.revisionKey, + parsedStorageConfig, + }); + } + + if (normalized.jobId) { + const loadedResult = await this.client.jobs.load(normalized.jobId); + const synced = await syncParseResultToParsedDocumentStorage({ + result: loadedResult, + storage: parsedStorageConfig.storage, + documentId: loadedResult.documentId ?? normalized.localDocumentId ?? normalized.jobId, + revisionKey: params.revisionKey ?? normalized.jobId, + chunkPageSize: parsedStorageConfig.limits?.chunkPageSize, + }); + return { + documentId: synced.documentId, + revisionKey: synced.revisionKey, + completed: true, + manifest: synced.snapshot?.manifest, + }; + } + + if (!normalized.localDocumentId) { + throw new ValidationError('localDocumentId, documentId, or jobId is required'); + } + + const { result } = await this.store.loadResult(normalized.localDocumentId); + const synced = await syncParseResultToParsedDocumentStorage({ + result, + storage: parsedStorageConfig.storage, + documentId: result.documentId ?? normalized.localDocumentId, + revisionKey: params.revisionKey ?? result.jobId, + chunkPageSize: parsedStorageConfig.limits?.chunkPageSize, + }); + return { + documentId: synced.documentId, + revisionKey: synced.revisionKey, + completed: true, + manifest: synced.snapshot?.manifest, + }; + } + + private async syncParseResultToConfiguredStorage(result: ParseResult): Promise<{ + result: ParseResult; + snapshot?: Awaited>['snapshot']; + }> { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return { result }; + } + + const synced = await syncParseResultToParsedDocumentStorage({ + result, + storage: parsedStorageConfig.storage, + chunkPageSize: parsedStorageConfig.limits?.chunkPageSize, + }); + return { + result: synced.result, + snapshot: synced.snapshot, + }; + } + + private async parseWithoutClientStorageAdapter( + params: KnowledgeParseParams, + ): Promise { + const parseParams: KnowledgeParseParams = { ...params }; + delete parseParams.localDocumentId; + delete parseParams.storageAdapter; + return this.client.parse(parseParams); } async cacheDocument(params: KnowledgeCacheDocumentParams): Promise { @@ -120,7 +287,7 @@ export class Knowledge { params.documentId, ); const jobId: string = await this.getPublishedDocumentJobId(params.documentId); - return this.cacheJobResult({ + return this.importJobResult({ jobId, localDocumentId: params.localDocumentId ?? @@ -169,7 +336,7 @@ export class Knowledge { } try { - const cached = await this.cacheJobResult({ + const cached = await this.importJobResult({ jobId, localDocumentId: trackedJob.localDocumentId, }); @@ -198,6 +365,19 @@ export class Knowledge { async getDocumentOutline( reference: string | KnowledgeDocumentReference, ): Promise { + const normalized = normalizeDocumentReference(reference); + if (normalized.documentId) { + return this.getRemoteDocumentOutline(normalized.documentId, normalized.revisionKey); + } + + if ( + normalized.localDocumentId && + looksLikeRemoteDocumentId(normalized.localDocumentId) && + !(await this.store.getDocument(normalized.localDocumentId)) + ) { + return this.getRemoteDocumentOutline(normalized.localDocumentId, normalized.revisionKey); + } + const { document, result } = await this.loadReadableResult(reference); const chunks = indexChunks(result); const sections = buildFlatSections(result, chunks); @@ -218,6 +398,20 @@ export class Knowledge { } async readChunks(params: KnowledgeReadParams): Promise { + validateReadParams(params); + const normalized = normalizeDocumentReference(params); + if (normalized.documentId) { + return this.readRemoteDocumentChunks(normalized.documentId, params); + } + + if ( + normalized.localDocumentId && + looksLikeRemoteDocumentId(normalized.localDocumentId) && + !(await this.store.getDocument(normalized.localDocumentId)) + ) { + return this.readRemoteDocumentChunks(normalized.localDocumentId, params); + } + const { document, result } = await this.loadReadableResult(params); const limit = clampLimit(params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT); const chunks = indexChunks(result).filter((chunk) => matchesReadScope(chunk, params)); @@ -240,6 +434,19 @@ export class Knowledge { throw new ValidationError('pattern is required'); } + const normalized = normalizeDocumentReference(params); + if (normalized.documentId) { + return this.grepRemoteDocumentChunks(normalized.documentId, params); + } + + if ( + normalized.localDocumentId && + looksLikeRemoteDocumentId(normalized.localDocumentId) && + !(await this.store.getDocument(normalized.localDocumentId)) + ) { + return this.grepRemoteDocumentChunks(normalized.localDocumentId, params); + } + const { document, result } = await this.loadReadableResult(params); const maxResults = clampLimit(params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT); const contextChars = params.contextChars ?? DEFAULT_CONTEXT_CHARS; @@ -336,7 +543,7 @@ export class Knowledge { } if (normalized.jobId) { - return this.cacheJobResult({ + return this.importJobResult({ jobId: normalized.jobId, localDocumentId: normalized.localDocumentId, }); @@ -361,105 +568,1568 @@ export class Knowledge { throw new Error(`Local Knowhere document not found: ${normalized.localDocumentId}`); } - private async findLocalDocumentByRemoteId( + private async readRemoteDocumentChunks( documentId: string, - ): Promise { - const documents = await this.store.listDocuments(); - return documents.find((document) => document.documentId === documentId); + params: KnowledgeReadParams, + ): Promise { + const explicitRevisionKey = params.revisionKey ?? params.jobId; + if (explicitRevisionKey) { + const stored = await this.tryReadChunksFromParsedStorage({ + documentId, + revisionKey: explicitRevisionKey, + params, + }); + if (stored) { + return stored; + } + } + + if (hasScanReadFilters(params)) { + return this.scanRemoteDocumentChunks(documentId, params); + } + + const pageParams = getReadPageParams(params); + const remotePage = await this.client.documents.listChunks(documentId, { + page: pageParams.page, + pageSize: pageParams.pageSize, + chunkType: params.chunkType, + includeAssetUrls: shouldRequestRemoteAssetUrls( + params.assetUrlPolicy, + this.parsedStorageConfig, + ), + }); + const revisionKey = getRemoteRevisionKey(remotePage); + const stored = await this.tryReadChunksFromParsedStorage({ + documentId, + revisionKey, + params, + }); + if (stored) { + return stored; + } + + const chunks = await this.toRemoteReadChunks({ + documentId, + revisionKey, + chunks: remotePage.chunks, + assetUrlPolicy: params.assetUrlPolicy, + }); + this.scheduleParsedStorageSync(documentId, revisionKey); + return { + document: createRemoteKnowledgeDocument({ + documentId, + response: remotePage, + revisionKey, + }), + chunks, + page: remotePage.pagination.page, + pageSize: remotePage.pagination.pageSize, + totalChunks: remotePage.pagination.total, + totalPages: remotePage.pagination.totalPages, + }; } - private async getPublishedDocumentJobId(documentId: string): Promise { - const response: DocumentChunkListResponse = await this.client.documents.listChunks(documentId, { + private async scanRemoteDocumentChunks( + documentId: string, + params: KnowledgeReadParams, + ): Promise { + const explicitRevisionKey = params.revisionKey ?? params.jobId; + if (explicitRevisionKey) { + const stored = await this.tryReadChunksFromParsedStorage({ + documentId, + revisionKey: explicitRevisionKey, + params, + }); + if (stored) { + return stored; + } + } + + const limit = clampLimit(params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT); + const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits); + const firstPage = await this.client.documents.listChunks(documentId, { page: 1, - pageSize: 1, + pageSize: limits.remotePageSize, + chunkType: params.chunkType, + includeAssetUrls: shouldRequestRemoteAssetUrls( + params.assetUrlPolicy, + this.parsedStorageConfig, + ), }); - if (!response.jobId) { - throw new Error( - `Cannot sync server document ${documentId}: current published job id was not returned.`, - ); + const revisionKey = getRemoteRevisionKey(firstPage); + const stored = await this.tryReadChunksFromParsedStorage({ + documentId, + revisionKey, + params, + }); + if (stored) { + return stored; } - return response.jobId; - } -} -function normalizeDocumentReference( - reference: string | KnowledgeDocumentReference, -): KnowledgeDocumentReference { - if (typeof reference === 'string') { - return { localDocumentId: reference }; + const selected: DocumentChunk[] = []; + let scannedPages = 0; + let page = 1; + let response = firstPage; + const startedAt = Date.now(); + + while (true) { + scannedPages += 1; + selected.push(...response.chunks.filter((chunk) => matchesRemoteReadScope(chunk, params))); + if ( + selected.length >= limit || + response.pagination.page >= response.pagination.totalPages || + scannedPages >= limits.outlineMaxPages || + Date.now() - startedAt >= limits.outlineDeadlineMs + ) { + break; + } + + page += 1; + response = await this.client.documents.listChunks(documentId, { + page, + pageSize: limits.remotePageSize, + chunkType: params.chunkType, + includeAssetUrls: shouldRequestRemoteAssetUrls( + params.assetUrlPolicy, + this.parsedStorageConfig, + ), + }); + } + + const chunks = await this.toRemoteReadChunks({ + documentId, + revisionKey, + chunks: selected.slice(0, limit), + assetUrlPolicy: params.assetUrlPolicy, + }); + this.scheduleParsedStorageSync(documentId, revisionKey); + return { + document: createRemoteKnowledgeDocument({ + documentId, + response: firstPage, + revisionKey, + }), + chunks, + nextChunk: selected.length > limit ? toChunkPosition(selected[limit]) : undefined, + totalChunks: firstPage.pagination.total, + totalPages: firstPage.pagination.totalPages, + }; } - return reference; -} -function looksLikeRemoteDocumentId(value: string): boolean { - return REMOTE_DOCUMENT_ID_PATTERN.test(value); -} + private async grepRemoteDocumentChunks( + documentId: string, + params: KnowledgeGrepParams, + ): Promise { + const explicitRevisionKey = params.revisionKey ?? params.jobId; + if (explicitRevisionKey && !params.continuationCursor) { + const stored = await this.tryGrepChunksFromParsedStorage({ + documentId, + revisionKey: explicitRevisionKey, + params, + }); + if (stored) { + return stored; + } + } -function createLocalDocumentIdForRemote(documentId: string): string { - if ( - SAFE_LOCAL_DOCUMENT_ID_PATTERN.test(documentId) && - !documentId.includes('..') && - !documentId.includes('/') && - !documentId.includes('\\') - ) { - return documentId; + const maxResults = clampLimit(params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT); + const contextChars = params.contextChars ?? DEFAULT_CONTEXT_CHARS; + const matcher = createMatcher(params); + const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits); + const cursor = parseContinuationCursor(params.continuationCursor); + const startPage = cursor?.documentId === documentId ? cursor.nextPage : 1; + const firstPage = await this.client.documents.listChunks(documentId, { + page: startPage, + pageSize: limits.remotePageSize, + chunkType: params.chunkType, + includeAssetUrls: false, + }); + const revisionKey = getRemoteRevisionKey(firstPage); + const stored = + startPage === 1 + ? await this.tryGrepChunksFromParsedStorage({ + documentId, + revisionKey, + params, + }) + : null; + if (stored) { + return stored; + } + + const matches: KnowledgeGrepMatch[] = []; + let scannedChunks = 0; + let scannedPages = 0; + let response = firstPage; + let page = startPage; + const startedAt = Date.now(); + + while (true) { + scannedPages += 1; + for (const [chunkIndex, chunk] of response.chunks.entries()) { + if ( + response.pagination.page === startPage && + cursor?.nextChunkIndex !== undefined && + chunkIndex < cursor.nextChunkIndex + ) { + continue; + } + if (!matchesGrepScope(toIndexedRemoteChunk(chunk), params)) { + continue; + } + scannedChunks += 1; + const chunkMatches = matcher(chunk.content ?? ''); + for (const [matchIndex, match] of chunkMatches.entries()) { + if ( + response.pagination.page === startPage && + cursor?.nextChunkIndex === chunkIndex && + cursor.nextMatchIndex !== undefined && + matchIndex < cursor.nextMatchIndex + ) { + continue; + } + matches.push({ + position: toChunkPosition(chunk), + chunkId: chunk.chunkId, + chunkType: chunk.chunkType, + sectionPath: chunk.sectionPath ?? '', + sourceChunkPath: chunk.sourceChunkPath ?? chunk.sectionPath ?? '', + filePath: chunk.filePath ?? undefined, + startOffset: match.startOffset, + endOffset: match.endOffset, + snippet: buildSnippet( + chunk.content ?? '', + match.startOffset, + match.endOffset, + contextChars, + ), + }); + if (matches.length >= maxResults) { + const continuationCursor = createGrepContinuationCursor({ + documentId, + revisionKey, + response, + chunkIndex, + matchIndex, + chunkMatchCount: chunkMatches.length, + }); + this.scheduleParsedStorageSync(documentId, revisionKey); + return { + document: createRemoteKnowledgeDocument({ + documentId, + response: firstPage, + revisionKey, + }), + matches, + scannedChunks, + truncated: continuationCursor !== undefined, + continuationCursor, + }; + } + } + } + + const shouldStop = + response.pagination.page >= response.pagination.totalPages || + scannedPages >= limits.grepMaxPages || + Date.now() - startedAt >= limits.grepDeadlineMs; + if (shouldStop) { + const truncated = response.pagination.page < response.pagination.totalPages; + this.scheduleParsedStorageSync(documentId, revisionKey); + return { + document: createRemoteKnowledgeDocument({ + documentId, + response: firstPage, + revisionKey, + }), + matches, + scannedChunks, + truncated, + continuationCursor: truncated + ? createContinuationCursor({ + documentId, + revisionKey, + nextPage: response.pagination.page + 1, + }) + : undefined, + }; + } + + page += 1; + response = await this.client.documents.listChunks(documentId, { + page, + pageSize: limits.remotePageSize, + chunkType: params.chunkType, + includeAssetUrls: false, + }); + } } - const hash = createHash('sha256').update(documentId).digest('hex').slice(0, 16); - return `remote_${hash}`; -} + private async getRemoteDocumentOutline( + documentId: string, + revisionKeyOverride?: string, + ): Promise { + if (revisionKeyOverride) { + const stored = await this.tryReadOutlineFromParsedStorage(documentId, revisionKeyOverride); + if (stored) { + return stored; + } + } -function indexChunks(result: ParseResult): IndexedKnowledgeChunk[] { - return result.chunks.map((chunk, index) => { - const filePath = getChunkFilePath(chunk); - const readableContent = getReadableChunkContent(chunk); + const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits); + const firstPage = await this.client.documents.listChunks(documentId, { + page: 1, + pageSize: limits.remotePageSize, + includeAssetUrls: false, + }); + const revisionKey = getRemoteRevisionKey(firstPage); + const stored = await this.tryReadOutlineFromParsedStorage(documentId, revisionKey); + if (stored) { + return stored; + } + + const chunks: IndexedKnowledgeChunk[] = []; + let page = 1; + let response = firstPage; + let scannedPages = 0; + const startedAt = Date.now(); + + while (true) { + scannedPages += 1; + chunks.push(...response.chunks.map(toIndexedRemoteChunk)); + if ( + response.pagination.page >= response.pagination.totalPages || + scannedPages >= limits.outlineMaxPages || + Date.now() - startedAt >= limits.outlineDeadlineMs + ) { + break; + } + page += 1; + response = await this.client.documents.listChunks(documentId, { + page, + pageSize: limits.remotePageSize, + includeAssetUrls: false, + }); + } + + const truncated = response.pagination.page < response.pagination.totalPages; + const sections = buildFlatSectionsFromIndexedChunks(chunks); + this.scheduleParsedStorageSync(documentId, revisionKey); return { - source: chunk, - position: index + 1, - chunkId: chunk.chunkId, - chunkType: chunk.type, - contentSource: chunk.contentSource, - content: chunk.content, - readableContent, - sectionPath: normalizeSectionPath(chunk.path, result.manifest.sourceFileName), - sourceChunkPath: chunk.path, - filePath, - pageNumbers: getChunkPageNumbers(chunk.metadata), - metadata: chunk.metadata, + document: createRemoteKnowledgeDocument({ + documentId, + response: firstPage, + revisionKey, + typeCounts: countIndexedTypes(chunks), + }), + totalChunks: firstPage.pagination.total, + typeCounts: countIndexedTypes(chunks), + sections, + sectionTree: nestSections(sections), + truncated, + continuationCursor: truncated + ? createContinuationCursor({ + documentId, + revisionKey, + nextPage: response.pagination.page + 1, + }) + : undefined, }; - }); -} - -function getReadableChunkContent(chunk: Chunk): string { - if (chunk.type !== 'page') { - return chunk.content; } - const summary = chunk.metadata.summary; - return typeof summary === 'string' && summary.trim().length > 0 ? summary : chunk.content; -} + private async tryReadChunksFromParsedStorage(params: { + documentId: string; + revisionKey: string; + params: KnowledgeReadParams; + }): Promise { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return null; + } -function getChunkPageNumbers(metadata: Record): number[] | undefined { - const value = metadata.pageNums; - if (!Array.isArray(value)) { - return undefined; - } + const manifest = await this.readFreshParsedStorageManifest( + params.documentId, + params.revisionKey, + ); + if (!manifest) { + return null; + } - const pageNumbers = value.filter( - (pageNumber): pageNumber is number => Number.isInteger(pageNumber) && pageNumber > 0, - ); - return pageNumbers.length > 0 - ? [...new Set(pageNumbers)].sort((left, right) => left - right) - : undefined; -} + const pageParams = getReadPageParams(params.params); + const indexedChunks = await this.readStoredSnapshotChunks({ + documentId: params.documentId, + revisionKey: params.revisionKey, + manifest, + chunkType: params.params.chunkType, + }); + if (!indexedChunks) { + return null; + } -function getChunkFilePath(chunk: Chunk): string | undefined { - if (chunk.type === 'image' || chunk.type === 'table') { - return chunk.filePath; + const scopedChunks = indexedChunks.filter((chunk) => matchesReadScope(chunk, params.params)); + const selected = hasScanReadFilters(params.params) + ? selectReadWindow( + scopedChunks, + params.params, + clampLimit(params.params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT), + ) + : scopedChunks.slice( + (pageParams.page - 1) * pageParams.pageSize, + pageParams.page * pageParams.pageSize, + ); + const visibleChunks = applyAssetUrlPolicyToReadChunks( + selected.map(toReadChunk), + params.params.assetUrlPolicy, + ); + const totalChunks = scopedChunks.length; + return { + document: createStoredKnowledgeDocument({ + documentId: params.documentId, + manifest, + revisionKey: params.revisionKey, + typeCounts: countIndexedTypes(indexedChunks), + }), + chunks: visibleChunks, + page: hasScanReadFilters(params.params) ? undefined : pageParams.page, + pageSize: hasScanReadFilters(params.params) ? undefined : pageParams.pageSize, + totalChunks, + totalPages: Math.max(1, Math.ceil(totalChunks / pageParams.pageSize)), + nextChunk: getNextChunkPosition(scopedChunks, selected), + }; } - const filePath = chunk.metadata.filePath; - return typeof filePath === 'string' ? filePath : undefined; + private async tryGrepChunksFromParsedStorage(params: { + documentId: string; + revisionKey: string; + params: KnowledgeGrepParams; + }): Promise { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return null; + } + + const manifest = await this.readFreshParsedStorageManifest( + params.documentId, + params.revisionKey, + ); + if (!manifest) { + return null; + } + + const indexedChunks = await this.readStoredSnapshotChunks({ + documentId: params.documentId, + revisionKey: params.revisionKey, + manifest, + chunkType: params.params.chunkType, + }); + if (!indexedChunks) { + return null; + } + + const maxResults = clampLimit(params.params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT); + const contextChars = params.params.contextChars ?? DEFAULT_CONTEXT_CHARS; + const matcher = createMatcher(params.params); + const scopedChunks = indexedChunks.filter((chunk) => matchesGrepScope(chunk, params.params)); + const matches: KnowledgeGrepMatch[] = []; + let scannedChunks = 0; + + for (const chunk of scopedChunks) { + scannedChunks += 1; + for (const match of matcher(chunk.content)) { + matches.push({ + position: chunk.position, + chunkId: chunk.chunkId, + chunkType: chunk.chunkType, + sectionPath: chunk.sectionPath, + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath, + startOffset: match.startOffset, + endOffset: match.endOffset, + snippet: buildSnippet(chunk.content, match.startOffset, match.endOffset, contextChars), + }); + if (matches.length >= maxResults) { + return { + document: createStoredKnowledgeDocument({ + documentId: params.documentId, + manifest, + revisionKey: params.revisionKey, + typeCounts: countIndexedTypes(indexedChunks), + }), + matches, + scannedChunks, + truncated: true, + }; + } + } + } + + return { + document: createStoredKnowledgeDocument({ + documentId: params.documentId, + manifest, + revisionKey: params.revisionKey, + typeCounts: countIndexedTypes(indexedChunks), + }), + matches, + scannedChunks, + truncated: false, + }; + } + + private async tryReadOutlineFromParsedStorage( + documentId: string, + revisionKey: string, + ): Promise { + const manifest = await this.readFreshParsedStorageManifest(documentId, revisionKey); + if (!manifest) { + return null; + } + + const indexedChunks = await this.readStoredSnapshotChunks({ + documentId, + revisionKey, + manifest, + }); + if (!indexedChunks) { + return null; + } + + const sections = buildFlatSectionsFromIndexedChunks(indexedChunks); + const typeCounts = countIndexedTypes(indexedChunks); + return { + document: createStoredKnowledgeDocument({ + documentId, + manifest, + revisionKey, + typeCounts, + }), + totalChunks: manifest.totalChunks, + typeCounts, + sections, + sectionTree: nestSections(sections), + truncated: false, + }; + } + + private async readFreshParsedStorageManifest( + documentId: string, + revisionKey: string, + ): Promise { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return null; + } + + const manifest = await parsedStorageConfig.storage.readManifest({ + documentId, + revisionKey, + }); + if (!manifest || !isFreshManifest(manifest, revisionKey)) { + return null; + } + return manifest; + } + + private async readStoredSnapshotChunks(params: { + documentId: string; + revisionKey: string; + manifest: KnowhereParsedSnapshotManifest; + chunkType?: KnowledgeChunkType; + }): Promise { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return null; + } + + const chunks: IndexedKnowledgeChunk[] = []; + for (const pageReference of params.manifest.chunkPages) { + const page = await parsedStorageConfig.storage.readChunkPage({ + documentId: params.documentId, + revisionKey: params.revisionKey, + page: pageReference.page, + chunkType: params.chunkType, + }); + if (!page) { + return null; + } + chunks.push( + ...page.chunks + .filter((chunk) => !params.chunkType || chunk.chunkType === params.chunkType) + .map(toIndexedSnapshotChunk), + ); + } + return chunks.sort((left, right) => left.position - right.position); + } + + private async toRemoteReadChunks(params: { + documentId: string; + revisionKey: string; + chunks: readonly DocumentChunk[]; + assetUrlPolicy: ParsedDocumentAssetUrlPolicy | undefined; + }): Promise { + const readChunks = params.chunks.map((chunk) => toRemoteReadChunk(chunk)); + if (params.assetUrlPolicy === 'durable') { + return this.hardenReadChunkAssetUrls({ + documentId: params.documentId, + revisionKey: params.revisionKey, + chunks: readChunks, + }); + } + return applyAssetUrlPolicyToReadChunks(readChunks, params.assetUrlPolicy); + } + + private async hardenReadChunkAssetUrls(params: { + documentId: string; + revisionKey: string; + chunks: readonly KnowledgeReadChunk[]; + }): Promise { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return applyAssetUrlPolicyToReadChunks(params.chunks, 'none'); + } + + const hardenedChunks: KnowledgeReadChunk[] = []; + for (const chunk of params.chunks) { + hardenedChunks.push( + await hardenReadChunkAssetUrls({ + storageConfig: parsedStorageConfig, + documentId: params.documentId, + revisionKey: params.revisionKey, + chunk, + }), + ); + } + return hardenedChunks; + } + + private scheduleParsedStorageSync(documentId: string, revisionKey: string): void { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return; + } + + const scheduler = parsedStorageConfig.scheduler ?? defaultParsedStorageScheduler; + const task = async (): Promise => { + try { + await this.syncRemoteDocumentToParsedStorage({ + documentId, + revisionKey, + parsedStorageConfig, + }); + } catch (error) { + await parsedStorageConfig.storage.writeSyncProgress({ + documentId, + revisionKey, + nextChunkPage: 1, + nextAssetIndex: 0, + status: 'failed', + updatedAt: new Date().toISOString(), + error: error instanceof Error ? error.message : String(error), + }); + } + }; + + void Promise.resolve(scheduler.schedule(task)).catch(() => undefined); + } + + private async syncRemoteDocumentToParsedStorage(params: { + documentId: string; + revisionKey?: string; + parsedStorageConfig: ParsedDocumentStorageConfig; + }): Promise { + const limits = getParsedStorageLimits(params.parsedStorageConfig.limits); + const existingProgress = params.revisionKey + ? await params.parsedStorageConfig.storage.readSyncProgress({ + documentId: params.documentId, + revisionKey: params.revisionKey, + }) + : null; + let nextPage = existingProgress?.status === 'running' ? existingProgress.nextChunkPage : 1; + let response = await this.client.documents.listChunks(params.documentId, { + page: nextPage, + pageSize: limits.remotePageSize, + includeAssetUrls: true, + }); + const revisionKey = params.revisionKey ?? getRemoteRevisionKey(response); + if (params.revisionKey && getRemoteRevisionKey(response) !== params.revisionKey) { + response = await this.client.documents.listChunks(params.documentId, { + page: 1, + pageSize: limits.remotePageSize, + includeAssetUrls: true, + }); + nextPage = 1; + } + + const totalPages = response.pagination.totalPages; + const chunkPages: KnowhereParsedSnapshotChunkPage[] = []; + let syncedPages = 0; + const startedAt = Date.now(); + + while (nextPage <= totalPages) { + if (syncedPages > 0) { + response = await this.client.documents.listChunks(params.documentId, { + page: nextPage, + pageSize: limits.remotePageSize, + includeAssetUrls: true, + }); + } + + const readChunks = await this.hardenReadChunkAssetUrls({ + documentId: params.documentId, + revisionKey, + chunks: response.chunks.map(toRemoteReadChunk), + }); + const page = toSnapshotChunkPageFromReadChunks({ + response, + revisionKey, + chunks: readChunks, + }); + await params.parsedStorageConfig.storage.writeChunkPage({ + documentId: params.documentId, + revisionKey, + page, + }); + chunkPages.push(page); + nextPage += 1; + syncedPages += 1; + await params.parsedStorageConfig.storage.writeSyncProgress({ + documentId: params.documentId, + revisionKey, + nextChunkPage: nextPage, + nextAssetIndex: 0, + status: 'running', + updatedAt: new Date().toISOString(), + }); + + if ( + syncedPages >= limits.maxPagesPerSync || + Date.now() - startedAt >= limits.syncDeadlineMs + ) { + return { + documentId: params.documentId, + revisionKey, + completed: false, + }; + } + } + + const manifest = createRemoteSnapshotManifest({ + response, + revisionKey, + chunkPages, + }); + await params.parsedStorageConfig.storage.writeManifest({ + documentId: params.documentId, + revisionKey, + manifest, + }); + await params.parsedStorageConfig.storage.writeSyncProgress({ + documentId: params.documentId, + revisionKey, + nextChunkPage: totalPages + 1, + nextAssetIndex: 0, + status: 'completed', + updatedAt: new Date().toISOString(), + }); + return { + documentId: params.documentId, + revisionKey, + completed: true, + manifest, + }; + } + + private requireParsedStorageConfig(): ParsedDocumentStorageConfig { + if (!this.parsedStorageConfig) { + throw new ValidationError('parsed document storage is not configured'); + } + return this.parsedStorageConfig; + } + + private async findLocalDocumentByRemoteId( + documentId: string, + ): Promise { + const documents = await this.store.listDocuments(); + return documents.find((document) => document.documentId === documentId); + } + + private async getPublishedDocumentJobId(documentId: string): Promise { + const response: DocumentChunkListResponse = await this.client.documents.listChunks(documentId, { + page: 1, + pageSize: 1, + }); + if (!response.jobId) { + throw new Error( + `Cannot sync server document ${documentId}: current published job id was not returned.`, + ); + } + return response.jobId; + } +} + +function normalizeDocumentReference( + reference: string | KnowledgeDocumentReference, +): KnowledgeDocumentReference { + if (typeof reference === 'string') { + return { localDocumentId: reference }; + } + return reference; +} + +function looksLikeRemoteDocumentId(value: string): boolean { + return REMOTE_DOCUMENT_ID_PATTERN.test(value); +} + +function createLocalDocumentIdForRemote(documentId: string): string { + if ( + SAFE_LOCAL_DOCUMENT_ID_PATTERN.test(documentId) && + !documentId.includes('..') && + !documentId.includes('/') && + !documentId.includes('\\') + ) { + return documentId; + } + + const hash = createHash('sha256').update(documentId).digest('hex').slice(0, 16); + return `remote_${hash}`; +} + +interface NormalizedParsedStorageLimits { + readonly chunkPageSize: number; + readonly remotePageSize: number; + readonly maxPagesPerSync: number; + readonly maxAssetsPerSync: number; + readonly syncDeadlineMs: number; + readonly grepMaxPages: number; + readonly grepDeadlineMs: number; + readonly outlineMaxPages: number; + readonly outlineDeadlineMs: number; +} + +interface ContinuationCursor { + readonly documentId: string; + readonly revisionKey: string; + readonly nextPage: number; + readonly nextChunkIndex?: number; + readonly nextMatchIndex?: number; +} + +const defaultParsedStorageScheduler = { + schedule(task: () => Promise): void { + void task(); + }, +}; + +function getParsedStorageLimits( + limits: ParsedDocumentStorageLimits | undefined, +): NormalizedParsedStorageLimits { + return { + chunkPageSize: clampLimit(limits?.chunkPageSize, DEFAULT_REMOTE_SCAN_PAGE_SIZE, MAX_PAGE_SIZE), + remotePageSize: clampLimit( + limits?.remotePageSize, + DEFAULT_REMOTE_SCAN_PAGE_SIZE, + MAX_PAGE_SIZE, + ), + maxPagesPerSync: clampLimit(limits?.maxPagesPerSync, DEFAULT_MAX_SYNC_PAGES, 1000), + maxAssetsPerSync: clampLimit(limits?.maxAssetsPerSync, DEFAULT_MAX_SYNC_ASSETS, 1000), + syncDeadlineMs: clampLimit(limits?.syncDeadlineMs, DEFAULT_SYNC_DEADLINE_MS, 60000), + grepMaxPages: clampLimit(limits?.grepMaxPages, DEFAULT_GREP_MAX_PAGES, 1000), + grepDeadlineMs: clampLimit(limits?.grepDeadlineMs, DEFAULT_GREP_DEADLINE_MS, 60000), + outlineMaxPages: clampLimit(limits?.outlineMaxPages, DEFAULT_OUTLINE_MAX_PAGES, 1000), + outlineDeadlineMs: clampLimit(limits?.outlineDeadlineMs, DEFAULT_OUTLINE_DEADLINE_MS, 60000), + }; +} + +function validateReadParams(params: KnowledgeReadParams): void { + const usesPagedDisplay = + params.page !== undefined || + params.pageSize !== undefined || + params.assetUrlPolicy !== undefined; + if (!usesPagedDisplay) { + return; + } + + if ( + params.sectionPath !== undefined || + params.startChunk !== undefined || + params.endChunk !== undefined || + params.chunkId !== undefined + ) { + throw new ValidationError( + 'page, pageSize, and assetUrlPolicy cannot be combined with sectionPath, startChunk, endChunk, or chunkId', + ); + } +} + +function hasScanReadFilters(params: KnowledgeReadParams): boolean { + return ( + params.sectionPath !== undefined || + params.startChunk !== undefined || + params.endChunk !== undefined || + params.chunkId !== undefined + ); +} + +function getReadPageParams(params: KnowledgeReadParams): { + readonly page: number; + readonly pageSize: number; +} { + return { + page: clampLimit(params.page, 1, Number.MAX_SAFE_INTEGER), + pageSize: clampLimit(params.pageSize ?? params.limit, DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE), + }; +} + +function shouldRequestRemoteAssetUrls( + assetUrlPolicy: ParsedDocumentAssetUrlPolicy | undefined, + parsedStorageConfig: ParsedDocumentStorageConfig | undefined, +): boolean { + return assetUrlPolicy === 'durable' && parsedStorageConfig !== undefined; +} + +function getRemoteRevisionKey(response: DocumentChunkListResponse): string { + const revisionKey = response.jobResultId ?? response.jobId; + if (!revisionKey) { + throw new Error( + `Cannot read server document ${response.documentId}: current published revision was not returned.`, + ); + } + return revisionKey; +} + +function createRemoteKnowledgeDocument(params: { + readonly documentId: string; + readonly response: DocumentChunkListResponse; + readonly revisionKey: string; + readonly typeCounts?: Record; +}): LocalKnowledgeDocument { + return { + localDocumentId: params.documentId, + jobId: params.response.jobId ?? params.revisionKey, + documentId: params.documentId, + namespace: params.response.namespace, + sourceFileName: params.documentId, + chunkCount: params.response.pagination.total, + typeCounts: params.typeCounts ?? countRemoteChunkTypes(params.response.chunks), + resultDirectoryPath: `remote:${params.documentId}`, + createdAt: new Date(0), + updatedAt: new Date(0), + }; +} + +function createStoredKnowledgeDocument(params: { + readonly documentId: string; + readonly manifest: KnowhereParsedSnapshotManifest; + readonly revisionKey: string; + readonly typeCounts: Record; +}): LocalKnowledgeDocument { + return { + localDocumentId: params.documentId, + jobId: params.manifest.jobId || params.revisionKey, + documentId: params.manifest.documentId ?? params.documentId, + namespace: params.manifest.namespace, + sourceFileName: params.manifest.sourceFileName, + chunkCount: params.manifest.totalChunks, + typeCounts: params.typeCounts, + resultDirectoryPath: `parsed-storage:${params.documentId}`, + createdAt: new Date(params.manifest.createdAt), + updatedAt: new Date(params.manifest.createdAt), + }; +} + +function isFreshManifest(manifest: KnowhereParsedSnapshotManifest, revisionKey: string): boolean { + return (manifest.revisionKey ?? manifest.jobId) === revisionKey; +} + +function toRemoteReadChunk(chunk: DocumentChunk): KnowledgeReadChunk { + const metadata = cloneMetadata(chunk.metadata); + return { + position: toChunkPosition(chunk), + chunkId: chunk.chunkId, + chunkType: chunk.chunkType, + contentSource: chunk.contentSource ?? undefined, + content: chunk.content ?? '', + readableContent: getReadableRemoteChunkContent(chunk, metadata), + sectionPath: chunk.sectionPath ?? '', + sourceChunkPath: chunk.sourceChunkPath ?? chunk.sectionPath ?? '', + filePath: chunk.filePath ?? undefined, + assetUrl: chunk.assetUrl ?? undefined, + pageNumbers: getChunkPageNumbers(metadata), + metadata, + }; +} + +function getReadableRemoteChunkContent( + chunk: DocumentChunk, + metadata: Record, +): string { + if (chunk.chunkType !== 'page') { + return chunk.content ?? ''; + } + + const summary = metadata.summary; + return typeof summary === 'string' && summary.trim().length > 0 ? summary : (chunk.content ?? ''); +} + +function toIndexedRemoteChunk(chunk: DocumentChunk): IndexedKnowledgeChunk { + const readChunk = toRemoteReadChunk(chunk); + return toIndexedReadChunk(readChunk); +} + +function toIndexedSnapshotChunk(chunk: KnowhereParsedSnapshotChunk): IndexedKnowledgeChunk { + return toIndexedReadChunk({ + position: chunk.sortOrder, + chunkId: chunk.chunkId, + chunkType: chunk.chunkType as KnowledgeChunkType, + contentSource: chunk.contentSource, + content: chunk.content, + readableContent: + chunk.chunkType === 'page' && + typeof chunk.metadata.summary === 'string' && + chunk.metadata.summary.trim().length > 0 + ? chunk.metadata.summary + : chunk.content, + sectionPath: chunk.sectionPath ?? '', + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath, + assetUrl: chunk.assetUrl, + pageNumbers: getChunkPageNumbers(chunk.metadata), + metadata: cloneMetadata(chunk.metadata), + }); +} + +function toIndexedReadChunk(chunk: KnowledgeReadChunk): IndexedKnowledgeChunk { + return { + source: createSyntheticChunk(chunk), + position: chunk.position, + chunkId: chunk.chunkId, + chunkType: chunk.chunkType, + contentSource: chunk.contentSource, + content: chunk.content, + readableContent: chunk.readableContent, + sectionPath: chunk.sectionPath, + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath, + assetUrl: chunk.assetUrl, + pageNumbers: chunk.pageNumbers, + metadata: chunk.metadata, + }; +} + +function createSyntheticChunk(chunk: KnowledgeReadChunk): Chunk { + return { + chunkId: chunk.chunkId, + type: chunk.chunkType, + contentSource: chunk.contentSource, + content: chunk.content, + path: chunk.sourceChunkPath, + metadata: chunk.metadata, + } as Chunk; +} + +function toChunkPosition(chunk: DocumentChunk | undefined): number { + return chunk ? chunk.sortOrder + 1 : 1; +} + +function buildFlatSectionsFromIndexedChunks(chunks: IndexedKnowledgeChunk[]): KnowledgeSection[] { + const byPath = new Map(); + for (const chunk of chunks) { + const path = chunk.sectionPath || chunk.sourceChunkPath; + const existing = byPath.get(path); + if (existing) { + addChunkToSection(existing, chunk); + } else { + byPath.set(path, createSectionFromChunk(path, chunk)); + } + } + return [...byPath.values()].sort(compareSections); +} + +function matchesRemoteReadScope(chunk: DocumentChunk, params: KnowledgeReadParams): boolean { + return matchesReadScope(toIndexedRemoteChunk(chunk), params); +} + +function getNextChunkPosition( + chunks: readonly IndexedKnowledgeChunk[], + selected: readonly IndexedKnowledgeChunk[], +): number | undefined { + const lastSelected = selected[selected.length - 1]; + if (!lastSelected || selected.length >= chunks.length) { + return undefined; + } + return chunks[chunks.indexOf(lastSelected) + 1]?.position; +} + +function applyAssetUrlPolicyToReadChunks( + chunks: readonly KnowledgeReadChunk[], + assetUrlPolicy: ParsedDocumentAssetUrlPolicy | undefined, +): KnowledgeReadChunk[] { + if (assetUrlPolicy !== 'none') { + return chunks.map((chunk) => ({ + ...chunk, + metadata: cloneMetadata(chunk.metadata), + })); + } + + return chunks.map((chunk) => ({ + ...chunk, + assetUrl: undefined, + metadata: removeAssetUrlsFromMetadata(chunk.metadata), + })); +} + +async function hardenReadChunkAssetUrls(params: { + readonly storageConfig: ParsedDocumentStorageConfig; + readonly documentId: string; + readonly revisionKey: string; + readonly chunk: KnowledgeReadChunk; +}): Promise { + const metadata = cloneMetadata(params.chunk.metadata); + const assetUrl = await hardenPrimaryAssetUrl(params, metadata); + const pageAssets = await hardenPageAssetUrls(params, metadata); + if (pageAssets) { + metadata.pageAssets = pageAssets; + } + + return { + ...params.chunk, + assetUrl, + metadata, + }; +} + +async function hardenPrimaryAssetUrl( + params: { + readonly storageConfig: ParsedDocumentStorageConfig; + readonly documentId: string; + readonly revisionKey: string; + readonly chunk: KnowledgeReadChunk; + }, + metadata: Record, +): Promise { + if (!params.chunk.assetUrl || !params.chunk.filePath) { + return undefined; + } + + return writeDurableAssetUrl({ + storageConfig: params.storageConfig, + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: params.chunk.filePath, + sourceUrl: params.chunk.assetUrl, + fallbackContentType: inferContentTypeFromPath(params.chunk.filePath), + metadata: { + chunkId: params.chunk.chunkId, + chunkType: params.chunk.chunkType, + sourcePath: params.chunk.filePath, + ...stringifyFlatMetadata(metadata), + }, + }); +} + +async function hardenPageAssetUrls( + params: { + readonly storageConfig: ParsedDocumentStorageConfig; + readonly documentId: string; + readonly revisionKey: string; + readonly chunk: KnowledgeReadChunk; + }, + metadata: Record, +): Promise { + const pageAssets = toUnknownArray(metadata.pageAssets); + if (!pageAssets) { + return undefined; + } + + const hardenedAssets: unknown[] = []; + for (const pageAsset of pageAssets) { + if (!isRecord(pageAsset)) { + hardenedAssets.push(pageAsset); + continue; + } + + const artifactRef = pageAsset.artifactRef; + const sourceUrl = pageAsset.assetUrl; + if (typeof artifactRef !== 'string' || typeof sourceUrl !== 'string') { + hardenedAssets.push(removeAssetUrlField(pageAsset)); + continue; + } + + const durableUrl = await writeDurableAssetUrl({ + storageConfig: params.storageConfig, + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: artifactRef, + sourceUrl, + fallbackContentType: + typeof pageAsset.contentType === 'string' + ? pageAsset.contentType + : inferContentTypeFromPath(artifactRef), + metadata: { + chunkId: params.chunk.chunkId, + chunkType: params.chunk.chunkType, + sourcePath: artifactRef, + }, + }); + const rest = removeAssetUrlField(pageAsset); + hardenedAssets.push(durableUrl ? { ...rest, assetUrl: durableUrl } : rest); + } + + return hardenedAssets; +} + +async function writeDurableAssetUrl(params: { + readonly storageConfig: ParsedDocumentStorageConfig; + readonly documentId: string; + readonly revisionKey: string; + readonly sourcePath: string; + readonly sourceUrl: string; + readonly fallbackContentType: string; + readonly metadata: Readonly>; +}): Promise { + const existingUrl = await params.storageConfig.storage.getAssetUrl({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: params.sourcePath, + }); + if (existingUrl) { + return existingUrl; + } + + try { + const fetched = await fetchAsset(params.sourceUrl, params.fallbackContentType); + const written = await params.storageConfig.storage.writeAsset({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: params.sourcePath, + body: fetched.body, + contentType: fetched.contentType, + metadata: params.metadata, + }); + return ( + written.url ?? + (await params.storageConfig.storage.getAssetUrl({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: params.sourcePath, + })) ?? + undefined + ); + } catch { + return undefined; + } +} + +async function fetchAsset( + sourceUrl: string, + fallbackContentType: string, +): Promise<{ + readonly body: Uint8Array; + readonly contentType: string; +}> { + const response = await fetch(sourceUrl); + if (!response.ok) { + throw new Error(`Unable to fetch parsed asset: ${response.status}`); + } + + return { + body: new Uint8Array(await response.arrayBuffer()), + contentType: response.headers.get('content-type') ?? fallbackContentType, + }; +} + +function removeAssetUrlsFromMetadata(metadata: Record): Record { + const cloned = cloneMetadata(metadata); + const pageAssets = toUnknownArray(cloned.pageAssets); + if (!pageAssets) { + return cloned; + } + + cloned.pageAssets = pageAssets.map((pageAsset) => { + if (!isRecord(pageAsset)) { + return pageAsset; + } + return removeAssetUrlField(pageAsset); + }); + return cloned; +} + +function removeAssetUrlField(value: Record): Record { + const cloned = { ...value }; + delete cloned.assetUrl; + return cloned; +} + +function toSnapshotChunkPageFromReadChunks(params: { + readonly response: DocumentChunkListResponse; + readonly revisionKey: string; + readonly chunks: readonly KnowledgeReadChunk[]; +}): KnowhereParsedSnapshotChunkPage { + return { + version: 1, + jobId: params.response.jobId ?? params.revisionKey, + revisionKey: params.revisionKey, + documentId: params.response.documentId, + namespace: params.response.namespace, + sourceFileName: params.response.documentId, + page: params.response.pagination.page, + pageSize: params.response.pagination.pageSize, + total: params.response.pagination.total, + totalPages: params.response.pagination.totalPages, + chunks: params.chunks.map(toSnapshotChunkFromReadChunk), + }; +} + +function toSnapshotChunkFromReadChunk(chunk: KnowledgeReadChunk): KnowhereParsedSnapshotChunk { + return { + id: chunk.chunkId, + chunkId: chunk.chunkId, + chunkType: chunk.chunkType, + contentSource: chunk.contentSource, + content: chunk.content, + sectionPath: chunk.sectionPath, + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath, + sortOrder: chunk.position, + metadata: cloneMetadata(chunk.metadata), + assetUrl: chunk.assetUrl, + }; +} + +function createRemoteSnapshotManifest(params: { + readonly response: DocumentChunkListResponse; + readonly revisionKey: string; + readonly chunkPages: readonly KnowhereParsedSnapshotChunkPage[]; +}): KnowhereParsedSnapshotManifest { + return { + version: 1, + kind: 'knowhere-parsed-result-snapshot', + jobId: params.response.jobId ?? params.revisionKey, + revisionKey: params.revisionKey, + documentId: params.response.documentId, + namespace: params.response.namespace, + sourceFileName: params.response.documentId, + totalChunks: params.response.pagination.total, + typeCounts: countSnapshotPagesTypes(params.chunkPages), + chunkPageSize: params.response.pagination.pageSize, + chunkPages: Array.from({ length: params.response.pagination.totalPages }, (_, index) => ({ + page: index + 1, + pageSize: params.response.pagination.pageSize, + chunkCount: + index + 1 === params.response.pagination.totalPages + ? params.response.pagination.total - + params.response.pagination.pageSize * (params.response.pagination.totalPages - 1) + : params.response.pagination.pageSize, + key: `parsed/chunks/page-${index + 1}.json`, + })), + assetUrlsByFilePath: {}, + createdAt: new Date().toISOString(), + }; +} + +function countSnapshotPagesTypes( + pages: readonly KnowhereParsedSnapshotChunkPage[], +): Record { + return pages + .flatMap((page) => page.chunks) + .reduce>( + (counts, chunk) => { + const chunkType = chunk.chunkType as KnowledgeChunkType; + counts[chunkType] += 1; + return counts; + }, + { text: 0, image: 0, table: 0, page: 0 }, + ); +} + +function countRemoteChunkTypes( + chunks: readonly DocumentChunk[], +): Record { + return chunks.reduce>( + (counts, chunk) => { + counts[chunk.chunkType] += 1; + return counts; + }, + { text: 0, image: 0, table: 0, page: 0 }, + ); +} + +function createContinuationCursor(cursor: ContinuationCursor): string { + return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64url'); +} + +function createGrepContinuationCursor(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly response: DocumentChunkListResponse; + readonly chunkIndex: number; + readonly matchIndex: number; + readonly chunkMatchCount: number; +}): string | undefined { + if (params.matchIndex + 1 < params.chunkMatchCount) { + return createContinuationCursor({ + documentId: params.documentId, + revisionKey: params.revisionKey, + nextPage: params.response.pagination.page, + nextChunkIndex: params.chunkIndex, + nextMatchIndex: params.matchIndex + 1, + }); + } + + if (params.chunkIndex + 1 < params.response.chunks.length) { + return createContinuationCursor({ + documentId: params.documentId, + revisionKey: params.revisionKey, + nextPage: params.response.pagination.page, + nextChunkIndex: params.chunkIndex + 1, + nextMatchIndex: 0, + }); + } + + if (params.response.pagination.page < params.response.pagination.totalPages) { + return createContinuationCursor({ + documentId: params.documentId, + revisionKey: params.revisionKey, + nextPage: params.response.pagination.page + 1, + nextChunkIndex: 0, + nextMatchIndex: 0, + }); + } + + return undefined; +} + +function parseContinuationCursor(value: string | undefined): ContinuationCursor | null { + if (!value) { + return null; + } + + try { + const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')) as unknown; + if (!isRecord(parsed)) { + return null; + } + const documentId = parsed.documentId; + const revisionKey = parsed.revisionKey; + const nextPage = parsed.nextPage; + const nextChunkIndex = parsed.nextChunkIndex; + const nextMatchIndex = parsed.nextMatchIndex; + if ( + typeof documentId !== 'string' || + typeof revisionKey !== 'string' || + typeof nextPage !== 'number' || + !Number.isInteger(nextPage) || + nextPage < 1 || + !isOptionalNonNegativeInteger(nextChunkIndex) || + !isOptionalNonNegativeInteger(nextMatchIndex) + ) { + return null; + } + return { + documentId, + revisionKey, + nextPage, + nextChunkIndex, + nextMatchIndex, + }; + } catch { + return null; + } +} + +function isOptionalNonNegativeInteger(value: unknown): value is number | undefined { + return ( + value === undefined || (typeof value === 'number' && Number.isInteger(value) && value >= 0) + ); +} + +function cloneMetadata(metadata: Record): Record { + return { ...metadata }; +} + +function stringifyFlatMetadata(metadata: Record): Record { + const entries = Object.entries(metadata).flatMap(([key, value]): Array<[string, string]> => { + if (typeof value === 'string') { + return [[key, value]]; + } + if (typeof value === 'number' || typeof value === 'boolean') { + return [[key, String(value)]]; + } + return []; + }); + return Object.fromEntries(entries); +} + +function toUnknownArray(value: unknown): readonly unknown[] | null { + return Array.isArray(value) ? (value as readonly unknown[]) : null; +} + +function inferContentTypeFromPath(filePath: string): string { + const extension = filePath.split('.').at(-1)?.toLowerCase(); + if (extension === 'png') return 'image/png'; + if (extension === 'jpg' || extension === 'jpeg') return 'image/jpeg'; + if (extension === 'html' || extension === 'htm') return 'text/html; charset=utf-8'; + return 'application/octet-stream'; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function indexChunks(result: ParseResult): IndexedKnowledgeChunk[] { + return result.chunks.map((chunk, index) => { + const filePath = getChunkFilePath(chunk); + const readableContent = getReadableChunkContent(chunk); + return { + source: chunk, + position: index + 1, + chunkId: chunk.chunkId, + chunkType: chunk.type, + contentSource: chunk.contentSource, + content: chunk.content, + readableContent, + sectionPath: normalizeSectionPath(chunk.path, result.manifest.sourceFileName), + sourceChunkPath: chunk.path, + filePath, + assetUrl: getChunkAssetUrl(chunk), + pageNumbers: getChunkPageNumbers(chunk.metadata), + metadata: chunk.metadata, + }; + }); +} + +function getReadableChunkContent(chunk: Chunk): string { + if (chunk.type !== 'page') { + return chunk.content; + } + + const summary = chunk.metadata.summary; + return typeof summary === 'string' && summary.trim().length > 0 ? summary : chunk.content; +} + +function getChunkPageNumbers(metadata: Record): number[] | undefined { + const values = [metadata.pageNums, metadata.page_nums].filter(Array.isArray); + const pageNumbers = values.flatMap((value) => + value.filter( + (pageNumber): pageNumber is number => Number.isInteger(pageNumber) && pageNumber > 0, + ), + ); + return pageNumbers.length > 0 + ? [...new Set(pageNumbers)].sort((left, right) => left - right) + : undefined; +} + +function getChunkFilePath(chunk: Chunk): string | undefined { + if (chunk.type === 'image' || chunk.type === 'table') { + return chunk.filePath; + } + + const filePath = chunk.metadata.filePath; + return typeof filePath === 'string' ? filePath : undefined; +} + +function getChunkAssetUrl(chunk: Chunk): string | undefined { + return chunk.type === 'image' || chunk.type === 'table' ? chunk.assetUrl : undefined; } function normalizeSectionPath(path: string, sourceFileName?: string): string { @@ -666,6 +2336,7 @@ function toReadChunk(chunk: IndexedKnowledgeChunk): KnowledgeReadChunk { sectionPath: chunk.sectionPath, sourceChunkPath: chunk.sourceChunkPath, filePath: chunk.filePath, + assetUrl: chunk.assetUrl, pageNumbers: chunk.pageNumbers, metadata: chunk.metadata, }; diff --git a/src/knowledge/types.ts b/src/knowledge/types.ts index 607be57..198dc3b 100644 --- a/src/knowledge/types.ts +++ b/src/knowledge/types.ts @@ -1,6 +1,13 @@ import type { ParseParams } from '../types/params.js'; import type { Job, JobResult } from '../types/job.js'; import type { Chunk, DocumentChunkType, ParseResult } from '../types/index.js'; +import type { + KnowhereAssetStorageOptions, + KnowhereParsedSnapshotManifest, + KnowhereParsedSnapshot, + ParsedDocumentAssetUrlPolicy, + ParsedDocumentStorageConfig, +} from '../types/storage.js'; export type KnowledgeChunkType = DocumentChunkType; @@ -35,6 +42,8 @@ export interface LocalKnowledgeDocument { export interface LocalKnowledgeParseResponse { document: LocalKnowledgeDocument; result: ParseResult; + assetUrlsByFilePath?: Readonly>; + parsedSnapshot?: KnowhereParsedSnapshot; } export interface KnowledgeAsyncParseParams extends ParseParams { @@ -72,12 +81,25 @@ export interface KnowledgeStartupRecoveryResponse { results: KnowledgeAsyncJobStatusResponse[]; } -export interface KnowledgeCacheJobResultParams { +export interface KnowledgeLoadJobResultParams { jobId: string; - localDocumentId?: string; verifyChecksum?: boolean; + storageAdapter?: KnowhereAssetStorageOptions; +} + +export interface KnowledgeImportJobResultParams extends KnowledgeLoadJobResultParams { + localDocumentId?: string; +} + +export interface KnowledgeJobResultResponse { + result: ParseResult; + assetUrlsByFilePath?: Readonly>; + parsedSnapshot?: KnowhereParsedSnapshot; } +/** @deprecated Use KnowledgeImportJobResultParams with importJobResult instead. */ +export type KnowledgeCacheJobResultParams = KnowledgeImportJobResultParams; + export interface KnowledgeCacheDocumentParams { /** Canonical Knowhere document identifier from published retrieval results. */ documentId: string; @@ -92,6 +114,19 @@ export interface KnowledgeDocumentReference { documentId?: string; /** Server parse job identifier whose completed result should be cached before reading. */ jobId?: string; + /** Current parsed revision key when the caller already has it. */ + revisionKey?: string; +} + +export type KnowledgeParsedStorageOptions = ParsedDocumentStorageConfig; + +export type KnowledgeSyncParsedDocumentParams = KnowledgeDocumentReference; + +export interface KnowledgeSyncParsedDocumentResponse { + documentId: string; + revisionKey: string; + completed: boolean; + manifest?: KnowhereParsedSnapshotManifest; } export interface KnowledgeSection { @@ -112,15 +147,20 @@ export interface KnowledgeOutline { typeCounts: Record; sections: KnowledgeSection[]; sectionTree: KnowledgeSection[]; + truncated?: boolean; + continuationCursor?: string; } export interface KnowledgeReadParams extends KnowledgeDocumentReference { + page?: number; + pageSize?: number; sectionPath?: string; startChunk?: number; endChunk?: number; chunkId?: string; chunkType?: KnowledgeChunkType; limit?: number; + assetUrlPolicy?: ParsedDocumentAssetUrlPolicy; } export interface KnowledgeReadChunk { @@ -135,6 +175,7 @@ export interface KnowledgeReadChunk { sectionPath: string; sourceChunkPath: string; filePath?: string; + assetUrl?: string; pageNumbers?: number[]; metadata: Record; } @@ -143,10 +184,15 @@ export interface KnowledgeReadResponse { document: LocalKnowledgeDocument; chunks: KnowledgeReadChunk[]; nextChunk?: number; + page?: number; + pageSize?: number; + totalChunks?: number; + totalPages?: number; } export interface KnowledgeGrepParams extends KnowledgeDocumentReference { pattern: string; + continuationCursor?: string; isRegex?: boolean; isCaseSensitive?: boolean; maxResults?: number; @@ -172,6 +218,7 @@ export interface KnowledgeGrepResponse { matches: KnowledgeGrepMatch[]; scannedChunks: number; truncated: boolean; + continuationCursor?: string; } export interface KnowledgeSearchParams { @@ -222,6 +269,7 @@ export interface IndexedKnowledgeChunk { sectionPath: string; sourceChunkPath: string; filePath?: string; + assetUrl?: string; pageNumbers?: number[]; metadata: Record; } diff --git a/src/lib/__tests__/result-parser.test.ts b/src/lib/__tests__/result-parser.test.ts index c527be0..06a6da9 100644 --- a/src/lib/__tests__/result-parser.test.ts +++ b/src/lib/__tests__/result-parser.test.ts @@ -459,6 +459,17 @@ describe('Result Parser', () => { summary: 'Summary for pages 4 through 6.', page_nums: [4, 5, 6], entities: [{ text: 'Knowhere', type: 'product' }], + page_assets: [ + { + page_num: 4, + artifact_ref: 'page_citation_assets/page-4.png', + asset_url: 'https://assets.example/page-4.png', + content_type: 'image/png', + width: 1200, + height: 1800, + source: 'knowhere-rendered-page-citation-source', + }, + ], }, }, ], @@ -474,9 +485,77 @@ describe('Result Parser', () => { expect(result.pageChunks[0].type).toBe('page'); expect(result.pageChunks[0].contentSource).toBe('summary'); expect(result.pageChunks[0].metadata.pageNums).toEqual([4, 5, 6]); + expect(result.pageChunks[0].metadata.pageAssets).toEqual([ + { + pageNum: 4, + artifactRef: 'page_citation_assets/page-4.png', + assetUrl: 'https://assets.example/page-4.png', + contentType: 'image/png', + width: 1200, + height: 1800, + source: 'knowhere-rendered-page-citation-source', + }, + ]); + expect(result.pageChunks[0]).not.toHaveProperty('pageAssets'); expect(result.textChunks).toHaveLength(0); }); + it('should ignore unsupported top-level page assets on page chunks', async () => { + const zip = new JSZip(); + zip.file( + 'manifest.json', + JSON.stringify({ + job_id: 'job-page-123', + source_file_name: 'manual.pdf', + statistics: { + total_chunks: 1, + text_chunks: 0, + image_chunks: 0, + table_chunks: 0, + page_chunks: 1, + }, + }), + ); + zip.file( + 'chunks.json', + JSON.stringify({ + chunks: [ + { + chunk_id: 'page-4', + type: 'page', + content: 'Summary for page 4.', + path: 'manual.pdf/Page 4', + metadata: { + summary: 'Summary for page 4.', + page_nums: [4], + }, + page_assets: [ + { + page_num: 4, + artifact_ref: 'page_citation_assets/page-4.png', + asset_url: 'https://assets.example/page-4.png', + content_type: 'image/png', + width: 1200, + height: 1800, + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + ], + }), + ); + const zipBuffer = await zip.generateAsync({ type: 'nodebuffer' }); + mockHttpClient.download.mockResolvedValue(zipBuffer); + + const result = await parseResult(mockHttpClient, 'https://s3.example.com/result.zip'); + + expect(result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(result.pageChunks[0].metadata).toEqual({ + summary: 'Summary for page 4.', + pageNums: [4], + }); + }); + it('should extract full markdown if present', async () => { const mockZipBuffer = await createMockResultZip({ includeFullMarkdown: true, @@ -803,6 +882,156 @@ describe('Result Parser', () => { expect(reloaded.rawZip.length).toBe(0); }); + it('should preserve server-provided page citation asset files when saving expanded results', async () => { + const zip = new JSZip(); + zip.file( + 'manifest.json', + JSON.stringify({ + job_id: 'job-page-assets', + source_file_name: 'manual.pdf', + statistics: { + total_chunks: 1, + text_chunks: 0, + image_chunks: 0, + table_chunks: 0, + page_chunks: 1, + }, + }), + ); + zip.file( + 'chunks.json', + JSON.stringify({ + chunks: [ + { + chunk_id: 'page-1', + type: 'page', + content: 'Page one summary.', + path: 'manual.pdf/Page 1', + metadata: { + page_nums: [1], + page_assets: [ + { + page_num: 1, + artifact_ref: 'page_citation_assets/page-1.png', + content_type: 'image/png', + width: 120, + height: 240, + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + }, + ], + }), + ); + zip.file('page_citation_assets/page-1.png', Buffer.from('server-page-png')); + const zipBuffer = await zip.generateAsync({ type: 'nodebuffer' }); + mockHttpClient.download.mockResolvedValue(zipBuffer); + + const result = await parseResult(mockHttpClient, 'https://s3.example.com/result.zip'); + + await saveExpandedParseResult(result, testOutputDir); + + const savedPageAsset = await fs.readFile( + join(testOutputDir, 'page_citation_assets/page-1.png'), + ); + const savedChunks = JSON.parse( + await fs.readFile(join(testOutputDir, 'chunks.json'), 'utf8'), + ) as { + chunks: [ + { + metadata: { + page_assets: Array>; + }; + }, + ]; + }; + + expect(savedPageAsset.toString()).toBe('server-page-png'); + expect(savedChunks.chunks[0].metadata.page_assets).toEqual([ + expect.objectContaining({ + artifact_ref: 'page_citation_assets/page-1.png', + }), + ]); + expect(savedChunks.chunks[0]).not.toHaveProperty('page_assets'); + }); + + it('should save rewritten page citation asset metadata after raw ZIP extraction', async () => { + const zip = new JSZip(); + zip.file( + 'manifest.json', + JSON.stringify({ + job_id: 'job-page-assets', + source_file_name: 'manual.pdf', + statistics: { + total_chunks: 1, + text_chunks: 0, + image_chunks: 0, + table_chunks: 0, + page_chunks: 1, + }, + }), + ); + zip.file( + 'chunks.json', + JSON.stringify({ + chunks: [ + { + chunk_id: 'page-1', + type: 'page', + content: 'Page one summary.', + path: 'manual.pdf/Page 1', + metadata: { + page_assets: [ + { + page_num: 1, + artifact_ref: 'page_citation_assets/page-1.png', + content_type: 'image/png', + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + }, + ], + }), + ); + zip.file('page_citation_assets/page-1.png', Buffer.from('server-page-png')); + mockHttpClient.download.mockResolvedValue(await zip.generateAsync({ type: 'nodebuffer' })); + const result = await parseResult(mockHttpClient, 'https://s3.example.com/result.zip'); + const pageAssets = result.pageChunks[0]?.metadata.pageAssets; + const firstPageAsset: unknown = Array.isArray(pageAssets) ? pageAssets[0] : undefined; + if ( + Array.isArray(pageAssets) && + typeof firstPageAsset === 'object' && + firstPageAsset !== null + ) { + pageAssets[0] = { + ...firstPageAsset, + assetUrl: 'https://blob.example/page_citation_assets/page-1.png', + }; + } + + await saveExpandedParseResult(result, testOutputDir); + + const savedChunks = JSON.parse( + await fs.readFile(join(testOutputDir, 'chunks.json'), 'utf8'), + ) as { + chunks: [ + { + metadata: { + page_assets: Array>; + }; + }, + ]; + }; + + expect(savedChunks.chunks[0].metadata.page_assets).toEqual([ + expect.objectContaining({ + asset_url: 'https://blob.example/page_citation_assets/page-1.png', + }), + ]); + }); + it('should expose jobId property', async () => { const mockZipBuffer = await createMockResultZip(); mockHttpClient.download.mockResolvedValue(mockZipBuffer); diff --git a/src/lib/result-parser.ts b/src/lib/result-parser.ts index 2628f39..2ad60aa 100644 --- a/src/lib/result-parser.ts +++ b/src/lib/result-parser.ts @@ -17,7 +17,7 @@ import type { } from '../types/result.js'; import type { LoadOptions } from '../types/params.js'; import { ChecksumError, KnowhereError } from '../errors/index.js'; -import { sanitizePath, getFileExtension, parseDates, keysToCamel } from './utils.js'; +import { sanitizePath, getFileExtension, parseDates, keysToCamel, keysToSnake } from './utils.js'; type RawChunk = { chunkId?: string; @@ -26,6 +26,7 @@ type RawChunk = { content?: string; path?: string; filePath?: string; + assetUrl?: string; metadata?: Record; }; @@ -226,12 +227,30 @@ export async function saveExpandedParseResult( if (result.rawZip.length > 0) { const didExtractZip = await tryExtractRawZip(result.rawZip, directory); if (didExtractZip) { + await writeExpandedParseResultMetadata(result, directory); return directory; } } await fs.mkdir(directory, { recursive: true }); + await writeExpandedParseResultMetadata(result, directory); + + for (const imageChunk of result.imageChunks) { + await writeBinaryAsset(directory, imageChunk.filePath, imageChunk.data); + } + + for (const tableChunk of result.tableChunks) { + await writeTextAsset(directory, tableChunk.filePath, tableChunk.html); + } + + return directory; +} + +async function writeExpandedParseResultMetadata( + result: ParseResult, + directory: string, +): Promise { await fs.writeFile(join(directory, 'manifest.json'), JSON.stringify(result.manifest, null, 2)); if (result.docNav) { @@ -240,7 +259,7 @@ export async function saveExpandedParseResult( await fs.writeFile( join(directory, 'chunks.json'), - JSON.stringify(serializeChunks(result.chunks), null, 2), + JSON.stringify(keysToSnake(serializeChunks(result.chunks)), null, 2), ); if (result.chunksSlim) { @@ -275,16 +294,6 @@ export async function saveExpandedParseResult( if (result.hierarchyViewHtml) { await fs.writeFile(join(directory, 'hierarchy_view.html'), result.hierarchyViewHtml); } - - for (const imageChunk of result.imageChunks) { - await writeBinaryAsset(directory, imageChunk.filePath, imageChunk.data); - } - - for (const tableChunk of result.tableChunks) { - await writeTextAsset(directory, tableChunk.filePath, tableChunk.html); - } - - return directory; } async function tryExtractRawZip(zipBuffer: Buffer, directory: string): Promise { @@ -468,13 +477,14 @@ function buildTextChunk(chunkData: RawChunk): TextChunk { } function buildPageChunk(chunkData: RawChunk): PageChunk { + const metadata = chunkData.metadata ?? {}; return { chunkId: chunkData.chunkId ?? '', type: 'page', contentSource: chunkData.contentSource ?? 'summary', content: chunkData.content ?? '', path: chunkData.path ?? '', - metadata: chunkData.metadata ?? {}, + metadata, }; } @@ -486,6 +496,7 @@ function buildImageChunk(chunkData: RawChunk, filePath: string, imageBuffer: Buf content: chunkData.content ?? '', path: chunkData.path ?? '', filePath, + assetUrl: chunkData.assetUrl, data: imageBuffer, metadata: chunkData.metadata ?? {}, @@ -507,6 +518,7 @@ function buildTableChunk(chunkData: RawChunk, filePath: string, html: string): T content: chunkData.content ?? '', path: chunkData.path ?? '', filePath, + assetUrl: chunkData.assetUrl, html, metadata: chunkData.metadata ?? {}, @@ -626,6 +638,7 @@ function serializeChunks(chunks: Chunk[]): { chunks: RawChunk[] } { if (chunk.type === 'image' || chunk.type === 'table') { rawChunk.filePath = chunk.filePath; + rawChunk.assetUrl = chunk.assetUrl; } return rawChunk; diff --git a/src/resources/__tests__/documents.test.ts b/src/resources/__tests__/documents.test.ts index 026c540..f135760 100644 --- a/src/resources/__tests__/documents.test.ts +++ b/src/resources/__tests__/documents.test.ts @@ -120,7 +120,19 @@ describe('Documents Resource', () => { sourceChunkPath: 'Chapter 1/Pages 4-6', filePath: null, sortOrder: 0, - metadata: { summary: 'Revenue rose across the covered pages.', pageNums: [4, 5, 6] }, + metadata: { + summary: 'Revenue rose across the covered pages.', + pageNums: [4, 5, 6], + pageAssets: [ + { + pageNum: 4, + artifactRef: 'page_citation_assets/page-4.png', + assetUrl: 'https://assets.example/page-4.png', + contentType: 'image/png', + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, assetUrl: null, createdAt: new Date('2026-04-27T04:00:00Z'), }, @@ -151,6 +163,12 @@ describe('Documents Resource', () => { expect(response.chunks[0]?.id).toBe('dchk-123'); expect(response.chunks[0]?.chunkType).toBe('page'); expect(response.chunks[0]?.contentSource).toBe('summary'); + expect(response.chunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + assetUrl: 'https://assets.example/page-4.png', + }), + ]); + expect(response.chunks[0]).not.toHaveProperty('pageAssets'); expect(response.pagination.totalPages).toBe(2); }); @@ -234,6 +252,39 @@ describe('Documents Resource', () => { expect(response.chunk.assetUrl).toBe('https://assets.example/figure-1.png'); }); + it('should get the page citation source through the canonical v2 route', async () => { + mockHttpClient.get.mockResolvedValue({ + documentId: 'doc-123', + namespace: 'support-center', + jobId: 'job-123', + jobResultId: 'jres-123', + variant: 'normalized_pdf', + fileName: 'report.pdf', + contentType: 'application/pdf', + url: 'https://assets.example/report.pdf', + expiresAt: new Date('2026-01-01T00:00:00.000Z'), + }); + + const source = await documents.getPageCitationSource('doc-123'); + + expect(mockHttpClient.get).toHaveBeenCalledWith( + '/v2/documents/doc-123/files/page-citation-source', + ); + expect(source).toMatchObject({ + documentId: 'doc-123', + jobResultId: 'jres-123', + contentType: 'application/pdf', + url: 'https://assets.example/report.pdf', + }); + }); + + it('should propagate page citation source API errors', async () => { + const error = new Error('not found'); + mockHttpClient.get.mockRejectedValue(error); + + await expect(documents.getPageCitationSource('doc-404')).rejects.toBe(error); + }); + it('should archive using the canonical route', async () => { mockHttpClient.post.mockResolvedValue({ documentId: 'doc-123', diff --git a/src/resources/__tests__/retrieval.test.ts b/src/resources/__tests__/retrieval.test.ts index 559b4c8..e7fb787 100644 --- a/src/resources/__tests__/retrieval.test.ts +++ b/src/resources/__tests__/retrieval.test.ts @@ -163,11 +163,22 @@ describe('Retrieval Resource', () => { { chunkId: 'chunk-1', documentId: 'doc-1', - chunkType: 'text', - sectionPath: 'Root', + chunkType: 'page', + sectionPath: 'Page 4', filePath: null, jobId: 'job-1', - assetUrl: 'https://example.com/1', + assetUrl: 'https://assets.example/page-4.png', + metadata: { + pageAssets: [ + { + pageNum: 4, + artifactRef: 'page_citation_assets/page-4.png', + assetUrl: 'https://assets.example/page-4.png', + contentType: 'image/png', + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, }, ], results: [], @@ -184,6 +195,12 @@ describe('Retrieval Resource', () => { expect(response.referencedChunks).toHaveLength(1); expect(referencedChunk?.chunkId).toBe('chunk-1'); expect(referencedChunk?.filePath).toBeNull(); + expect(referencedChunk?.metadata?.pageAssets).toEqual([ + expect.objectContaining({ + assetUrl: 'https://assets.example/page-4.png', + }), + ]); + expect(referencedChunk).not.toHaveProperty('pageAssets'); expect(response.decisionTrace).toHaveLength(2); expect(response.decisionTrace![0]).toHaveProperty('phase', 'discovery'); expect(response.decisionTrace![1]).toHaveProperty('phase', 'terminal'); diff --git a/src/resources/documents.ts b/src/resources/documents.ts index 88b6960..64263ff 100644 --- a/src/resources/documents.ts +++ b/src/resources/documents.ts @@ -5,6 +5,7 @@ import type { DocumentChunkListParams, DocumentChunkListResponse, DocumentChunkResponse, + DocumentPageCitationSource, DocumentListParams, DocumentListResponse, } from '../types/document.js'; @@ -65,6 +66,15 @@ export class Documents extends BaseResource { ); } + /** + * Get a signed normalized source-file URL for compatibility and debugging. + */ + async getPageCitationSource(documentId: string): Promise { + return this.httpClient.get( + this.endpoint(`/documents/${documentId}/files/page-citation-source`), + ); + } + /** * Archive one canonical document by ID. */ diff --git a/src/storage/__tests__/asset-storage.test.ts b/src/storage/__tests__/asset-storage.test.ts new file mode 100644 index 0000000..0eca3e5 --- /dev/null +++ b/src/storage/__tests__/asset-storage.test.ts @@ -0,0 +1,394 @@ +import JSZip from 'jszip'; +import { describe, expect, it, vi } from 'vitest'; + +import { storeParseResultAssets } from '../asset-storage.js'; +import type { + KnowhereAssetStorageAdapter, + KnowhereAssetStorageHead, + KnowhereAssetStorageObject, + KnowhereAssetStorageWriteResult, +} from '../../types/storage.js'; +import type { + Chunk, + ImageChunk, + PageChunk, + ParseResult, + TableChunk, + TextChunk, +} from '../../types/result.js'; + +describe('storeParseResultAssets', () => { + it('stores image, table, and page citation assets and rewrites display URLs', async () => { + const result = await createParseResultWithAssets(); + const writes: Array<{ + readonly key: string; + readonly bodyText: string; + readonly contentType: string; + }> = []; + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => { + writes.push({ + key: input.key, + bodyText: Buffer.from(input.body).toString('utf8'), + contentType: input.contentType, + }); + return Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }); + }, + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + + const stored = await storeParseResultAssets(result, { + adapter, + keyPrefix: 'workspaces/workspace-1/sources/source-1/parsed-result', + }); + + expect(stored.assetUrlsByFilePath).toEqual({ + 'images/chart.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', + 'page_citation_assets/page-1.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + 'tables/revenue.html': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/tables/revenue.html', + }); + expect( + writes.filter((write) => write.contentType !== 'application/json; charset=utf-8'), + ).toEqual([ + { + key: 'workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', + bodyText: 'chart-image', + contentType: 'image/png', + }, + { + key: 'workspaces/workspace-1/sources/source-1/parsed-result/tables/revenue.html', + bodyText: 'Revenue
', + contentType: 'text/html; charset=utf-8', + }, + { + key: 'workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + bodyText: 'page-one-image', + contentType: 'image/png', + }, + ]); + expect(stored.result.imageChunks[0]?.assetUrl).toBe( + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', + ); + expect(stored.result.tableChunks[0]?.assetUrl).toBe( + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/tables/revenue.html', + ); + expect(stored.result.pageChunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + expect(stored.result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(stored.snapshot?.manifest).toMatchObject({ + version: 1, + kind: 'knowhere-parsed-result-snapshot', + jobId: 'job-1', + sourceFileName: 'report.pdf', + totalChunks: 3, + chunkPageSize: 200, + assetUrlsByFilePath: { + 'images/chart.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', + 'page_citation_assets/page-1.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + 'tables/revenue.html': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/tables/revenue.html', + }, + }); + expect(stored.snapshot?.manifestKey).toBe( + 'workspaces/workspace-1/sources/source-1/parsed-result/manifest/current.json', + ); + expect(stored.snapshot?.indexKey).toBe( + 'workspaces/workspace-1/sources/source-1/parsed-result/index.json', + ); + expect(stored.snapshot?.manifest.chunkPages).toEqual([ + expect.objectContaining({ + page: 1, + chunkCount: 3, + key: 'workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + url: 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + }), + ]); + const chunkPageWrite = writes.find((write) => write.key.endsWith('/chunks/page-1.json')); + expect(chunkPageWrite).toBeDefined(); + const chunkPage = JSON.parse(chunkPageWrite?.bodyText ?? '{}') as { + readonly chunks: readonly { + readonly chunkId: string; + readonly chunkType: string; + readonly assetUrl?: string; + readonly metadata: Record; + }[]; + }; + expect(chunkPage.chunks.map((chunk) => chunk.chunkId)).toEqual([ + 'image-1', + 'table-1', + 'page-1', + ]); + expect(chunkPage.chunks[0]).toMatchObject({ + chunkType: 'image', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', + }); + expect(chunkPage.chunks[2]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + }); + + it('stores paged chunk snapshots even when a result has no media assets', async () => { + const result = createParseResult([ + { + chunkId: 'text-1', + type: 'text', + content: 'Alpha', + path: 'report/Intro', + metadata: { summary: 'Intro' }, + }, + { + chunkId: 'text-2', + type: 'text', + content: 'Beta', + path: 'report/Body', + metadata: {}, + }, + ]); + const writes: Array<{ + readonly key: string; + readonly bodyText: string; + readonly contentType: string; + }> = []; + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => { + writes.push({ + key: input.key, + bodyText: Buffer.from(input.body).toString('utf8'), + contentType: input.contentType, + }); + return Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }); + }, + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + + const stored = await storeParseResultAssets(result, { + adapter, + keyPrefix: 'parsed-result', + chunkPageSize: 1, + }); + + expect(stored.assetUrlsByFilePath).toEqual({}); + expect(stored.snapshot?.manifest.chunkPages).toEqual([ + expect.objectContaining({ page: 1, chunkCount: 1, key: 'parsed-result/chunks/page-1.json' }), + expect.objectContaining({ page: 2, chunkCount: 1, key: 'parsed-result/chunks/page-2.json' }), + ]); + expect(writes.map((write) => write.key)).toEqual([ + 'parsed-result/chunks/page-1.json', + 'parsed-result/chunks/page-2.json', + 'parsed-result/manifest/current.json', + 'parsed-result/index.json', + ]); + expect(writes.every((write) => write.contentType === 'application/json; charset=utf-8')).toBe( + true, + ); + }); + + it('uses existing storage URLs when skipExisting is enabled', async () => { + const result = createParseResult([ + { + chunkId: 'image-1', + type: 'image', + content: 'Chart summary', + path: 'report/images/chart.png', + filePath: 'images/chart.png', + data: Buffer.from('chart-image'), + metadata: {}, + format: 'png', + save: vi.fn(), + }, + ]); + const headObject = vi.fn( + (key: string): Promise => + Promise.resolve({ + key, + url: `https://blob.example/existing/${key}`, + }), + ); + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => + Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }), + ); + const adapter: KnowhereAssetStorageAdapter = { + headObject, + writeObject, + }; + + const stored = await storeParseResultAssets(result, { + adapter, + keyPrefix: 'parsed-result', + }); + + expect( + writeObject.mock.calls.filter( + ([input]) => input.contentType !== 'application/json; charset=utf-8', + ), + ).toEqual([]); + expect(stored.assetUrlsByFilePath).toEqual({ + 'images/chart.png': 'https://blob.example/existing/parsed-result/images/chart.png', + }); + expect(stored.result.imageChunks[0]?.assetUrl).toBe( + 'https://blob.example/existing/parsed-result/images/chart.png', + ); + expect(stored.snapshot?.manifestUrl).toBe( + 'https://blob.example/parsed-result/manifest/current.json', + ); + }); + + it('rejects unsafe storage key prefixes and ignores unsafe asset refs', async () => { + const result = createParseResult([ + { + chunkId: 'image-1', + type: 'image', + content: 'Chart summary', + path: 'report/images/chart.png', + filePath: '../chart.png', + data: Buffer.from('chart-image'), + metadata: {}, + format: 'png', + save: vi.fn(), + }, + ]); + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => + Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }), + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + + await expect( + storeParseResultAssets(result, { + adapter, + keyPrefix: '../parsed-result', + }), + ).rejects.toThrow(/keyPrefix/); + + const stored = await storeParseResultAssets(result, { + adapter, + keyPrefix: 'parsed-result', + }); + + expect(writeObject).toHaveBeenCalledTimes(3); + expect(stored.assetUrlsByFilePath).toEqual({}); + expect(stored.snapshot?.manifest.totalChunks).toBe(1); + }); +}); + +async function createParseResultWithAssets(): Promise { + const zip = new JSZip(); + zip.file('page_citation_assets/page-1.png', Buffer.from('page-one-image')); + + return createParseResult( + [ + { + chunkId: 'image-1', + type: 'image', + content: 'Chart summary', + path: 'report/images/chart.png', + filePath: 'images/chart.png', + data: Buffer.from('chart-image'), + metadata: {}, + format: 'png', + save: vi.fn(), + }, + { + chunkId: 'table-1', + type: 'table', + content: 'Revenue
', + path: 'report/tables/revenue.html', + filePath: 'tables/revenue.html', + html: 'Revenue
', + metadata: {}, + save: vi.fn(), + }, + { + chunkId: 'page-1', + type: 'page', + content: 'Page one summary', + contentSource: 'summary', + path: 'report/Page 1', + metadata: { + pageNums: [1], + pageAssets: [ + { + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + contentType: 'image/png', + width: 120, + height: 240, + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + }, + ], + Buffer.from(await zip.generateAsync({ type: 'arraybuffer' })), + ); +} + +function createParseResult( + chunks: readonly Chunk[], + rawZip: Buffer = Buffer.alloc(0), +): ParseResult { + return { + manifest: { + version: '2.0', + jobId: 'job-1', + sourceFileName: 'report.pdf', + statistics: { + totalChunks: chunks.length, + textChunks: chunks.filter((chunk) => chunk.type === 'text').length, + imageChunks: chunks.filter((chunk) => chunk.type === 'image').length, + tableChunks: chunks.filter((chunk) => chunk.type === 'table').length, + pageChunks: chunks.filter((chunk) => chunk.type === 'page').length, + }, + }, + chunks: [...chunks], + rawZip, + textChunks: chunks.filter((chunk): chunk is TextChunk => chunk.type === 'text'), + imageChunks: chunks.filter((chunk): chunk is ImageChunk => chunk.type === 'image'), + tableChunks: chunks.filter((chunk): chunk is TableChunk => chunk.type === 'table'), + pageChunks: chunks.filter((chunk): chunk is PageChunk => chunk.type === 'page'), + jobId: 'job-1', + statistics: { + totalChunks: chunks.length, + textChunks: chunks.filter((chunk) => chunk.type === 'text').length, + imageChunks: chunks.filter((chunk) => chunk.type === 'image').length, + tableChunks: chunks.filter((chunk) => chunk.type === 'table').length, + pageChunks: chunks.filter((chunk) => chunk.type === 'page').length, + }, + getChunk: (chunkId: string) => chunks.find((chunk) => chunk.chunkId === chunkId), + save: vi.fn(), + }; +} diff --git a/src/storage/asset-storage.ts b/src/storage/asset-storage.ts new file mode 100644 index 0000000..7f88b98 --- /dev/null +++ b/src/storage/asset-storage.ts @@ -0,0 +1,546 @@ +import JSZip from 'jszip'; + +import type { + ChunkMetadata, + Chunk, + ImageChunk, + PageChunk, + ParseResult, + TableChunk, + TextChunk, +} from '../types/result.js'; +import type { PageCitationAsset } from '../types/page-citation-assets.js'; +import type { + KnowhereAssetStorageAdapter, + KnowhereAssetStorageBody, + KnowhereAssetStorageOptions, + KnowhereAssetStorageResult, + KnowhereParsedSnapshot, + KnowhereParsedSnapshotChunk, + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotChunkPageReference, + KnowhereParsedSnapshotManifest, +} from '../types/storage.js'; + +interface StorageAsset { + readonly sourcePath: string; + readonly storageKey: string; + readonly body: KnowhereAssetStorageBody; + readonly contentType: string; + readonly metadata: Readonly>; +} + +const pageCitationAssetsMetadataKey = 'pageAssets'; +const defaultImageContentType = 'application/octet-stream'; +const tableContentType = 'text/html; charset=utf-8'; +const snapshotManifestKey = 'manifest/current.json'; +const snapshotIndexKey = 'index.json'; +const defaultSnapshotChunkPageSize = 200; +const snapshotJsonContentType = 'application/json; charset=utf-8'; + +export async function storeParseResultAssets( + result: ParseResult, + options?: KnowhereAssetStorageOptions, +): Promise { + if (!options) { + return { + result, + assetUrlsByFilePath: {}, + }; + } + + const normalizedKeyPrefix = normalizeStorageKeyPrefix(options.keyPrefix); + const assets = dedupeStorageAssets(await collectStorageAssets(result, normalizedKeyPrefix)); + const assetUrlsByFilePath: Record = {}; + + for (const asset of assets) { + const storedUrl = await storeAsset(asset, options.adapter, options.skipExisting ?? true); + if (storedUrl) { + assetUrlsByFilePath[asset.sourcePath] = storedUrl; + } + } + + const normalizedResult = + Object.keys(assetUrlsByFilePath).length === 0 + ? result + : rewriteResultAssetUrls(result, assetUrlsByFilePath); + const snapshot = await storeParsedSnapshot({ + result: normalizedResult, + assetUrlsByFilePath, + keyPrefix: normalizedKeyPrefix, + options, + }); + + return { + result: normalizedResult, + assetUrlsByFilePath, + snapshot, + }; +} + +async function storeParsedSnapshot(input: { + readonly result: ParseResult; + readonly assetUrlsByFilePath: Readonly>; + readonly keyPrefix: string; + readonly options: KnowhereAssetStorageOptions; +}): Promise { + const pageSize = normalizeSnapshotChunkPageSize(input.options.chunkPageSize); + const totalChunks = input.result.chunks.length; + const totalPages = Math.max(1, Math.ceil(totalChunks / pageSize)); + const chunkPageReferences: KnowhereParsedSnapshotChunkPageReference[] = []; + const chunkPageUrlsByPage: Record = {}; + + for (let page = 1; page <= totalPages; page += 1) { + const pageChunks = input.result.chunks.slice((page - 1) * pageSize, page * pageSize); + const pageKey = `${input.keyPrefix}/chunks/page-${page}.json`; + const chunkPage: KnowhereParsedSnapshotChunkPage = { + version: 1, + jobId: input.result.jobId, + revisionKey: input.options.revisionKey ?? input.result.jobId, + documentId: input.result.documentId, + namespace: input.result.namespace, + sourceFileName: input.result.manifest.sourceFileName, + page, + pageSize, + total: totalChunks, + totalPages, + chunks: pageChunks.map((chunk, index) => + toSnapshotChunk(chunk, (page - 1) * pageSize + index + 1), + ), + }; + const pageUrl = await writeJsonObject({ + key: pageKey, + value: chunkPage, + adapter: input.options.adapter, + }); + if (pageUrl) chunkPageUrlsByPage[page] = pageUrl; + chunkPageReferences.push({ + page, + pageSize, + chunkCount: pageChunks.length, + key: pageKey, + url: pageUrl, + }); + } + + const manifest: KnowhereParsedSnapshotManifest = { + version: 1, + kind: 'knowhere-parsed-result-snapshot', + jobId: input.result.jobId, + revisionKey: input.options.revisionKey ?? input.result.jobId, + documentId: input.result.documentId, + namespace: input.result.namespace, + sourceFileName: input.result.manifest.sourceFileName, + totalChunks, + typeCounts: countSnapshotChunkTypes(input.result.chunks), + chunkPageSize: pageSize, + chunkPages: chunkPageReferences, + assetUrlsByFilePath: input.assetUrlsByFilePath, + createdAt: new Date().toISOString(), + }; + const manifestKey = `${input.keyPrefix}/${snapshotManifestKey}`; + const indexKey = `${input.keyPrefix}/${snapshotIndexKey}`; + const manifestUrl = await writeJsonObject({ + key: manifestKey, + value: manifest, + adapter: input.options.adapter, + }); + const indexUrl = await writeJsonObject({ + key: indexKey, + value: manifest, + adapter: input.options.adapter, + }); + + return { + manifest, + manifestKey, + manifestUrl, + indexKey, + indexUrl, + chunkPageUrlsByPage, + }; +} + +function countSnapshotChunkTypes( + chunks: readonly Chunk[], +): Readonly> { + return chunks.reduce>( + (counts, chunk) => { + counts[chunk.type] += 1; + return counts; + }, + { text: 0, image: 0, table: 0, page: 0 }, + ); +} + +async function writeJsonObject(input: { + readonly key: string; + readonly value: unknown; + readonly adapter: KnowhereAssetStorageAdapter; +}): Promise { + const result = await input.adapter.writeObject({ + key: input.key, + body: Buffer.from(JSON.stringify(input.value, null, 2), 'utf8'), + contentType: snapshotJsonContentType, + metadata: { + kind: 'knowhere-parsed-result-snapshot', + }, + }); + return result.url ?? (await input.adapter.getObjectUrl?.(result.key)) ?? undefined; +} + +function normalizeSnapshotChunkPageSize(value: number | undefined): number { + if (!Number.isInteger(value) || value === undefined) return defaultSnapshotChunkPageSize; + return Math.min(Math.max(value, 1), 1000); +} + +function toSnapshotChunk(chunk: Chunk, position: number): KnowhereParsedSnapshotChunk { + const filePath = getSnapshotChunkFilePath(chunk); + return { + id: chunk.chunkId, + chunkId: chunk.chunkId, + chunkType: chunk.type, + contentSource: chunk.contentSource, + content: chunk.content, + sectionPath: normalizeSnapshotSectionPath(chunk.path), + sourceChunkPath: chunk.path, + filePath, + sortOrder: position, + metadata: cloneChunkMetadata(chunk.metadata), + assetUrl: getSnapshotChunkAssetUrl(chunk), + }; +} + +function normalizeSnapshotSectionPath(path: string): string | undefined { + return path.length > 0 ? path : undefined; +} + +function getSnapshotChunkFilePath(chunk: Chunk): string | undefined { + if (chunk.type === 'image' || chunk.type === 'table') { + return chunk.filePath; + } + const filePath = chunk.metadata.filePath; + return typeof filePath === 'string' ? filePath : undefined; +} + +function getSnapshotChunkAssetUrl(chunk: Chunk): string | undefined { + return chunk.type === 'image' || chunk.type === 'table' ? chunk.assetUrl : undefined; +} + +function cloneChunkMetadata(metadata: ChunkMetadata): Record { + return { ...metadata }; +} + +async function collectStorageAssets( + result: ParseResult, + keyPrefix: string, +): Promise { + const resultZip = await loadResultZipForPageCitationAssets(result); + return [ + ...result.imageChunks.flatMap((chunk) => collectImageStorageAsset(chunk, keyPrefix)), + ...result.tableChunks.flatMap((chunk) => collectTableStorageAsset(chunk, keyPrefix)), + ...(resultZip + ? await collectPageCitationStorageAssets(result.pageChunks, keyPrefix, resultZip) + : []), + ]; +} + +function collectImageStorageAsset(chunk: ImageChunk, keyPrefix: string): StorageAsset[] { + const sourcePath = normalizeResultAssetPath(chunk.filePath); + if (!sourcePath) return []; + + return [ + { + sourcePath, + storageKey: `${keyPrefix}/${sourcePath}`, + body: chunk.data, + contentType: getImageContentType(chunk.filePath), + metadata: { + chunkId: chunk.chunkId, + chunkType: chunk.type, + sourcePath, + }, + }, + ]; +} + +function collectTableStorageAsset(chunk: TableChunk, keyPrefix: string): StorageAsset[] { + const sourcePath = normalizeResultAssetPath(chunk.filePath); + if (!sourcePath) return []; + + return [ + { + sourcePath, + storageKey: `${keyPrefix}/${sourcePath}`, + body: Buffer.from(chunk.html, 'utf8'), + contentType: tableContentType, + metadata: { + chunkId: chunk.chunkId, + chunkType: chunk.type, + sourcePath, + }, + }, + ]; +} + +async function collectPageCitationStorageAssets( + chunks: readonly PageChunk[], + keyPrefix: string, + resultZip: JSZip, +): Promise { + const assets: StorageAsset[] = []; + + for (const chunk of chunks) { + const pageAssets = parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]); + for (const pageAsset of pageAssets) { + const sourcePath = normalizeResultAssetPath(pageAsset.artifactRef); + if (!sourcePath) continue; + + const zipEntry = resultZip.file(sourcePath); + if (!zipEntry) continue; + + assets.push({ + sourcePath, + storageKey: `${keyPrefix}/${sourcePath}`, + body: await zipEntry.async('uint8array'), + contentType: pageAsset.contentType, + metadata: getPageCitationAssetMetadata(chunk, pageAsset, sourcePath), + }); + } + } + + return assets; +} + +function getPageCitationAssetMetadata( + chunk: PageChunk, + pageAsset: PageCitationAsset, + sourcePath: string, +): Readonly> { + return { + chunkId: chunk.chunkId, + chunkType: chunk.type, + pageNum: String(pageAsset.pageNum), + sourcePath, + }; +} + +async function loadResultZipForPageCitationAssets(result: ParseResult): Promise { + if (!hasPageCitationAssetReferences(result) || result.rawZip.length === 0) { + return null; + } + + try { + return await JSZip.loadAsync(result.rawZip); + } catch (cause) { + throw new Error('Unable to read page citation assets from the Knowhere result ZIP.', { + cause, + }); + } +} + +function hasPageCitationAssetReferences(result: ParseResult): boolean { + return result.pageChunks.some( + (chunk) => parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]).length > 0, + ); +} + +function dedupeStorageAssets(assets: readonly StorageAsset[]): StorageAsset[] { + const assetsBySourcePath = new Map(); + + for (const asset of assets) { + if (!assetsBySourcePath.has(asset.sourcePath)) { + assetsBySourcePath.set(asset.sourcePath, asset); + } + } + + return [...assetsBySourcePath.values()]; +} + +async function storeAsset( + asset: StorageAsset, + adapter: KnowhereAssetStorageAdapter, + skipExisting: boolean, +): Promise { + if (skipExisting && adapter.headObject) { + const existing = await adapter.headObject(asset.storageKey); + if (existing) { + const existingUrl = existing.url ?? (await adapter.getObjectUrl?.(asset.storageKey)); + if (existingUrl) return existingUrl; + } + } + + const writeResult = await adapter.writeObject({ + key: asset.storageKey, + body: asset.body, + contentType: asset.contentType, + metadata: asset.metadata, + }); + return writeResult.url ?? (await adapter.getObjectUrl?.(writeResult.key)) ?? null; +} + +function rewriteResultAssetUrls( + result: ParseResult, + assetUrlsByFilePath: Readonly>, +): ParseResult { + const chunks = result.chunks.map((chunk) => rewriteChunkAssetUrls(chunk, assetUrlsByFilePath)); + result.chunks.splice(0, result.chunks.length, ...chunks); + refreshStaticChunkCollections(result); + return result; +} + +function rewriteChunkAssetUrls( + chunk: Chunk, + assetUrlsByPath: Readonly>, +): Chunk { + if (chunk.type === 'image' || chunk.type === 'table') { + const sourcePath = normalizeResultAssetPath(chunk.filePath); + const assetUrl = sourcePath ? assetUrlsByPath[sourcePath] : undefined; + if (!assetUrl) return chunk; + return { + ...chunk, + assetUrl, + }; + } + + if (chunk.type === 'page') { + const pageAssets = parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]); + if (pageAssets.length === 0) return chunk; + + const rewrittenPageAssets = pageAssets.map((pageAsset) => { + const sourcePath = normalizeResultAssetPath(pageAsset.artifactRef); + const assetUrl = sourcePath ? assetUrlsByPath[sourcePath] : undefined; + return assetUrl ? { ...pageAsset, assetUrl } : pageAsset; + }); + + return { + ...chunk, + metadata: { + ...chunk.metadata, + [pageCitationAssetsMetadataKey]: rewrittenPageAssets, + }, + }; + } + + return chunk; +} + +function parsePageCitationAssets(value: unknown): PageCitationAsset[] { + if (!Array.isArray(value)) return []; + + return value.flatMap((item): PageCitationAsset[] => { + if (!isRecord(item)) return []; + const pageNum = item.pageNum; + const artifactRef = item.artifactRef; + const contentType = item.contentType; + const source = item.source; + + if (typeof pageNum !== 'number' || !Number.isSafeInteger(pageNum) || pageNum <= 0) return []; + if (typeof artifactRef !== 'string') return []; + if (contentType !== 'image/png' && contentType !== 'image/jpeg') return []; + if (source !== 'knowhere-rendered-page-citation-source') return []; + + return [ + { + pageNum, + artifactRef, + assetUrl: typeof item.assetUrl === 'string' ? item.assetUrl : undefined, + contentType, + width: typeof item.width === 'number' ? item.width : undefined, + height: typeof item.height === 'number' ? item.height : undefined, + source, + }, + ]; + }); +} + +function normalizeStorageKeyPrefix(value: string): string { + const normalized = normalizeStoragePath(value, { allowDotPrefix: false }); + if (!normalized) { + throw new Error('storageAdapter keyPrefix must be a non-empty relative POSIX path.'); + } + return normalized; +} + +function normalizeResultAssetPath(value: string | undefined): string | null { + return normalizeStoragePath(value, { allowDotPrefix: true }); +} + +function normalizeStoragePath( + value: string | undefined, + options: { readonly allowDotPrefix: boolean }, +): string | null { + if (!value) return null; + const trimmed = value.trim(); + if (trimmed.length === 0 || trimmed.includes('\0')) return null; + if (trimmed.startsWith('/') || /^[A-Za-z]:/.test(trimmed)) return null; + + const normalized = options.allowDotPrefix + ? trimmed.replaceAll('\\', '/').replace(/^\.\/+/, '') + : trimmed.replaceAll('\\', '/'); + + if (normalized.length === 0) return null; + + const parts = normalized.split('/'); + if (parts.some((part) => part.length === 0 || part === '.' || part === '..')) return null; + + return parts.join('/'); +} + +type StaticChunkCollections = { + textChunks: TextChunk[]; + imageChunks: ImageChunk[]; + tableChunks: TableChunk[]; + pageChunks: PageChunk[]; +}; + +function refreshStaticChunkCollections(result: ParseResult): void { + replaceStaticChunkCollection( + result, + 'textChunks', + result.chunks.filter((chunk): chunk is TextChunk => chunk.type === 'text'), + ); + replaceStaticChunkCollection( + result, + 'imageChunks', + result.chunks.filter((chunk): chunk is ImageChunk => chunk.type === 'image'), + ); + replaceStaticChunkCollection( + result, + 'tableChunks', + result.chunks.filter((chunk): chunk is TableChunk => chunk.type === 'table'), + ); + replaceStaticChunkCollection( + result, + 'pageChunks', + result.chunks.filter((chunk): chunk is PageChunk => chunk.type === 'page'), + ); +} + +function replaceStaticChunkCollection( + result: ParseResult, + property: Key, + value: StaticChunkCollections[Key], +): void { + const descriptor = Object.getOwnPropertyDescriptor(result, property); + if (!descriptor || descriptor.get || descriptor.writable !== true) return; + + Object.defineProperty(result, property, { + ...descriptor, + value, + }); +} + +function getImageContentType(filePath: string): string { + const extension = filePath.split('.').at(-1)?.toLowerCase(); + if (extension === 'jpg' || extension === 'jpeg') return 'image/jpeg'; + if (extension === 'png') return 'image/png'; + if (extension === 'webp') return 'image/webp'; + if (extension === 'gif') return 'image/gif'; + if (extension === 'svg') return 'image/svg+xml'; + return defaultImageContentType; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} diff --git a/src/storage/disk-parsed-document-storage.ts b/src/storage/disk-parsed-document-storage.ts new file mode 100644 index 0000000..58fc0c9 --- /dev/null +++ b/src/storage/disk-parsed-document-storage.ts @@ -0,0 +1,208 @@ +import { createHash } from 'crypto'; +import { promises as fs } from 'fs'; +import path from 'path'; + +import type { + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, + ParsedDocumentStorage, + ParsedDocumentSyncProgress, +} from '../types/storage.js'; + +export class DiskParsedDocumentStorage implements ParsedDocumentStorage { + private readonly rootDirectory: string; + + constructor(rootDirectory: string) { + this.rootDirectory = rootDirectory; + } + + async readManifest(params: { + readonly documentId: string; + readonly revisionKey: string; + }): Promise { + return this.readOptionalJson( + this.getManifestPath(params.documentId, params.revisionKey), + ); + } + + async writeManifest(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly manifest: KnowhereParsedSnapshotManifest; + }): Promise { + await this.writeJson( + this.getManifestPath(params.documentId, params.revisionKey), + params.manifest, + ); + } + + async readChunkPage(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly page: number; + }): Promise { + return this.readOptionalJson( + this.getChunkPagePath(params.documentId, params.revisionKey, params.page), + ); + } + + async writeChunkPage(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly page: KnowhereParsedSnapshotChunkPage; + }): Promise { + await this.writeJson( + this.getChunkPagePath(params.documentId, params.revisionKey, params.page.page), + params.page, + ); + } + + async writeAsset(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly sourcePath: string; + readonly body: Uint8Array; + readonly contentType: string; + readonly metadata?: Readonly>; + }): Promise<{ + readonly sourcePath: string; + readonly url?: string; + }> { + const assetPath = this.getAssetPath(params.documentId, params.revisionKey, params.sourcePath); + await fs.mkdir(path.dirname(assetPath), { recursive: true }); + await fs.writeFile(assetPath, params.body); + await this.writeJson(`${assetPath}.metadata.json`, { + contentType: params.contentType, + metadata: params.metadata ?? {}, + }); + return { + sourcePath: params.sourcePath, + url: toFileUrl(assetPath), + }; + } + + async getAssetUrl(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly sourcePath: string; + }): Promise { + const assetPath = this.getAssetPath(params.documentId, params.revisionKey, params.sourcePath); + try { + await fs.access(assetPath); + return toFileUrl(assetPath); + } catch (error) { + if (isMissingFileError(error)) { + return null; + } + throw error; + } + } + + async readSyncProgress(params: { + readonly documentId: string; + readonly revisionKey: string; + }): Promise { + return this.readOptionalJson( + this.getProgressPath(params.documentId, params.revisionKey), + ); + } + + async writeSyncProgress(params: ParsedDocumentSyncProgress): Promise { + await this.writeJson(this.getProgressPath(params.documentId, params.revisionKey), params); + } + + private getManifestPath(documentId: string, revisionKey: string): string { + return path.join( + this.getRevisionDirectory(documentId, revisionKey), + 'manifest', + 'current.json', + ); + } + + private getChunkPagePath(documentId: string, revisionKey: string, page: number): string { + return path.join( + this.getRevisionDirectory(documentId, revisionKey), + 'chunks', + `page-${page}.json`, + ); + } + + private getProgressPath(documentId: string, revisionKey: string): string { + return path.join(this.getRevisionDirectory(documentId, revisionKey), 'sync-progress.json'); + } + + private getAssetPath(documentId: string, revisionKey: string, sourcePath: string): string { + const normalizedSourcePath = normalizeRelativeStoragePath(sourcePath); + const assetPath = path.resolve( + this.getRevisionDirectory(documentId, revisionKey), + 'assets', + normalizedSourcePath, + ); + const assetsDirectory = path.resolve( + this.getRevisionDirectory(documentId, revisionKey), + 'assets', + ); + if (!isPathInsideDirectory(assetPath, assetsDirectory)) { + throw new Error(`Parsed asset path resolves outside storage: ${sourcePath}`); + } + return assetPath; + } + + private getRevisionDirectory(documentId: string, revisionKey: string): string { + return path.join(this.rootDirectory, hashPathPart(documentId), hashPathPart(revisionKey)); + } + + private async readOptionalJson(filePath: string): Promise { + try { + return JSON.parse(await fs.readFile(filePath, 'utf8')) as T; + } catch (error) { + if (isMissingFileError(error)) { + return null; + } + throw error; + } + } + + private async writeJson(filePath: string, value: unknown): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, JSON.stringify(value, null, 2), { + encoding: 'utf8', + flag: 'w', + }); + } +} + +function hashPathPart(value: string): string { + return createHash('sha256').update(value).digest('hex').slice(0, 24); +} + +function normalizeRelativeStoragePath(value: string): string { + const normalized = value.replaceAll('\\', '/').replace(/^\.\/+/, ''); + if ( + normalized.length === 0 || + normalized.includes('\0') || + normalized.startsWith('/') || + normalized.split('/').some((part) => part.length === 0 || part === '.' || part === '..') + ) { + throw new Error(`Invalid parsed storage path: ${value}`); + } + return normalized; +} + +function isPathInsideDirectory(filePath: string, directory: string): boolean { + const relative = path.relative(directory, filePath); + return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative)); +} + +function toFileUrl(filePath: string): string { + return `file://${filePath}`; +} + +function isMissingFileError(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code?: string }).code === 'ENOENT' + ); +} diff --git a/src/storage/parsed-document-storage.ts b/src/storage/parsed-document-storage.ts new file mode 100644 index 0000000..e9f6a55 --- /dev/null +++ b/src/storage/parsed-document-storage.ts @@ -0,0 +1,173 @@ +import { storeParseResultAssets } from './asset-storage.js'; +import type { ParseResult } from '../types/result.js'; +import type { + KnowhereAssetStorageAdapter, + KnowhereAssetStorageHead, + KnowhereAssetStorageObject, + KnowhereAssetStorageWriteResult, + KnowhereParsedSnapshot, + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, + ParsedDocumentStorage, +} from '../types/storage.js'; + +const parsedStorageKeyPrefix = 'parsed'; +const manifestStoragePath = 'manifest/current.json'; +const indexStoragePath = 'index.json'; +const chunkPagePathPattern = /^chunks\/page-(\d+)\.json$/; + +export async function syncParseResultToParsedDocumentStorage(params: { + readonly result: ParseResult; + readonly storage: ParsedDocumentStorage; + readonly documentId?: string; + readonly revisionKey?: string; + readonly chunkPageSize?: number; +}): Promise<{ + readonly result: ParseResult; + readonly snapshot?: KnowhereParsedSnapshot; + readonly documentId: string; + readonly revisionKey: string; +}> { + const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId; + const revisionKey = params.revisionKey ?? params.result.jobId; + const adapter = createParsedDocumentObjectStorageAdapter({ + storage: params.storage, + documentId, + revisionKey, + }); + const stored = await storeParseResultAssets(params.result, { + adapter, + keyPrefix: parsedStorageKeyPrefix, + chunkPageSize: params.chunkPageSize, + revisionKey, + }); + return { + result: stored.result, + snapshot: stored.snapshot, + documentId, + revisionKey, + }; +} + +function createParsedDocumentObjectStorageAdapter(params: { + readonly storage: ParsedDocumentStorage; + readonly documentId: string; + readonly revisionKey: string; +}): KnowhereAssetStorageAdapter { + return { + writeObject: (input) => writeParsedStorageObject(params, input), + headObject: (key) => headParsedStorageObject(params, key), + getObjectUrl: (key) => getParsedStorageObjectUrl(params, key), + }; +} + +async function writeParsedStorageObject( + params: { + readonly storage: ParsedDocumentStorage; + readonly documentId: string; + readonly revisionKey: string; + }, + input: KnowhereAssetStorageObject, +): Promise { + const storagePath = stripParsedStorageKeyPrefix(input.key); + if (storagePath === manifestStoragePath) { + const manifest = JSON.parse( + Buffer.from(input.body).toString('utf8'), + ) as KnowhereParsedSnapshotManifest; + await params.storage.writeManifest({ + documentId: params.documentId, + revisionKey: params.revisionKey, + manifest, + }); + return { key: input.key }; + } + + if (storagePath === indexStoragePath) { + return { key: input.key }; + } + + const chunkPageMatch = storagePath.match(chunkPagePathPattern); + if (chunkPageMatch) { + const page = JSON.parse( + Buffer.from(input.body).toString('utf8'), + ) as KnowhereParsedSnapshotChunkPage; + await params.storage.writeChunkPage({ + documentId: params.documentId, + revisionKey: params.revisionKey, + page, + }); + return { key: input.key }; + } + + const stored = await params.storage.writeAsset({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: storagePath, + body: input.body, + contentType: input.contentType, + metadata: input.metadata, + }); + return { + key: input.key, + url: stored.url, + }; +} + +async function headParsedStorageObject( + params: { + readonly storage: ParsedDocumentStorage; + readonly documentId: string; + readonly revisionKey: string; + }, + key: string, +): Promise { + const storagePath = stripParsedStorageKeyPrefix(key); + if ( + storagePath === manifestStoragePath || + storagePath === indexStoragePath || + chunkPagePathPattern.test(storagePath) + ) { + return null; + } + + const url = await params.storage.getAssetUrl({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: storagePath, + }); + return url + ? { + key, + url, + } + : null; +} + +async function getParsedStorageObjectUrl( + params: { + readonly storage: ParsedDocumentStorage; + readonly documentId: string; + readonly revisionKey: string; + }, + key: string, +): Promise { + const storagePath = stripParsedStorageKeyPrefix(key); + if ( + storagePath === manifestStoragePath || + storagePath === indexStoragePath || + chunkPagePathPattern.test(storagePath) + ) { + return null; + } + + return params.storage.getAssetUrl({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: storagePath, + }); +} + +function stripParsedStorageKeyPrefix(key: string): string { + const prefix = `${parsedStorageKeyPrefix}/`; + return key.startsWith(prefix) ? key.slice(prefix.length) : key; +} diff --git a/src/types/document.ts b/src/types/document.ts index 959ada8..86df509 100644 --- a/src/types/document.ts +++ b/src/types/document.ts @@ -89,7 +89,7 @@ export interface DocumentChunkListParams { pageSize?: number; /** Optional chunk type filter */ chunkType?: DocumentChunkType; - /** Set true to include 7-day asset URLs for media chunks */ + /** Set true to include 7-day asset URLs for media and page citation assets */ includeAssetUrls?: boolean; } @@ -97,7 +97,7 @@ export interface DocumentChunkListParams { * Query parameters for GET /v2/documents/{document_id}/chunks/{document_chunk_id}. */ export interface DocumentChunkGetParams { - /** Set true to include 7-day asset URLs for media chunks */ + /** Set true to include 7-day asset URLs for media and page citation assets */ includeAssetUrls?: boolean; } @@ -127,12 +127,27 @@ export interface DocumentChunk { sortOrder: number; /** Chunk metadata returned by the API */ metadata: Record; - /** 7-day asset URL for media chunks when available */ + /** 7-day asset URL for media or page citation assets when available */ assetUrl?: string | null; /** Chunk creation timestamp */ createdAt?: Date; } +/** + * Response from GET /v2/documents/{document_id}/files/page-citation-source. + */ +export interface DocumentPageCitationSource { + documentId: string; + namespace?: string; + jobId?: string; + jobResultId?: string; + variant?: string; + fileName: string; + contentType: string; + url: string; + expiresAt?: Date; +} + /** * Response from GET /v2/documents/{document_id}/chunks. */ diff --git a/src/types/index.ts b/src/types/index.ts index 7ec1387..fae5e68 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -5,4 +5,6 @@ export * from './params.js'; export * from './result.js'; export * from './retrieval.js'; export * from './document.js'; +export * from './page-citation-assets.js'; +export * from './storage.js'; export * from '../knowledge/types.js'; diff --git a/src/types/page-citation-assets.ts b/src/types/page-citation-assets.ts new file mode 100644 index 0000000..27f60e5 --- /dev/null +++ b/src/types/page-citation-assets.ts @@ -0,0 +1,13 @@ +export type PageCitationAssetContentType = 'image/png' | 'image/jpeg'; + +export type PageCitationAssetSource = 'knowhere-rendered-page-citation-source'; + +export interface PageCitationAsset { + pageNum: number; + artifactRef: string; + assetUrl?: string; + contentType: PageCitationAssetContentType; + width?: number; + height?: number; + source: PageCitationAssetSource; +} diff --git a/src/types/params.ts b/src/types/params.ts index 296fddb..278af74 100644 --- a/src/types/params.ts +++ b/src/types/params.ts @@ -1,4 +1,5 @@ import type { ReadStream } from 'fs'; +import type { KnowhereAssetStorageOptions } from './storage.js'; /** * Parsing model options @@ -147,6 +148,11 @@ export interface ParseParams { pollTimeout?: number; /** Whether to verify ZIP checksum (default: true) */ verifyChecksum?: boolean; + /** + * Optional asset storage adapter used to copy parsed media and page citation + * assets into application-owned storage before the result is returned. + */ + storageAdapter?: KnowhereAssetStorageOptions; /** Webhook configuration */ webhook?: WebhookConfig; /** Upload progress callback */ diff --git a/src/types/result.ts b/src/types/result.ts index af4ff06..123d1e6 100644 --- a/src/types/result.ts +++ b/src/types/result.ts @@ -194,6 +194,8 @@ export interface ImageChunk extends BaseChunk { type: 'image'; /** Relative file path in ZIP */ filePath: string; + /** Application-owned or server-provided display URL for the image asset. */ + assetUrl?: string; /** Image data buffer */ data: Buffer; /** Image format (derived from file extension) */ @@ -209,6 +211,8 @@ export interface TableChunk extends BaseChunk { type: 'table'; /** Relative file path in ZIP */ filePath: string; + /** Application-owned or server-provided display URL for the table asset. */ + assetUrl?: string; /** HTML representation */ html: string; /** Save table HTML to disk */ diff --git a/src/types/storage.ts b/src/types/storage.ts new file mode 100644 index 0000000..39f4b63 --- /dev/null +++ b/src/types/storage.ts @@ -0,0 +1,190 @@ +import type { ParseResult } from './result.js'; +import type { DocumentChunkType } from './document.js'; + +export type KnowhereAssetStorageBody = Uint8Array; + +export interface KnowhereAssetStorageObject { + readonly key: string; + readonly body: KnowhereAssetStorageBody; + readonly contentType: string; + readonly metadata?: Readonly>; +} + +export interface KnowhereAssetStorageHead { + readonly key: string; + readonly contentType?: string; + readonly contentLength?: number; + readonly metadata?: Readonly>; + readonly url?: string; +} + +export interface KnowhereAssetStorageWriteResult { + readonly key: string; + readonly url?: string; +} + +export interface KnowhereAssetStorageAdapter { + writeObject(input: KnowhereAssetStorageObject): Promise; + headObject?(key: string): Promise; + getObjectUrl?(key: string): Promise; +} + +export interface KnowhereAssetStorageOptions { + readonly adapter: KnowhereAssetStorageAdapter; + readonly keyPrefix: string; + readonly skipExisting?: boolean; + readonly chunkPageSize?: number; + readonly revisionKey?: string; +} + +export interface KnowhereParsedSnapshotChunk { + readonly id: string; + readonly chunkId: string; + readonly chunkType: string; + readonly contentSource?: string; + readonly content: string; + readonly sectionPath?: string; + readonly sourceChunkPath: string; + readonly filePath?: string; + readonly sortOrder: number; + readonly metadata: Record; + readonly assetUrl?: string; +} + +export interface KnowhereParsedSnapshotChunkPage { + readonly version: 1; + readonly jobId: string; + readonly revisionKey?: string; + readonly documentId?: string; + readonly namespace?: string; + readonly sourceFileName: string; + readonly page: number; + readonly pageSize: number; + readonly total: number; + readonly totalPages: number; + readonly chunks: readonly KnowhereParsedSnapshotChunk[]; +} + +export interface KnowhereParsedSnapshotChunkPageReference { + readonly page: number; + readonly pageSize: number; + readonly chunkCount: number; + readonly key: string; + readonly url?: string; +} + +export interface KnowhereParsedSnapshotManifest { + readonly version: 1; + readonly kind: 'knowhere-parsed-result-snapshot'; + readonly jobId: string; + readonly revisionKey?: string; + readonly documentId?: string; + readonly namespace?: string; + readonly sourceFileName: string; + readonly totalChunks: number; + readonly typeCounts?: Readonly>>; + readonly chunkPageSize: number; + readonly chunkPages: readonly KnowhereParsedSnapshotChunkPageReference[]; + readonly assetUrlsByFilePath: Readonly>; + readonly createdAt: string; +} + +export interface KnowhereParsedSnapshot { + readonly manifest: KnowhereParsedSnapshotManifest; + readonly manifestKey: string; + readonly manifestUrl?: string; + readonly indexKey: string; + readonly indexUrl?: string; + readonly chunkPageUrlsByPage: Readonly>; +} + +export interface KnowhereAssetStorageResult { + readonly result: ParseResult; + readonly assetUrlsByFilePath: Readonly>; + readonly snapshot?: KnowhereParsedSnapshot; +} + +export type ParsedDocumentAssetUrlPolicy = 'none' | 'durable'; + +export interface ParsedDocumentStorageDocument { + readonly documentId: string; + readonly revisionKey: string; +} + +export type ParsedDocumentStorageManifestParams = ParsedDocumentStorageDocument; + +export interface ParsedDocumentStorageChunkPageParams extends ParsedDocumentStorageDocument { + readonly page: number; + readonly chunkType?: DocumentChunkType; +} + +export interface ParsedDocumentStorageAsset { + readonly sourcePath: string; + readonly body: KnowhereAssetStorageBody; + readonly contentType: string; + readonly metadata?: Readonly>; +} + +export interface ParsedDocumentStorageAssetParams extends ParsedDocumentStorageDocument { + readonly sourcePath: string; +} + +export interface ParsedDocumentSyncProgress { + readonly documentId: string; + readonly revisionKey: string; + readonly nextChunkPage: number; + readonly nextAssetIndex: number; + readonly status: 'running' | 'completed' | 'failed'; + readonly updatedAt: string; + readonly error?: string; +} + +export interface ParsedDocumentStorage { + readManifest( + params: ParsedDocumentStorageManifestParams, + ): Promise; + writeManifest(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly manifest: KnowhereParsedSnapshotManifest; + }): Promise; + readChunkPage( + params: ParsedDocumentStorageChunkPageParams, + ): Promise; + writeChunkPage(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly page: KnowhereParsedSnapshotChunkPage; + }): Promise; + writeAsset(params: ParsedDocumentStorageDocument & ParsedDocumentStorageAsset): Promise<{ + readonly sourcePath: string; + readonly url?: string; + }>; + getAssetUrl(params: ParsedDocumentStorageAssetParams): Promise; + readSyncProgress( + params: ParsedDocumentStorageDocument, + ): Promise; + writeSyncProgress(params: ParsedDocumentSyncProgress): Promise; +} + +export interface ParsedDocumentSyncScheduler { + schedule(task: () => Promise): void | Promise; +} + +export interface ParsedDocumentStorageLimits { + readonly chunkPageSize?: number; + readonly remotePageSize?: number; + readonly maxPagesPerSync?: number; + readonly maxAssetsPerSync?: number; + readonly syncDeadlineMs?: number; + readonly grepMaxPages?: number; + readonly grepDeadlineMs?: number; + readonly outlineMaxPages?: number; + readonly outlineDeadlineMs?: number; +} + +export interface ParsedDocumentStorageConfig { + readonly storage: ParsedDocumentStorage; + readonly scheduler?: ParsedDocumentSyncScheduler; + readonly limits?: ParsedDocumentStorageLimits; +} diff --git a/vitest.config.mts b/vitest.config.mts index cfc4f2a..1a34869 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -1,13 +1,15 @@ -import { defineConfig } from 'vitest/config'; +import { configDefaults, defineConfig } from 'vitest/config'; export default defineConfig({ test: { globals: true, environment: 'node', + exclude: [...configDefaults.exclude, '.repos/**'], coverage: { provider: 'v8', reporter: ['text', 'json', 'html', 'json-summary'], exclude: [ + '.repos/**', 'node_modules/', 'dist/', '**/*.test.ts',