From 12e1c4c8e924127a279bd7c66a708070c4c0e9d6 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Fri, 3 Jul 2026 02:06:48 +0800 Subject: [PATCH 01/10] feat: add page citation asset generation --- .gitattributes | 1 + .gitignore | 1 + package.json | 22 +- pnpm-lock.yaml | 638 +++++++++++- scripts/prepare-effect.sh | 13 + src/client.ts | 18 +- src/errors/index.ts | 3 + src/errors/page-citation-assets.ts | 12 + src/index.ts | 25 + src/knowledge/__tests__/knowledge.test.ts | 201 ++++ src/knowledge/knowledge.ts | 45 +- src/knowledge/types.ts | 12 +- src/lib/result-parser.ts | 10 +- .../__fixtures__/tiny.pdf | Bin 0 -> 429 bytes .../__tests__/page-renderer-pdfjs.test.ts | 26 + .../__tests__/workflow.test.mts | 498 +++++++++ src/page-citation-assets/workflow-loader.ts | 62 ++ src/page-citation-assets/workflow.mts | 951 ++++++++++++++++++ src/page-renderer-pdfjs.ts | 141 +++ src/resources/__tests__/documents.test.ts | 33 + src/resources/documents.ts | 10 + src/storage/__tests__/local-storage.test.ts | 81 ++ src/storage/local-storage.ts | 259 +++++ src/types/document.ts | 19 + src/types/index.ts | 1 + src/types/page-citation-assets.ts | 135 +++ src/types/params.ts | 3 + src/types/pdfjs-dist-build-pdf.d.ts | 7 + src/types/result.ts | 6 + vitest.config.mts | 4 +- 30 files changed, 3204 insertions(+), 33 deletions(-) create mode 100644 .gitattributes create mode 100755 scripts/prepare-effect.sh create mode 100644 src/errors/page-citation-assets.ts create mode 100644 src/page-citation-assets/__fixtures__/tiny.pdf create mode 100644 src/page-citation-assets/__tests__/page-renderer-pdfjs.test.ts create mode 100644 src/page-citation-assets/__tests__/workflow.test.mts create mode 100644 src/page-citation-assets/workflow-loader.ts create mode 100644 src/page-citation-assets/workflow.mts create mode 100644 src/page-renderer-pdfjs.ts create mode 100644 src/storage/__tests__/local-storage.test.ts create mode 100644 src/storage/local-storage.ts create mode 100644 src/types/page-citation-assets.ts create mode 100644 src/types/pdfjs-dist-build-pdf.d.ts diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d72fd52 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.pdf binary diff --git a/.gitignore b/.gitignore index 6be33cb..ecc2c5a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Dependencies node_modules/ yarn.lock +.repos/effect # Build output dist/ diff --git a/package.json b/package.json index 936d54f..7f0dfe3 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,11 @@ "types": "./dist/index.d.ts", "import": "./dist/index.mjs", "require": "./dist/index.js" + }, + "./page-renderer-pdfjs": { + "types": "./dist/page-renderer-pdfjs.d.ts", + "import": "./dist/page-renderer-pdfjs.mjs", + "require": "./dist/page-renderer-pdfjs.js" } }, "files": [ @@ -39,16 +44,16 @@ "LICENSE" ], "engines": { - "node": ">=20.19.0", + "node": ">=22.13.0", "npm": ">=10.0.0", "pnpm": ">=9.0.0" }, "scripts": { - "build": "tsup src/index.ts --format cjs,esm --dts --clean", + "build": "tsup src/index.ts src/page-renderer-pdfjs.ts src/page-citation-assets/workflow.mts --format cjs,esm --dts --clean", "build:all": "pnpm --include-workspace-root -r build", "changeset": "changeset", "changeset:publish": "changeset publish", - "dev": "tsup src/index.ts --format cjs,esm --dts --watch", + "dev": "tsup src/index.ts src/page-renderer-pdfjs.ts src/page-citation-assets/workflow.mts --format cjs,esm --dts --watch", "test": "vitest", "test:all": "pnpm --include-workspace-root -r exec vitest run", "test:ci": "vitest run --coverage", @@ -66,15 +71,22 @@ "typecheck": "tsc --noEmit", "typecheck:all": "pnpm run build && pnpm --include-workspace-root -r typecheck", "check:all": "pnpm run lint:all && pnpm run typecheck:all && pnpm run test:all", - "prepublishOnly": "pnpm run lint && pnpm run typecheck && pnpm run test:ci && pnpm run build" + "prepublishOnly": "pnpm run lint && pnpm run typecheck && pnpm run test:ci && pnpm run build", + "prepare": "./scripts/prepare-effect.sh" }, "dependencies": { + "@effect/platform-node": "4.0.0-beta.93", + "@napi-rs/canvas": "^1.0.2", "axios": "^1.15.2", - "jszip": "^3.10.0" + "effect": "4.0.0-beta.93", + "jszip": "^3.10.0", + "pdfjs-dist": "^6.1.200", + "piscina": "^5.2.0" }, "devDependencies": { "@changesets/changelog-github": "^0.6.0", "@changesets/cli": "^2.30.0", + "@effect/vitest": "4.0.0-beta.93", "@eslint/js": "^10.0.1", "@types/node": "^25.2.3", "@typescript-eslint/eslint-plugin": "^8.55.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 14c47ba..96698a1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,12 +8,27 @@ importers: .: dependencies: + '@effect/platform-node': + specifier: 4.0.0-beta.93 + version: 4.0.0-beta.93(effect@4.0.0-beta.93)(ioredis@5.11.1) + '@napi-rs/canvas': + specifier: ^1.0.2 + version: 1.0.2 axios: specifier: ^1.15.2 version: 1.18.1 + effect: + specifier: 4.0.0-beta.93 + version: 4.0.0-beta.93 jszip: specifier: ^3.10.0 version: 3.10.1 + pdfjs-dist: + specifier: ^6.1.200 + version: 6.1.200 + piscina: + specifier: ^5.2.0 + version: 5.2.0 devDependencies: '@changesets/changelog-github': specifier: ^0.6.0 @@ -21,6 +36,9 @@ importers: '@changesets/cli': specifier: ^2.30.0 version: 2.31.0(@types/node@25.9.4) + '@effect/vitest': + specifier: 4.0.0-beta.93 + version: 4.0.0-beta.93(effect@4.0.0-beta.93)(vitest@4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0))) '@eslint/js': specifier: ^10.0.1 version: 10.0.1(eslint@10.5.0) @@ -44,7 +62,7 @@ importers: version: 3.8.4 tsup: specifier: ^8.0.0 - version: 8.5.1(postcss@8.5.15)(typescript@5.9.3) + version: 8.5.1(postcss@8.5.15)(typescript@5.9.3)(yaml@2.9.0) typescript: specifier: ^5.3.0 version: 5.9.3 @@ -53,7 +71,7 @@ importers: version: 8.62.0(eslint@10.5.0)(typescript@5.9.3) vitest: specifier: ^4.0.18 - version: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)) + version: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)) packages/mcp: dependencies: @@ -72,13 +90,13 @@ importers: version: 25.9.4 tsup: specifier: ^8.0.0 - version: 8.5.1(postcss@8.5.15)(typescript@5.9.3) + version: 8.5.1(postcss@8.5.15)(typescript@5.9.3)(yaml@2.9.0) typescript: specifier: ^5.3.0 version: 5.9.3 vitest: specifier: ^4.0.18 - version: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)) + version: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)) packages: @@ -168,6 +186,25 @@ packages: '@changesets/write@0.4.0': resolution: {integrity: sha512-CdTLvIOPiCNuH71pyDu3rA+Q0n65cmAbXnwWH84rKGiFumFzkmHNT8KHTMEchcxN+Kl8I54xGUhJ7l3E7X396Q==} + '@effect/platform-node-shared@4.0.0-beta.93': + resolution: {integrity: sha512-XUqZ2u5GglBqY8q2jj4Q7GjN5K/enedk8auZM9rY/l5a/myaQTrQp3QnvpIK4/Yg0WFjLGuctGPMKWRk3OLIrA==} + engines: {node: '>=18.0.0'} + peerDependencies: + effect: ^4.0.0-beta.93 + + '@effect/platform-node@4.0.0-beta.93': + resolution: {integrity: sha512-QagsCGR0ZOXaCQqS5qGR2mcDng4LiP2bYhiiX1D6UC8cT9vsusVVOHiJWn8CupeDx+yVnPcu81QmA/SDt6GM1w==} + engines: {node: '>=18.0.0'} + peerDependencies: + effect: ^4.0.0-beta.93 + ioredis: ^5.7.0 + + '@effect/vitest@4.0.0-beta.93': + resolution: {integrity: sha512-gMAnZ9PiMeJMDED9s0jWgCOhc2JccrTCxowhur/KriImsHnHIRj4VG/vK0xLw0Axe4AkTWzXNdRsFrYOjBTl3A==} + peerDependencies: + effect: ^4.0.0-beta.93 + vitest: ^3.0.0 || ^4.0.0 + '@emnapi/core@1.11.1': resolution: {integrity: sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==} @@ -407,6 +444,9 @@ packages: '@types/node': optional: true + '@ioredis/commands@1.10.0': + resolution: {integrity: sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==} + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -436,6 +476,212 @@ packages: '@cfworker/json-schema': optional: true + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': + resolution: {integrity: sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': + resolution: {integrity: sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': + resolution: {integrity: sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': + resolution: {integrity: sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': + resolution: {integrity: sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': + resolution: {integrity: sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==} + cpu: [x64] + os: [win32] + + '@napi-rs/canvas-android-arm64@1.0.2': + resolution: {integrity: sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/canvas-darwin-arm64@1.0.2': + resolution: {integrity: sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/canvas-darwin-x64@1.0.2': + resolution: {integrity: sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/canvas-linux-arm-gnueabihf@1.0.2': + resolution: {integrity: sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/canvas-linux-arm64-gnu@1.0.2': + resolution: {integrity: sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/canvas-linux-arm64-musl@1.0.2': + resolution: {integrity: sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/canvas-linux-riscv64-gnu@1.0.2': + resolution: {integrity: sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + + '@napi-rs/canvas-linux-x64-gnu@1.0.2': + resolution: {integrity: sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/canvas-linux-x64-musl@1.0.2': + resolution: {integrity: sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/canvas-win32-arm64-msvc@1.0.2': + resolution: {integrity: sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/canvas-win32-x64-msvc@1.0.2': + resolution: {integrity: sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/canvas@1.0.2': + resolution: {integrity: sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ==} + engines: {node: '>= 10'} + + '@napi-rs/nice-android-arm-eabi@1.1.1': + resolution: {integrity: sha512-kjirL3N6TnRPv5iuHw36wnucNqXAO46dzK9oPb0wj076R5Xm8PfUVA9nAFB5ZNMmfJQJVKACAPd/Z2KYMppthw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@napi-rs/nice-android-arm64@1.1.1': + resolution: {integrity: sha512-blG0i7dXgbInN5urONoUCNf+DUEAavRffrO7fZSeoRMJc5qD+BJeNcpr54msPF6qfDD6kzs9AQJogZvT2KD5nw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/nice-darwin-arm64@1.1.1': + resolution: {integrity: sha512-s/E7w45NaLqTGuOjC2p96pct4jRfo61xb9bU1unM/MJ/RFkKlJyJDx7OJI/O0ll/hrfpqKopuAFDV8yo0hfT7A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/nice-darwin-x64@1.1.1': + resolution: {integrity: sha512-dGoEBnVpsdcC+oHHmW1LRK5eiyzLwdgNQq3BmZIav+9/5WTZwBYX7r5ZkQC07Nxd3KHOCkgbHSh4wPkH1N1LiQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/nice-freebsd-x64@1.1.1': + resolution: {integrity: sha512-kHv4kEHAylMYmlNwcQcDtXjklYp4FCf0b05E+0h6nDHsZ+F0bDe04U/tXNOqrx5CmIAth4vwfkjjUmp4c4JktQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/nice-linux-arm-gnueabihf@1.1.1': + resolution: {integrity: sha512-E1t7K0efyKXZDoZg1LzCOLxgolxV58HCkaEkEvIYQx12ht2pa8hoBo+4OB3qh7e+QiBlp1SRf+voWUZFxyhyqg==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/nice-linux-arm64-gnu@1.1.1': + resolution: {integrity: sha512-CIKLA12DTIZlmTaaKhQP88R3Xao+gyJxNWEn04wZwC2wmRapNnxCUZkVwggInMJvtVElA+D4ZzOU5sX4jV+SmQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/nice-linux-arm64-musl@1.1.1': + resolution: {integrity: sha512-+2Rzdb3nTIYZ0YJF43qf2twhqOCkiSrHx2Pg6DJaCPYhhaxbLcdlV8hCRMHghQ+EtZQWGNcS2xF4KxBhSGeutg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/nice-linux-ppc64-gnu@1.1.1': + resolution: {integrity: sha512-4FS8oc0GeHpwvv4tKciKkw3Y4jKsL7FRhaOeiPei0X9T4Jd619wHNe4xCLmN2EMgZoeGg+Q7GY7BsvwKpL22Tg==} + engines: {node: '>= 10'} + cpu: [ppc64] + os: [linux] + + '@napi-rs/nice-linux-riscv64-gnu@1.1.1': + resolution: {integrity: sha512-HU0nw9uD4FO/oGCCk409tCi5IzIZpH2agE6nN4fqpwVlCn5BOq0MS1dXGjXaG17JaAvrlpV5ZeyZwSon10XOXw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + + '@napi-rs/nice-linux-s390x-gnu@1.1.1': + resolution: {integrity: sha512-2YqKJWWl24EwrX0DzCQgPLKQBxYDdBxOHot1KWEq7aY2uYeX+Uvtv4I8xFVVygJDgf6/92h9N3Y43WPx8+PAgQ==} + engines: {node: '>= 10'} + cpu: [s390x] + os: [linux] + + '@napi-rs/nice-linux-x64-gnu@1.1.1': + resolution: {integrity: sha512-/gaNz3R92t+dcrfCw/96pDopcmec7oCcAQ3l/M+Zxr82KT4DljD37CpgrnXV+pJC263JkW572pdbP3hP+KjcIg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/nice-linux-x64-musl@1.1.1': + resolution: {integrity: sha512-xScCGnyj/oppsNPMnevsBe3pvNaoK7FGvMjT35riz9YdhB2WtTG47ZlbxtOLpjeO9SqqQ2J2igCmz6IJOD5JYw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/nice-openharmony-arm64@1.1.1': + resolution: {integrity: sha512-6uJPRVwVCLDeoOaNyeiW0gp2kFIM4r7PL2MczdZQHkFi9gVlgm+Vn+V6nTWRcu856mJ2WjYJiumEajfSm7arPQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [openharmony] + + '@napi-rs/nice-win32-arm64-msvc@1.1.1': + resolution: {integrity: sha512-uoTb4eAvM5B2aj/z8j+Nv8OttPf2m+HVx3UjA5jcFxASvNhQriyCQF1OB1lHL43ZhW+VwZlgvjmP5qF3+59atA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/nice-win32-ia32-msvc@1.1.1': + resolution: {integrity: sha512-CNQqlQT9MwuCsg1Vd/oKXiuH+TcsSPJmlAFc5frFyX/KkOh0UpBLEj7aoY656d5UKZQMQFP7vJNa1DNUNORvug==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/nice-win32-x64-msvc@1.1.1': + resolution: {integrity: sha512-vB+4G/jBQCAh0jelMTY3+kgFy00Hlx2f2/1zjMoH821IbplbWZOkLiTYXQkygNTzQJTq5cvwBDgn2ppHD+bglQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/nice@1.1.1': + resolution: {integrity: sha512-xJIPs+bYuc9ASBl+cvGsKbGrJmS6fAKaSZCnT0lhahT5rhA2VVy9/EcIgd2JhtEuFOJNx7UHNn/qiTPTY4nrQw==} + engines: {node: '>= 10'} + '@napi-rs/wasm-runtime@1.1.6': resolution: {integrity: sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==} peerDependencies: @@ -701,6 +947,9 @@ packages: '@types/node@25.9.4': resolution: {integrity: sha512-dszCsrKb5U7ZsVZBWiHFklTloVl0mSEnWH/iZXfZUlI4rzCUnsvGmgqfuVRHL54ugE7/wRuxEIXRa2iMZ+BG6g==} + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@typescript-eslint/eslint-plugin@8.62.0': resolution: {integrity: sha512-o+mpz7EYiMzXoySXiKmzlabIvTVqUuK5yLrAedRPRDA0IpPFMUV1IXt6OqljIxX/kumN6EjUYp41Hqelh6p/Dw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -917,6 +1166,10 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} + cluster-key-slot@1.1.1: + resolution: {integrity: sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==} + engines: {node: '>=0.10.0'} + combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -985,6 +1238,10 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -1012,6 +1269,9 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + effect@4.0.0-beta.93: + resolution: {integrity: sha512-wNS5MKFa3C42uBfIDik2oJ78lhpoYz2hN4oBR0229BeeDCIrkg/FiOvoiPGdCVlWa7MEKxEL5I0f8AILVHSD9A==} + encodeurl@2.0.0: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} @@ -1130,6 +1390,10 @@ packages: extendable-error@0.1.7: resolution: {integrity: sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==} + fast-check@4.8.0: + resolution: {integrity: sha512-GOJ158CUMnN6cSahsv4+ExARvIDuzzinFjkp0E9WtiBa5zcVeLozVkWaE4IzFcc+Y48Wp1EDlUZsXRyAztQcSg==} + engines: {node: '>=12.17.0'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -1170,6 +1434,9 @@ packages: resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} engines: {node: '>= 18.0.0'} + find-my-way-ts@0.1.6: + resolution: {integrity: sha512-a85L9ZoXtNAey3Y6Z+eBWW658kO/MwR7zIafkIUPUMf3isZG0NCs2pjW2wtjxAKuJPxMAsHUIP4ZPGv0o5gyTA==} + find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} @@ -1309,6 +1576,14 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + ini@7.0.0: + resolution: {integrity: sha512-ifK0CgjALofS5bkrcTy4RaQ9Vx2Knf/eLeIO+NaswQEpH1UblrtTSCIvN71qQDMq0PeQ/SSPojvEJp9vvvfr+w==} + engines: {node: ^22.22.2 || ^24.15.0 || >=26.0.0} + + ioredis@5.11.1: + resolution: {integrity: sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==} + engines: {node: '>=12.22.0'} + ip-address@10.2.0: resolution: {integrity: sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==} engines: {node: '>= 12'} @@ -1400,6 +1675,9 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + kubernetes-types@1.30.0: + resolution: {integrity: sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==} + levn@0.4.1: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} @@ -1545,6 +1823,11 @@ packages: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} + mime@4.1.0: + resolution: {integrity: sha512-X5ju04+cAzsojXKes0B/S4tcYtFAJ6tTMuSPBEn9CPGlrWr8Fiw7qYeLT0XyH80HSoAoqWCaz+MWKh22P7G1cw==} + engines: {node: '>=16'} + hasBin: true + minimatch@10.2.5: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} @@ -1559,6 +1842,16 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + msgpackr-extract@3.0.4: + resolution: {integrity: sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==} + hasBin: true + + msgpackr@2.0.4: + resolution: {integrity: sha512-o1C5KRmuRt+apqMr1HuGSqWStZoRBUpEsCsl15uM9VdAF1qHLtvMOU2En747EnTyEl6c4pzPewRMFF31s1CNbA==} + + multipasta@0.2.7: + resolution: {integrity: sha512-KPA58d68KgGil15oDqXjkUBEBYc00XvbPj5/X+dyzeo/lWm9Nc25pQRlf1D+gv4OpK7NM0J1odrbu9JNNGvynA==} + mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} @@ -1583,6 +1876,10 @@ packages: encoding: optional: true + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -1665,6 +1962,10 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pdfjs-dist@6.1.200: + resolution: {integrity: sha512-o8MolyzirkkLrcdsae/HEOiIcXWI7DS5zGpvqW8xTC2YUsW30rltFw2bDGvw/fskUdEMrQm2br68jzDS5BH2vw==} + engines: {node: '>=22.13.0 || >=24'} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -1684,6 +1985,10 @@ packages: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} + piscina@5.2.0: + resolution: {integrity: sha512-DszUCKeVN/5G5QKo6jAVHL8fmKnkJvQ0ACiVgY7YGCq3TUB2oznAOayvZPIAdEThvhczkXR+qm3IHsNXpFCYfA==} + engines: {node: '>=20.x'} + pkce-challenge@5.0.1: resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} engines: {node: '>=16.20.0'} @@ -1742,6 +2047,9 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + pure-rand@8.4.1: + resolution: {integrity: sha512-c58R2+SPFcSIPXoU834QN/KPDDOSd8sXcSrqf6e83Me6Rrp1EYkxukkjXMVrKvKaADs1SOyNkWdfvLf6zY8qLQ==} + qs@6.15.2: resolution: {integrity: sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==} engines: {node: '>=0.6'} @@ -1771,6 +2079,14 @@ packages: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + require-from-string@2.0.2: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} @@ -1877,6 +2193,9 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + statuses@2.0.2: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} @@ -1941,6 +2260,10 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + toml@4.1.2: + resolution: {integrity: sha512-m0vXfHODcw3gk+KONAOlVQ5yNHc3yS3B1ybM3HS1vqDoS0RWTDDVBVVTYi8hH0k+2OM1vmo9fb1WX9EVqjqfHA==} + engines: {node: '>=20'} + tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} @@ -2005,6 +2328,10 @@ packages: undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici@8.6.0: + resolution: {integrity: sha512-l2FlC6I510GawyEd1qgcE/okihKrzy+BRTEBlu6T0fdbM9m5yxtIH5Oa3ysRsH0zC4EhmWUEaSDsy2QngBeRlw==} + engines: {node: '>=22.19.0'} + universalify@0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} @@ -2019,6 +2346,10 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + uuid@14.0.1: + resolution: {integrity: sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==} + hasBin: true + vary@1.1.2: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} @@ -2130,6 +2461,23 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -2319,6 +2667,31 @@ snapshots: human-id: 4.2.0 prettier: 2.8.8 + '@effect/platform-node-shared@4.0.0-beta.93(effect@4.0.0-beta.93)': + dependencies: + '@types/ws': 8.18.1 + effect: 4.0.0-beta.93 + ws: 8.21.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@effect/platform-node@4.0.0-beta.93(effect@4.0.0-beta.93)(ioredis@5.11.1)': + dependencies: + '@effect/platform-node-shared': 4.0.0-beta.93(effect@4.0.0-beta.93) + effect: 4.0.0-beta.93 + ioredis: 5.11.1 + mime: 4.1.0 + undici: 8.6.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@effect/vitest@4.0.0-beta.93(effect@4.0.0-beta.93)(vitest@4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)))': + dependencies: + effect: 4.0.0-beta.93 + vitest: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)) + '@emnapi/core@1.11.1': dependencies: '@emnapi/wasi-threads': 1.2.2 @@ -2474,6 +2847,8 @@ snapshots: optionalDependencies: '@types/node': 25.9.4 + '@ioredis/commands@1.10.0': {} + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -2526,6 +2901,143 @@ snapshots: transitivePeerDependencies: - supports-color + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': + optional: true + + '@napi-rs/canvas-android-arm64@1.0.2': + optional: true + + '@napi-rs/canvas-darwin-arm64@1.0.2': + optional: true + + '@napi-rs/canvas-darwin-x64@1.0.2': + optional: true + + '@napi-rs/canvas-linux-arm-gnueabihf@1.0.2': + optional: true + + '@napi-rs/canvas-linux-arm64-gnu@1.0.2': + optional: true + + '@napi-rs/canvas-linux-arm64-musl@1.0.2': + optional: true + + '@napi-rs/canvas-linux-riscv64-gnu@1.0.2': + optional: true + + '@napi-rs/canvas-linux-x64-gnu@1.0.2': + optional: true + + '@napi-rs/canvas-linux-x64-musl@1.0.2': + optional: true + + '@napi-rs/canvas-win32-arm64-msvc@1.0.2': + optional: true + + '@napi-rs/canvas-win32-x64-msvc@1.0.2': + optional: true + + '@napi-rs/canvas@1.0.2': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 1.0.2 + '@napi-rs/canvas-darwin-arm64': 1.0.2 + '@napi-rs/canvas-darwin-x64': 1.0.2 + '@napi-rs/canvas-linux-arm-gnueabihf': 1.0.2 + '@napi-rs/canvas-linux-arm64-gnu': 1.0.2 + '@napi-rs/canvas-linux-arm64-musl': 1.0.2 + '@napi-rs/canvas-linux-riscv64-gnu': 1.0.2 + '@napi-rs/canvas-linux-x64-gnu': 1.0.2 + '@napi-rs/canvas-linux-x64-musl': 1.0.2 + '@napi-rs/canvas-win32-arm64-msvc': 1.0.2 + '@napi-rs/canvas-win32-x64-msvc': 1.0.2 + + '@napi-rs/nice-android-arm-eabi@1.1.1': + optional: true + + '@napi-rs/nice-android-arm64@1.1.1': + optional: true + + '@napi-rs/nice-darwin-arm64@1.1.1': + optional: true + + '@napi-rs/nice-darwin-x64@1.1.1': + optional: true + + '@napi-rs/nice-freebsd-x64@1.1.1': + optional: true + + '@napi-rs/nice-linux-arm-gnueabihf@1.1.1': + optional: true + + '@napi-rs/nice-linux-arm64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-arm64-musl@1.1.1': + optional: true + + '@napi-rs/nice-linux-ppc64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-riscv64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-s390x-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-x64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-x64-musl@1.1.1': + optional: true + + '@napi-rs/nice-openharmony-arm64@1.1.1': + optional: true + + '@napi-rs/nice-win32-arm64-msvc@1.1.1': + optional: true + + '@napi-rs/nice-win32-ia32-msvc@1.1.1': + optional: true + + '@napi-rs/nice-win32-x64-msvc@1.1.1': + optional: true + + '@napi-rs/nice@1.1.1': + optionalDependencies: + '@napi-rs/nice-android-arm-eabi': 1.1.1 + '@napi-rs/nice-android-arm64': 1.1.1 + '@napi-rs/nice-darwin-arm64': 1.1.1 + '@napi-rs/nice-darwin-x64': 1.1.1 + '@napi-rs/nice-freebsd-x64': 1.1.1 + '@napi-rs/nice-linux-arm-gnueabihf': 1.1.1 + '@napi-rs/nice-linux-arm64-gnu': 1.1.1 + '@napi-rs/nice-linux-arm64-musl': 1.1.1 + '@napi-rs/nice-linux-ppc64-gnu': 1.1.1 + '@napi-rs/nice-linux-riscv64-gnu': 1.1.1 + '@napi-rs/nice-linux-s390x-gnu': 1.1.1 + '@napi-rs/nice-linux-x64-gnu': 1.1.1 + '@napi-rs/nice-linux-x64-musl': 1.1.1 + '@napi-rs/nice-openharmony-arm64': 1.1.1 + '@napi-rs/nice-win32-arm64-msvc': 1.1.1 + '@napi-rs/nice-win32-ia32-msvc': 1.1.1 + '@napi-rs/nice-win32-x64-msvc': 1.1.1 + optional: true + '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)': dependencies: '@emnapi/core': 1.11.1 @@ -2699,6 +3211,10 @@ snapshots: dependencies: undici-types: 7.24.6 + '@types/ws@8.18.1': + dependencies: + '@types/node': 25.9.4 + '@typescript-eslint/eslint-plugin@8.62.0(@typescript-eslint/parser@8.62.0(eslint@10.5.0)(typescript@5.9.3))(eslint@10.5.0)(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -2802,7 +3318,7 @@ snapshots: obug: 2.1.3 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)) + vitest: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)) '@vitest/expect@4.1.9': dependencies: @@ -2813,13 +3329,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.9(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7))': + '@vitest/mocker@4.1.9(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.9 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.1.0(@types/node@25.9.4)(esbuild@0.27.7) + vite: 8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0) '@vitest/pretty-format@4.1.9': dependencies: @@ -2969,6 +3485,8 @@ snapshots: dependencies: readdirp: 4.1.2 + cluster-key-slot@1.1.1: {} + combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 @@ -3014,6 +3532,8 @@ snapshots: delayed-stream@1.0.0: {} + denque@2.1.0: {} + depd@2.0.0: {} detect-indent@6.1.0: {} @@ -3034,6 +3554,19 @@ snapshots: ee-first@1.1.1: {} + effect@4.0.0-beta.93: + dependencies: + '@standard-schema/spec': 1.1.0 + fast-check: 4.8.0 + find-my-way-ts: 0.1.6 + ini: 7.0.0 + kubernetes-types: 1.30.0 + msgpackr: 2.0.4 + multipasta: 0.2.7 + toml: 4.1.2 + uuid: 14.0.1 + yaml: 2.9.0 + encodeurl@2.0.0: {} enquirer@2.4.1: @@ -3211,6 +3744,10 @@ snapshots: extendable-error@0.1.7: {} + fast-check@4.8.0: + dependencies: + pure-rand: 8.4.1 + fast-deep-equal@3.1.3: {} fast-glob@3.3.3: @@ -3254,6 +3791,8 @@ snapshots: transitivePeerDependencies: - supports-color + find-my-way-ts@0.1.6: {} + find-up@4.1.0: dependencies: locate-path: 5.0.0 @@ -3394,6 +3933,20 @@ snapshots: inherits@2.0.4: {} + ini@7.0.0: {} + + ioredis@5.11.1: + dependencies: + '@ioredis/commands': 1.10.0 + cluster-key-slot: 1.1.1 + debug: 4.4.3 + denque: 2.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + ip-address@10.2.0: {} ipaddr.js@1.9.1: {} @@ -3471,6 +4024,8 @@ snapshots: dependencies: json-buffer: 3.0.1 + kubernetes-types@1.30.0: {} + levn@0.4.1: dependencies: prelude-ls: 1.2.1 @@ -3584,6 +4139,8 @@ snapshots: dependencies: mime-db: 1.54.0 + mime@4.1.0: {} + minimatch@10.2.5: dependencies: brace-expansion: 5.0.6 @@ -3599,6 +4156,24 @@ snapshots: ms@2.1.3: {} + msgpackr-extract@3.0.4: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.4 + optional: true + + msgpackr@2.0.4: + optionalDependencies: + msgpackr-extract: 3.0.4 + + multipasta@0.2.7: {} + mz@2.7.0: dependencies: any-promise: 1.3.0 @@ -3615,6 +4190,11 @@ snapshots: dependencies: whatwg-url: 5.0.0 + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.1.2 + optional: true + object-assign@4.1.1: {} object-inspect@1.13.4: {} @@ -3682,6 +4262,10 @@ snapshots: pathe@2.0.3: {} + pdfjs-dist@6.1.200: + optionalDependencies: + '@napi-rs/canvas': 1.0.2 + picocolors@1.1.1: {} picomatch@2.3.2: {} @@ -3692,6 +4276,10 @@ snapshots: pirates@4.0.7: {} + piscina@5.2.0: + optionalDependencies: + '@napi-rs/nice': 1.1.1 + pkce-challenge@5.0.1: {} pkg-types@1.3.1: @@ -3700,11 +4288,12 @@ snapshots: mlly: 1.8.2 pathe: 2.0.3 - postcss-load-config@6.0.1(postcss@8.5.15): + postcss-load-config@6.0.1(postcss@8.5.15)(yaml@2.9.0): dependencies: lilconfig: 3.1.3 optionalDependencies: postcss: 8.5.15 + yaml: 2.9.0 postcss@8.5.15: dependencies: @@ -3729,6 +4318,8 @@ snapshots: punycode@2.3.1: {} + pure-rand@8.4.1: {} + qs@6.15.2: dependencies: side-channel: 1.1.1 @@ -3765,6 +4356,12 @@ snapshots: readdirp@4.1.2: {} + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + require-from-string@2.0.2: {} resolve-from@5.0.0: {} @@ -3925,6 +4522,8 @@ snapshots: stackback@0.0.2: {} + standard-as-callback@2.1.0: {} + statuses@2.0.2: {} std-env@4.1.0: {} @@ -3982,6 +4581,8 @@ snapshots: toidentifier@1.0.1: {} + toml@4.1.2: {} + tr46@0.0.3: {} tree-kill@1.2.2: {} @@ -3995,7 +4596,7 @@ snapshots: tslib@2.8.1: optional: true - tsup@8.5.1(postcss@8.5.15)(typescript@5.9.3): + tsup@8.5.1(postcss@8.5.15)(typescript@5.9.3)(yaml@2.9.0): dependencies: bundle-require: 5.1.0(esbuild@0.27.7) cac: 6.7.14 @@ -4006,7 +4607,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.5.15) + postcss-load-config: 6.0.1(postcss@8.5.15)(yaml@2.9.0) resolve-from: 5.0.0 rollup: 4.62.2 source-map: 0.7.6 @@ -4050,6 +4651,8 @@ snapshots: undici-types@7.24.6: {} + undici@8.6.0: {} + universalify@0.1.2: {} unpipe@1.0.0: {} @@ -4060,9 +4663,11 @@ snapshots: util-deprecate@1.0.2: {} + uuid@14.0.1: {} + vary@1.1.2: {} - vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7): + vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -4073,11 +4678,12 @@ snapshots: '@types/node': 25.9.4 esbuild: 0.27.7 fsevents: 2.3.3 + yaml: 2.9.0 - vitest@4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)): + vitest@4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.9 - '@vitest/mocker': 4.1.9(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)) + '@vitest/mocker': 4.1.9(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.9 '@vitest/runner': 4.1.9 '@vitest/snapshot': 4.1.9 @@ -4094,7 +4700,7 @@ snapshots: tinyexec: 1.2.4 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vite: 8.1.0(@types/node@25.9.4)(esbuild@0.27.7) + vite: 8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 25.9.4 @@ -4122,6 +4728,10 @@ snapshots: wrappy@1.0.2: {} + ws@8.21.0: {} + + yaml@2.9.0: {} + yocto-queue@0.1.0: {} zod-to-json-schema@3.25.2(zod@3.25.76): diff --git a/scripts/prepare-effect.sh b/scripts/prepare-effect.sh new file mode 100755 index 0000000..24a9d31 --- /dev/null +++ b/scripts/prepare-effect.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +set -eu + +repo_dir=".repos/effect" +repo_url="https://github.com/Effect-TS/effect-smol" + +if [ -d "$repo_dir/.git" ]; then + exit 0 +fi + +mkdir -p ".repos" +git clone "$repo_url" "$repo_dir" diff --git a/src/client.ts b/src/client.ts index 7b21b0e..f863f2b 100644 --- a/src/client.ts +++ b/src/client.ts @@ -13,6 +13,7 @@ import { Knowledge } from './knowledge/index.js'; import { DEFAULT_BASE_URL, ENV } from './constants.js'; import { ValidationError } from './errors/index.js'; import { enrichParseResult } from './lib/utils.js'; +import { loadPageCitationAssetWorkflow } from './page-citation-assets/workflow-loader.js'; function inferFileName(file: ParseParams['file'], explicitFileName?: string): string | undefined { if (explicitFileName) { @@ -145,7 +146,22 @@ export class Knowhere { verifyChecksum: params.verifyChecksum, }); - return enrichParseResult(result, jobResult); + const enrichedResult = enrichParseResult(result, { + namespace: jobResult.namespace, + documentId: jobResult.documentId ?? params.documentId, + }); + + if (!params.pageCitationAssets) { + return enrichedResult; + } + + const workflow = await loadPageCitationAssetWorkflow(); + return workflow.enrichParseResultWithPageCitationAssets({ + result: enrichedResult, + options: params.pageCitationAssets, + documents: this.documents, + fallbackDocumentId: jobResult.documentId ?? params.documentId, + }); } /** diff --git a/src/errors/index.ts b/src/errors/index.ts index 24284fc..a63cb9d 100644 --- a/src/errors/index.ts +++ b/src/errors/index.ts @@ -6,3 +6,6 @@ export * from './api-errors.js'; // Job errors export * from './job-errors.js'; + +// Page citation asset errors +export * from './page-citation-assets.js'; diff --git a/src/errors/page-citation-assets.ts b/src/errors/page-citation-assets.ts new file mode 100644 index 0000000..6f64e7e --- /dev/null +++ b/src/errors/page-citation-assets.ts @@ -0,0 +1,12 @@ +import { KnowhereError } from './base.js'; +import type { PageCitationAssetWarning } from '../types/index.js'; + +export class PageCitationAssetGenerationError extends KnowhereError { + readonly warnings: readonly PageCitationAssetWarning[]; + + constructor(message: string, warnings: readonly PageCitationAssetWarning[]) { + super(message); + this.name = 'PageCitationAssetGenerationError'; + this.warnings = warnings; + } +} diff --git a/src/index.ts b/src/index.ts index ea39b30..21287f0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,7 @@ export type { DocumentChunkPagination, DocumentChunkResponse, DocumentChunkType, + DocumentPageCitationSource, DocumentListPagination, DocumentListParams, DocumentListResponse, @@ -49,6 +50,24 @@ export type { ParseParams, UploadProgress, PollProgress, + PageCitationAsset, + PageCitationAssetCurrentIndex, + PageCitationAssetIndex, + PageCitationAssetMimeType, + PageCitationAssetsOptions, + PageCitationAssetSource, + PageCitationAssetTimeoutOptions, + PageCitationAssetWarning, + PageCitationAssetWarningCode, + KnowhereSdkStorage, + KnowhereSdkStorageBody, + KnowhereSdkStorageHead, + KnowhereSdkStorageObject, + KnowhereSdkStorageReadResult, + KnowhereSdkStorageWriteResult, + PageRenderer, + RenderedPage, + RenderPageInput, // Result types ParseResult, Manifest, @@ -109,8 +128,14 @@ export { GatewayTimeoutError, // Job errors JobFailedError, + PageCitationAssetGenerationError, } from './errors/index.js'; // Resources (for advanced usage) export { Jobs, Retrieval, Documents } from './resources/index.js'; export { Knowledge, LocalKnowledgeStore } from './knowledge/index.js'; +export { + LocalKnowhereSdkStorage, + createLocalKnowhereSdkStorage, +} from './storage/local-storage.js'; +export type { LocalKnowhereSdkStorageOptions } from './storage/local-storage.js'; diff --git a/src/knowledge/__tests__/knowledge.test.ts b/src/knowledge/__tests__/knowledge.test.ts index c589116..255dee5 100644 --- a/src/knowledge/__tests__/knowledge.test.ts +++ b/src/knowledge/__tests__/knowledge.test.ts @@ -6,8 +6,16 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; import { Knowledge } from '../knowledge.js'; import type { Knowhere } from '../../client.js'; import type { + Chunk, DocumentChunkListResponse, + KnowhereSdkStorage, + KnowhereSdkStorageHead, + KnowhereSdkStorageObject, + KnowhereSdkStorageWriteResult, + PageRenderer, ParseResult, + RenderedPage, + RenderPageInput, TextChunk, TableChunk, } from '../../types/index.js'; @@ -17,6 +25,7 @@ describe('Knowledge', () => { afterEach(async () => { vi.restoreAllMocks(); + vi.unstubAllGlobals(); await Promise.all( tempDirectories.map((directory) => rm(directory, { recursive: true, force: true })), ); @@ -427,6 +436,57 @@ describe('Knowledge', () => { expect(savedTable).toBe('
Revenue
'); }); + it('should persist page assets from parse results into local read chunks', async () => { + const knowledge = await createKnowledgeWithCachedResult(createPageParseResultWithAssets()); + + const read = await knowledge.readChunks({ + localDocumentId: 'local-report', + chunkType: 'page', + limit: 1, + }); + + expect(read.chunks[0]?.pageAssets?.[0]).toMatchObject({ + pageNum: 1, + key: 'page-assets/doc-1/page-1.png', + width: 120, + height: 240, + }); + }); + + it('should enrich and cache completed job results with page assets', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = createPageParseResult(); + const { client, jobsLoad, documentsGetPageCitationSource } = createClient(parseResult); + const knowledge = new Knowledge(client, { cacheDirectory }); + const storage = new MemoryStorage(); + const renderer = new FakeRenderer(); + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(new Response(new Uint8Array([37, 80, 68, 70]))), + ); + + const cached = await knowledge.cacheJobResult({ + jobId: 'job-1', + localDocumentId: 'local-report', + pageCitationAssets: { storage, renderer }, + }); + const read = await knowledge.readChunks({ + localDocumentId: 'local-report', + chunkType: 'page', + limit: 1, + }); + + expect(jobsLoad).toHaveBeenCalledWith('job-1', { verifyChecksum: undefined }); + expect(documentsGetPageCitationSource).toHaveBeenCalledWith('doc-1'); + expect(renderer.renderedPageNums).toEqual([1]); + expect(cached.pageCitationAssetWarnings).toBeUndefined(); + expect(read.chunks[0]?.pageAssets?.[0]).toMatchObject({ + pageNum: 1, + width: 101, + height: 201, + }); + }); + async function createKnowledgeWithCachedResult( parseResult = createParseResult(), ): Promise { @@ -454,6 +514,7 @@ function createClient(parseResult: ParseResult): { jobsGet: ReturnType; jobsLoad: ReturnType; documentsListChunks: ReturnType; + documentsGetPageCitationSource: ReturnType; retrievalQuery: ReturnType; } { const parse = vi.fn().mockResolvedValue(parseResult); @@ -507,6 +568,17 @@ function createClient(parseResult: ParseResult): { }, }), ); + const documentsGetPageCitationSource = vi.fn().mockResolvedValue({ + documentId: 'doc-1', + namespace: 'support-center', + jobId: 'job-1', + jobResultId: 'jres-1', + variant: 'normalized_pdf', + fileName: 'report.pdf', + contentType: 'application/pdf', + url: 'https://assets.example/source.pdf', + expiresAt: new Date('2026-01-01T00:00:00.000Z'), + }); const retrievalQuery = vi.fn().mockResolvedValue({ namespace: 'support-center', query: 'margin', @@ -545,6 +617,7 @@ function createClient(parseResult: ParseResult): { }, documents: { listChunks: documentsListChunks, + getPageCitationSource: documentsGetPageCitationSource, }, retrieval: { query: retrievalQuery, @@ -555,10 +628,63 @@ function createClient(parseResult: ParseResult): { jobsGet, jobsLoad, documentsListChunks, + documentsGetPageCitationSource, retrievalQuery, }; } +class MemoryStorage implements KnowhereSdkStorage { + private readonly objects = new Map< + string, + { + readonly body: Uint8Array; + readonly contentType?: string; + readonly metadata?: Readonly>; + } + >(); + + headObject(key: string): Promise { + const object = this.objects.get(key); + return Promise.resolve( + object + ? { + key, + contentType: object.contentType, + contentLength: object.body.byteLength, + metadata: object.metadata, + } + : null, + ); + } + + writeObject(input: KnowhereSdkStorageObject): Promise { + this.objects.set(input.key, { + body: input.body instanceof Uint8Array ? input.body : new Uint8Array(), + contentType: input.contentType, + metadata: input.metadata, + }); + return Promise.resolve({ key: input.key, url: `memory://${input.key}` }); + } + + getObjectUrl(key: string): Promise { + return Promise.resolve(this.objects.has(key) ? `memory://${key}` : null); + } +} + +class FakeRenderer implements PageRenderer { + readonly renderedPageNums: number[] = []; + + renderPage(input: RenderPageInput): Promise { + this.renderedPageNums.push(input.pageNum); + return Promise.resolve({ + body: new Uint8Array([input.pageNum]), + mimeType: input.format, + width: 100 + input.pageNum, + height: 200 + input.pageNum, + }); + } +} + async function expectFileExists(filePath: string): Promise { await expect(access(filePath)).resolves.toBeUndefined(); } @@ -675,3 +801,78 @@ function createParseResultWithFullPaths(): ParseResult { }; return result; } + +function createPageParseResult(chunks: Chunk[] = [createPageChunk()]): ParseResult { + const pageChunks = chunks.filter((chunk): chunk is Extract => chunk.type === 'page'); + + return { + manifest: { + version: '2.0', + jobId: 'job-1', + sourceFileName: 'report.pdf', + statistics: { + totalChunks: chunks.length, + textChunks: chunks.filter((chunk) => chunk.type === 'text').length, + imageChunks: chunks.filter((chunk) => chunk.type === 'image').length, + tableChunks: chunks.filter((chunk) => chunk.type === 'table').length, + pageChunks: pageChunks.length, + }, + }, + chunks, + rawZip: Buffer.from('not-used-in-tests'), + namespace: 'support-center', + documentId: 'doc-1', + textChunks: chunks.filter((chunk): chunk is TextChunk => chunk.type === 'text'), + imageChunks: [], + tableChunks: [], + pageChunks, + jobId: 'job-1', + statistics: { + totalChunks: chunks.length, + textChunks: chunks.filter((chunk) => chunk.type === 'text').length, + imageChunks: chunks.filter((chunk) => chunk.type === 'image').length, + tableChunks: chunks.filter((chunk) => chunk.type === 'table').length, + pageChunks: pageChunks.length, + }, + getChunk: (chunkId: string) => chunks.find((chunk) => chunk.chunkId === chunkId), + save: vi.fn(), + }; +} + +function createPageChunk(): Chunk { + return { + chunkId: 'page-1', + type: 'page', + content: 'Page one summary.', + contentSource: 'summary', + path: 'report.pdf/Page 1', + metadata: { summary: 'Page one summary.', pageNums: [1] }, + }; +} + +function createPageParseResultWithAssets(): ParseResult { + const chunks: Chunk[] = [ + { + chunkId: 'page-1', + type: 'page', + content: 'Page one summary.', + contentSource: 'summary', + path: 'report.pdf/Page 1', + metadata: { summary: 'Page one summary.', pageNums: [1] }, + pageAssets: [ + { + pageNum: 1, + key: 'page-assets/doc-1/page-1.png', + assetUrl: 'https://assets.example/page-1.png', + mimeType: 'image/png', + width: 120, + height: 240, + source: 'client-rendered-pdf-page', + variant: 'default', + }, + ], + }, + ]; + + return createPageParseResult(chunks); +} diff --git a/src/knowledge/knowledge.ts b/src/knowledge/knowledge.ts index edecc3b..35acce6 100644 --- a/src/knowledge/knowledge.ts +++ b/src/knowledge/knowledge.ts @@ -10,6 +10,7 @@ import type { } from '../types/index.js'; import { ValidationError } from '../errors/index.js'; import { LocalKnowledgeStore } from './local-store.js'; +import { loadPageCitationAssetWorkflow } from '../page-citation-assets/workflow-loader.js'; import type { IndexedKnowledgeChunk, KnowledgeAsyncCacheResult, @@ -64,7 +65,11 @@ export class Knowledge { const document = await this.store.saveResult(result, { localDocumentId: params.localDocumentId, }); - return { document, result }; + return { + document, + result, + pageCitationAssetWarnings: result.pageCitationAssetWarnings, + }; } async startParse(params: KnowledgeAsyncParseParams): Promise { @@ -102,13 +107,20 @@ export class Knowledge { async cacheJobResult( params: KnowledgeCacheJobResultParams, ): Promise { - const result = await this.client.jobs.load(params.jobId, { + const loadedResult = await this.client.jobs.load(params.jobId, { verifyChecksum: params.verifyChecksum, }); + const result = params.pageCitationAssets + ? await this.enrichResultWithPageCitationAssets(loadedResult, params.pageCitationAssets) + : loadedResult; const document = await this.store.saveResult(result, { localDocumentId: params.localDocumentId, }); - return { document, result }; + return { + document, + result, + pageCitationAssetWarnings: result.pageCitationAssetWarnings, + }; } async cacheDocument(params: KnowledgeCacheDocumentParams): Promise { @@ -323,6 +335,19 @@ export class Knowledge { return documents.filter((document) => requested.has(document.localDocumentId)); } + private async enrichResultWithPageCitationAssets( + result: ParseResult, + options: NonNullable, + ): Promise { + const workflow = await loadPageCitationAssetWorkflow(); + return workflow.enrichParseResultWithPageCitationAssets({ + result, + options, + documents: this.client.documents, + fallbackDocumentId: result.documentId, + }); + } + private async loadReadableResult(reference: string | KnowledgeDocumentReference): Promise<{ document: LocalKnowledgeDocument; result: ParseResult; @@ -425,6 +450,7 @@ function indexChunks(result: ParseResult): IndexedKnowledgeChunk[] { sourceChunkPath: chunk.path, filePath, pageNumbers: getChunkPageNumbers(chunk.metadata), + pageAssets: chunk.type === 'page' ? chunk.pageAssets : undefined, metadata: chunk.metadata, }; }); @@ -440,13 +466,11 @@ function getReadableChunkContent(chunk: Chunk): string { } function getChunkPageNumbers(metadata: Record): number[] | undefined { - const value = metadata.pageNums; - if (!Array.isArray(value)) { - return undefined; - } - - const pageNumbers = value.filter( - (pageNumber): pageNumber is number => Number.isInteger(pageNumber) && pageNumber > 0, + const values = [metadata.pageNums, metadata.page_nums].filter(Array.isArray); + const pageNumbers = values.flatMap((value) => + value.filter( + (pageNumber): pageNumber is number => Number.isInteger(pageNumber) && pageNumber > 0, + ), ); return pageNumbers.length > 0 ? [...new Set(pageNumbers)].sort((left, right) => left - right) @@ -667,6 +691,7 @@ function toReadChunk(chunk: IndexedKnowledgeChunk): KnowledgeReadChunk { sourceChunkPath: chunk.sourceChunkPath, filePath: chunk.filePath, pageNumbers: chunk.pageNumbers, + pageAssets: chunk.pageAssets, metadata: chunk.metadata, }; } diff --git a/src/knowledge/types.ts b/src/knowledge/types.ts index 607be57..caa6be5 100644 --- a/src/knowledge/types.ts +++ b/src/knowledge/types.ts @@ -1,6 +1,12 @@ import type { ParseParams } from '../types/params.js'; import type { Job, JobResult } from '../types/job.js'; -import type { Chunk, DocumentChunkType, ParseResult } from '../types/index.js'; +import type { + Chunk, + DocumentChunkType, + PageCitationAsset, + PageCitationAssetWarning, + ParseResult, +} from '../types/index.js'; export type KnowledgeChunkType = DocumentChunkType; @@ -35,6 +41,7 @@ export interface LocalKnowledgeDocument { export interface LocalKnowledgeParseResponse { document: LocalKnowledgeDocument; result: ParseResult; + pageCitationAssetWarnings?: readonly PageCitationAssetWarning[]; } export interface KnowledgeAsyncParseParams extends ParseParams { @@ -76,6 +83,7 @@ export interface KnowledgeCacheJobResultParams { jobId: string; localDocumentId?: string; verifyChecksum?: boolean; + pageCitationAssets?: ParseParams['pageCitationAssets']; } export interface KnowledgeCacheDocumentParams { @@ -136,6 +144,7 @@ export interface KnowledgeReadChunk { sourceChunkPath: string; filePath?: string; pageNumbers?: number[]; + pageAssets?: readonly PageCitationAsset[]; metadata: Record; } @@ -223,5 +232,6 @@ export interface IndexedKnowledgeChunk { sourceChunkPath: string; filePath?: string; pageNumbers?: number[]; + pageAssets?: readonly PageCitationAsset[]; metadata: Record; } diff --git a/src/lib/result-parser.ts b/src/lib/result-parser.ts index 2628f39..58a41df 100644 --- a/src/lib/result-parser.ts +++ b/src/lib/result-parser.ts @@ -15,6 +15,7 @@ import type { SlimChunk, DocNav, } from '../types/result.js'; +import type { PageCitationAsset } from '../types/page-citation-assets.js'; import type { LoadOptions } from '../types/params.js'; import { ChecksumError, KnowhereError } from '../errors/index.js'; import { sanitizePath, getFileExtension, parseDates, keysToCamel } from './utils.js'; @@ -27,6 +28,7 @@ type RawChunk = { path?: string; filePath?: string; metadata?: Record; + pageAssets?: readonly PageCitationAsset[]; }; type ChunkPayload = RawChunk[] | { chunks?: RawChunk[] }; @@ -223,7 +225,7 @@ export async function saveExpandedParseResult( result: ParseResult, directory: string, ): Promise { - if (result.rawZip.length > 0) { + if (result.rawZip.length > 0 && !hasEnrichedPageAssets(result)) { const didExtractZip = await tryExtractRawZip(result.rawZip, directory); if (didExtractZip) { return directory; @@ -475,6 +477,7 @@ function buildPageChunk(chunkData: RawChunk): PageChunk { content: chunkData.content ?? '', path: chunkData.path ?? '', metadata: chunkData.metadata ?? {}, + pageAssets: chunkData.pageAssets, }; } @@ -622,6 +625,7 @@ function serializeChunks(chunks: Chunk[]): { chunks: RawChunk[] } { content: chunk.content, path: chunk.path, metadata: chunk.metadata, + pageAssets: chunk.type === 'page' ? chunk.pageAssets : undefined, }; if (chunk.type === 'image' || chunk.type === 'table') { @@ -633,6 +637,10 @@ function serializeChunks(chunks: Chunk[]): { chunks: RawChunk[] } { }; } +function hasEnrichedPageAssets(result: ParseResult): boolean { + return result.chunks.some((chunk) => chunk.type === 'page' && chunk.pageAssets !== undefined); +} + async function readRequiredTextFile(directory: string, fileName: string): Promise { try { return await fs.readFile(join(directory, fileName), 'utf8'); diff --git a/src/page-citation-assets/__fixtures__/tiny.pdf b/src/page-citation-assets/__fixtures__/tiny.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e74ed808d1ef5d65186bf981975a7bac93ba0754 GIT binary patch literal 429 zcmZXQPfx=z48`yH6z|d_r0GZjY3hMLy8vP;+@>7@^-9r5P?|}DPmhxhI@BnKg}=w(-R1&__N{)0r!`$0}$3yx^Ov*}zbbvsfOorrz!RE8An=Rdv)Q z0>54E$+qE*1$N&DPv%3!DGtxWJbO { + it('renders a tiny PDF fixture page to PNG', async () => { + const renderer = createPdfJsPageRenderer(); + const source = await readFile( + path.join(process.cwd(), 'src', 'page-citation-assets', '__fixtures__', 'tiny.pdf'), + ); + + const rendered = await renderer.renderPage({ + source, + pageNum: 1, + format: 'image/png', + scale: 1, + }); + + expect(rendered.mimeType).toBe('image/png'); + expect(rendered.width).toBeGreaterThan(0); + expect(rendered.height).toBeGreaterThan(0); + expect(Buffer.from(rendered.body).subarray(0, 8).toString('hex')).toBe('89504e470d0a1a0a'); + }); +}); diff --git a/src/page-citation-assets/__tests__/workflow.test.mts b/src/page-citation-assets/__tests__/workflow.test.mts new file mode 100644 index 0000000..e2a677a --- /dev/null +++ b/src/page-citation-assets/__tests__/workflow.test.mts @@ -0,0 +1,498 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; + +import { PageCitationAssetGenerationError } from '../../errors/index.js'; +import { enrichParseResultWithPageCitationAssets } from '../workflow.mjs'; +import type { + Chunk, + DocumentPageCitationSource, + KnowhereSdkStorage, + KnowhereSdkStorageHead, + KnowhereSdkStorageObject, + KnowhereSdkStorageReadResult, + KnowhereSdkStorageWriteResult, + PageRenderer, + ParseResult, + RenderPageInput, + RenderedPage, +} from '../../types/index.js'; + +describe('page citation asset workflow', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('renders and attaches assets for page chunks only', async () => { + const storage = new MemoryStorage(); + const renderer = new FakeRenderer(); + const documents = createDocumentsClient(); + stubSourceFetch(); + + const result = await enrichParseResultWithPageCitationAssets({ + result: createParseResult([ + createPageChunk('page-1', [1]), + createTextChunk('text-1'), + createImageChunk('image-1'), + createTableChunk('table-1'), + createPageChunk('page-2', [2]), + ]), + documents, + options: { storage, renderer }, + }); + + expect(renderer.renderedPageNums).toEqual([1, 2]); + expect(getPageChunks(result)[0]?.pageAssets?.[0]).toMatchObject({ + pageNum: 1, + mimeType: 'image/png', + width: 101, + height: 201, + }); + expect(result.textChunks[0]).not.toHaveProperty('pageAssets'); + expect(result.imageChunks[0]).not.toHaveProperty('pageAssets'); + expect(result.tableChunks[0]).not.toHaveProperty('pageAssets'); + expect(storage.getJsonObjects().some((entry) => entry.key.endsWith('/index.json'))).toBe(true); + }); + + it('attaches cached assets without rendering or fetching the source', async () => { + const storage = new MemoryStorage(); + const renderer = new FakeRenderer(); + const result = createParseResult([createPageChunk('page-1', [1])]); + const key = expectedAssetKey(result, 1); + storage.seedObject(key, { + contentType: 'image/png', + metadata: { + pageNum: '1', + width: '300', + height: '400', + mimeType: 'image/png', + }, + }); + const documents = createDocumentsClient(); + const fetchMock = stubSourceFetch(); + + const enriched = await enrichParseResultWithPageCitationAssets({ + result, + documents, + options: { storage, renderer }, + }); + + expect(renderer.renderedPageNums).toEqual([]); + expect(documents.calls).toEqual([]); + expect(fetchMock).not.toHaveBeenCalled(); + expect(getPageChunks(enriched)[0]?.pageAssets?.[0]).toMatchObject({ + pageNum: 1, + width: 300, + height: 400, + }); + }); + + it('returns warnings and preserves parse success for partial render failures', async () => { + const storage = new MemoryStorage(); + const renderer = new FakeRenderer({ failedPageNums: [2] }); + stubSourceFetch(); + + const result = await enrichParseResultWithPageCitationAssets({ + result: createParseResult([createPageChunk('page-1', [1]), createPageChunk('page-2', [2])]), + documents: createDocumentsClient(), + options: { storage, renderer }, + }); + + expect(getPageChunks(result)[0]?.pageAssets).toHaveLength(1); + expect(getPageChunks(result)[1]?.pageAssets).toBeUndefined(); + expect(result.pageCitationAssetWarnings?.map((warning) => warning.code)).toContain( + 'render_failed', + ); + }); + + it('returns warnings and preserves parse success for source failures', async () => { + const storage = new MemoryStorage(); + const renderer = new FakeRenderer(); + const documents = { + getPageCitationSource: vi.fn().mockRejectedValue(new Error('source missing')), + }; + + const result = await enrichParseResultWithPageCitationAssets({ + result: createParseResult([createPageChunk('page-1', [1])]), + documents, + options: { storage, renderer }, + }); + + expect(documents.getPageCitationSource).toHaveBeenCalledWith('doc-1'); + expect(renderer.renderedPageNums).toEqual([]); + expect(getPageChunks(result)[0]?.pageAssets).toBeUndefined(); + expect(result.pageCitationAssetWarnings?.[0]).toMatchObject({ + code: 'source_fetch_failed', + }); + }); + + it('returns warnings and preserves parse success for storage write failures', async () => { + const storage = new WriteFailingStorage(); + const renderer = new FakeRenderer(); + stubSourceFetch(); + + const result = await enrichParseResultWithPageCitationAssets({ + result: createParseResult([createPageChunk('page-1', [1])]), + documents: createDocumentsClient(), + options: { storage, renderer }, + }); + + expect(renderer.renderedPageNums).toEqual([1]); + expect(getPageChunks(result)[0]?.pageAssets).toBeUndefined(); + expect(result.pageCitationAssetWarnings?.[0]).toMatchObject({ + code: 'storage_failed', + }); + }); + + it('attaches written assets when storage URLs are unavailable', async () => { + const storage = new UrlFailingStorage(); + const renderer = new FakeRenderer(); + stubSourceFetch(); + + const result = await enrichParseResultWithPageCitationAssets({ + result: createParseResult([createPageChunk('page-1', [1])]), + documents: createDocumentsClient(), + options: { storage, renderer }, + }); + + expect(getPageChunks(result)[0]?.pageAssets?.[0]).toMatchObject({ + pageNum: 1, + assetUrl: undefined, + key: expectedAssetKey(result, 1), + }); + expect(result.pageCitationAssetWarnings).toBeUndefined(); + }); + + it('fails in strict mode when generation returns warnings', async () => { + const storage = new MemoryStorage(); + const renderer = new FakeRenderer({ failedPageNums: [1] }); + stubSourceFetch(); + + await expect( + enrichParseResultWithPageCitationAssets({ + result: createParseResult([createPageChunk('page-1', [1])]), + documents: createDocumentsClient(), + options: { storage, renderer, strict: true }, + }), + ).rejects.toBeInstanceOf(PageCitationAssetGenerationError); + }); + + it('limits new renders after cache checks and writes only concrete assets to the index', async () => { + const storage = new MemoryStorage(); + const renderer = new FakeRenderer(); + stubSourceFetch(); + + const result = await enrichParseResultWithPageCitationAssets({ + result: createParseResult([createPageChunk('page-1', [1]), createPageChunk('page-2', [2])]), + documents: createDocumentsClient(), + options: { storage, renderer, maxPagesToRenderPerRun: 1 }, + }); + const index = storage + .getJsonObjects() + .find((entry) => entry.key.endsWith('/index.json'))?.value; + + expect(renderer.renderedPageNums).toEqual([1]); + expect(result.pageCitationAssetWarnings?.map((warning) => warning.code)).toContain( + 'render_limit_exceeded', + ); + expect(index).toMatchObject({ + version: 1, + assets: [expect.objectContaining({ pageNum: 1 })], + }); + expect(JSON.stringify(index)).not.toContain('status'); + expect(JSON.stringify(index)).not.toContain('placeholder'); + }); + + it('warns when documentId is missing for page chunks', async () => { + const result = await enrichParseResultWithPageCitationAssets({ + result: createParseResult([createPageChunk('page-1', [1])], { documentId: undefined }), + documents: createDocumentsClient(), + options: { storage: new MemoryStorage(), renderer: new FakeRenderer() }, + }); + + expect(result.pageCitationAssetWarnings?.[0]).toMatchObject({ + code: 'missing_document_id', + }); + }); +}); + +class MemoryStorage implements KnowhereSdkStorage { + private readonly objects = new Map< + string, + { + readonly body: Uint8Array; + readonly contentType?: string; + readonly metadata?: Readonly>; + } + >(); + + headObject(key: string): Promise { + const object = this.objects.get(key); + if (!object) { + return Promise.resolve(null); + } + return Promise.resolve({ + key, + contentType: object.contentType, + contentLength: object.body.byteLength, + metadata: object.metadata, + }); + } + + async writeObject(input: KnowhereSdkStorageObject): Promise { + this.objects.set(input.key, { + body: await readBody(input.body), + contentType: input.contentType, + metadata: input.metadata, + }); + return { + key: input.key, + url: `memory://${input.key}`, + }; + } + + getObjectUrl(key: string): Promise { + return Promise.resolve(this.objects.has(key) ? `memory://${key}` : null); + } + + readObject(key: string): Promise { + const object = this.objects.get(key); + if (!object) { + return Promise.resolve(null); + } + return Promise.resolve({ + body: object.body, + contentType: object.contentType, + metadata: object.metadata, + }); + } + + seedObject( + key: string, + options: { contentType?: string; metadata?: Readonly> }, + ): void { + this.objects.set(key, { + body: new Uint8Array([1]), + contentType: options.contentType, + metadata: options.metadata, + }); + } + + getJsonObjects(): Array<{ key: string; value: unknown }> { + return [...this.objects.entries()] + .filter((entry) => entry[1].contentType === 'application/json') + .map(([key, object]) => { + const value: unknown = JSON.parse(Buffer.from(object.body).toString('utf8')); + return { key, value }; + }); + } +} + +class WriteFailingStorage extends MemoryStorage { + writeObject(): Promise { + return Promise.reject(new Error('write failed')); + } +} + +class UrlFailingStorage extends MemoryStorage { + async writeObject(input: KnowhereSdkStorageObject): Promise { + await super.writeObject(input); + return { key: input.key }; + } + + getObjectUrl(): Promise { + return Promise.reject(new Error('url failed')); + } +} + +class FakeRenderer implements PageRenderer { + readonly renderedPageNums: number[] = []; + private readonly failedPageNums: ReadonlySet; + + constructor(options: { failedPageNums?: readonly number[] } = {}) { + this.failedPageNums = new Set(options.failedPageNums ?? []); + } + + renderPage(input: RenderPageInput): Promise { + this.renderedPageNums.push(input.pageNum); + if (this.failedPageNums.has(input.pageNum)) { + throw new Error(`failed page ${input.pageNum}`); + } + return Promise.resolve({ + body: new Uint8Array([input.pageNum]), + mimeType: input.format, + width: 100 + input.pageNum, + height: 200 + input.pageNum, + }); + } +} + +function createDocumentsClient(): { + calls: string[]; + getPageCitationSource(documentId: string): Promise; +} { + const calls: string[] = []; + return { + calls, + getPageCitationSource(documentId: string): Promise { + calls.push(documentId); + return Promise.resolve({ + documentId: 'doc-1', + namespace: 'support-center', + jobId: 'job-1', + jobResultId: 'jres-1', + variant: 'normalized_pdf', + fileName: 'report.pdf', + contentType: 'application/pdf', + url: 'https://assets.example/source.pdf', + expiresAt: new Date('2026-01-01T00:00:00.000Z'), + }); + }, + }; +} + +function stubSourceFetch(): ReturnType { + const fetchMock = vi.fn().mockResolvedValue( + new Response(new Uint8Array([37, 80, 68, 70])), + ); + vi.stubGlobal('fetch', fetchMock); + return fetchMock; +} + +function createParseResult( + chunks: Chunk[], + options: { documentId?: string } = {}, +): ParseResult { + const textChunks = chunks.filter( + (chunk): chunk is Extract => chunk.type === 'text', + ); + const imageChunks = chunks.filter( + (chunk): chunk is Extract => chunk.type === 'image', + ); + const tableChunks = chunks.filter( + (chunk): chunk is Extract => chunk.type === 'table', + ); + const pageChunks = chunks.filter( + (chunk): chunk is Extract => chunk.type === 'page', + ); + + return { + manifest: { + version: '2.0', + jobId: 'job-1', + sourceFileName: 'report.pdf', + statistics: { + totalChunks: chunks.length, + textChunks: textChunks.length, + imageChunks: imageChunks.length, + tableChunks: tableChunks.length, + pageChunks: pageChunks.length, + }, + }, + chunks, + rawZip: Buffer.alloc(0), + namespace: 'support-center', + documentId: 'documentId' in options ? options.documentId : 'doc-1', + textChunks, + imageChunks, + tableChunks, + pageChunks, + jobId: 'job-1', + statistics: { + totalChunks: chunks.length, + textChunks: textChunks.length, + imageChunks: imageChunks.length, + tableChunks: tableChunks.length, + pageChunks: pageChunks.length, + }, + getChunk: (chunkId: string) => chunks.find((chunk) => chunk.chunkId === chunkId), + save: vi.fn(), + }; +} + +function getPageChunks(result: ParseResult): Array> { + return result.chunks.filter( + (chunk): chunk is Extract => chunk.type === 'page', + ); +} + +function createPageChunk(chunkId: string, pageNums: readonly number[]): Chunk { + return { + chunkId, + type: 'page', + content: `Summary ${chunkId}`, + contentSource: 'summary', + path: `report.pdf/${chunkId}`, + metadata: { pageNums: [...pageNums] }, + }; +} + +function createTextChunk(chunkId: string): Chunk { + return { + chunkId, + type: 'text', + content: `Text ${chunkId}`, + contentSource: 'content', + path: `report.pdf/${chunkId}`, + metadata: {}, + }; +} + +function createImageChunk(chunkId: string): Chunk { + return { + chunkId, + type: 'image', + content: `Image ${chunkId}`, + contentSource: 'caption', + path: `report.pdf/${chunkId}`, + filePath: `images/${chunkId}.png`, + data: Buffer.from([1]), + metadata: { pageNums: [99] }, + format: 'png', + save: vi.fn(), + }; +} + +function createTableChunk(chunkId: string): Chunk { + return { + chunkId, + type: 'table', + content: `Table ${chunkId}`, + contentSource: 'summary', + path: `report.pdf/${chunkId}`, + filePath: `tables/${chunkId}.html`, + html: '
', + metadata: { page_nums: [100] }, + save: vi.fn(), + }; +} + +function expectedAssetKey(result: ParseResult, pageNum: number): string { + return [ + 'page-citation-assets', + 'documents', + result.documentId, + 'jobs', + result.jobId, + 'variants', + 'default', + 'scale-1', + `page-${String(pageNum).padStart(6, '0')}.png`, + ].join('/'); +} + +async function readBody(body: Uint8Array | ReadableStream | Blob): Promise { + if (body instanceof Uint8Array) { + return body; + } + if (body instanceof Blob) { + return new Uint8Array(await body.arrayBuffer()); + } + const reader = body.getReader(); + const chunks: Uint8Array[] = []; + while (true) { + const result = await reader.read(); + if (result.done) { + return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); + } + chunks.push(result.value); + } +} diff --git a/src/page-citation-assets/workflow-loader.ts b/src/page-citation-assets/workflow-loader.ts new file mode 100644 index 0000000..ab6dd07 --- /dev/null +++ b/src/page-citation-assets/workflow-loader.ts @@ -0,0 +1,62 @@ +import type { + DocumentPageCitationSource, + PageCitationAssetsOptions, + ParseResult, +} from '../types/index.js'; + +type PageCitationWorkflowInput = { + result: ParseResult; + options: PageCitationAssetsOptions; + documents: { + getPageCitationSource(documentId: string): Promise; + }; + fallbackDocumentId?: string; +}; + +type PageCitationWorkflowModule = { + enrichParseResultWithPageCitationAssets(input: PageCitationWorkflowInput): Promise; +}; + +export async function loadPageCitationAssetWorkflow(): Promise { + const sourceLayoutPath: string = './workflow.mjs'; + const bundledLayoutPath: string = './page-citation-assets/workflow.mjs'; + + try { + return await importWorkflowModule(sourceLayoutPath); + } catch (error) { + if (!isModuleNotFoundError(error)) { + throw error; + } + return importWorkflowModule(bundledLayoutPath); + } +} + +async function importWorkflowModule(modulePath: string): Promise { + const importedModule: unknown = await import(modulePath); + if (isPageCitationWorkflowModule(importedModule)) { + return importedModule; + } + + throw new Error('Page citation asset workflow module did not expose the expected API'); +} + +function isPageCitationWorkflowModule(value: unknown): value is PageCitationWorkflowModule { + return ( + typeof value === 'object' && + value !== null && + 'enrichParseResultWithPageCitationAssets' in value && + typeof value.enrichParseResultWithPageCitationAssets === 'function' + ); +} + +function isModuleNotFoundError(error: unknown): boolean { + if (!(error instanceof Error)) { + return false; + } + + return ( + error.message.includes('Cannot find module') || + error.message.includes('ERR_MODULE_NOT_FOUND') || + error.message.includes('Failed to resolve') + ); +} diff --git a/src/page-citation-assets/workflow.mts b/src/page-citation-assets/workflow.mts new file mode 100644 index 0000000..23734c3 --- /dev/null +++ b/src/page-citation-assets/workflow.mts @@ -0,0 +1,951 @@ +import { createHash } from 'crypto'; + +import { Context, Data, Effect, Layer } from 'effect'; + +import { PageCitationAssetGenerationError } from '../errors/index.js'; +import { createLocalKnowhereSdkStorage } from '../storage/local-storage.js'; +import type { + Chunk, + DocumentPageCitationSource, + KnowhereSdkStorage, + KnowhereSdkStorageHead, + KnowhereSdkStorageObject, + KnowhereSdkStorageWriteResult, + PageCitationAsset, + PageCitationAssetCurrentIndex, + PageCitationAssetIndex, + PageCitationAssetMimeType, + PageCitationAssetSource, + PageCitationAssetsOptions, + PageCitationAssetWarning, + PageCitationAssetWarningCode, + PageRenderer, + ParseResult, + RenderedPage, + RenderPageInput, +} from '../types/index.js'; + +const DEFAULT_PAGE_CITATION_ASSET_LIMITS = { + maxPagesToRenderPerRun: 25, + totalMs: 120_000, + sourceFetchMs: 30_000, + pageRenderMs: 30_000, + storageOperationMs: 15_000, +} as const; + +const DEFAULT_VARIANT = 'default'; +const DEFAULT_FORMAT: PageCitationAssetMimeType = 'image/png'; +const DEFAULT_SCALE = 1; +const PAGE_ASSET_SOURCE: PageCitationAssetSource = 'client-rendered-pdf-page'; + +export type DocumentPageCitationSourceClient = { + getPageCitationSource(documentId: string): Promise; +}; + +export interface PageCitationWorkflowInput { + result: ParseResult; + options: PageCitationAssetsOptions; + documents: DocumentPageCitationSourceClient; + fallbackDocumentId?: string; +} + +interface PageCitationWorkflowOutput { + result: ParseResult; + warnings: readonly PageCitationAssetWarning[]; +} + +interface NormalizedPageCitationOptions { + storage: KnowhereSdkStorage; + renderer?: PageRenderer; + variant: string; + format: PageCitationAssetMimeType; + scale: number; + quality?: number; + strict: boolean; + maxPagesToRenderPerRun: number; + totalMs: number; + sourceFetchMs: number; + pageRenderMs: number; + storageOperationMs: number; +} + +interface PageRequest { + pageNum: number; + key: string; + chunkIds: readonly string[]; +} + +interface PageAssetContext { + documentId: string; + namespace?: string; + jobId: string; + jobResultId?: string; + variant: string; + format: PageCitationAssetMimeType; + scale: number; +} + +interface SourceFile { + source: DocumentPageCitationSource; + body: Uint8Array; +} + +type PageCitationSourceServiceShape = { + readonly getSource: ( + documentId: string, + ) => Effect.Effect; +}; + +type PageCitationRendererServiceShape = { + readonly renderPage: ( + input: RenderPageInput, + ) => Effect.Effect; +}; + +type PageCitationStorageServiceShape = { + readonly headObject: ( + key: string, + ) => Effect.Effect; + readonly writeObject: ( + input: KnowhereSdkStorageObject, + ) => Effect.Effect; + readonly getObjectUrl: ( + key: string, + ) => Effect.Effect; +}; + +class PageCitationSourceError extends Data.TaggedError('PageCitationSourceError')<{ + readonly operation: 'resolve_source' | 'fetch_source'; + readonly documentId: string; + readonly cause: unknown; +}> {} + +class PageCitationRenderError extends Data.TaggedError('PageCitationRenderError')<{ + readonly pageNum: number; + readonly cause: unknown; +}> {} + +class PageCitationStorageError extends Data.TaggedError('PageCitationStorageError')<{ + readonly operation: 'head' | 'write' | 'url'; + readonly key: string; + readonly cause: unknown; +}> {} + +class PageCitationRendererUnavailableError extends Data.TaggedError( + 'PageCitationRendererUnavailableError', +)<{ + readonly cause: unknown; +}> {} + +class PageCitationSourceService extends Context.Service< + PageCitationSourceService, + PageCitationSourceServiceShape +>()('Knowhere/PageCitationSourceService') {} + +class PageCitationRendererService extends Context.Service< + PageCitationRendererService, + PageCitationRendererServiceShape +>()('Knowhere/PageCitationRendererService') {} + +class PageCitationStorageService extends Context.Service< + PageCitationStorageService, + PageCitationStorageServiceShape +>()('Knowhere/PageCitationStorageService') {} + +export async function enrichParseResultWithPageCitationAssets( + input: PageCitationWorkflowInput, +): Promise { + const options = normalizeOptions(input.options); + const program = generatePageCitationAssets({ + ...input, + options, + }).pipe( + Effect.provide([ + createSourceLayer(input.documents, options.sourceFetchMs), + createRendererLayer(options), + createStorageLayer(options.storage, options.storageOperationMs), + ]), + ); + + const output = await runPromiseWithTimeout( + () => Effect.runPromise(program), + options.totalMs, + 'Page citation asset generation timed out', + ).catch((error: unknown): PageCitationWorkflowOutput => { + if (options.strict) { + throw error; + } + + return { + result: input.result, + warnings: [ + createWarning({ + code: 'render_failed', + message: 'Page citation asset generation timed out or failed before completing.', + documentId: input.result.documentId ?? input.fallbackDocumentId, + jobId: input.result.jobId, + cause: getErrorMessage(error), + }), + ], + }; + }); + + if (output.warnings.length > 0) { + output.result.pageCitationAssetWarnings = output.warnings; + } + + if (options.strict && output.warnings.length > 0) { + throw new PageCitationAssetGenerationError( + 'Page citation asset generation failed in strict mode.', + output.warnings, + ); + } + + return output.result; +} + +const generatePageCitationAssets = Effect.fn('Knowhere.generatePageCitationAssets')( + function*( + input: Omit & { + readonly options: NormalizedPageCitationOptions; + }, + ) { + const pageChunks = input.result.chunks.filter((chunk) => chunk.type === 'page'); + if (pageChunks.length === 0) { + return { result: input.result, warnings: [] }; + } + + const documentId = input.result.documentId ?? input.fallbackDocumentId; + if (!documentId) { + const warning = createWarning({ + code: 'missing_document_id', + message: 'documentId is required to generate page citation assets for page chunks.', + jobId: input.result.jobId, + }); + return { result: input.result, warnings: [warning] }; + } + + const warnings: PageCitationAssetWarning[] = []; + const context: PageAssetContext = { + documentId, + namespace: input.result.namespace, + jobId: input.result.jobId, + variant: input.options.variant, + format: input.options.format, + scale: input.options.scale, + }; + const pageRequests = buildPageRequests(input.result.chunks, context, warnings); + + if (pageRequests.length === 0) { + return { result: input.result, warnings }; + } + + const storage = yield* PageCitationStorageService; + const assetByPageNum = new Map(); + const missingRequests: PageRequest[] = []; + + for (const request of pageRequests) { + const cachedAsset = yield* readCachedAsset(storage, request, context); + if (cachedAsset) { + assetByPageNum.set(request.pageNum, cachedAsset); + } else { + missingRequests.push(request); + } + } + + const renderableRequests = missingRequests.slice(0, input.options.maxPagesToRenderPerRun); + const skippedRequests = missingRequests.slice(input.options.maxPagesToRenderPerRun); + for (const skipped of skippedRequests) { + warnings.push( + createWarning({ + code: 'render_limit_exceeded', + message: `Page ${skipped.pageNum} was not rendered because maxPagesToRenderPerRun was reached.`, + documentId, + jobId: context.jobId, + pageNum: skipped.pageNum, + key: skipped.key, + }), + ); + } + + if (renderableRequests.length > 0) { + const sourceResult = yield* resolveSourceFile(documentId).pipe( + Effect.match({ + onFailure: (error) => { + warnings.push(sourceErrorToWarning(error, context)); + return undefined; + }, + onSuccess: (source) => source, + }), + ); + + if (sourceResult) { + context.namespace = sourceResult.source.namespace ?? context.namespace; + context.jobResultId = sourceResult.source.jobResultId; + yield* renderMissingAssets({ + sourceFile: sourceResult, + pageRequests: renderableRequests, + context, + options: input.options, + warnings, + assetByPageNum, + }); + } + } + + const enrichedChunks = attachAssetsToChunks(input.result.chunks, assetByPageNum); + input.result.chunks.splice(0, input.result.chunks.length, ...enrichedChunks); + + const concreteAssets = [...assetByPageNum.values()].sort( + (left, right) => left.pageNum - right.pageNum, + ); + if (concreteAssets.length > 0) { + yield* writeAssetIndexes(storage, context, concreteAssets).pipe( + Effect.match({ + onFailure: (error) => { + warnings.push(storageErrorToWarning(error, 'index_write_failed', context)); + return undefined; + }, + onSuccess: () => undefined, + }), + ); + } + + return { + result: input.result, + warnings, + }; + }, +); + +const resolveSourceFile = Effect.fn('Knowhere.resolvePageCitationSource')( + function*(documentId: string) { + const source = yield* PageCitationSourceService; + return yield* source.getSource(documentId); + }, +); + +const renderMissingAssets = Effect.fn('Knowhere.renderMissingPageCitationAssets')( + function*(input: { + readonly sourceFile: SourceFile; + readonly pageRequests: readonly PageRequest[]; + readonly context: PageAssetContext; + readonly options: NormalizedPageCitationOptions; + readonly warnings: PageCitationAssetWarning[]; + readonly assetByPageNum: Map; + }) { + const renderer = yield* PageCitationRendererService; + const storage = yield* PageCitationStorageService; + + for (const request of input.pageRequests) { + const rendered = yield* renderer + .renderPage({ + source: input.sourceFile.body, + pageNum: request.pageNum, + format: input.options.format, + scale: input.options.scale, + quality: input.options.quality, + }) + .pipe( + Effect.match({ + onFailure: (error) => { + input.warnings.push(renderErrorToWarning(error, request, input.context)); + return undefined; + }, + onSuccess: (page) => page, + }), + ); + + if (!rendered) { + continue; + } + + const asset = yield* writeRenderedPage(storage, request, rendered, input.context).pipe( + Effect.match({ + onFailure: (error) => { + input.warnings.push(storageErrorToWarning(error, 'storage_failed', input.context)); + return undefined; + }, + onSuccess: (writtenAsset) => writtenAsset, + }), + ); + + if (asset) { + input.assetByPageNum.set(request.pageNum, asset); + } + } + }, +); + +function createSourceLayer( + documents: DocumentPageCitationSourceClient, + sourceFetchTimeoutMs: number, +): Layer.Layer { + return Layer.succeed(PageCitationSourceService, { + getSource: Effect.fn('PageCitationSourceService.getSource')(function*(documentId: string) { + const source = yield* Effect.tryPromise({ + try: () => + runPromiseWithTimeout( + () => documents.getPageCitationSource(documentId), + sourceFetchTimeoutMs, + `Timed out resolving page citation source for document ${documentId}`, + ), + catch: (cause) => + new PageCitationSourceError({ + operation: 'resolve_source', + documentId, + cause, + }), + }); + + const body = yield* Effect.tryPromise({ + try: () => fetchSourceBytes(source.url, sourceFetchTimeoutMs), + catch: (cause) => + new PageCitationSourceError({ + operation: 'fetch_source', + documentId, + cause, + }), + }); + + return { source, body }; + }), + }); +} + +function createRendererLayer( + options: NormalizedPageCitationOptions, +): Layer.Layer { + let cachedRenderer: PageRenderer | undefined = options.renderer; + + return Layer.succeed(PageCitationRendererService, { + renderPage: Effect.fn('PageCitationRendererService.renderPage')( + function*(input: RenderPageInput) { + const renderer = yield* Effect.tryPromise({ + try: async () => { + cachedRenderer ??= await createDefaultPageRenderer(); + return cachedRenderer; + }, + catch: (cause) => new PageCitationRendererUnavailableError({ cause }), + }); + + return yield* Effect.tryPromise({ + try: () => + runPromiseWithTimeout( + () => renderer.renderPage(input), + options.pageRenderMs, + `Timed out rendering PDF page ${input.pageNum}`, + ), + catch: (cause) => + new PageCitationRenderError({ + pageNum: input.pageNum, + cause, + }), + }); + }, + ), + }); +} + +function createStorageLayer( + storage: KnowhereSdkStorage, + storageOperationTimeoutMs: number, +): Layer.Layer { + return Layer.succeed(PageCitationStorageService, { + headObject: Effect.fn('PageCitationStorageService.headObject')(function*(key: string) { + return yield* Effect.tryPromise({ + try: () => + runPromiseWithTimeout( + () => storage.headObject(key), + storageOperationTimeoutMs, + `Timed out reading storage metadata for ${key}`, + ), + catch: (cause) => + new PageCitationStorageError({ + operation: 'head', + key, + cause, + }), + }); + }), + writeObject: Effect.fn('PageCitationStorageService.writeObject')( + function*(input: KnowhereSdkStorageObject) { + return yield* Effect.tryPromise({ + try: () => + runPromiseWithTimeout( + () => storage.writeObject(input), + storageOperationTimeoutMs, + `Timed out writing storage object ${input.key}`, + ), + catch: (cause) => + new PageCitationStorageError({ + operation: 'write', + key: input.key, + cause, + }), + }); + }, + ), + getObjectUrl: Effect.fn('PageCitationStorageService.getObjectUrl')(function*(key: string) { + if (!storage.getObjectUrl) { + return undefined; + } + + return yield* Effect.tryPromise({ + try: () => + runPromiseWithTimeout( + () => storage.getObjectUrl?.(key) ?? Promise.resolve(null), + storageOperationTimeoutMs, + `Timed out resolving storage URL for ${key}`, + ), + catch: (cause) => + new PageCitationStorageError({ + operation: 'url', + key, + cause, + }), + }).pipe(Effect.map((url) => url ?? undefined)); + }), + }); +} + +function normalizeOptions(options: PageCitationAssetsOptions): NormalizedPageCitationOptions { + return { + storage: options.storage ?? createLocalKnowhereSdkStorage(), + renderer: options.renderer, + variant: options.variant ?? DEFAULT_VARIANT, + format: options.format ?? DEFAULT_FORMAT, + scale: options.scale ?? DEFAULT_SCALE, + quality: options.quality, + strict: options.strict ?? false, + maxPagesToRenderPerRun: + options.maxPagesToRenderPerRun ?? DEFAULT_PAGE_CITATION_ASSET_LIMITS.maxPagesToRenderPerRun, + totalMs: options.timeouts?.totalMs ?? DEFAULT_PAGE_CITATION_ASSET_LIMITS.totalMs, + sourceFetchMs: + options.timeouts?.sourceFetchMs ?? DEFAULT_PAGE_CITATION_ASSET_LIMITS.sourceFetchMs, + pageRenderMs: + options.timeouts?.pageRenderMs ?? DEFAULT_PAGE_CITATION_ASSET_LIMITS.pageRenderMs, + storageOperationMs: + options.timeouts?.storageOperationMs ?? + DEFAULT_PAGE_CITATION_ASSET_LIMITS.storageOperationMs, + }; +} + +function buildPageRequests( + chunks: readonly Chunk[], + context: PageAssetContext, + warnings: PageCitationAssetWarning[], +): PageRequest[] { + const chunkIdsByPageNum = new Map(); + + for (const chunk of chunks) { + if (chunk.type !== 'page') { + continue; + } + + const pageNumbers = getChunkPageNumbers(chunk.metadata); + if (pageNumbers.invalidCount > 0) { + warnings.push( + createWarning({ + code: 'invalid_page_number', + message: `${pageNumbers.invalidCount} invalid page number value(s) were ignored.`, + documentId: context.documentId, + jobId: context.jobId, + chunkId: chunk.chunkId, + }), + ); + } + + for (const pageNum of pageNumbers.validPageNumbers) { + const chunkIds = chunkIdsByPageNum.get(pageNum) ?? []; + chunkIds.push(chunk.chunkId); + chunkIdsByPageNum.set(pageNum, chunkIds); + } + } + + return [...chunkIdsByPageNum.entries()] + .sort(([left], [right]) => left - right) + .map(([pageNum, chunkIds]) => ({ + pageNum, + chunkIds, + key: createPageAssetKey(context, pageNum), + })); +} + +function getChunkPageNumbers(metadata: Record): { + validPageNumbers: readonly number[]; + invalidCount: number; +} { + const values = [metadata.pageNums, metadata.page_nums].filter(isReadonlyUnknownArray); + const validPageNumbers = new Set(); + let invalidCount = 0; + + for (const value of values) { + for (const pageNum of value) { + if (typeof pageNum === 'number' && Number.isInteger(pageNum) && pageNum > 0) { + validPageNumbers.add(pageNum); + } else { + invalidCount += 1; + } + } + } + + return { + validPageNumbers: [...validPageNumbers].sort((left, right) => left - right), + invalidCount, + }; +} + +function isReadonlyUnknownArray(value: unknown): value is readonly unknown[] { + return Array.isArray(value); +} + +const readCachedAsset = Effect.fn('Knowhere.readCachedPageCitationAsset')( + function*( + storage: PageCitationStorageServiceShape, + request: PageRequest, + context: PageAssetContext, + ) { + const head = yield* storage.headObject(request.key).pipe( + Effect.match({ + onFailure: () => null, + onSuccess: (metadata) => metadata, + }), + ); + + if (!head) { + return undefined; + } + + const metadata = head.metadata ?? {}; + const width = readPositiveInteger(metadata.width); + const height = readPositiveInteger(metadata.height); + const mimeType = normalizeMimeType(metadata.mimeType ?? head.contentType); + + if (!width || !height || !mimeType) { + return undefined; + } + + const assetUrl = yield* storage.getObjectUrl(request.key).pipe( + Effect.match({ + onFailure: () => undefined, + onSuccess: (url) => url, + }), + ); + + return { + pageNum: request.pageNum, + key: request.key, + assetUrl, + mimeType, + width, + height, + source: PAGE_ASSET_SOURCE, + variant: context.variant, + }; + }, +); + +const writeRenderedPage = Effect.fn('Knowhere.writeRenderedPageCitationAsset')( + function*( + storage: PageCitationStorageServiceShape, + request: PageRequest, + rendered: RenderedPage, + context: PageAssetContext, + ) { + const writeResult = yield* storage.writeObject({ + key: request.key, + body: rendered.body, + contentType: rendered.mimeType, + metadata: { + pageNum: String(request.pageNum), + width: String(rendered.width), + height: String(rendered.height), + mimeType: rendered.mimeType, + source: PAGE_ASSET_SOURCE, + variant: context.variant, + documentId: context.documentId, + jobId: context.jobId, + }, + }); + const assetUrl = + writeResult.url ?? + (yield* storage.getObjectUrl(request.key).pipe( + Effect.match({ + onFailure: () => undefined, + onSuccess: (url) => url, + }), + )); + + return { + pageNum: request.pageNum, + key: request.key, + assetUrl, + mimeType: rendered.mimeType, + width: rendered.width, + height: rendered.height, + source: PAGE_ASSET_SOURCE, + variant: context.variant, + }; + }, +); + +function writeAssetIndexes( + storage: PageCitationStorageServiceShape, + context: PageAssetContext, + assets: readonly PageCitationAsset[], +): Effect.Effect { + const generatedAt = new Date().toISOString(); + const indexKey = createPageAssetIndexKey(context); + const currentKey = createPageAssetCurrentIndexKey(context); + const index: PageCitationAssetIndex = { + version: 1, + documentId: context.documentId, + namespace: context.namespace, + jobId: context.jobId, + jobResultId: context.jobResultId, + variant: context.variant, + generatedAt, + assets, + }; + const current: PageCitationAssetCurrentIndex = { + version: 1, + documentId: context.documentId, + namespace: context.namespace, + jobId: context.jobId, + jobResultId: context.jobResultId, + variant: context.variant, + indexKey, + updatedAt: generatedAt, + }; + + return Effect.gen(function*() { + yield* storage.writeObject({ + key: indexKey, + body: Buffer.from(JSON.stringify(index, null, 2)), + contentType: 'application/json', + metadata: { + documentId: context.documentId, + jobId: context.jobId, + variant: context.variant, + }, + }); + yield* storage.writeObject({ + key: currentKey, + body: Buffer.from(JSON.stringify(current, null, 2)), + contentType: 'application/json', + metadata: { + documentId: context.documentId, + jobId: context.jobId, + variant: context.variant, + }, + }); + }); +} + +function attachAssetsToChunks( + chunks: readonly Chunk[], + assetByPageNum: ReadonlyMap, +): Chunk[] { + return chunks.map((chunk) => { + if (chunk.type !== 'page') { + return chunk; + } + + const assets = getChunkPageNumbers(chunk.metadata).validPageNumbers + .map((pageNum) => assetByPageNum.get(pageNum)) + .filter((asset): asset is PageCitationAsset => asset !== undefined); + + if (assets.length === 0) { + return chunk; + } + + return { + ...chunk, + pageAssets: assets, + }; + }); +} + +function createPageAssetKey(context: PageAssetContext, pageNum: number): string { + const extension = context.format === 'image/png' ? 'png' : 'jpg'; + return [ + 'page-citation-assets', + 'documents', + toSafeKeySegment(context.documentId), + 'jobs', + toSafeKeySegment(context.jobId), + 'variants', + toSafeKeySegment(context.variant), + `scale-${toSafeKeySegment(String(context.scale))}`, + `page-${String(pageNum).padStart(6, '0')}.${extension}`, + ].join('/'); +} + +function createPageAssetIndexKey(context: PageAssetContext): string { + return [ + 'page-citation-assets', + 'documents', + toSafeKeySegment(context.documentId), + 'jobs', + toSafeKeySegment(context.jobId), + 'variants', + toSafeKeySegment(context.variant), + 'index.json', + ].join('/'); +} + +function createPageAssetCurrentIndexKey(context: PageAssetContext): string { + return [ + 'page-citation-assets', + 'documents', + toSafeKeySegment(context.documentId), + 'current.json', + ].join('/'); +} + +function toSafeKeySegment(value: string): string { + const normalized = value.replace(/[^A-Za-z0-9._-]+/g, '_').replace(/^_+|_+$/g, ''); + if (normalized.length > 0 && normalized === value) { + return normalized; + } + + const hash = createHash('sha256').update(value).digest('hex').slice(0, 12); + return `${(normalized || 'value').slice(0, 48)}-${hash}`; +} + +async function createDefaultPageRenderer(): Promise { + const rendererModule = await import('../page-renderer-pdfjs.js'); + return rendererModule.createPdfJsPageRenderer(); +} + +async function fetchSourceBytes(url: string, timeoutMs: number): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { signal: controller.signal }); + if (!response.ok) { + throw new Error(`Source fetch failed with HTTP ${response.status}`); + } + return new Uint8Array(await response.arrayBuffer()); + } finally { + clearTimeout(timeout); + } +} + +function createWarning(params: { + code: PageCitationAssetWarningCode; + message: string; + documentId?: string; + jobId?: string; + chunkId?: string; + pageNum?: number; + key?: string; + cause?: string; +}): PageCitationAssetWarning { + return { + code: params.code, + message: params.message, + documentId: params.documentId, + jobId: params.jobId, + chunkId: params.chunkId, + pageNum: params.pageNum, + key: params.key, + cause: params.cause, + }; +} + +function sourceErrorToWarning( + error: PageCitationSourceError, + context: PageAssetContext, +): PageCitationAssetWarning { + return createWarning({ + code: 'source_fetch_failed', + message: `Unable to ${error.operation === 'resolve_source' ? 'resolve' : 'fetch'} page citation source.`, + documentId: context.documentId, + jobId: context.jobId, + cause: getErrorMessage(error.cause), + }); +} + +function renderErrorToWarning( + error: PageCitationRenderError | PageCitationRendererUnavailableError, + request: PageRequest, + context: PageAssetContext, +): PageCitationAssetWarning { + if (error._tag === 'PageCitationRendererUnavailableError') { + return createWarning({ + code: 'renderer_unavailable', + message: 'The page citation renderer could not be loaded.', + documentId: context.documentId, + jobId: context.jobId, + pageNum: request.pageNum, + key: request.key, + cause: getErrorMessage(error.cause), + }); + } + + return createWarning({ + code: 'render_failed', + message: `Unable to render PDF page ${request.pageNum}.`, + documentId: context.documentId, + jobId: context.jobId, + pageNum: request.pageNum, + key: request.key, + cause: getErrorMessage(error.cause), + }); +} + +function storageErrorToWarning( + error: PageCitationStorageError, + code: 'storage_failed' | 'index_write_failed', + context: PageAssetContext, +): PageCitationAssetWarning { + return createWarning({ + code, + message: + code === 'index_write_failed' + ? 'Unable to write the page citation asset index.' + : `Unable to ${error.operation} page citation asset storage object.`, + documentId: context.documentId, + jobId: context.jobId, + key: error.key, + cause: getErrorMessage(error.cause), + }); +} + +function readPositiveInteger(value: string | undefined): number | undefined { + if (value === undefined) { + return undefined; + } + + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; +} + +function normalizeMimeType(value: string | undefined): PageCitationAssetMimeType | undefined { + if (value === 'image/png' || value === 'image/jpeg') { + return value; + } + return undefined; +} + +function getErrorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + return String(error); +} + +function runPromiseWithTimeout( + promiseFactory: () => Promise, + timeoutMs: number, + timeoutMessage: string, +): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error(timeoutMessage)), timeoutMs); + void promiseFactory() + .then(resolve, reject) + .finally(() => clearTimeout(timeout)); + }); +} diff --git a/src/page-renderer-pdfjs.ts b/src/page-renderer-pdfjs.ts new file mode 100644 index 0000000..90183f1 --- /dev/null +++ b/src/page-renderer-pdfjs.ts @@ -0,0 +1,141 @@ +import { createCanvas, DOMMatrix, ImageData, Path2D } from '@napi-rs/canvas'; + +import type { PageRenderer, RenderedPage, RenderPageInput } from './types/index.js'; + +interface PdfJsModule { + getDocument(params: PdfDocumentInitParameters): PdfLoadingTask; +} + +interface PdfDocumentInitParameters { + data: Uint8Array; + useSystemFonts: boolean; +} + +interface PdfLoadingTask { + promise: Promise; + destroy(): Promise; +} + +interface PdfDocument { + getPage(pageNum: number): Promise; + cleanup(): Promise; +} + +interface PdfPage { + getViewport(params: { scale: number }): PdfViewport; + render(params: PdfRenderParameters): PdfRenderTask; + cleanup(): void; +} + +interface PdfViewport { + width: number; + height: number; +} + +interface PdfRenderParameters { + canvas: null; + canvasContext: object; + viewport: PdfViewport; + background: string; +} + +interface PdfRenderTask { + promise: Promise; +} + +export interface PdfJsPageRendererOptions { + background?: string; +} + +export function createPdfJsPageRenderer(options: PdfJsPageRendererOptions = {}): PageRenderer { + const background = options.background ?? 'rgb(255,255,255)'; + + return { + async renderPage(input: RenderPageInput): Promise { + throwIfAborted(input.signal); + const pdfjs = await loadPdfJsModule(); + const loadingTask = pdfjs.getDocument({ + data: new Uint8Array(input.source), + useSystemFonts: true, + }); + const document = await loadingTask.promise; + + try { + throwIfAborted(input.signal); + const page = await document.getPage(input.pageNum); + const viewport = page.getViewport({ scale: input.scale }); + const width = Math.ceil(viewport.width); + const height = Math.ceil(viewport.height); + const canvas = createCanvas(width, height); + const context = canvas.getContext('2d'); + + throwIfAborted(input.signal); + await page.render({ + canvas: null, + canvasContext: context, + viewport, + background, + }).promise; + page.cleanup(); + + const body = + input.format === 'image/png' + ? canvas.toBuffer('image/png') + : canvas.toBuffer('image/jpeg', input.quality); + + return { + body, + mimeType: input.format, + width, + height, + }; + } finally { + await document.cleanup(); + await loadingTask.destroy(); + } + }, + }; +} + +async function loadPdfJsModule(): Promise { + installPdfJsDomPolyfills(); + const importedModule: unknown = await import('pdfjs-dist/legacy/build/pdf.mjs'); + if (isPdfJsModule(importedModule)) { + return importedModule; + } + + throw new Error('pdfjs-dist/build/pdf.mjs did not expose getDocument'); +} + +function installPdfJsDomPolyfills(): void { + installGlobalValue('DOMMatrix', DOMMatrix); + installGlobalValue('ImageData', ImageData); + installGlobalValue('Path2D', Path2D); +} + +function installGlobalValue(name: string, value: unknown): void { + if (name in globalThis) { + return; + } + + Object.defineProperty(globalThis, name, { + value, + configurable: true, + writable: true, + }); +} + +function isPdfJsModule(value: unknown): value is PdfJsModule { + return ( + typeof value === 'object' && + value !== null && + 'getDocument' in value && + typeof value.getDocument === 'function' + ); +} + +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) { + throw new Error('PDF page render aborted'); + } +} diff --git a/src/resources/__tests__/documents.test.ts b/src/resources/__tests__/documents.test.ts index 026c540..3669ed0 100644 --- a/src/resources/__tests__/documents.test.ts +++ b/src/resources/__tests__/documents.test.ts @@ -234,6 +234,39 @@ describe('Documents Resource', () => { expect(response.chunk.assetUrl).toBe('https://assets.example/figure-1.png'); }); + it('should get the page citation source through the canonical v2 route', async () => { + mockHttpClient.get.mockResolvedValue({ + documentId: 'doc-123', + namespace: 'support-center', + jobId: 'job-123', + jobResultId: 'jres-123', + variant: 'normalized_pdf', + fileName: 'report.pdf', + contentType: 'application/pdf', + url: 'https://assets.example/report.pdf', + expiresAt: new Date('2026-01-01T00:00:00.000Z'), + }); + + const source = await documents.getPageCitationSource('doc-123'); + + expect(mockHttpClient.get).toHaveBeenCalledWith( + '/v2/documents/doc-123/files/page-citation-source', + ); + expect(source).toMatchObject({ + documentId: 'doc-123', + jobResultId: 'jres-123', + contentType: 'application/pdf', + url: 'https://assets.example/report.pdf', + }); + }); + + it('should propagate page citation source API errors', async () => { + const error = new Error('not found'); + mockHttpClient.get.mockRejectedValue(error); + + await expect(documents.getPageCitationSource('doc-404')).rejects.toBe(error); + }); + it('should archive using the canonical route', async () => { mockHttpClient.post.mockResolvedValue({ documentId: 'doc-123', diff --git a/src/resources/documents.ts b/src/resources/documents.ts index 88b6960..ffc8cc3 100644 --- a/src/resources/documents.ts +++ b/src/resources/documents.ts @@ -5,6 +5,7 @@ import type { DocumentChunkListParams, DocumentChunkListResponse, DocumentChunkResponse, + DocumentPageCitationSource, DocumentListParams, DocumentListResponse, } from '../types/document.js'; @@ -65,6 +66,15 @@ export class Documents extends BaseResource { ); } + /** + * Get a signed source file URL suitable for SDK-side page citation rendering. + */ + async getPageCitationSource(documentId: string): Promise { + return this.httpClient.get( + this.endpoint(`/documents/${documentId}/files/page-citation-source`), + ); + } + /** * Archive one canonical document by ID. */ diff --git a/src/storage/__tests__/local-storage.test.ts b/src/storage/__tests__/local-storage.test.ts new file mode 100644 index 0000000..65612b9 --- /dev/null +++ b/src/storage/__tests__/local-storage.test.ts @@ -0,0 +1,81 @@ +import { mkdtemp, readFile, rm } from 'fs/promises'; +import { tmpdir } from 'os'; +import path from 'path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { LocalKnowhereSdkStorage } from '../local-storage.js'; + +describe('LocalKnowhereSdkStorage', () => { + const tempDirectories: string[] = []; + + afterEach(async () => { + await Promise.all( + tempDirectories.map((directory) => rm(directory, { recursive: true, force: true })), + ); + tempDirectories.length = 0; + }); + + it('writes objects with metadata sidecars and optional public URLs', async () => { + const rootDirectory = await createTempDirectory(tempDirectories); + const storage = new LocalKnowhereSdkStorage({ + rootDirectory, + publicBaseUrl: 'https://cdn.example/assets/', + }); + + const write = await storage.writeObject({ + key: 'documents/doc-1/page.png', + body: new TextEncoder().encode('image'), + contentType: 'image/png', + metadata: { width: '120', height: '240' }, + }); + const head = await storage.headObject('documents/doc-1/page.png'); + const read = await storage.readObject('documents/doc-1/page.png'); + const sidecar = await readFile( + path.join(rootDirectory, 'documents', 'doc-1', 'page.png.metadata.json'), + 'utf8', + ); + + expect(write.url).toBe('https://cdn.example/assets/documents/doc-1/page.png'); + expect(head).toMatchObject({ + contentType: 'image/png', + metadata: { width: '120', height: '240' }, + }); + expect(Buffer.from(read?.body ?? new Uint8Array()).toString('utf8')).toBe('image'); + expect(JSON.parse(sidecar)).toMatchObject({ + contentType: 'image/png', + metadata: { width: '120', height: '240' }, + }); + }); + + it('rejects unsafe keys', async () => { + const rootDirectory = await createTempDirectory(tempDirectories); + const storage = new LocalKnowhereSdkStorage({ rootDirectory }); + + await expect( + storage.writeObject({ + key: '../escape.txt', + body: new Uint8Array([1]), + }), + ).rejects.toThrow(/relative POSIX path/); + await expect(storage.headObject('/absolute.txt')).rejects.toThrow(/relative POSIX path/); + }); + + it('deletes prefixes inside the storage root', async () => { + const rootDirectory = await createTempDirectory(tempDirectories); + const storage = new LocalKnowhereSdkStorage({ rootDirectory }); + + await storage.writeObject({ + key: 'documents/doc-1/page.png', + body: new Uint8Array([1]), + }); + await storage.deletePrefix('documents/doc-1'); + + expect(await storage.headObject('documents/doc-1/page.png')).toBeNull(); + }); +}); + +async function createTempDirectory(tempDirectories: string[]): Promise { + const directory = await mkdtemp(path.join(tmpdir(), 'knowhere-storage-')); + tempDirectories.push(directory); + return directory; +} diff --git a/src/storage/local-storage.ts b/src/storage/local-storage.ts new file mode 100644 index 0000000..dd79277 --- /dev/null +++ b/src/storage/local-storage.ts @@ -0,0 +1,259 @@ +import os from 'os'; +import path from 'path'; +import { promises as fs } from 'fs'; + +import type { + KnowhereSdkStorage, + KnowhereSdkStorageHead, + KnowhereSdkStorageObject, + KnowhereSdkStorageReadResult, + KnowhereSdkStorageWriteResult, +} from '../types/page-citation-assets.js'; + +const DEFAULT_STORAGE_DIRECTORY = path.join( + os.homedir(), + '.knowhere-node-sdk', + 'page-citation-assets', +); + +interface StoredObjectMetadata { + contentType?: string; + metadata?: Readonly>; +} + +export interface LocalKnowhereSdkStorageOptions { + rootDirectory?: string; + publicBaseUrl?: string; +} + +export class LocalKnowhereSdkStorage implements KnowhereSdkStorage { + private readonly rootDirectory: string; + private readonly publicBaseUrl?: string; + + constructor(options: LocalKnowhereSdkStorageOptions = {}) { + this.rootDirectory = path.resolve(options.rootDirectory ?? DEFAULT_STORAGE_DIRECTORY); + this.publicBaseUrl = options.publicBaseUrl?.replace(/\/+$/, ''); + } + + async headObject(key: string): Promise { + const objectPath = this.resolveObjectPath(key); + try { + const stat = await fs.stat(objectPath); + const storedMetadata = await this.readMetadata(objectPath); + return { + key, + contentType: storedMetadata.contentType, + contentLength: stat.size, + metadata: storedMetadata.metadata, + }; + } catch (error) { + if (isMissingFileError(error)) { + return null; + } + throw error; + } + } + + async writeObject(input: KnowhereSdkStorageObject): Promise { + const objectPath = this.resolveObjectPath(input.key); + await fs.mkdir(path.dirname(objectPath), { recursive: true }); + await fs.writeFile(objectPath, await readStorageBody(input.body)); + await this.writeMetadata(objectPath, { + contentType: input.contentType, + metadata: input.metadata, + }); + const url = await this.getObjectUrl(input.key); + return { + key: input.key, + url: url ?? undefined, + }; + } + + getObjectUrl(key: string): Promise { + this.resolveObjectPath(key); + if (!this.publicBaseUrl) { + return Promise.resolve(null); + } + + return Promise.resolve( + `${this.publicBaseUrl}/${key.split('/').map(encodeURIComponent).join('/')}`, + ); + } + + async readObject(key: string): Promise { + const objectPath = this.resolveObjectPath(key); + try { + const body = await fs.readFile(objectPath); + const storedMetadata = await this.readMetadata(objectPath); + return { + body, + contentType: storedMetadata.contentType, + metadata: storedMetadata.metadata, + }; + } catch (error) { + if (isMissingFileError(error)) { + return null; + } + throw error; + } + } + + async deleteObject(key: string): Promise { + const objectPath = this.resolveObjectPath(key); + await fs.rm(objectPath, { force: true }); + await fs.rm(this.getMetadataPath(objectPath), { force: true }); + } + + async deletePrefix(prefix: string): Promise { + const prefixPath = this.resolvePrefixPath(prefix); + await fs.rm(prefixPath, { recursive: true, force: true }); + } + + private resolveObjectPath(key: string): string { + validateStorageKey(key); + const objectPath = path.resolve(this.rootDirectory, key); + if (!isPathInsideDirectory(objectPath, this.rootDirectory)) { + throw new Error(`Storage key resolves outside the storage root: ${key}`); + } + return objectPath; + } + + private resolvePrefixPath(prefix: string): string { + validateStoragePrefix(prefix); + const prefixPath = path.resolve(this.rootDirectory, prefix); + if (!isPathInsideDirectory(prefixPath, this.rootDirectory)) { + throw new Error(`Storage prefix resolves outside the storage root: ${prefix}`); + } + return prefixPath; + } + + private getMetadataPath(objectPath: string): string { + return `${objectPath}.metadata.json`; + } + + private async readMetadata(objectPath: string): Promise { + try { + const raw = await fs.readFile(this.getMetadataPath(objectPath), 'utf8'); + return normalizeStoredMetadata(JSON.parse(raw)); + } catch (error) { + if (isMissingFileError(error)) { + return {}; + } + throw error; + } + } + + private async writeMetadata( + objectPath: string, + metadata: StoredObjectMetadata, + ): Promise { + if (!metadata.contentType && !metadata.metadata) { + await fs.rm(this.getMetadataPath(objectPath), { force: true }); + return; + } + + await fs.writeFile(this.getMetadataPath(objectPath), JSON.stringify(metadata, null, 2)); + } +} + +export function createLocalKnowhereSdkStorage( + options?: LocalKnowhereSdkStorageOptions, +): LocalKnowhereSdkStorage { + return new LocalKnowhereSdkStorage(options); +} + +function validateStorageKey(key: string): void { + if ( + key.length === 0 || + key.startsWith('/') || + key.includes('\\') || + key.includes('\0') || + path.posix.normalize(key) !== key || + key.split('/').some((segment) => segment.length === 0 || segment === '..' || segment === '.') + ) { + throw new Error( + 'Storage key must be a relative POSIX path without empty, traversal, or absolute segments', + ); + } +} + +function validateStoragePrefix(prefix: string): void { + if (prefix.length === 0) { + throw new Error('Storage prefix is required'); + } + validateStorageKey(prefix.endsWith('/') ? `${prefix}__prefix_marker__` : prefix); +} + +async function readStorageBody(body: Uint8Array | ReadableStream | Blob): Promise { + if (body instanceof Uint8Array) { + return Buffer.from(body); + } + + if (isBlob(body)) { + return Buffer.from(await body.arrayBuffer()); + } + + return readReadableStream(body); +} + +function isBlob(value: unknown): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +async function readReadableStream(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const chunks: Buffer[] = []; + + while (true) { + const result = await reader.read(); + if (result.done) { + return Buffer.concat(chunks); + } + chunks.push(Buffer.from(result.value)); + } +} + +function normalizeStoredMetadata(value: unknown): StoredObjectMetadata { + if (!isPlainRecord(value)) { + return {}; + } + + return { + contentType: typeof value.contentType === 'string' ? value.contentType : undefined, + metadata: normalizeMetadataMap(value.metadata), + }; +} + +function normalizeMetadataMap(value: unknown): Readonly> | undefined { + if (!isPlainRecord(value)) { + return undefined; + } + + const metadata: Record = {}; + for (const [key, metadataValue] of Object.entries(value)) { + if (typeof metadataValue === 'string') { + metadata[key] = metadataValue; + } + } + + return Object.keys(metadata).length > 0 ? metadata : undefined; +} + +function isPlainRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && value.constructor === Object; +} + +function isPathInsideDirectory(targetPath: string, parentDirectory: string): boolean { + const relativePath = path.relative(parentDirectory, targetPath); + return ( + relativePath.length === 0 || (!relativePath.startsWith('..') && !path.isAbsolute(relativePath)) + ); +} + +function isMissingFileError(error: unknown): boolean { + return hasErrorCode(error) && error.code === 'ENOENT'; +} + +function hasErrorCode(error: unknown): error is { readonly code: unknown } { + return typeof error === 'object' && error !== null && 'code' in error; +} diff --git a/src/types/document.ts b/src/types/document.ts index 959ada8..3934b77 100644 --- a/src/types/document.ts +++ b/src/types/document.ts @@ -1,3 +1,5 @@ +import type { PageCitationAsset } from './page-citation-assets.js'; + /** * Canonical document state returned by document lifecycle endpoints. */ @@ -129,10 +131,27 @@ export interface DocumentChunk { metadata: Record; /** 7-day asset URL for media chunks when available */ assetUrl?: string | null; + /** Rendered source-page citation assets attached by SDK-side generation when available. */ + pageAssets?: readonly PageCitationAsset[]; /** Chunk creation timestamp */ createdAt?: Date; } +/** + * Response from GET /v2/documents/{document_id}/files/page-citation-source. + */ +export interface DocumentPageCitationSource { + documentId: string; + namespace?: string; + jobId?: string; + jobResultId?: string; + variant?: string; + fileName: string; + contentType: string; + url: string; + expiresAt?: Date; +} + /** * Response from GET /v2/documents/{document_id}/chunks. */ diff --git a/src/types/index.ts b/src/types/index.ts index 7ec1387..81cb84c 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -5,4 +5,5 @@ export * from './params.js'; export * from './result.js'; export * from './retrieval.js'; export * from './document.js'; +export * from './page-citation-assets.js'; export * from '../knowledge/types.js'; diff --git a/src/types/page-citation-assets.ts b/src/types/page-citation-assets.ts new file mode 100644 index 0000000..16dbc20 --- /dev/null +++ b/src/types/page-citation-assets.ts @@ -0,0 +1,135 @@ +export type PageCitationAssetMimeType = 'image/png' | 'image/jpeg'; + +export type PageCitationAssetSource = 'client-rendered-pdf-page'; + +export interface PageCitationAsset { + pageNum: number; + key: string; + assetUrl?: string; + mimeType: PageCitationAssetMimeType; + width: number; + height: number; + source: PageCitationAssetSource; + variant: string; +} + +export type PageCitationAssetWarningCode = + | 'missing_document_id' + | 'invalid_page_number' + | 'source_fetch_failed' + | 'renderer_unavailable' + | 'render_failed' + | 'storage_failed' + | 'render_limit_exceeded' + | 'index_write_failed'; + +export interface PageCitationAssetWarning { + code: PageCitationAssetWarningCode; + message: string; + documentId?: string; + jobId?: string; + chunkId?: string; + pageNum?: number; + key?: string; + cause?: string; +} + +export interface PageCitationAssetTimeoutOptions { + totalMs?: number; + sourceFetchMs?: number; + pageRenderMs?: number; + storageOperationMs?: number; +} + +export interface PageCitationAssetsOptions { + storage?: KnowhereSdkStorage; + renderer?: PageRenderer; + variant?: string; + format?: PageCitationAssetMimeType; + scale?: number; + quality?: number; + strict?: boolean; + maxPagesToRenderPerRun?: number; + timeouts?: PageCitationAssetTimeoutOptions; +} + +export type KnowhereSdkStorageBody = Uint8Array | ReadableStream | Blob; + +export interface KnowhereSdkStorageObject { + key: string; + contentType?: string; + body: KnowhereSdkStorageBody; + metadata?: Readonly>; +} + +export interface KnowhereSdkStorageHead { + key: string; + contentType?: string; + contentLength?: number; + metadata?: Readonly>; +} + +export interface KnowhereSdkStorageWriteResult { + key: string; + url?: string; +} + +export interface KnowhereSdkStorageReadResult { + body: Uint8Array; + contentType?: string; + metadata?: Readonly>; +} + +export interface KnowhereSdkStorage { + headObject(key: string): Promise; + writeObject(input: KnowhereSdkStorageObject): Promise; + getObjectUrl?( + key: string, + options?: { expiresInSeconds?: number }, + ): Promise; + readObject?(key: string): Promise; + deleteObject?(key: string): Promise; + deletePrefix?(prefix: string): Promise; +} + +export interface RenderPageInput { + source: Uint8Array; + pageNum: number; + format: PageCitationAssetMimeType; + scale: number; + quality?: number; + signal?: AbortSignal; +} + +export interface RenderedPage { + body: Uint8Array; + mimeType: PageCitationAssetMimeType; + width: number; + height: number; +} + +export interface PageRenderer { + renderPage(input: RenderPageInput): Promise; +} + +export interface PageCitationAssetIndex { + version: 1; + documentId: string; + namespace?: string; + jobId: string; + jobResultId?: string; + variant: string; + generatedAt: string; + assets: readonly PageCitationAsset[]; +} + +export interface PageCitationAssetCurrentIndex { + version: 1; + documentId: string; + namespace?: string; + jobId: string; + jobResultId?: string; + variant: string; + indexKey: string; + updatedAt: string; +} diff --git a/src/types/params.ts b/src/types/params.ts index 296fddb..960654c 100644 --- a/src/types/params.ts +++ b/src/types/params.ts @@ -1,4 +1,5 @@ import type { ReadStream } from 'fs'; +import type { PageCitationAssetsOptions } from './page-citation-assets.js'; /** * Parsing model options @@ -155,6 +156,8 @@ export interface ParseParams { onPollProgress?: (status: PollProgress) => void; /** Abort signal */ signal?: AbortSignal; + /** Generate rendered page citation assets for page chunks. */ + pageCitationAssets?: PageCitationAssetsOptions; } /** diff --git a/src/types/pdfjs-dist-build-pdf.d.ts b/src/types/pdfjs-dist-build-pdf.d.ts new file mode 100644 index 0000000..656837f --- /dev/null +++ b/src/types/pdfjs-dist-build-pdf.d.ts @@ -0,0 +1,7 @@ +declare module 'pdfjs-dist/build/pdf.mjs' { + export const getDocument: unknown; +} + +declare module 'pdfjs-dist/legacy/build/pdf.mjs' { + export const getDocument: unknown; +} diff --git a/src/types/result.ts b/src/types/result.ts index af4ff06..89e7d96 100644 --- a/src/types/result.ts +++ b/src/types/result.ts @@ -1,3 +1,5 @@ +import type { PageCitationAsset, PageCitationAssetWarning } from './page-citation-assets.js'; + /** * Statistics about the parsed document */ @@ -220,6 +222,8 @@ export interface TableChunk extends BaseChunk { */ export interface PageChunk extends BaseChunk { type: 'page'; + /** Rendered source-page citation assets attached by the SDK when requested. */ + pageAssets?: readonly PageCitationAsset[]; } /** @@ -241,6 +245,8 @@ export interface ParseResult { fullMarkdown?: string; /** Raw ZIP buffer */ rawZip: Buffer; + /** Non-fatal page citation asset diagnostics emitted by SDK-side generation. */ + pageCitationAssetWarnings?: readonly PageCitationAssetWarning[]; // Legacy — the current worker no longer emits these files /** @deprecated Current worker no longer emits chunks_slim.json */ diff --git a/vitest.config.mts b/vitest.config.mts index cfc4f2a..1a34869 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -1,13 +1,15 @@ -import { defineConfig } from 'vitest/config'; +import { configDefaults, defineConfig } from 'vitest/config'; export default defineConfig({ test: { globals: true, environment: 'node', + exclude: [...configDefaults.exclude, '.repos/**'], coverage: { provider: 'v8', reporter: ['text', 'json', 'html', 'json-summary'], exclude: [ + '.repos/**', 'node_modules/', 'dist/', '**/*.test.ts', From 928758a259854f93b44fa8cb7c5b221f1134e639 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Fri, 3 Jul 2026 02:17:44 +0800 Subject: [PATCH 02/10] fix: require document id for page citation assets --- .../__tests__/workflow.test.mts | 34 ++++++++++---- src/page-citation-assets/workflow.mts | 44 +++++++++++++++---- 2 files changed, 60 insertions(+), 18 deletions(-) diff --git a/src/page-citation-assets/__tests__/workflow.test.mts b/src/page-citation-assets/__tests__/workflow.test.mts index e2a677a..ffec22a 100644 --- a/src/page-citation-assets/__tests__/workflow.test.mts +++ b/src/page-citation-assets/__tests__/workflow.test.mts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; -import { PageCitationAssetGenerationError } from '../../errors/index.js'; +import { PageCitationAssetGenerationError, ValidationError } from '../../errors/index.js'; import { enrichParseResultWithPageCitationAssets } from '../workflow.mjs'; import type { Chunk, @@ -201,16 +201,32 @@ describe('page citation asset workflow', () => { expect(JSON.stringify(index)).not.toContain('placeholder'); }); - it('warns when documentId is missing for page chunks', async () => { - const result = await enrichParseResultWithPageCitationAssets({ - result: createParseResult([createPageChunk('page-1', [1])], { documentId: undefined }), - documents: createDocumentsClient(), - options: { storage: new MemoryStorage(), renderer: new FakeRenderer() }, - }); + it('rejects when documentId is missing for page chunks', async () => { + await expect( + enrichParseResultWithPageCitationAssets({ + result: createParseResult([createPageChunk('page-1', [1])], { documentId: undefined }), + documents: createDocumentsClient(), + options: { storage: new MemoryStorage(), renderer: new FakeRenderer() }, + }), + ).rejects.toBeInstanceOf(ValidationError); + }); - expect(result.pageCitationAssetWarnings?.[0]).toMatchObject({ - code: 'missing_document_id', + it('does not require documentId when no page chunks are present', async () => { + const documents: ReturnType = createDocumentsClient(); + const renderer: FakeRenderer = new FakeRenderer(); + + const result: ParseResult = await enrichParseResultWithPageCitationAssets({ + result: createParseResult( + [createTextChunk('text-1'), createImageChunk('image-1'), createTableChunk('table-1')], + { documentId: undefined }, + ), + documents, + options: { storage: new MemoryStorage(), renderer }, }); + + expect(result.pageCitationAssetWarnings).toBeUndefined(); + expect(documents.calls).toEqual([]); + expect(renderer.renderedPageNums).toEqual([]); }); }); diff --git a/src/page-citation-assets/workflow.mts b/src/page-citation-assets/workflow.mts index 23734c3..f3b0a84 100644 --- a/src/page-citation-assets/workflow.mts +++ b/src/page-citation-assets/workflow.mts @@ -2,7 +2,7 @@ import { createHash } from 'crypto'; import { Context, Data, Effect, Layer } from 'effect'; -import { PageCitationAssetGenerationError } from '../errors/index.js'; +import { PageCitationAssetGenerationError, ValidationError } from '../errors/index.js'; import { createLocalKnowhereSdkStorage } from '../storage/local-storage.js'; import type { Chunk, @@ -37,6 +37,8 @@ const DEFAULT_VARIANT = 'default'; const DEFAULT_FORMAT: PageCitationAssetMimeType = 'image/png'; const DEFAULT_SCALE = 1; const PAGE_ASSET_SOURCE: PageCitationAssetSource = 'client-rendered-pdf-page'; +const MISSING_DOCUMENT_ID_MESSAGE: string = + 'documentId is required to generate page citation assets for page chunks.'; export type DocumentPageCitationSourceClient = { getPageCitationSource(documentId: string): Promise; @@ -155,6 +157,8 @@ class PageCitationStorageService extends Context.Service< export async function enrichParseResultWithPageCitationAssets( input: PageCitationWorkflowInput, ): Promise { + assertPageCitationAssetDocumentId(input); + const options = normalizeOptions(input.options); const program = generatePageCitationAssets({ ...input, @@ -210,19 +214,18 @@ const generatePageCitationAssets = Effect.fn('Knowhere.generatePageCitationAsset readonly options: NormalizedPageCitationOptions; }, ) { - const pageChunks = input.result.chunks.filter((chunk) => chunk.type === 'page'); + const pageChunks: Array> = + input.result.chunks.filter(isPageChunk); if (pageChunks.length === 0) { return { result: input.result, warnings: [] }; } - const documentId = input.result.documentId ?? input.fallbackDocumentId; + const documentId: string | undefined = getPageCitationAssetDocumentId( + input.result, + input.fallbackDocumentId, + ); if (!documentId) { - const warning = createWarning({ - code: 'missing_document_id', - message: 'documentId is required to generate page citation assets for page chunks.', - jobId: input.result.jobId, - }); - return { result: input.result, warnings: [warning] }; + throw new ValidationError(MISSING_DOCUMENT_ID_MESSAGE); } const warnings: PageCitationAssetWarning[] = []; @@ -318,6 +321,29 @@ const generatePageCitationAssets = Effect.fn('Knowhere.generatePageCitationAsset }, ); +function assertPageCitationAssetDocumentId(input: PageCitationWorkflowInput): void { + if (!input.result.chunks.some(isPageChunk)) { + return; + } + + if (getPageCitationAssetDocumentId(input.result, input.fallbackDocumentId)) { + return; + } + + throw new ValidationError(MISSING_DOCUMENT_ID_MESSAGE); +} + +function getPageCitationAssetDocumentId( + result: ParseResult, + fallbackDocumentId: string | undefined, +): string | undefined { + return result.documentId ?? fallbackDocumentId; +} + +function isPageChunk(chunk: Chunk): chunk is Extract { + return chunk.type === 'page'; +} + const resolveSourceFile = Effect.fn('Knowhere.resolvePageCitationSource')( function*(documentId: string) { const source = yield* PageCitationSourceService; From 763e7729896c9ae8cb735ccbaedfe344306031e9 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Fri, 3 Jul 2026 03:01:56 +0800 Subject: [PATCH 03/10] Add Piscina-backed PDF page renderer --- README.md | 118 +++++++++- package.json | 4 +- .../__tests__/page-renderer-pdfjs.test.ts | 26 ++- .../__tests__/workflow.test.mts | 43 ++++ src/page-citation-assets/workflow.mts | 62 +++++- src/page-renderer-pdfjs-worker.ts | 138 ++++++++++++ src/page-renderer-pdfjs.ts | 209 +++++++++--------- 7 files changed, 472 insertions(+), 128 deletions(-) create mode 100644 src/page-renderer-pdfjs-worker.ts diff --git a/README.md b/README.md index 0b09f67..80a9447 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ npm install @ontos-ai/knowhere-sdk **Requirements:** -- Node.js >= 20.19.0 +- Node.js >= 22.13.0 - npm >= 10.0.0 - TypeScript >= 5.0 (optional, for type checking) @@ -145,6 +145,122 @@ const result = await client.parse({ }); ``` +### Page Citation Assets + +Page citation asset generation is opt-in and runs only when parsed results are +materialized through `parse(...)` or `knowledge.cacheJobResult(...)`. Providing +`pageCitationAssets` enables generation for page chunks only; text, image, and +table chunks never trigger page rendering even when they contain page metadata. + +```typescript +import Knowhere, { createLocalKnowhereSdkStorage } from '@ontos-ai/knowhere-sdk'; +import { createPdfJsPageRenderer } from '@ontos-ai/knowhere-sdk/page-renderer-pdfjs'; + +const client = new Knowhere({ apiKey: process.env.KNOWHERE_API_KEY }); +const storage = createLocalKnowhereSdkStorage({ + rootDirectory: '.knowhere/page-citation-assets', + publicBaseUrl: 'https://cdn.example.com/knowhere', +}); +const renderer = createPdfJsPageRenderer({ + maxThreads: 1, +}); + +try { + const result = await client.parse({ + file: './manual.pdf', + pageCitationAssets: { + storage, + renderer, + maxPagesToRenderPerRun: 25, + strict: false, + }, + }); + + for (const chunk of result.pageChunks) { + console.log(chunk.pageAssets); + } + + console.log(result.pageCitationAssetWarnings); +} finally { + await renderer.close(); +} +``` + +`pageAssets` entries are concrete stored page images. The durable identity is +`key`; `assetUrl` is optional access metadata returned by the storage adapter +and may be public, signed, or absent. + +```typescript +type PageCitationAsset = { + pageNum: number; + key: string; + assetUrl?: string; + mimeType: 'image/png' | 'image/jpeg'; + width: number; + height: number; + source: 'client-rendered-pdf-page'; + variant: string; +}; +``` + +When page chunks exist and `pageCitationAssets` is provided, the parsed result +must have a `documentId` so the SDK can call +`documents.getPageCitationSource(documentId)` and fetch the normalized PDF +source. Source fetch, render, and storage failures are tolerant by default and +surface as `pageCitationAssetWarnings`; set `strict: true` to fail on generation +warnings. + +`knowledge.cacheJobResult(...)` uses the same option and persists enriched +chunks so `knowledge.readChunks(...)` returns `pageAssets` later. + +```typescript +await client.knowledge.cacheJobResult({ + jobId: 'job_123', + localDocumentId: 'manual', + pageCitationAssets: { + storage, + renderer, + }, +}); +``` + +Custom object storage backends can implement `KnowhereSdkStorage`. `headObject` +is required so the SDK can attach cached assets without downloading image +bodies. + +```typescript +import type { + KnowhereSdkStorage, + KnowhereSdkStorageHead, + KnowhereSdkStorageObject, + KnowhereSdkStorageReadResult, + KnowhereSdkStorageWriteResult, +} from '@ontos-ai/knowhere-sdk'; + +class BlobStorage implements KnowhereSdkStorage { + headObject(key: string): Promise { + throw new Error('implement metadata lookup'); + } + + writeObject(input: KnowhereSdkStorageObject): Promise { + throw new Error('implement object write'); + } + + getObjectUrl(key: string): Promise { + throw new Error('return a public or signed URL when available'); + } + + readObject(key: string): Promise { + throw new Error('implement object read when needed'); + } +} +``` + +The built-in PDF.js renderer is exported from +`@ontos-ai/knowhere-sdk/page-renderer-pdfjs` to keep the root import lightweight. +It uses Piscina worker threads internally with `maxThreads: 1` by default and +caps explicit `maxThreads` values at `2` for serverless-friendly CPU isolation. + ### Low-Level API For granular control over the job lifecycle: diff --git a/package.json b/package.json index 7f0dfe3..601c833 100644 --- a/package.json +++ b/package.json @@ -49,11 +49,11 @@ "pnpm": ">=9.0.0" }, "scripts": { - "build": "tsup src/index.ts src/page-renderer-pdfjs.ts src/page-citation-assets/workflow.mts --format cjs,esm --dts --clean", + "build": "tsup src/index.ts src/page-renderer-pdfjs.ts src/page-renderer-pdfjs-worker.ts src/page-citation-assets/workflow.mts --format cjs,esm --dts --clean", "build:all": "pnpm --include-workspace-root -r build", "changeset": "changeset", "changeset:publish": "changeset publish", - "dev": "tsup src/index.ts src/page-renderer-pdfjs.ts src/page-citation-assets/workflow.mts --format cjs,esm --dts --watch", + "dev": "tsup src/index.ts src/page-renderer-pdfjs.ts src/page-renderer-pdfjs-worker.ts src/page-citation-assets/workflow.mts --format cjs,esm --dts --watch", "test": "vitest", "test:all": "pnpm --include-workspace-root -r exec vitest run", "test:ci": "vitest run --coverage", diff --git a/src/page-citation-assets/__tests__/page-renderer-pdfjs.test.ts b/src/page-citation-assets/__tests__/page-renderer-pdfjs.test.ts index 9620f47..ed1421b 100644 --- a/src/page-citation-assets/__tests__/page-renderer-pdfjs.test.ts +++ b/src/page-citation-assets/__tests__/page-renderer-pdfjs.test.ts @@ -6,21 +6,25 @@ import { createPdfJsPageRenderer } from '../../page-renderer-pdfjs.js'; describe('createPdfJsPageRenderer', () => { it('renders a tiny PDF fixture page to PNG', async () => { - const renderer = createPdfJsPageRenderer(); + const renderer = createPdfJsPageRenderer({ maxThreads: 1 }); const source = await readFile( path.join(process.cwd(), 'src', 'page-citation-assets', '__fixtures__', 'tiny.pdf'), ); - const rendered = await renderer.renderPage({ - source, - pageNum: 1, - format: 'image/png', - scale: 1, - }); + try { + const rendered = await renderer.renderPage({ + source, + pageNum: 1, + format: 'image/png', + scale: 1, + }); - expect(rendered.mimeType).toBe('image/png'); - expect(rendered.width).toBeGreaterThan(0); - expect(rendered.height).toBeGreaterThan(0); - expect(Buffer.from(rendered.body).subarray(0, 8).toString('hex')).toBe('89504e470d0a1a0a'); + expect(rendered.mimeType).toBe('image/png'); + expect(rendered.width).toBeGreaterThan(0); + expect(rendered.height).toBeGreaterThan(0); + expect(Buffer.from(rendered.body).subarray(0, 8).toString('hex')).toBe('89504e470d0a1a0a'); + } finally { + await renderer.close(); + } }); }); diff --git a/src/page-citation-assets/__tests__/workflow.test.mts b/src/page-citation-assets/__tests__/workflow.test.mts index ffec22a..24f1905 100644 --- a/src/page-citation-assets/__tests__/workflow.test.mts +++ b/src/page-citation-assets/__tests__/workflow.test.mts @@ -18,6 +18,7 @@ import type { describe('page citation asset workflow', () => { afterEach(() => { + vi.doUnmock('../../page-renderer-pdfjs.js'); vi.unstubAllGlobals(); }); @@ -228,6 +229,39 @@ describe('page citation asset workflow', () => { expect(documents.calls).toEqual([]); expect(renderer.renderedPageNums).toEqual([]); }); + + it('does not close a caller-provided closeable renderer', async () => { + const storage = new MemoryStorage(); + const renderer = new CloseableFakeRenderer(); + stubSourceFetch(); + + await enrichParseResultWithPageCitationAssets({ + result: createParseResult([createPageChunk('page-1', [1])]), + documents: createDocumentsClient(), + options: { storage, renderer }, + }); + + expect(renderer.renderedPageNums).toEqual([1]); + expect(renderer.closeCalls).toBe(0); + }); + + it('closes the SDK-created default renderer after generation', async () => { + const storage = new MemoryStorage(); + const renderer = new CloseableFakeRenderer(); + vi.doMock('../../page-renderer-pdfjs.js', () => ({ + createPdfJsPageRenderer: (): CloseableFakeRenderer => renderer, + })); + stubSourceFetch(); + + await enrichParseResultWithPageCitationAssets({ + result: createParseResult([createPageChunk('page-1', [1])]), + documents: createDocumentsClient(), + options: { storage }, + }); + + expect(renderer.renderedPageNums).toEqual([1]); + expect(renderer.closeCalls).toBe(1); + }); }); class MemoryStorage implements KnowhereSdkStorage { @@ -341,6 +375,15 @@ class FakeRenderer implements PageRenderer { } } +class CloseableFakeRenderer extends FakeRenderer { + closeCalls: number = 0; + + close(): Promise { + this.closeCalls += 1; + return Promise.resolve(); + } +} + function createDocumentsClient(): { calls: string[]; getPageCitationSource(documentId: string): Promise; diff --git a/src/page-citation-assets/workflow.mts b/src/page-citation-assets/workflow.mts index f3b0a84..676b970 100644 --- a/src/page-citation-assets/workflow.mts +++ b/src/page-citation-assets/workflow.mts @@ -71,6 +71,15 @@ interface NormalizedPageCitationOptions { storageOperationMs: number; } +interface CloseablePageRenderer extends PageRenderer { + close(): Promise; +} + +interface PageCitationRendererHandle { + getRenderer(): Promise; + closeOwnedRenderer(): Promise; +} + interface PageRequest { pageNum: number; key: string; @@ -160,15 +169,17 @@ export async function enrichParseResultWithPageCitationAssets( assertPageCitationAssetDocumentId(input); const options = normalizeOptions(input.options); + const rendererHandle: PageCitationRendererHandle = createRendererHandle(options.renderer); const program = generatePageCitationAssets({ ...input, options, }).pipe( Effect.provide([ createSourceLayer(input.documents, options.sourceFetchMs), - createRendererLayer(options), + createRendererLayer(options, rendererHandle), createStorageLayer(options.storage, options.storageOperationMs), ]), + Effect.ensuring(closeOwnedRenderer(rendererHandle)), ); const output = await runPromiseWithTimeout( @@ -441,17 +452,13 @@ function createSourceLayer( function createRendererLayer( options: NormalizedPageCitationOptions, + rendererHandle: PageCitationRendererHandle, ): Layer.Layer { - let cachedRenderer: PageRenderer | undefined = options.renderer; - return Layer.succeed(PageCitationRendererService, { renderPage: Effect.fn('PageCitationRendererService.renderPage')( function*(input: RenderPageInput) { const renderer = yield* Effect.tryPromise({ - try: async () => { - cachedRenderer ??= await createDefaultPageRenderer(); - return cachedRenderer; - }, + try: () => rendererHandle.getRenderer(), catch: (cause) => new PageCitationRendererUnavailableError({ cause }), }); @@ -473,6 +480,43 @@ function createRendererLayer( }); } +function createRendererHandle(renderer: PageRenderer | undefined): PageCitationRendererHandle { + let cachedRenderer: PageRenderer | undefined = renderer; + let ownedRenderer: CloseablePageRenderer | undefined; + + return { + async getRenderer(): Promise { + if (cachedRenderer) { + return cachedRenderer; + } + + const defaultRenderer: PageRenderer = await createDefaultPageRenderer(); + cachedRenderer = defaultRenderer; + ownedRenderer = isCloseablePageRenderer(defaultRenderer) ? defaultRenderer : undefined; + return defaultRenderer; + }, + + async closeOwnedRenderer(): Promise { + if (!ownedRenderer) { + return; + } + + const rendererToClose: CloseablePageRenderer = ownedRenderer; + ownedRenderer = undefined; + await rendererToClose.close(); + }, + }; +} + +function closeOwnedRenderer( + rendererHandle: PageCitationRendererHandle, +): Effect.Effect { + return Effect.tryPromise({ + try: () => rendererHandle.closeOwnedRenderer(), + catch: (cause) => cause, + }).pipe(Effect.orElseSucceed(() => undefined)); +} + function createStorageLayer( storage: KnowhereSdkStorage, storageOperationTimeoutMs: number, @@ -845,6 +889,10 @@ async function createDefaultPageRenderer(): Promise { return rendererModule.createPdfJsPageRenderer(); } +function isCloseablePageRenderer(renderer: PageRenderer): renderer is CloseablePageRenderer { + return 'close' in renderer && typeof renderer.close === 'function'; +} + async function fetchSourceBytes(url: string, timeoutMs: number): Promise { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); diff --git a/src/page-renderer-pdfjs-worker.ts b/src/page-renderer-pdfjs-worker.ts new file mode 100644 index 0000000..fe47244 --- /dev/null +++ b/src/page-renderer-pdfjs-worker.ts @@ -0,0 +1,138 @@ +import { Canvas, createCanvas, DOMMatrix, ImageData, Path2D } from '@napi-rs/canvas'; + +interface PdfJsPageRenderTaskInput { + source: Uint8Array; + pageNum: number; + format: 'image/png' | 'image/jpeg'; + scale: number; + background: string; + quality?: number; +} + +interface PdfJsPageRenderTaskOutput { + body: Uint8Array; + mimeType: 'image/png' | 'image/jpeg'; + width: number; + height: number; +} + +interface PdfJsModule { + getDocument(params: PdfDocumentInitParameters): PdfLoadingTask; +} + +interface PdfDocumentInitParameters { + data: Uint8Array; + useSystemFonts: boolean; +} + +interface PdfLoadingTask { + promise: Promise; + destroy(): Promise; +} + +interface PdfDocument { + getPage(pageNum: number): Promise; + cleanup(): Promise; +} + +interface PdfPage { + getViewport(params: { scale: number }): PdfViewport; + render(params: PdfRenderParameters): PdfRenderTask; + cleanup(): void; +} + +interface PdfViewport { + width: number; + height: number; +} + +interface PdfRenderParameters { + canvas: null; + canvasContext: object; + viewport: PdfViewport; + background: string; +} + +interface PdfRenderTask { + promise: Promise; +} + +export async function renderPage( + input: PdfJsPageRenderTaskInput, +): Promise { + const pdfjs: PdfJsModule = await loadPdfJsModule(); + const loadingTask: PdfLoadingTask = pdfjs.getDocument({ + data: new Uint8Array(input.source), + useSystemFonts: true, + }); + const document: PdfDocument = await loadingTask.promise; + + try { + const page: PdfPage = await document.getPage(input.pageNum); + const viewport: PdfViewport = page.getViewport({ scale: input.scale }); + const width: number = Math.ceil(viewport.width); + const height: number = Math.ceil(viewport.height); + const canvas: Canvas = createCanvas(width, height); + const context: object = canvas.getContext('2d'); + + await page.render({ + canvas: null, + canvasContext: context, + viewport, + background: input.background, + }).promise; + page.cleanup(); + + const body: Uint8Array = + input.format === 'image/png' + ? canvas.toBuffer('image/png') + : canvas.toBuffer('image/jpeg', input.quality); + + return { + body, + mimeType: input.format, + width, + height, + }; + } finally { + await document.cleanup(); + await loadingTask.destroy(); + } +} + +async function loadPdfJsModule(): Promise { + installPdfJsDomPolyfills(); + const importedModule: unknown = await import('pdfjs-dist/legacy/build/pdf.mjs'); + if (isPdfJsModule(importedModule)) { + return importedModule; + } + + throw new Error('pdfjs-dist/build/pdf.mjs did not expose getDocument'); +} + +function installPdfJsDomPolyfills(): void { + installGlobalValue('DOMMatrix', DOMMatrix); + installGlobalValue('ImageData', ImageData); + installGlobalValue('Path2D', Path2D); +} + +function installGlobalValue(name: string, value: unknown): void { + if (name in globalThis) { + return; + } + + Object.defineProperty(globalThis, name, { + value, + configurable: true, + writable: true, + }); +} + +function isPdfJsModule(value: unknown): value is PdfJsModule { + return ( + typeof value === 'object' && + value !== null && + 'getDocument' in value && + typeof value.getDocument === 'function' + ); +} diff --git a/src/page-renderer-pdfjs.ts b/src/page-renderer-pdfjs.ts index 90183f1..1ad4466 100644 --- a/src/page-renderer-pdfjs.ts +++ b/src/page-renderer-pdfjs.ts @@ -1,137 +1,132 @@ -import { createCanvas, DOMMatrix, ImageData, Path2D } from '@napi-rs/canvas'; - -import type { PageRenderer, RenderedPage, RenderPageInput } from './types/index.js'; - -interface PdfJsModule { - getDocument(params: PdfDocumentInitParameters): PdfLoadingTask; -} - -interface PdfDocumentInitParameters { - data: Uint8Array; - useSystemFonts: boolean; -} - -interface PdfLoadingTask { - promise: Promise; - destroy(): Promise; -} - -interface PdfDocument { - getPage(pageNum: number): Promise; - cleanup(): Promise; -} - -interface PdfPage { - getViewport(params: { scale: number }): PdfViewport; - render(params: PdfRenderParameters): PdfRenderTask; - cleanup(): void; -} - -interface PdfViewport { - width: number; - height: number; -} - -interface PdfRenderParameters { - canvas: null; - canvasContext: object; - viewport: PdfViewport; +import { existsSync } from 'fs'; +import { createRequire } from 'module'; +import path from 'path'; +import { pathToFileURL } from 'url'; + +import Piscina from 'piscina'; + +import type { RenderedPage, RenderPageInput } from './types/index.js'; + +const DEFAULT_BACKGROUND: string = 'rgb(255,255,255)'; +const DEFAULT_MAX_THREADS: number = 1; +const DEFAULT_IDLE_TIMEOUT_MS: number = 30_000; +const DEFAULT_CONCURRENT_TASKS_PER_WORKER: number = 1; +const PACKAGE_RENDERER_SUBPATH: string = '@ontos-ai/knowhere-sdk/page-renderer-pdfjs'; +const WORKER_BASENAME: string = 'page-renderer-pdfjs-worker'; +const WORKER_TASK_NAME: string = 'renderPage'; + +interface PdfJsPageRenderTaskInput { + source: Uint8Array; + pageNum: number; + format: 'image/png' | 'image/jpeg'; + scale: number; background: string; + quality?: number; } -interface PdfRenderTask { - promise: Promise; +export interface PdfJsPageRenderer { + renderPage(input: RenderPageInput): Promise; + close(): Promise; + destroy(): Promise; } export interface PdfJsPageRendererOptions { background?: string; + maxThreads?: number; + idleTimeoutMs?: number; + workerFile?: string | URL; } -export function createPdfJsPageRenderer(options: PdfJsPageRendererOptions = {}): PageRenderer { - const background = options.background ?? 'rgb(255,255,255)'; +export function createPdfJsPageRenderer( + options: PdfJsPageRendererOptions = {}, +): PdfJsPageRenderer { + const background: string = options.background ?? DEFAULT_BACKGROUND; + const pool: Piscina = createWorkerPool(options); return { async renderPage(input: RenderPageInput): Promise { throwIfAborted(input.signal); - const pdfjs = await loadPdfJsModule(); - const loadingTask = pdfjs.getDocument({ - data: new Uint8Array(input.source), - useSystemFonts: true, - }); - const document = await loadingTask.promise; - - try { - throwIfAborted(input.signal); - const page = await document.getPage(input.pageNum); - const viewport = page.getViewport({ scale: input.scale }); - const width = Math.ceil(viewport.width); - const height = Math.ceil(viewport.height); - const canvas = createCanvas(width, height); - const context = canvas.getContext('2d'); - - throwIfAborted(input.signal); - await page.render({ - canvas: null, - canvasContext: context, - viewport, - background, - }).promise; - page.cleanup(); - - const body = - input.format === 'image/png' - ? canvas.toBuffer('image/png') - : canvas.toBuffer('image/jpeg', input.quality); - - return { - body, - mimeType: input.format, - width, - height, - }; - } finally { - await document.cleanup(); - await loadingTask.destroy(); + const taskInput: PdfJsPageRenderTaskInput = { + source: new Uint8Array(input.source), + pageNum: input.pageNum, + format: input.format, + scale: input.scale, + background, + quality: input.quality, + }; + + if (input.signal) { + return pool.run(taskInput, { signal: input.signal }); } + + return pool.run(taskInput); + }, + + close(): Promise { + return pool.close(); + }, + + destroy(): Promise { + return pool.destroy(); }, }; } -async function loadPdfJsModule(): Promise { - installPdfJsDomPolyfills(); - const importedModule: unknown = await import('pdfjs-dist/legacy/build/pdf.mjs'); - if (isPdfJsModule(importedModule)) { - return importedModule; +function createWorkerPool( + options: PdfJsPageRendererOptions, +): Piscina { + return new Piscina({ + filename: resolveWorkerFilename(options.workerFile), + name: WORKER_TASK_NAME, + minThreads: 0, + maxThreads: normalizeMaxThreads(options.maxThreads), + idleTimeout: options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS, + concurrentTasksPerWorker: DEFAULT_CONCURRENT_TASKS_PER_WORKER, + }); +} + +function normalizeMaxThreads(maxThreads: number | undefined): number { + if (maxThreads === undefined) { + return DEFAULT_MAX_THREADS; } - throw new Error('pdfjs-dist/build/pdf.mjs did not expose getDocument'); + return Math.max(1, Math.min(2, Math.floor(maxThreads))); } -function installPdfJsDomPolyfills(): void { - installGlobalValue('DOMMatrix', DOMMatrix); - installGlobalValue('ImageData', ImageData); - installGlobalValue('Path2D', Path2D); -} +function resolveWorkerFilename(workerFile: string | URL | undefined): string { + if (workerFile instanceof URL) { + return workerFile.href; + } -function installGlobalValue(name: string, value: unknown): void { - if (name in globalThis) { - return; + if (workerFile) { + return workerFile; } - Object.defineProperty(globalThis, name, { - value, - configurable: true, - writable: true, - }); + const bundledWorkerFilename: string | undefined = resolveBundledWorkerFilename(); + if (bundledWorkerFilename) { + return bundledWorkerFilename; + } + + const sourceWorkerPath: string = path.resolve(process.cwd(), 'src', `${WORKER_BASENAME}.ts`); + if (existsSync(sourceWorkerPath)) { + return pathToFileURL(sourceWorkerPath).href; + } + + throw new Error('Unable to resolve the PDF.js page renderer worker file.'); } -function isPdfJsModule(value: unknown): value is PdfJsModule { - return ( - typeof value === 'object' && - value !== null && - 'getDocument' in value && - typeof value.getDocument === 'function' - ); +function resolveBundledWorkerFilename(): string | undefined { + try { + const projectRequire: NodeRequire = createRequire(path.join(process.cwd(), 'package.json')); + const rendererFilename: string = projectRequire.resolve(PACKAGE_RENDERER_SUBPATH); + const workerFilename: string = path.join( + path.dirname(rendererFilename), + `${WORKER_BASENAME}.js`, + ); + return existsSync(workerFilename) ? pathToFileURL(workerFilename).href : undefined; + } catch { + return undefined; + } } function throwIfAborted(signal: AbortSignal | undefined): void { From afef48e3777efdc40634d5e889c5d2bdb9dbbac0 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Fri, 3 Jul 2026 17:40:37 +0800 Subject: [PATCH 04/10] Use server page citation assets --- README.md | 116 +- package.json | 20 +- pnpm-lock.yaml | 604 +--------- scripts/prepare-effect.sh | 13 - src/client.ts | 15 +- src/errors/index.ts | 3 - src/errors/page-citation-assets.ts | 12 - src/index.ts | 23 +- src/knowledge/__tests__/knowledge.test.ts | 93 +- src/knowledge/knowledge.ts | 20 +- src/knowledge/types.ts | 3 +- src/lib/__tests__/result-parser.test.ts | 31 + src/lib/result-parser.ts | 71 +- .../__fixtures__/tiny.pdf | Bin 429 -> 0 bytes .../__tests__/page-renderer-pdfjs.test.ts | 30 - .../__tests__/workflow.test.mts | 557 --------- src/page-citation-assets/workflow-loader.ts | 62 - src/page-citation-assets/workflow.mts | 1025 ----------------- src/page-renderer-pdfjs-worker.ts | 138 --- src/page-renderer-pdfjs.ts | 136 --- src/resources/__tests__/documents.test.ts | 26 +- src/resources/__tests__/retrieval.test.ts | 29 +- src/resources/documents.ts | 2 +- src/storage/__tests__/local-storage.test.ts | 81 -- src/storage/local-storage.ts | 259 ----- src/types/document.ts | 8 +- src/types/page-citation-assets.ts | 136 +-- src/types/params.ts | 9 +- src/types/pdfjs-dist-build-pdf.d.ts | 7 - src/types/result.ts | 6 +- src/types/retrieval.ts | 6 + 31 files changed, 227 insertions(+), 3314 deletions(-) delete mode 100755 scripts/prepare-effect.sh delete mode 100644 src/errors/page-citation-assets.ts delete mode 100644 src/page-citation-assets/__fixtures__/tiny.pdf delete mode 100644 src/page-citation-assets/__tests__/page-renderer-pdfjs.test.ts delete mode 100644 src/page-citation-assets/__tests__/workflow.test.mts delete mode 100644 src/page-citation-assets/workflow-loader.ts delete mode 100644 src/page-citation-assets/workflow.mts delete mode 100644 src/page-renderer-pdfjs-worker.ts delete mode 100644 src/page-renderer-pdfjs.ts delete mode 100644 src/storage/__tests__/local-storage.test.ts delete mode 100644 src/storage/local-storage.ts delete mode 100644 src/types/pdfjs-dist-build-pdf.d.ts diff --git a/README.md b/README.md index 80a9447..9a96123 100644 --- a/README.md +++ b/README.md @@ -147,119 +147,55 @@ const result = await client.parse({ ### Page Citation Assets -Page citation asset generation is opt-in and runs only when parsed results are -materialized through `parse(...)` or `knowledge.cacheJobResult(...)`. Providing -`pageCitationAssets` enables generation for page chunks only; text, image, and -table chunks never trigger page rendering even when they contain page metadata. +Page citation assets are generated by Knowhere during page-memory parsing. The +SDK does not render PDF pages locally; it preserves the page image descriptors +returned by the server on page chunks and retrieval metadata. ```typescript -import Knowhere, { createLocalKnowhereSdkStorage } from '@ontos-ai/knowhere-sdk'; -import { createPdfJsPageRenderer } from '@ontos-ai/knowhere-sdk/page-renderer-pdfjs'; +import Knowhere from '@ontos-ai/knowhere-sdk'; const client = new Knowhere({ apiKey: process.env.KNOWHERE_API_KEY }); -const storage = createLocalKnowhereSdkStorage({ - rootDirectory: '.knowhere/page-citation-assets', - publicBaseUrl: 'https://cdn.example.com/knowhere', -}); -const renderer = createPdfJsPageRenderer({ - maxThreads: 1, -}); -try { - const result = await client.parse({ - file: './manual.pdf', - pageCitationAssets: { - storage, - renderer, - maxPagesToRenderPerRun: 25, - strict: false, - }, - }); - - for (const chunk of result.pageChunks) { - console.log(chunk.pageAssets); - } +const result = await client.parse({ + file: './manual.pdf', +}); - console.log(result.pageCitationAssetWarnings); -} finally { - await renderer.close(); +for (const chunk of result.pageChunks) { + console.log(chunk.pageAssets); } ``` -`pageAssets` entries are concrete stored page images. The durable identity is -`key`; `assetUrl` is optional access metadata returned by the storage adapter -and may be public, signed, or absent. +`pageAssets` entries point at concrete page images stored with the Knowhere +result. `artifactRef` is the durable result artifact path; `assetUrl` is an +optional server-generated access URL. ```typescript type PageCitationAsset = { pageNum: number; - key: string; + artifactRef: string; assetUrl?: string; - mimeType: 'image/png' | 'image/jpeg'; - width: number; - height: number; - source: 'client-rendered-pdf-page'; - variant: string; + contentType: 'image/png' | 'image/jpeg'; + width?: number; + height?: number; + source: + | 'knowhere-rendered-page-citation-source' + | 'client-rendered-page-citation-source'; }; ``` -When page chunks exist and `pageCitationAssets` is provided, the parsed result -must have a `documentId` so the SDK can call -`documents.getPageCitationSource(documentId)` and fetch the normalized PDF -source. Source fetch, render, and storage failures are tolerant by default and -surface as `pageCitationAssetWarnings`; set `strict: true` to fail on generation -warnings. - -`knowledge.cacheJobResult(...)` uses the same option and persists enriched -chunks so `knowledge.readChunks(...)` returns `pageAssets` later. +Document chunk APIs can include signed page image URLs when requested: ```typescript -await client.knowledge.cacheJobResult({ - jobId: 'job_123', - localDocumentId: 'manual', - pageCitationAssets: { - storage, - renderer, - }, +const chunks = await client.documents.listChunks('doc_123', { + chunkType: 'page', + includeAssetUrls: true, }); -``` - -Custom object storage backends can implement `KnowhereSdkStorage`. `headObject` -is required so the SDK can attach cached assets without downloading image -bodies. - -```typescript -import type { - KnowhereSdkStorage, - KnowhereSdkStorageHead, - KnowhereSdkStorageObject, - KnowhereSdkStorageReadResult, - KnowhereSdkStorageWriteResult, -} from '@ontos-ai/knowhere-sdk'; - -class BlobStorage implements KnowhereSdkStorage { - headObject(key: string): Promise { - throw new Error('implement metadata lookup'); - } - - writeObject(input: KnowhereSdkStorageObject): Promise { - throw new Error('implement object write'); - } - getObjectUrl(key: string): Promise { - throw new Error('return a public or signed URL when available'); - } - - readObject(key: string): Promise { - throw new Error('implement object read when needed'); - } -} +console.log(chunks.chunks[0]?.pageAssets?.[0]?.assetUrl); ``` -The built-in PDF.js renderer is exported from -`@ontos-ai/knowhere-sdk/page-renderer-pdfjs` to keep the root import lightweight. -It uses Piscina worker threads internally with `maxThreads: 1` by default and -caps explicit `maxThreads` values at `2` for serverless-friendly CPU isolation. +`parse(...)` and `knowledge.cacheJobResult(...)` still accept the deprecated +`pageCitationAssets` option for source compatibility, but it is ignored. ### Low-Level API diff --git a/package.json b/package.json index 601c833..9492eac 100644 --- a/package.json +++ b/package.json @@ -31,11 +31,6 @@ "types": "./dist/index.d.ts", "import": "./dist/index.mjs", "require": "./dist/index.js" - }, - "./page-renderer-pdfjs": { - "types": "./dist/page-renderer-pdfjs.d.ts", - "import": "./dist/page-renderer-pdfjs.mjs", - "require": "./dist/page-renderer-pdfjs.js" } }, "files": [ @@ -49,11 +44,11 @@ "pnpm": ">=9.0.0" }, "scripts": { - "build": "tsup src/index.ts src/page-renderer-pdfjs.ts src/page-renderer-pdfjs-worker.ts src/page-citation-assets/workflow.mts --format cjs,esm --dts --clean", + "build": "tsup src/index.ts --format cjs,esm --dts --clean", "build:all": "pnpm --include-workspace-root -r build", "changeset": "changeset", "changeset:publish": "changeset publish", - "dev": "tsup src/index.ts src/page-renderer-pdfjs.ts src/page-renderer-pdfjs-worker.ts src/page-citation-assets/workflow.mts --format cjs,esm --dts --watch", + "dev": "tsup src/index.ts --format cjs,esm --dts --watch", "test": "vitest", "test:all": "pnpm --include-workspace-root -r exec vitest run", "test:ci": "vitest run --coverage", @@ -71,22 +66,15 @@ "typecheck": "tsc --noEmit", "typecheck:all": "pnpm run build && pnpm --include-workspace-root -r typecheck", "check:all": "pnpm run lint:all && pnpm run typecheck:all && pnpm run test:all", - "prepublishOnly": "pnpm run lint && pnpm run typecheck && pnpm run test:ci && pnpm run build", - "prepare": "./scripts/prepare-effect.sh" + "prepublishOnly": "pnpm run lint && pnpm run typecheck && pnpm run test:ci && pnpm run build" }, "dependencies": { - "@effect/platform-node": "4.0.0-beta.93", - "@napi-rs/canvas": "^1.0.2", "axios": "^1.15.2", - "effect": "4.0.0-beta.93", - "jszip": "^3.10.0", - "pdfjs-dist": "^6.1.200", - "piscina": "^5.2.0" + "jszip": "^3.10.0" }, "devDependencies": { "@changesets/changelog-github": "^0.6.0", "@changesets/cli": "^2.30.0", - "@effect/vitest": "4.0.0-beta.93", "@eslint/js": "^10.0.1", "@types/node": "^25.2.3", "@typescript-eslint/eslint-plugin": "^8.55.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 96698a1..bc78944 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,27 +8,12 @@ importers: .: dependencies: - '@effect/platform-node': - specifier: 4.0.0-beta.93 - version: 4.0.0-beta.93(effect@4.0.0-beta.93)(ioredis@5.11.1) - '@napi-rs/canvas': - specifier: ^1.0.2 - version: 1.0.2 axios: specifier: ^1.15.2 version: 1.18.1 - effect: - specifier: 4.0.0-beta.93 - version: 4.0.0-beta.93 jszip: specifier: ^3.10.0 version: 3.10.1 - pdfjs-dist: - specifier: ^6.1.200 - version: 6.1.200 - piscina: - specifier: ^5.2.0 - version: 5.2.0 devDependencies: '@changesets/changelog-github': specifier: ^0.6.0 @@ -36,9 +21,6 @@ importers: '@changesets/cli': specifier: ^2.30.0 version: 2.31.0(@types/node@25.9.4) - '@effect/vitest': - specifier: 4.0.0-beta.93 - version: 4.0.0-beta.93(effect@4.0.0-beta.93)(vitest@4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0))) '@eslint/js': specifier: ^10.0.1 version: 10.0.1(eslint@10.5.0) @@ -186,25 +168,6 @@ packages: '@changesets/write@0.4.0': resolution: {integrity: sha512-CdTLvIOPiCNuH71pyDu3rA+Q0n65cmAbXnwWH84rKGiFumFzkmHNT8KHTMEchcxN+Kl8I54xGUhJ7l3E7X396Q==} - '@effect/platform-node-shared@4.0.0-beta.93': - resolution: {integrity: sha512-XUqZ2u5GglBqY8q2jj4Q7GjN5K/enedk8auZM9rY/l5a/myaQTrQp3QnvpIK4/Yg0WFjLGuctGPMKWRk3OLIrA==} - engines: {node: '>=18.0.0'} - peerDependencies: - effect: ^4.0.0-beta.93 - - '@effect/platform-node@4.0.0-beta.93': - resolution: {integrity: sha512-QagsCGR0ZOXaCQqS5qGR2mcDng4LiP2bYhiiX1D6UC8cT9vsusVVOHiJWn8CupeDx+yVnPcu81QmA/SDt6GM1w==} - engines: {node: '>=18.0.0'} - peerDependencies: - effect: ^4.0.0-beta.93 - ioredis: ^5.7.0 - - '@effect/vitest@4.0.0-beta.93': - resolution: {integrity: sha512-gMAnZ9PiMeJMDED9s0jWgCOhc2JccrTCxowhur/KriImsHnHIRj4VG/vK0xLw0Axe4AkTWzXNdRsFrYOjBTl3A==} - peerDependencies: - effect: ^4.0.0-beta.93 - vitest: ^3.0.0 || ^4.0.0 - '@emnapi/core@1.11.1': resolution: {integrity: sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==} @@ -444,9 +407,6 @@ packages: '@types/node': optional: true - '@ioredis/commands@1.10.0': - resolution: {integrity: sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==} - '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -476,212 +436,6 @@ packages: '@cfworker/json-schema': optional: true - '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': - resolution: {integrity: sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==} - cpu: [arm64] - os: [darwin] - - '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': - resolution: {integrity: sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==} - cpu: [x64] - os: [darwin] - - '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': - resolution: {integrity: sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==} - cpu: [arm64] - os: [linux] - - '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': - resolution: {integrity: sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==} - cpu: [arm] - os: [linux] - - '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': - resolution: {integrity: sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==} - cpu: [x64] - os: [linux] - - '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': - resolution: {integrity: sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==} - cpu: [x64] - os: [win32] - - '@napi-rs/canvas-android-arm64@1.0.2': - resolution: {integrity: sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [android] - - '@napi-rs/canvas-darwin-arm64@1.0.2': - resolution: {integrity: sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [darwin] - - '@napi-rs/canvas-darwin-x64@1.0.2': - resolution: {integrity: sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A==} - engines: {node: '>= 10'} - cpu: [x64] - os: [darwin] - - '@napi-rs/canvas-linux-arm-gnueabihf@1.0.2': - resolution: {integrity: sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ==} - engines: {node: '>= 10'} - cpu: [arm] - os: [linux] - - '@napi-rs/canvas-linux-arm64-gnu@1.0.2': - resolution: {integrity: sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - - '@napi-rs/canvas-linux-arm64-musl@1.0.2': - resolution: {integrity: sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - - '@napi-rs/canvas-linux-riscv64-gnu@1.0.2': - resolution: {integrity: sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw==} - engines: {node: '>= 10'} - cpu: [riscv64] - os: [linux] - - '@napi-rs/canvas-linux-x64-gnu@1.0.2': - resolution: {integrity: sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - - '@napi-rs/canvas-linux-x64-musl@1.0.2': - resolution: {integrity: sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - - '@napi-rs/canvas-win32-arm64-msvc@1.0.2': - resolution: {integrity: sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [win32] - - '@napi-rs/canvas-win32-x64-msvc@1.0.2': - resolution: {integrity: sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A==} - engines: {node: '>= 10'} - cpu: [x64] - os: [win32] - - '@napi-rs/canvas@1.0.2': - resolution: {integrity: sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ==} - engines: {node: '>= 10'} - - '@napi-rs/nice-android-arm-eabi@1.1.1': - resolution: {integrity: sha512-kjirL3N6TnRPv5iuHw36wnucNqXAO46dzK9oPb0wj076R5Xm8PfUVA9nAFB5ZNMmfJQJVKACAPd/Z2KYMppthw==} - engines: {node: '>= 10'} - cpu: [arm] - os: [android] - - '@napi-rs/nice-android-arm64@1.1.1': - resolution: {integrity: sha512-blG0i7dXgbInN5urONoUCNf+DUEAavRffrO7fZSeoRMJc5qD+BJeNcpr54msPF6qfDD6kzs9AQJogZvT2KD5nw==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [android] - - '@napi-rs/nice-darwin-arm64@1.1.1': - resolution: {integrity: sha512-s/E7w45NaLqTGuOjC2p96pct4jRfo61xb9bU1unM/MJ/RFkKlJyJDx7OJI/O0ll/hrfpqKopuAFDV8yo0hfT7A==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [darwin] - - '@napi-rs/nice-darwin-x64@1.1.1': - resolution: {integrity: sha512-dGoEBnVpsdcC+oHHmW1LRK5eiyzLwdgNQq3BmZIav+9/5WTZwBYX7r5ZkQC07Nxd3KHOCkgbHSh4wPkH1N1LiQ==} - engines: {node: '>= 10'} - cpu: [x64] - os: [darwin] - - '@napi-rs/nice-freebsd-x64@1.1.1': - resolution: {integrity: sha512-kHv4kEHAylMYmlNwcQcDtXjklYp4FCf0b05E+0h6nDHsZ+F0bDe04U/tXNOqrx5CmIAth4vwfkjjUmp4c4JktQ==} - engines: {node: '>= 10'} - cpu: [x64] - os: [freebsd] - - '@napi-rs/nice-linux-arm-gnueabihf@1.1.1': - resolution: {integrity: sha512-E1t7K0efyKXZDoZg1LzCOLxgolxV58HCkaEkEvIYQx12ht2pa8hoBo+4OB3qh7e+QiBlp1SRf+voWUZFxyhyqg==} - engines: {node: '>= 10'} - cpu: [arm] - os: [linux] - - '@napi-rs/nice-linux-arm64-gnu@1.1.1': - resolution: {integrity: sha512-CIKLA12DTIZlmTaaKhQP88R3Xao+gyJxNWEn04wZwC2wmRapNnxCUZkVwggInMJvtVElA+D4ZzOU5sX4jV+SmQ==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - - '@napi-rs/nice-linux-arm64-musl@1.1.1': - resolution: {integrity: sha512-+2Rzdb3nTIYZ0YJF43qf2twhqOCkiSrHx2Pg6DJaCPYhhaxbLcdlV8hCRMHghQ+EtZQWGNcS2xF4KxBhSGeutg==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - - '@napi-rs/nice-linux-ppc64-gnu@1.1.1': - resolution: {integrity: sha512-4FS8oc0GeHpwvv4tKciKkw3Y4jKsL7FRhaOeiPei0X9T4Jd619wHNe4xCLmN2EMgZoeGg+Q7GY7BsvwKpL22Tg==} - engines: {node: '>= 10'} - cpu: [ppc64] - os: [linux] - - '@napi-rs/nice-linux-riscv64-gnu@1.1.1': - resolution: {integrity: sha512-HU0nw9uD4FO/oGCCk409tCi5IzIZpH2agE6nN4fqpwVlCn5BOq0MS1dXGjXaG17JaAvrlpV5ZeyZwSon10XOXw==} - engines: {node: '>= 10'} - cpu: [riscv64] - os: [linux] - - '@napi-rs/nice-linux-s390x-gnu@1.1.1': - resolution: {integrity: sha512-2YqKJWWl24EwrX0DzCQgPLKQBxYDdBxOHot1KWEq7aY2uYeX+Uvtv4I8xFVVygJDgf6/92h9N3Y43WPx8+PAgQ==} - engines: {node: '>= 10'} - cpu: [s390x] - os: [linux] - - '@napi-rs/nice-linux-x64-gnu@1.1.1': - resolution: {integrity: sha512-/gaNz3R92t+dcrfCw/96pDopcmec7oCcAQ3l/M+Zxr82KT4DljD37CpgrnXV+pJC263JkW572pdbP3hP+KjcIg==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - - '@napi-rs/nice-linux-x64-musl@1.1.1': - resolution: {integrity: sha512-xScCGnyj/oppsNPMnevsBe3pvNaoK7FGvMjT35riz9YdhB2WtTG47ZlbxtOLpjeO9SqqQ2J2igCmz6IJOD5JYw==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - - '@napi-rs/nice-openharmony-arm64@1.1.1': - resolution: {integrity: sha512-6uJPRVwVCLDeoOaNyeiW0gp2kFIM4r7PL2MczdZQHkFi9gVlgm+Vn+V6nTWRcu856mJ2WjYJiumEajfSm7arPQ==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [openharmony] - - '@napi-rs/nice-win32-arm64-msvc@1.1.1': - resolution: {integrity: sha512-uoTb4eAvM5B2aj/z8j+Nv8OttPf2m+HVx3UjA5jcFxASvNhQriyCQF1OB1lHL43ZhW+VwZlgvjmP5qF3+59atA==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [win32] - - '@napi-rs/nice-win32-ia32-msvc@1.1.1': - resolution: {integrity: sha512-CNQqlQT9MwuCsg1Vd/oKXiuH+TcsSPJmlAFc5frFyX/KkOh0UpBLEj7aoY656d5UKZQMQFP7vJNa1DNUNORvug==} - engines: {node: '>= 10'} - cpu: [ia32] - os: [win32] - - '@napi-rs/nice-win32-x64-msvc@1.1.1': - resolution: {integrity: sha512-vB+4G/jBQCAh0jelMTY3+kgFy00Hlx2f2/1zjMoH821IbplbWZOkLiTYXQkygNTzQJTq5cvwBDgn2ppHD+bglQ==} - engines: {node: '>= 10'} - cpu: [x64] - os: [win32] - - '@napi-rs/nice@1.1.1': - resolution: {integrity: sha512-xJIPs+bYuc9ASBl+cvGsKbGrJmS6fAKaSZCnT0lhahT5rhA2VVy9/EcIgd2JhtEuFOJNx7UHNn/qiTPTY4nrQw==} - engines: {node: '>= 10'} - '@napi-rs/wasm-runtime@1.1.6': resolution: {integrity: sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==} peerDependencies: @@ -947,9 +701,6 @@ packages: '@types/node@25.9.4': resolution: {integrity: sha512-dszCsrKb5U7ZsVZBWiHFklTloVl0mSEnWH/iZXfZUlI4rzCUnsvGmgqfuVRHL54ugE7/wRuxEIXRa2iMZ+BG6g==} - '@types/ws@8.18.1': - resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} - '@typescript-eslint/eslint-plugin@8.62.0': resolution: {integrity: sha512-o+mpz7EYiMzXoySXiKmzlabIvTVqUuK5yLrAedRPRDA0IpPFMUV1IXt6OqljIxX/kumN6EjUYp41Hqelh6p/Dw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -1166,10 +917,6 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} - cluster-key-slot@1.1.1: - resolution: {integrity: sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==} - engines: {node: '>=0.10.0'} - combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -1238,10 +985,6 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - denque@2.1.0: - resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} - engines: {node: '>=0.10'} - depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -1269,9 +1012,6 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - effect@4.0.0-beta.93: - resolution: {integrity: sha512-wNS5MKFa3C42uBfIDik2oJ78lhpoYz2hN4oBR0229BeeDCIrkg/FiOvoiPGdCVlWa7MEKxEL5I0f8AILVHSD9A==} - encodeurl@2.0.0: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} @@ -1390,10 +1130,6 @@ packages: extendable-error@0.1.7: resolution: {integrity: sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==} - fast-check@4.8.0: - resolution: {integrity: sha512-GOJ158CUMnN6cSahsv4+ExARvIDuzzinFjkp0E9WtiBa5zcVeLozVkWaE4IzFcc+Y48Wp1EDlUZsXRyAztQcSg==} - engines: {node: '>=12.17.0'} - fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -1434,9 +1170,6 @@ packages: resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} engines: {node: '>= 18.0.0'} - find-my-way-ts@0.1.6: - resolution: {integrity: sha512-a85L9ZoXtNAey3Y6Z+eBWW658kO/MwR7zIafkIUPUMf3isZG0NCs2pjW2wtjxAKuJPxMAsHUIP4ZPGv0o5gyTA==} - find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} @@ -1576,14 +1309,6 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - ini@7.0.0: - resolution: {integrity: sha512-ifK0CgjALofS5bkrcTy4RaQ9Vx2Knf/eLeIO+NaswQEpH1UblrtTSCIvN71qQDMq0PeQ/SSPojvEJp9vvvfr+w==} - engines: {node: ^22.22.2 || ^24.15.0 || >=26.0.0} - - ioredis@5.11.1: - resolution: {integrity: sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==} - engines: {node: '>=12.22.0'} - ip-address@10.2.0: resolution: {integrity: sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==} engines: {node: '>= 12'} @@ -1675,9 +1400,6 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} - kubernetes-types@1.30.0: - resolution: {integrity: sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==} - levn@0.4.1: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} @@ -1823,11 +1545,6 @@ packages: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} - mime@4.1.0: - resolution: {integrity: sha512-X5ju04+cAzsojXKes0B/S4tcYtFAJ6tTMuSPBEn9CPGlrWr8Fiw7qYeLT0XyH80HSoAoqWCaz+MWKh22P7G1cw==} - engines: {node: '>=16'} - hasBin: true - minimatch@10.2.5: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} @@ -1842,16 +1559,6 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - msgpackr-extract@3.0.4: - resolution: {integrity: sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==} - hasBin: true - - msgpackr@2.0.4: - resolution: {integrity: sha512-o1C5KRmuRt+apqMr1HuGSqWStZoRBUpEsCsl15uM9VdAF1qHLtvMOU2En747EnTyEl6c4pzPewRMFF31s1CNbA==} - - multipasta@0.2.7: - resolution: {integrity: sha512-KPA58d68KgGil15oDqXjkUBEBYc00XvbPj5/X+dyzeo/lWm9Nc25pQRlf1D+gv4OpK7NM0J1odrbu9JNNGvynA==} - mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} @@ -1876,10 +1583,6 @@ packages: encoding: optional: true - node-gyp-build-optional-packages@5.2.2: - resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} - hasBin: true - object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -1962,10 +1665,6 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - pdfjs-dist@6.1.200: - resolution: {integrity: sha512-o8MolyzirkkLrcdsae/HEOiIcXWI7DS5zGpvqW8xTC2YUsW30rltFw2bDGvw/fskUdEMrQm2br68jzDS5BH2vw==} - engines: {node: '>=22.13.0 || >=24'} - picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -1985,10 +1684,6 @@ packages: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} - piscina@5.2.0: - resolution: {integrity: sha512-DszUCKeVN/5G5QKo6jAVHL8fmKnkJvQ0ACiVgY7YGCq3TUB2oznAOayvZPIAdEThvhczkXR+qm3IHsNXpFCYfA==} - engines: {node: '>=20.x'} - pkce-challenge@5.0.1: resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} engines: {node: '>=16.20.0'} @@ -2047,9 +1742,6 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} - pure-rand@8.4.1: - resolution: {integrity: sha512-c58R2+SPFcSIPXoU834QN/KPDDOSd8sXcSrqf6e83Me6Rrp1EYkxukkjXMVrKvKaADs1SOyNkWdfvLf6zY8qLQ==} - qs@6.15.2: resolution: {integrity: sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==} engines: {node: '>=0.6'} @@ -2079,14 +1771,6 @@ packages: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} - redis-errors@1.2.0: - resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} - engines: {node: '>=4'} - - redis-parser@3.0.0: - resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} - engines: {node: '>=4'} - require-from-string@2.0.2: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} @@ -2193,9 +1877,6 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - standard-as-callback@2.1.0: - resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} - statuses@2.0.2: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} @@ -2260,10 +1941,6 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} - toml@4.1.2: - resolution: {integrity: sha512-m0vXfHODcw3gk+KONAOlVQ5yNHc3yS3B1ybM3HS1vqDoS0RWTDDVBVVTYi8hH0k+2OM1vmo9fb1WX9EVqjqfHA==} - engines: {node: '>=20'} - tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} @@ -2328,10 +2005,6 @@ packages: undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} - undici@8.6.0: - resolution: {integrity: sha512-l2FlC6I510GawyEd1qgcE/okihKrzy+BRTEBlu6T0fdbM9m5yxtIH5Oa3ysRsH0zC4EhmWUEaSDsy2QngBeRlw==} - engines: {node: '>=22.19.0'} - universalify@0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} @@ -2346,10 +2019,6 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} - uuid@14.0.1: - resolution: {integrity: sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==} - hasBin: true - vary@1.1.2: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} @@ -2461,18 +2130,6 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - ws@8.21.0: - resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - yaml@2.9.0: resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} engines: {node: '>= 14.6'} @@ -2667,31 +2324,6 @@ snapshots: human-id: 4.2.0 prettier: 2.8.8 - '@effect/platform-node-shared@4.0.0-beta.93(effect@4.0.0-beta.93)': - dependencies: - '@types/ws': 8.18.1 - effect: 4.0.0-beta.93 - ws: 8.21.0 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - '@effect/platform-node@4.0.0-beta.93(effect@4.0.0-beta.93)(ioredis@5.11.1)': - dependencies: - '@effect/platform-node-shared': 4.0.0-beta.93(effect@4.0.0-beta.93) - effect: 4.0.0-beta.93 - ioredis: 5.11.1 - mime: 4.1.0 - undici: 8.6.0 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - '@effect/vitest@4.0.0-beta.93(effect@4.0.0-beta.93)(vitest@4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)))': - dependencies: - effect: 4.0.0-beta.93 - vitest: 4.1.9(@types/node@25.9.4)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0)) - '@emnapi/core@1.11.1': dependencies: '@emnapi/wasi-threads': 1.2.2 @@ -2847,8 +2479,6 @@ snapshots: optionalDependencies: '@types/node': 25.9.4 - '@ioredis/commands@1.10.0': {} - '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -2901,143 +2531,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': - optional: true - - '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': - optional: true - - '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': - optional: true - - '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': - optional: true - - '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': - optional: true - - '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': - optional: true - - '@napi-rs/canvas-android-arm64@1.0.2': - optional: true - - '@napi-rs/canvas-darwin-arm64@1.0.2': - optional: true - - '@napi-rs/canvas-darwin-x64@1.0.2': - optional: true - - '@napi-rs/canvas-linux-arm-gnueabihf@1.0.2': - optional: true - - '@napi-rs/canvas-linux-arm64-gnu@1.0.2': - optional: true - - '@napi-rs/canvas-linux-arm64-musl@1.0.2': - optional: true - - '@napi-rs/canvas-linux-riscv64-gnu@1.0.2': - optional: true - - '@napi-rs/canvas-linux-x64-gnu@1.0.2': - optional: true - - '@napi-rs/canvas-linux-x64-musl@1.0.2': - optional: true - - '@napi-rs/canvas-win32-arm64-msvc@1.0.2': - optional: true - - '@napi-rs/canvas-win32-x64-msvc@1.0.2': - optional: true - - '@napi-rs/canvas@1.0.2': - optionalDependencies: - '@napi-rs/canvas-android-arm64': 1.0.2 - '@napi-rs/canvas-darwin-arm64': 1.0.2 - '@napi-rs/canvas-darwin-x64': 1.0.2 - '@napi-rs/canvas-linux-arm-gnueabihf': 1.0.2 - '@napi-rs/canvas-linux-arm64-gnu': 1.0.2 - '@napi-rs/canvas-linux-arm64-musl': 1.0.2 - '@napi-rs/canvas-linux-riscv64-gnu': 1.0.2 - '@napi-rs/canvas-linux-x64-gnu': 1.0.2 - '@napi-rs/canvas-linux-x64-musl': 1.0.2 - '@napi-rs/canvas-win32-arm64-msvc': 1.0.2 - '@napi-rs/canvas-win32-x64-msvc': 1.0.2 - - '@napi-rs/nice-android-arm-eabi@1.1.1': - optional: true - - '@napi-rs/nice-android-arm64@1.1.1': - optional: true - - '@napi-rs/nice-darwin-arm64@1.1.1': - optional: true - - '@napi-rs/nice-darwin-x64@1.1.1': - optional: true - - '@napi-rs/nice-freebsd-x64@1.1.1': - optional: true - - '@napi-rs/nice-linux-arm-gnueabihf@1.1.1': - optional: true - - '@napi-rs/nice-linux-arm64-gnu@1.1.1': - optional: true - - '@napi-rs/nice-linux-arm64-musl@1.1.1': - optional: true - - '@napi-rs/nice-linux-ppc64-gnu@1.1.1': - optional: true - - '@napi-rs/nice-linux-riscv64-gnu@1.1.1': - optional: true - - '@napi-rs/nice-linux-s390x-gnu@1.1.1': - optional: true - - '@napi-rs/nice-linux-x64-gnu@1.1.1': - optional: true - - '@napi-rs/nice-linux-x64-musl@1.1.1': - optional: true - - '@napi-rs/nice-openharmony-arm64@1.1.1': - optional: true - - '@napi-rs/nice-win32-arm64-msvc@1.1.1': - optional: true - - '@napi-rs/nice-win32-ia32-msvc@1.1.1': - optional: true - - '@napi-rs/nice-win32-x64-msvc@1.1.1': - optional: true - - '@napi-rs/nice@1.1.1': - optionalDependencies: - '@napi-rs/nice-android-arm-eabi': 1.1.1 - '@napi-rs/nice-android-arm64': 1.1.1 - '@napi-rs/nice-darwin-arm64': 1.1.1 - '@napi-rs/nice-darwin-x64': 1.1.1 - '@napi-rs/nice-freebsd-x64': 1.1.1 - '@napi-rs/nice-linux-arm-gnueabihf': 1.1.1 - '@napi-rs/nice-linux-arm64-gnu': 1.1.1 - '@napi-rs/nice-linux-arm64-musl': 1.1.1 - '@napi-rs/nice-linux-ppc64-gnu': 1.1.1 - '@napi-rs/nice-linux-riscv64-gnu': 1.1.1 - '@napi-rs/nice-linux-s390x-gnu': 1.1.1 - '@napi-rs/nice-linux-x64-gnu': 1.1.1 - '@napi-rs/nice-linux-x64-musl': 1.1.1 - '@napi-rs/nice-openharmony-arm64': 1.1.1 - '@napi-rs/nice-win32-arm64-msvc': 1.1.1 - '@napi-rs/nice-win32-ia32-msvc': 1.1.1 - '@napi-rs/nice-win32-x64-msvc': 1.1.1 - optional: true - '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)': dependencies: '@emnapi/core': 1.11.1 @@ -3211,10 +2704,6 @@ snapshots: dependencies: undici-types: 7.24.6 - '@types/ws@8.18.1': - dependencies: - '@types/node': 25.9.4 - '@typescript-eslint/eslint-plugin@8.62.0(@typescript-eslint/parser@8.62.0(eslint@10.5.0)(typescript@5.9.3))(eslint@10.5.0)(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -3485,8 +2974,6 @@ snapshots: dependencies: readdirp: 4.1.2 - cluster-key-slot@1.1.1: {} - combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 @@ -3532,8 +3019,6 @@ snapshots: delayed-stream@1.0.0: {} - denque@2.1.0: {} - depd@2.0.0: {} detect-indent@6.1.0: {} @@ -3554,19 +3039,6 @@ snapshots: ee-first@1.1.1: {} - effect@4.0.0-beta.93: - dependencies: - '@standard-schema/spec': 1.1.0 - fast-check: 4.8.0 - find-my-way-ts: 0.1.6 - ini: 7.0.0 - kubernetes-types: 1.30.0 - msgpackr: 2.0.4 - multipasta: 0.2.7 - toml: 4.1.2 - uuid: 14.0.1 - yaml: 2.9.0 - encodeurl@2.0.0: {} enquirer@2.4.1: @@ -3744,10 +3216,6 @@ snapshots: extendable-error@0.1.7: {} - fast-check@4.8.0: - dependencies: - pure-rand: 8.4.1 - fast-deep-equal@3.1.3: {} fast-glob@3.3.3: @@ -3791,8 +3259,6 @@ snapshots: transitivePeerDependencies: - supports-color - find-my-way-ts@0.1.6: {} - find-up@4.1.0: dependencies: locate-path: 5.0.0 @@ -3933,20 +3399,6 @@ snapshots: inherits@2.0.4: {} - ini@7.0.0: {} - - ioredis@5.11.1: - dependencies: - '@ioredis/commands': 1.10.0 - cluster-key-slot: 1.1.1 - debug: 4.4.3 - denque: 2.1.0 - redis-errors: 1.2.0 - redis-parser: 3.0.0 - standard-as-callback: 2.1.0 - transitivePeerDependencies: - - supports-color - ip-address@10.2.0: {} ipaddr.js@1.9.1: {} @@ -4024,8 +3476,6 @@ snapshots: dependencies: json-buffer: 3.0.1 - kubernetes-types@1.30.0: {} - levn@0.4.1: dependencies: prelude-ls: 1.2.1 @@ -4139,8 +3589,6 @@ snapshots: dependencies: mime-db: 1.54.0 - mime@4.1.0: {} - minimatch@10.2.5: dependencies: brace-expansion: 5.0.6 @@ -4156,24 +3604,6 @@ snapshots: ms@2.1.3: {} - msgpackr-extract@3.0.4: - dependencies: - node-gyp-build-optional-packages: 5.2.2 - optionalDependencies: - '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.4 - '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.4 - '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.4 - '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.4 - '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.4 - '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.4 - optional: true - - msgpackr@2.0.4: - optionalDependencies: - msgpackr-extract: 3.0.4 - - multipasta@0.2.7: {} - mz@2.7.0: dependencies: any-promise: 1.3.0 @@ -4190,11 +3620,6 @@ snapshots: dependencies: whatwg-url: 5.0.0 - node-gyp-build-optional-packages@5.2.2: - dependencies: - detect-libc: 2.1.2 - optional: true - object-assign@4.1.1: {} object-inspect@1.13.4: {} @@ -4262,10 +3687,6 @@ snapshots: pathe@2.0.3: {} - pdfjs-dist@6.1.200: - optionalDependencies: - '@napi-rs/canvas': 1.0.2 - picocolors@1.1.1: {} picomatch@2.3.2: {} @@ -4276,10 +3697,6 @@ snapshots: pirates@4.0.7: {} - piscina@5.2.0: - optionalDependencies: - '@napi-rs/nice': 1.1.1 - pkce-challenge@5.0.1: {} pkg-types@1.3.1: @@ -4318,8 +3735,6 @@ snapshots: punycode@2.3.1: {} - pure-rand@8.4.1: {} - qs@6.15.2: dependencies: side-channel: 1.1.1 @@ -4356,12 +3771,6 @@ snapshots: readdirp@4.1.2: {} - redis-errors@1.2.0: {} - - redis-parser@3.0.0: - dependencies: - redis-errors: 1.2.0 - require-from-string@2.0.2: {} resolve-from@5.0.0: {} @@ -4522,8 +3931,6 @@ snapshots: stackback@0.0.2: {} - standard-as-callback@2.1.0: {} - statuses@2.0.2: {} std-env@4.1.0: {} @@ -4581,8 +3988,6 @@ snapshots: toidentifier@1.0.1: {} - toml@4.1.2: {} - tr46@0.0.3: {} tree-kill@1.2.2: {} @@ -4651,8 +4056,6 @@ snapshots: undici-types@7.24.6: {} - undici@8.6.0: {} - universalify@0.1.2: {} unpipe@1.0.0: {} @@ -4663,8 +4066,6 @@ snapshots: util-deprecate@1.0.2: {} - uuid@14.0.1: {} - vary@1.1.2: {} vite@8.1.0(@types/node@25.9.4)(esbuild@0.27.7)(yaml@2.9.0): @@ -4728,9 +4129,8 @@ snapshots: wrappy@1.0.2: {} - ws@8.21.0: {} - - yaml@2.9.0: {} + yaml@2.9.0: + optional: true yocto-queue@0.1.0: {} diff --git a/scripts/prepare-effect.sh b/scripts/prepare-effect.sh deleted file mode 100755 index 24a9d31..0000000 --- a/scripts/prepare-effect.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env sh - -set -eu - -repo_dir=".repos/effect" -repo_url="https://github.com/Effect-TS/effect-smol" - -if [ -d "$repo_dir/.git" ]; then - exit 0 -fi - -mkdir -p ".repos" -git clone "$repo_url" "$repo_dir" diff --git a/src/client.ts b/src/client.ts index f863f2b..6b284a3 100644 --- a/src/client.ts +++ b/src/client.ts @@ -13,7 +13,6 @@ import { Knowledge } from './knowledge/index.js'; import { DEFAULT_BASE_URL, ENV } from './constants.js'; import { ValidationError } from './errors/index.js'; import { enrichParseResult } from './lib/utils.js'; -import { loadPageCitationAssetWorkflow } from './page-citation-assets/workflow-loader.js'; function inferFileName(file: ParseParams['file'], explicitFileName?: string): string | undefined { if (explicitFileName) { @@ -146,22 +145,10 @@ export class Knowhere { verifyChecksum: params.verifyChecksum, }); - const enrichedResult = enrichParseResult(result, { + return enrichParseResult(result, { namespace: jobResult.namespace, documentId: jobResult.documentId ?? params.documentId, }); - - if (!params.pageCitationAssets) { - return enrichedResult; - } - - const workflow = await loadPageCitationAssetWorkflow(); - return workflow.enrichParseResultWithPageCitationAssets({ - result: enrichedResult, - options: params.pageCitationAssets, - documents: this.documents, - fallbackDocumentId: jobResult.documentId ?? params.documentId, - }); } /** diff --git a/src/errors/index.ts b/src/errors/index.ts index a63cb9d..24284fc 100644 --- a/src/errors/index.ts +++ b/src/errors/index.ts @@ -6,6 +6,3 @@ export * from './api-errors.js'; // Job errors export * from './job-errors.js'; - -// Page citation asset errors -export * from './page-citation-assets.js'; diff --git a/src/errors/page-citation-assets.ts b/src/errors/page-citation-assets.ts deleted file mode 100644 index 6f64e7e..0000000 --- a/src/errors/page-citation-assets.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { KnowhereError } from './base.js'; -import type { PageCitationAssetWarning } from '../types/index.js'; - -export class PageCitationAssetGenerationError extends KnowhereError { - readonly warnings: readonly PageCitationAssetWarning[]; - - constructor(message: string, warnings: readonly PageCitationAssetWarning[]) { - super(message); - this.name = 'PageCitationAssetGenerationError'; - this.warnings = warnings; - } -} diff --git a/src/index.ts b/src/index.ts index 21287f0..27ef222 100644 --- a/src/index.ts +++ b/src/index.ts @@ -51,23 +51,8 @@ export type { UploadProgress, PollProgress, PageCitationAsset, - PageCitationAssetCurrentIndex, - PageCitationAssetIndex, - PageCitationAssetMimeType, - PageCitationAssetsOptions, + PageCitationAssetContentType, PageCitationAssetSource, - PageCitationAssetTimeoutOptions, - PageCitationAssetWarning, - PageCitationAssetWarningCode, - KnowhereSdkStorage, - KnowhereSdkStorageBody, - KnowhereSdkStorageHead, - KnowhereSdkStorageObject, - KnowhereSdkStorageReadResult, - KnowhereSdkStorageWriteResult, - PageRenderer, - RenderedPage, - RenderPageInput, // Result types ParseResult, Manifest, @@ -128,14 +113,8 @@ export { GatewayTimeoutError, // Job errors JobFailedError, - PageCitationAssetGenerationError, } from './errors/index.js'; // Resources (for advanced usage) export { Jobs, Retrieval, Documents } from './resources/index.js'; export { Knowledge, LocalKnowledgeStore } from './knowledge/index.js'; -export { - LocalKnowhereSdkStorage, - createLocalKnowhereSdkStorage, -} from './storage/local-storage.js'; -export type { LocalKnowhereSdkStorageOptions } from './storage/local-storage.js'; diff --git a/src/knowledge/__tests__/knowledge.test.ts b/src/knowledge/__tests__/knowledge.test.ts index 255dee5..dad9aaa 100644 --- a/src/knowledge/__tests__/knowledge.test.ts +++ b/src/knowledge/__tests__/knowledge.test.ts @@ -8,14 +8,7 @@ import type { Knowhere } from '../../client.js'; import type { Chunk, DocumentChunkListResponse, - KnowhereSdkStorage, - KnowhereSdkStorageHead, - KnowhereSdkStorageObject, - KnowhereSdkStorageWriteResult, - PageRenderer, ParseResult, - RenderedPage, - RenderPageInput, TextChunk, TableChunk, } from '../../types/index.js'; @@ -447,28 +440,22 @@ describe('Knowledge', () => { expect(read.chunks[0]?.pageAssets?.[0]).toMatchObject({ pageNum: 1, - key: 'page-assets/doc-1/page-1.png', + artifactRef: 'page_citation_assets/page-1.png', width: 120, height: 240, }); }); - it('should enrich and cache completed job results with page assets', async () => { + it('should cache server-provided page assets without SDK-side rendering', async () => { const cacheDirectory = await createTempDirectory(); - const parseResult = createPageParseResult(); + const parseResult = createPageParseResultWithAssets(); const { client, jobsLoad, documentsGetPageCitationSource } = createClient(parseResult); const knowledge = new Knowledge(client, { cacheDirectory }); - const storage = new MemoryStorage(); - const renderer = new FakeRenderer(); - vi.stubGlobal( - 'fetch', - vi.fn().mockResolvedValue(new Response(new Uint8Array([37, 80, 68, 70]))), - ); const cached = await knowledge.cacheJobResult({ jobId: 'job-1', localDocumentId: 'local-report', - pageCitationAssets: { storage, renderer }, + pageCitationAssets: { deprecated: true }, }); const read = await knowledge.readChunks({ localDocumentId: 'local-report', @@ -477,13 +464,16 @@ describe('Knowledge', () => { }); expect(jobsLoad).toHaveBeenCalledWith('job-1', { verifyChecksum: undefined }); - expect(documentsGetPageCitationSource).toHaveBeenCalledWith('doc-1'); - expect(renderer.renderedPageNums).toEqual([1]); - expect(cached.pageCitationAssetWarnings).toBeUndefined(); + expect(documentsGetPageCitationSource).not.toHaveBeenCalled(); + expect(cached.result.pageChunks[0]?.pageAssets?.[0]).toMatchObject({ + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + }); expect(read.chunks[0]?.pageAssets?.[0]).toMatchObject({ pageNum: 1, - width: 101, - height: 201, + artifactRef: 'page_citation_assets/page-1.png', + width: 120, + height: 240, }); }); @@ -633,58 +623,6 @@ function createClient(parseResult: ParseResult): { }; } -class MemoryStorage implements KnowhereSdkStorage { - private readonly objects = new Map< - string, - { - readonly body: Uint8Array; - readonly contentType?: string; - readonly metadata?: Readonly>; - } - >(); - - headObject(key: string): Promise { - const object = this.objects.get(key); - return Promise.resolve( - object - ? { - key, - contentType: object.contentType, - contentLength: object.body.byteLength, - metadata: object.metadata, - } - : null, - ); - } - - writeObject(input: KnowhereSdkStorageObject): Promise { - this.objects.set(input.key, { - body: input.body instanceof Uint8Array ? input.body : new Uint8Array(), - contentType: input.contentType, - metadata: input.metadata, - }); - return Promise.resolve({ key: input.key, url: `memory://${input.key}` }); - } - - getObjectUrl(key: string): Promise { - return Promise.resolve(this.objects.has(key) ? `memory://${key}` : null); - } -} - -class FakeRenderer implements PageRenderer { - readonly renderedPageNums: number[] = []; - - renderPage(input: RenderPageInput): Promise { - this.renderedPageNums.push(input.pageNum); - return Promise.resolve({ - body: new Uint8Array([input.pageNum]), - mimeType: input.format, - width: 100 + input.pageNum, - height: 200 + input.pageNum, - }); - } -} - async function expectFileExists(filePath: string): Promise { await expect(access(filePath)).resolves.toBeUndefined(); } @@ -862,13 +800,12 @@ function createPageParseResultWithAssets(): ParseResult { pageAssets: [ { pageNum: 1, - key: 'page-assets/doc-1/page-1.png', + artifactRef: 'page_citation_assets/page-1.png', assetUrl: 'https://assets.example/page-1.png', - mimeType: 'image/png', + contentType: 'image/png', width: 120, height: 240, - source: 'client-rendered-pdf-page', - variant: 'default', + source: 'knowhere-rendered-page-citation-source', }, ], }, diff --git a/src/knowledge/knowledge.ts b/src/knowledge/knowledge.ts index 35acce6..93b4e7b 100644 --- a/src/knowledge/knowledge.ts +++ b/src/knowledge/knowledge.ts @@ -10,7 +10,6 @@ import type { } from '../types/index.js'; import { ValidationError } from '../errors/index.js'; import { LocalKnowledgeStore } from './local-store.js'; -import { loadPageCitationAssetWorkflow } from '../page-citation-assets/workflow-loader.js'; import type { IndexedKnowledgeChunk, KnowledgeAsyncCacheResult, @@ -68,7 +67,6 @@ export class Knowledge { return { document, result, - pageCitationAssetWarnings: result.pageCitationAssetWarnings, }; } @@ -110,16 +108,13 @@ export class Knowledge { const loadedResult = await this.client.jobs.load(params.jobId, { verifyChecksum: params.verifyChecksum, }); - const result = params.pageCitationAssets - ? await this.enrichResultWithPageCitationAssets(loadedResult, params.pageCitationAssets) - : loadedResult; + const result = loadedResult; const document = await this.store.saveResult(result, { localDocumentId: params.localDocumentId, }); return { document, result, - pageCitationAssetWarnings: result.pageCitationAssetWarnings, }; } @@ -335,19 +330,6 @@ export class Knowledge { return documents.filter((document) => requested.has(document.localDocumentId)); } - private async enrichResultWithPageCitationAssets( - result: ParseResult, - options: NonNullable, - ): Promise { - const workflow = await loadPageCitationAssetWorkflow(); - return workflow.enrichParseResultWithPageCitationAssets({ - result, - options, - documents: this.client.documents, - fallbackDocumentId: result.documentId, - }); - } - private async loadReadableResult(reference: string | KnowledgeDocumentReference): Promise<{ document: LocalKnowledgeDocument; result: ParseResult; diff --git a/src/knowledge/types.ts b/src/knowledge/types.ts index caa6be5..e4443ea 100644 --- a/src/knowledge/types.ts +++ b/src/knowledge/types.ts @@ -4,7 +4,6 @@ import type { Chunk, DocumentChunkType, PageCitationAsset, - PageCitationAssetWarning, ParseResult, } from '../types/index.js'; @@ -41,7 +40,6 @@ export interface LocalKnowledgeDocument { export interface LocalKnowledgeParseResponse { document: LocalKnowledgeDocument; result: ParseResult; - pageCitationAssetWarnings?: readonly PageCitationAssetWarning[]; } export interface KnowledgeAsyncParseParams extends ParseParams { @@ -83,6 +81,7 @@ export interface KnowledgeCacheJobResultParams { jobId: string; localDocumentId?: string; verifyChecksum?: boolean; + /** @deprecated Ignored. Page citation assets are provided by the server. */ pageCitationAssets?: ParseParams['pageCitationAssets']; } diff --git a/src/lib/__tests__/result-parser.test.ts b/src/lib/__tests__/result-parser.test.ts index c527be0..f888b91 100644 --- a/src/lib/__tests__/result-parser.test.ts +++ b/src/lib/__tests__/result-parser.test.ts @@ -459,7 +459,29 @@ describe('Result Parser', () => { summary: 'Summary for pages 4 through 6.', page_nums: [4, 5, 6], entities: [{ text: 'Knowhere', type: 'product' }], + page_assets: [ + { + page_num: 4, + artifact_ref: 'page_citation_assets/page-4.png', + asset_url: 'https://assets.example/page-4.png', + content_type: 'image/png', + width: 1200, + height: 1800, + source: 'knowhere-rendered-page-citation-source', + }, + ], }, + page_assets: [ + { + page_num: 4, + artifact_ref: 'page_citation_assets/page-4.png', + asset_url: 'https://assets.example/page-4.png', + content_type: 'image/png', + width: 1200, + height: 1800, + source: 'knowhere-rendered-page-citation-source', + }, + ], }, ], }), @@ -474,6 +496,15 @@ describe('Result Parser', () => { expect(result.pageChunks[0].type).toBe('page'); expect(result.pageChunks[0].contentSource).toBe('summary'); expect(result.pageChunks[0].metadata.pageNums).toEqual([4, 5, 6]); + expect(result.pageChunks[0].pageAssets?.[0]).toEqual({ + pageNum: 4, + artifactRef: 'page_citation_assets/page-4.png', + assetUrl: 'https://assets.example/page-4.png', + contentType: 'image/png', + width: 1200, + height: 1800, + source: 'knowhere-rendered-page-citation-source', + }); expect(result.textChunks).toHaveLength(0); }); diff --git a/src/lib/result-parser.ts b/src/lib/result-parser.ts index 58a41df..295dc1e 100644 --- a/src/lib/result-parser.ts +++ b/src/lib/result-parser.ts @@ -470,14 +470,15 @@ function buildTextChunk(chunkData: RawChunk): TextChunk { } function buildPageChunk(chunkData: RawChunk): PageChunk { + const metadata = chunkData.metadata ?? {}; return { chunkId: chunkData.chunkId ?? '', type: 'page', contentSource: chunkData.contentSource ?? 'summary', content: chunkData.content ?? '', path: chunkData.path ?? '', - metadata: chunkData.metadata ?? {}, - pageAssets: chunkData.pageAssets, + metadata, + pageAssets: normalizePageAssets(chunkData.pageAssets ?? metadata.pageAssets), }; } @@ -618,14 +619,18 @@ async function processDirectoryChunk(directory: string, chunkData: RawChunk): Pr function serializeChunks(chunks: Chunk[]): { chunks: RawChunk[] } { return { chunks: chunks.map((chunk): RawChunk => { + const pageAssets = chunk.type === 'page' ? chunk.pageAssets : undefined; const rawChunk: RawChunk = { chunkId: chunk.chunkId, type: chunk.type, contentSource: chunk.contentSource, content: chunk.content, path: chunk.path, - metadata: chunk.metadata, - pageAssets: chunk.type === 'page' ? chunk.pageAssets : undefined, + metadata: + chunk.type === 'page' && pageAssets + ? { ...chunk.metadata, pageAssets } + : chunk.metadata, + pageAssets, }; if (chunk.type === 'image' || chunk.type === 'table') { @@ -641,6 +646,64 @@ function hasEnrichedPageAssets(result: ParseResult): boolean { return result.chunks.some((chunk) => chunk.type === 'page' && chunk.pageAssets !== undefined); } +function normalizePageAssets(value: unknown): readonly PageCitationAsset[] | undefined { + if (!Array.isArray(value)) { + return undefined; + } + const assets = value.flatMap((item): PageCitationAsset[] => { + if (!isRecord(item)) { + return []; + } + const pageNum = getPositiveInteger(item.pageNum); + const artifactRef = getString(item.artifactRef); + const contentType = getPageCitationAssetContentType(item.contentType); + const source = getPageCitationAssetSource(item.source); + if (!pageNum || !artifactRef || !contentType || !source) { + return []; + } + return [ + { + pageNum, + artifactRef, + assetUrl: getString(item.assetUrl), + contentType, + width: getPositiveInteger(item.width), + height: getPositiveInteger(item.height), + source, + }, + ]; + }); + return assets.length > 0 ? assets : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function getString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value : undefined; +} + +function getPositiveInteger(value: unknown): number | undefined { + if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) { + return undefined; + } + return value; +} + +function getPageCitationAssetContentType( + value: unknown, +): PageCitationAsset['contentType'] | undefined { + return value === 'image/png' || value === 'image/jpeg' ? value : undefined; +} + +function getPageCitationAssetSource(value: unknown): PageCitationAsset['source'] | undefined { + return value === 'knowhere-rendered-page-citation-source' || + value === 'client-rendered-page-citation-source' + ? value + : undefined; +} + async function readRequiredTextFile(directory: string, fileName: string): Promise { try { return await fs.readFile(join(directory, fileName), 'utf8'); diff --git a/src/page-citation-assets/__fixtures__/tiny.pdf b/src/page-citation-assets/__fixtures__/tiny.pdf deleted file mode 100644 index e74ed808d1ef5d65186bf981975a7bac93ba0754..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZXQPfx=z48`yH6z|d_r0GZjY3hMLy8vP;+@>7@^-9r5P?|}DPmhxhI@BnKg}=w(-R1&__N{)0r!`$0}$3yx^Ov*}zbbvsfOorrz!RE8An=Rdv)Q z0>54E$+qE*1$N&DPv%3!DGtxWJbO { - it('renders a tiny PDF fixture page to PNG', async () => { - const renderer = createPdfJsPageRenderer({ maxThreads: 1 }); - const source = await readFile( - path.join(process.cwd(), 'src', 'page-citation-assets', '__fixtures__', 'tiny.pdf'), - ); - - try { - const rendered = await renderer.renderPage({ - source, - pageNum: 1, - format: 'image/png', - scale: 1, - }); - - expect(rendered.mimeType).toBe('image/png'); - expect(rendered.width).toBeGreaterThan(0); - expect(rendered.height).toBeGreaterThan(0); - expect(Buffer.from(rendered.body).subarray(0, 8).toString('hex')).toBe('89504e470d0a1a0a'); - } finally { - await renderer.close(); - } - }); -}); diff --git a/src/page-citation-assets/__tests__/workflow.test.mts b/src/page-citation-assets/__tests__/workflow.test.mts deleted file mode 100644 index 24f1905..0000000 --- a/src/page-citation-assets/__tests__/workflow.test.mts +++ /dev/null @@ -1,557 +0,0 @@ -import { describe, it, expect, vi, afterEach } from 'vitest'; - -import { PageCitationAssetGenerationError, ValidationError } from '../../errors/index.js'; -import { enrichParseResultWithPageCitationAssets } from '../workflow.mjs'; -import type { - Chunk, - DocumentPageCitationSource, - KnowhereSdkStorage, - KnowhereSdkStorageHead, - KnowhereSdkStorageObject, - KnowhereSdkStorageReadResult, - KnowhereSdkStorageWriteResult, - PageRenderer, - ParseResult, - RenderPageInput, - RenderedPage, -} from '../../types/index.js'; - -describe('page citation asset workflow', () => { - afterEach(() => { - vi.doUnmock('../../page-renderer-pdfjs.js'); - vi.unstubAllGlobals(); - }); - - it('renders and attaches assets for page chunks only', async () => { - const storage = new MemoryStorage(); - const renderer = new FakeRenderer(); - const documents = createDocumentsClient(); - stubSourceFetch(); - - const result = await enrichParseResultWithPageCitationAssets({ - result: createParseResult([ - createPageChunk('page-1', [1]), - createTextChunk('text-1'), - createImageChunk('image-1'), - createTableChunk('table-1'), - createPageChunk('page-2', [2]), - ]), - documents, - options: { storage, renderer }, - }); - - expect(renderer.renderedPageNums).toEqual([1, 2]); - expect(getPageChunks(result)[0]?.pageAssets?.[0]).toMatchObject({ - pageNum: 1, - mimeType: 'image/png', - width: 101, - height: 201, - }); - expect(result.textChunks[0]).not.toHaveProperty('pageAssets'); - expect(result.imageChunks[0]).not.toHaveProperty('pageAssets'); - expect(result.tableChunks[0]).not.toHaveProperty('pageAssets'); - expect(storage.getJsonObjects().some((entry) => entry.key.endsWith('/index.json'))).toBe(true); - }); - - it('attaches cached assets without rendering or fetching the source', async () => { - const storage = new MemoryStorage(); - const renderer = new FakeRenderer(); - const result = createParseResult([createPageChunk('page-1', [1])]); - const key = expectedAssetKey(result, 1); - storage.seedObject(key, { - contentType: 'image/png', - metadata: { - pageNum: '1', - width: '300', - height: '400', - mimeType: 'image/png', - }, - }); - const documents = createDocumentsClient(); - const fetchMock = stubSourceFetch(); - - const enriched = await enrichParseResultWithPageCitationAssets({ - result, - documents, - options: { storage, renderer }, - }); - - expect(renderer.renderedPageNums).toEqual([]); - expect(documents.calls).toEqual([]); - expect(fetchMock).not.toHaveBeenCalled(); - expect(getPageChunks(enriched)[0]?.pageAssets?.[0]).toMatchObject({ - pageNum: 1, - width: 300, - height: 400, - }); - }); - - it('returns warnings and preserves parse success for partial render failures', async () => { - const storage = new MemoryStorage(); - const renderer = new FakeRenderer({ failedPageNums: [2] }); - stubSourceFetch(); - - const result = await enrichParseResultWithPageCitationAssets({ - result: createParseResult([createPageChunk('page-1', [1]), createPageChunk('page-2', [2])]), - documents: createDocumentsClient(), - options: { storage, renderer }, - }); - - expect(getPageChunks(result)[0]?.pageAssets).toHaveLength(1); - expect(getPageChunks(result)[1]?.pageAssets).toBeUndefined(); - expect(result.pageCitationAssetWarnings?.map((warning) => warning.code)).toContain( - 'render_failed', - ); - }); - - it('returns warnings and preserves parse success for source failures', async () => { - const storage = new MemoryStorage(); - const renderer = new FakeRenderer(); - const documents = { - getPageCitationSource: vi.fn().mockRejectedValue(new Error('source missing')), - }; - - const result = await enrichParseResultWithPageCitationAssets({ - result: createParseResult([createPageChunk('page-1', [1])]), - documents, - options: { storage, renderer }, - }); - - expect(documents.getPageCitationSource).toHaveBeenCalledWith('doc-1'); - expect(renderer.renderedPageNums).toEqual([]); - expect(getPageChunks(result)[0]?.pageAssets).toBeUndefined(); - expect(result.pageCitationAssetWarnings?.[0]).toMatchObject({ - code: 'source_fetch_failed', - }); - }); - - it('returns warnings and preserves parse success for storage write failures', async () => { - const storage = new WriteFailingStorage(); - const renderer = new FakeRenderer(); - stubSourceFetch(); - - const result = await enrichParseResultWithPageCitationAssets({ - result: createParseResult([createPageChunk('page-1', [1])]), - documents: createDocumentsClient(), - options: { storage, renderer }, - }); - - expect(renderer.renderedPageNums).toEqual([1]); - expect(getPageChunks(result)[0]?.pageAssets).toBeUndefined(); - expect(result.pageCitationAssetWarnings?.[0]).toMatchObject({ - code: 'storage_failed', - }); - }); - - it('attaches written assets when storage URLs are unavailable', async () => { - const storage = new UrlFailingStorage(); - const renderer = new FakeRenderer(); - stubSourceFetch(); - - const result = await enrichParseResultWithPageCitationAssets({ - result: createParseResult([createPageChunk('page-1', [1])]), - documents: createDocumentsClient(), - options: { storage, renderer }, - }); - - expect(getPageChunks(result)[0]?.pageAssets?.[0]).toMatchObject({ - pageNum: 1, - assetUrl: undefined, - key: expectedAssetKey(result, 1), - }); - expect(result.pageCitationAssetWarnings).toBeUndefined(); - }); - - it('fails in strict mode when generation returns warnings', async () => { - const storage = new MemoryStorage(); - const renderer = new FakeRenderer({ failedPageNums: [1] }); - stubSourceFetch(); - - await expect( - enrichParseResultWithPageCitationAssets({ - result: createParseResult([createPageChunk('page-1', [1])]), - documents: createDocumentsClient(), - options: { storage, renderer, strict: true }, - }), - ).rejects.toBeInstanceOf(PageCitationAssetGenerationError); - }); - - it('limits new renders after cache checks and writes only concrete assets to the index', async () => { - const storage = new MemoryStorage(); - const renderer = new FakeRenderer(); - stubSourceFetch(); - - const result = await enrichParseResultWithPageCitationAssets({ - result: createParseResult([createPageChunk('page-1', [1]), createPageChunk('page-2', [2])]), - documents: createDocumentsClient(), - options: { storage, renderer, maxPagesToRenderPerRun: 1 }, - }); - const index = storage - .getJsonObjects() - .find((entry) => entry.key.endsWith('/index.json'))?.value; - - expect(renderer.renderedPageNums).toEqual([1]); - expect(result.pageCitationAssetWarnings?.map((warning) => warning.code)).toContain( - 'render_limit_exceeded', - ); - expect(index).toMatchObject({ - version: 1, - assets: [expect.objectContaining({ pageNum: 1 })], - }); - expect(JSON.stringify(index)).not.toContain('status'); - expect(JSON.stringify(index)).not.toContain('placeholder'); - }); - - it('rejects when documentId is missing for page chunks', async () => { - await expect( - enrichParseResultWithPageCitationAssets({ - result: createParseResult([createPageChunk('page-1', [1])], { documentId: undefined }), - documents: createDocumentsClient(), - options: { storage: new MemoryStorage(), renderer: new FakeRenderer() }, - }), - ).rejects.toBeInstanceOf(ValidationError); - }); - - it('does not require documentId when no page chunks are present', async () => { - const documents: ReturnType = createDocumentsClient(); - const renderer: FakeRenderer = new FakeRenderer(); - - const result: ParseResult = await enrichParseResultWithPageCitationAssets({ - result: createParseResult( - [createTextChunk('text-1'), createImageChunk('image-1'), createTableChunk('table-1')], - { documentId: undefined }, - ), - documents, - options: { storage: new MemoryStorage(), renderer }, - }); - - expect(result.pageCitationAssetWarnings).toBeUndefined(); - expect(documents.calls).toEqual([]); - expect(renderer.renderedPageNums).toEqual([]); - }); - - it('does not close a caller-provided closeable renderer', async () => { - const storage = new MemoryStorage(); - const renderer = new CloseableFakeRenderer(); - stubSourceFetch(); - - await enrichParseResultWithPageCitationAssets({ - result: createParseResult([createPageChunk('page-1', [1])]), - documents: createDocumentsClient(), - options: { storage, renderer }, - }); - - expect(renderer.renderedPageNums).toEqual([1]); - expect(renderer.closeCalls).toBe(0); - }); - - it('closes the SDK-created default renderer after generation', async () => { - const storage = new MemoryStorage(); - const renderer = new CloseableFakeRenderer(); - vi.doMock('../../page-renderer-pdfjs.js', () => ({ - createPdfJsPageRenderer: (): CloseableFakeRenderer => renderer, - })); - stubSourceFetch(); - - await enrichParseResultWithPageCitationAssets({ - result: createParseResult([createPageChunk('page-1', [1])]), - documents: createDocumentsClient(), - options: { storage }, - }); - - expect(renderer.renderedPageNums).toEqual([1]); - expect(renderer.closeCalls).toBe(1); - }); -}); - -class MemoryStorage implements KnowhereSdkStorage { - private readonly objects = new Map< - string, - { - readonly body: Uint8Array; - readonly contentType?: string; - readonly metadata?: Readonly>; - } - >(); - - headObject(key: string): Promise { - const object = this.objects.get(key); - if (!object) { - return Promise.resolve(null); - } - return Promise.resolve({ - key, - contentType: object.contentType, - contentLength: object.body.byteLength, - metadata: object.metadata, - }); - } - - async writeObject(input: KnowhereSdkStorageObject): Promise { - this.objects.set(input.key, { - body: await readBody(input.body), - contentType: input.contentType, - metadata: input.metadata, - }); - return { - key: input.key, - url: `memory://${input.key}`, - }; - } - - getObjectUrl(key: string): Promise { - return Promise.resolve(this.objects.has(key) ? `memory://${key}` : null); - } - - readObject(key: string): Promise { - const object = this.objects.get(key); - if (!object) { - return Promise.resolve(null); - } - return Promise.resolve({ - body: object.body, - contentType: object.contentType, - metadata: object.metadata, - }); - } - - seedObject( - key: string, - options: { contentType?: string; metadata?: Readonly> }, - ): void { - this.objects.set(key, { - body: new Uint8Array([1]), - contentType: options.contentType, - metadata: options.metadata, - }); - } - - getJsonObjects(): Array<{ key: string; value: unknown }> { - return [...this.objects.entries()] - .filter((entry) => entry[1].contentType === 'application/json') - .map(([key, object]) => { - const value: unknown = JSON.parse(Buffer.from(object.body).toString('utf8')); - return { key, value }; - }); - } -} - -class WriteFailingStorage extends MemoryStorage { - writeObject(): Promise { - return Promise.reject(new Error('write failed')); - } -} - -class UrlFailingStorage extends MemoryStorage { - async writeObject(input: KnowhereSdkStorageObject): Promise { - await super.writeObject(input); - return { key: input.key }; - } - - getObjectUrl(): Promise { - return Promise.reject(new Error('url failed')); - } -} - -class FakeRenderer implements PageRenderer { - readonly renderedPageNums: number[] = []; - private readonly failedPageNums: ReadonlySet; - - constructor(options: { failedPageNums?: readonly number[] } = {}) { - this.failedPageNums = new Set(options.failedPageNums ?? []); - } - - renderPage(input: RenderPageInput): Promise { - this.renderedPageNums.push(input.pageNum); - if (this.failedPageNums.has(input.pageNum)) { - throw new Error(`failed page ${input.pageNum}`); - } - return Promise.resolve({ - body: new Uint8Array([input.pageNum]), - mimeType: input.format, - width: 100 + input.pageNum, - height: 200 + input.pageNum, - }); - } -} - -class CloseableFakeRenderer extends FakeRenderer { - closeCalls: number = 0; - - close(): Promise { - this.closeCalls += 1; - return Promise.resolve(); - } -} - -function createDocumentsClient(): { - calls: string[]; - getPageCitationSource(documentId: string): Promise; -} { - const calls: string[] = []; - return { - calls, - getPageCitationSource(documentId: string): Promise { - calls.push(documentId); - return Promise.resolve({ - documentId: 'doc-1', - namespace: 'support-center', - jobId: 'job-1', - jobResultId: 'jres-1', - variant: 'normalized_pdf', - fileName: 'report.pdf', - contentType: 'application/pdf', - url: 'https://assets.example/source.pdf', - expiresAt: new Date('2026-01-01T00:00:00.000Z'), - }); - }, - }; -} - -function stubSourceFetch(): ReturnType { - const fetchMock = vi.fn().mockResolvedValue( - new Response(new Uint8Array([37, 80, 68, 70])), - ); - vi.stubGlobal('fetch', fetchMock); - return fetchMock; -} - -function createParseResult( - chunks: Chunk[], - options: { documentId?: string } = {}, -): ParseResult { - const textChunks = chunks.filter( - (chunk): chunk is Extract => chunk.type === 'text', - ); - const imageChunks = chunks.filter( - (chunk): chunk is Extract => chunk.type === 'image', - ); - const tableChunks = chunks.filter( - (chunk): chunk is Extract => chunk.type === 'table', - ); - const pageChunks = chunks.filter( - (chunk): chunk is Extract => chunk.type === 'page', - ); - - return { - manifest: { - version: '2.0', - jobId: 'job-1', - sourceFileName: 'report.pdf', - statistics: { - totalChunks: chunks.length, - textChunks: textChunks.length, - imageChunks: imageChunks.length, - tableChunks: tableChunks.length, - pageChunks: pageChunks.length, - }, - }, - chunks, - rawZip: Buffer.alloc(0), - namespace: 'support-center', - documentId: 'documentId' in options ? options.documentId : 'doc-1', - textChunks, - imageChunks, - tableChunks, - pageChunks, - jobId: 'job-1', - statistics: { - totalChunks: chunks.length, - textChunks: textChunks.length, - imageChunks: imageChunks.length, - tableChunks: tableChunks.length, - pageChunks: pageChunks.length, - }, - getChunk: (chunkId: string) => chunks.find((chunk) => chunk.chunkId === chunkId), - save: vi.fn(), - }; -} - -function getPageChunks(result: ParseResult): Array> { - return result.chunks.filter( - (chunk): chunk is Extract => chunk.type === 'page', - ); -} - -function createPageChunk(chunkId: string, pageNums: readonly number[]): Chunk { - return { - chunkId, - type: 'page', - content: `Summary ${chunkId}`, - contentSource: 'summary', - path: `report.pdf/${chunkId}`, - metadata: { pageNums: [...pageNums] }, - }; -} - -function createTextChunk(chunkId: string): Chunk { - return { - chunkId, - type: 'text', - content: `Text ${chunkId}`, - contentSource: 'content', - path: `report.pdf/${chunkId}`, - metadata: {}, - }; -} - -function createImageChunk(chunkId: string): Chunk { - return { - chunkId, - type: 'image', - content: `Image ${chunkId}`, - contentSource: 'caption', - path: `report.pdf/${chunkId}`, - filePath: `images/${chunkId}.png`, - data: Buffer.from([1]), - metadata: { pageNums: [99] }, - format: 'png', - save: vi.fn(), - }; -} - -function createTableChunk(chunkId: string): Chunk { - return { - chunkId, - type: 'table', - content: `Table ${chunkId}`, - contentSource: 'summary', - path: `report.pdf/${chunkId}`, - filePath: `tables/${chunkId}.html`, - html: '
', - metadata: { page_nums: [100] }, - save: vi.fn(), - }; -} - -function expectedAssetKey(result: ParseResult, pageNum: number): string { - return [ - 'page-citation-assets', - 'documents', - result.documentId, - 'jobs', - result.jobId, - 'variants', - 'default', - 'scale-1', - `page-${String(pageNum).padStart(6, '0')}.png`, - ].join('/'); -} - -async function readBody(body: Uint8Array | ReadableStream | Blob): Promise { - if (body instanceof Uint8Array) { - return body; - } - if (body instanceof Blob) { - return new Uint8Array(await body.arrayBuffer()); - } - const reader = body.getReader(); - const chunks: Uint8Array[] = []; - while (true) { - const result = await reader.read(); - if (result.done) { - return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); - } - chunks.push(result.value); - } -} diff --git a/src/page-citation-assets/workflow-loader.ts b/src/page-citation-assets/workflow-loader.ts deleted file mode 100644 index ab6dd07..0000000 --- a/src/page-citation-assets/workflow-loader.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { - DocumentPageCitationSource, - PageCitationAssetsOptions, - ParseResult, -} from '../types/index.js'; - -type PageCitationWorkflowInput = { - result: ParseResult; - options: PageCitationAssetsOptions; - documents: { - getPageCitationSource(documentId: string): Promise; - }; - fallbackDocumentId?: string; -}; - -type PageCitationWorkflowModule = { - enrichParseResultWithPageCitationAssets(input: PageCitationWorkflowInput): Promise; -}; - -export async function loadPageCitationAssetWorkflow(): Promise { - const sourceLayoutPath: string = './workflow.mjs'; - const bundledLayoutPath: string = './page-citation-assets/workflow.mjs'; - - try { - return await importWorkflowModule(sourceLayoutPath); - } catch (error) { - if (!isModuleNotFoundError(error)) { - throw error; - } - return importWorkflowModule(bundledLayoutPath); - } -} - -async function importWorkflowModule(modulePath: string): Promise { - const importedModule: unknown = await import(modulePath); - if (isPageCitationWorkflowModule(importedModule)) { - return importedModule; - } - - throw new Error('Page citation asset workflow module did not expose the expected API'); -} - -function isPageCitationWorkflowModule(value: unknown): value is PageCitationWorkflowModule { - return ( - typeof value === 'object' && - value !== null && - 'enrichParseResultWithPageCitationAssets' in value && - typeof value.enrichParseResultWithPageCitationAssets === 'function' - ); -} - -function isModuleNotFoundError(error: unknown): boolean { - if (!(error instanceof Error)) { - return false; - } - - return ( - error.message.includes('Cannot find module') || - error.message.includes('ERR_MODULE_NOT_FOUND') || - error.message.includes('Failed to resolve') - ); -} diff --git a/src/page-citation-assets/workflow.mts b/src/page-citation-assets/workflow.mts deleted file mode 100644 index 676b970..0000000 --- a/src/page-citation-assets/workflow.mts +++ /dev/null @@ -1,1025 +0,0 @@ -import { createHash } from 'crypto'; - -import { Context, Data, Effect, Layer } from 'effect'; - -import { PageCitationAssetGenerationError, ValidationError } from '../errors/index.js'; -import { createLocalKnowhereSdkStorage } from '../storage/local-storage.js'; -import type { - Chunk, - DocumentPageCitationSource, - KnowhereSdkStorage, - KnowhereSdkStorageHead, - KnowhereSdkStorageObject, - KnowhereSdkStorageWriteResult, - PageCitationAsset, - PageCitationAssetCurrentIndex, - PageCitationAssetIndex, - PageCitationAssetMimeType, - PageCitationAssetSource, - PageCitationAssetsOptions, - PageCitationAssetWarning, - PageCitationAssetWarningCode, - PageRenderer, - ParseResult, - RenderedPage, - RenderPageInput, -} from '../types/index.js'; - -const DEFAULT_PAGE_CITATION_ASSET_LIMITS = { - maxPagesToRenderPerRun: 25, - totalMs: 120_000, - sourceFetchMs: 30_000, - pageRenderMs: 30_000, - storageOperationMs: 15_000, -} as const; - -const DEFAULT_VARIANT = 'default'; -const DEFAULT_FORMAT: PageCitationAssetMimeType = 'image/png'; -const DEFAULT_SCALE = 1; -const PAGE_ASSET_SOURCE: PageCitationAssetSource = 'client-rendered-pdf-page'; -const MISSING_DOCUMENT_ID_MESSAGE: string = - 'documentId is required to generate page citation assets for page chunks.'; - -export type DocumentPageCitationSourceClient = { - getPageCitationSource(documentId: string): Promise; -}; - -export interface PageCitationWorkflowInput { - result: ParseResult; - options: PageCitationAssetsOptions; - documents: DocumentPageCitationSourceClient; - fallbackDocumentId?: string; -} - -interface PageCitationWorkflowOutput { - result: ParseResult; - warnings: readonly PageCitationAssetWarning[]; -} - -interface NormalizedPageCitationOptions { - storage: KnowhereSdkStorage; - renderer?: PageRenderer; - variant: string; - format: PageCitationAssetMimeType; - scale: number; - quality?: number; - strict: boolean; - maxPagesToRenderPerRun: number; - totalMs: number; - sourceFetchMs: number; - pageRenderMs: number; - storageOperationMs: number; -} - -interface CloseablePageRenderer extends PageRenderer { - close(): Promise; -} - -interface PageCitationRendererHandle { - getRenderer(): Promise; - closeOwnedRenderer(): Promise; -} - -interface PageRequest { - pageNum: number; - key: string; - chunkIds: readonly string[]; -} - -interface PageAssetContext { - documentId: string; - namespace?: string; - jobId: string; - jobResultId?: string; - variant: string; - format: PageCitationAssetMimeType; - scale: number; -} - -interface SourceFile { - source: DocumentPageCitationSource; - body: Uint8Array; -} - -type PageCitationSourceServiceShape = { - readonly getSource: ( - documentId: string, - ) => Effect.Effect; -}; - -type PageCitationRendererServiceShape = { - readonly renderPage: ( - input: RenderPageInput, - ) => Effect.Effect; -}; - -type PageCitationStorageServiceShape = { - readonly headObject: ( - key: string, - ) => Effect.Effect; - readonly writeObject: ( - input: KnowhereSdkStorageObject, - ) => Effect.Effect; - readonly getObjectUrl: ( - key: string, - ) => Effect.Effect; -}; - -class PageCitationSourceError extends Data.TaggedError('PageCitationSourceError')<{ - readonly operation: 'resolve_source' | 'fetch_source'; - readonly documentId: string; - readonly cause: unknown; -}> {} - -class PageCitationRenderError extends Data.TaggedError('PageCitationRenderError')<{ - readonly pageNum: number; - readonly cause: unknown; -}> {} - -class PageCitationStorageError extends Data.TaggedError('PageCitationStorageError')<{ - readonly operation: 'head' | 'write' | 'url'; - readonly key: string; - readonly cause: unknown; -}> {} - -class PageCitationRendererUnavailableError extends Data.TaggedError( - 'PageCitationRendererUnavailableError', -)<{ - readonly cause: unknown; -}> {} - -class PageCitationSourceService extends Context.Service< - PageCitationSourceService, - PageCitationSourceServiceShape ->()('Knowhere/PageCitationSourceService') {} - -class PageCitationRendererService extends Context.Service< - PageCitationRendererService, - PageCitationRendererServiceShape ->()('Knowhere/PageCitationRendererService') {} - -class PageCitationStorageService extends Context.Service< - PageCitationStorageService, - PageCitationStorageServiceShape ->()('Knowhere/PageCitationStorageService') {} - -export async function enrichParseResultWithPageCitationAssets( - input: PageCitationWorkflowInput, -): Promise { - assertPageCitationAssetDocumentId(input); - - const options = normalizeOptions(input.options); - const rendererHandle: PageCitationRendererHandle = createRendererHandle(options.renderer); - const program = generatePageCitationAssets({ - ...input, - options, - }).pipe( - Effect.provide([ - createSourceLayer(input.documents, options.sourceFetchMs), - createRendererLayer(options, rendererHandle), - createStorageLayer(options.storage, options.storageOperationMs), - ]), - Effect.ensuring(closeOwnedRenderer(rendererHandle)), - ); - - const output = await runPromiseWithTimeout( - () => Effect.runPromise(program), - options.totalMs, - 'Page citation asset generation timed out', - ).catch((error: unknown): PageCitationWorkflowOutput => { - if (options.strict) { - throw error; - } - - return { - result: input.result, - warnings: [ - createWarning({ - code: 'render_failed', - message: 'Page citation asset generation timed out or failed before completing.', - documentId: input.result.documentId ?? input.fallbackDocumentId, - jobId: input.result.jobId, - cause: getErrorMessage(error), - }), - ], - }; - }); - - if (output.warnings.length > 0) { - output.result.pageCitationAssetWarnings = output.warnings; - } - - if (options.strict && output.warnings.length > 0) { - throw new PageCitationAssetGenerationError( - 'Page citation asset generation failed in strict mode.', - output.warnings, - ); - } - - return output.result; -} - -const generatePageCitationAssets = Effect.fn('Knowhere.generatePageCitationAssets')( - function*( - input: Omit & { - readonly options: NormalizedPageCitationOptions; - }, - ) { - const pageChunks: Array> = - input.result.chunks.filter(isPageChunk); - if (pageChunks.length === 0) { - return { result: input.result, warnings: [] }; - } - - const documentId: string | undefined = getPageCitationAssetDocumentId( - input.result, - input.fallbackDocumentId, - ); - if (!documentId) { - throw new ValidationError(MISSING_DOCUMENT_ID_MESSAGE); - } - - const warnings: PageCitationAssetWarning[] = []; - const context: PageAssetContext = { - documentId, - namespace: input.result.namespace, - jobId: input.result.jobId, - variant: input.options.variant, - format: input.options.format, - scale: input.options.scale, - }; - const pageRequests = buildPageRequests(input.result.chunks, context, warnings); - - if (pageRequests.length === 0) { - return { result: input.result, warnings }; - } - - const storage = yield* PageCitationStorageService; - const assetByPageNum = new Map(); - const missingRequests: PageRequest[] = []; - - for (const request of pageRequests) { - const cachedAsset = yield* readCachedAsset(storage, request, context); - if (cachedAsset) { - assetByPageNum.set(request.pageNum, cachedAsset); - } else { - missingRequests.push(request); - } - } - - const renderableRequests = missingRequests.slice(0, input.options.maxPagesToRenderPerRun); - const skippedRequests = missingRequests.slice(input.options.maxPagesToRenderPerRun); - for (const skipped of skippedRequests) { - warnings.push( - createWarning({ - code: 'render_limit_exceeded', - message: `Page ${skipped.pageNum} was not rendered because maxPagesToRenderPerRun was reached.`, - documentId, - jobId: context.jobId, - pageNum: skipped.pageNum, - key: skipped.key, - }), - ); - } - - if (renderableRequests.length > 0) { - const sourceResult = yield* resolveSourceFile(documentId).pipe( - Effect.match({ - onFailure: (error) => { - warnings.push(sourceErrorToWarning(error, context)); - return undefined; - }, - onSuccess: (source) => source, - }), - ); - - if (sourceResult) { - context.namespace = sourceResult.source.namespace ?? context.namespace; - context.jobResultId = sourceResult.source.jobResultId; - yield* renderMissingAssets({ - sourceFile: sourceResult, - pageRequests: renderableRequests, - context, - options: input.options, - warnings, - assetByPageNum, - }); - } - } - - const enrichedChunks = attachAssetsToChunks(input.result.chunks, assetByPageNum); - input.result.chunks.splice(0, input.result.chunks.length, ...enrichedChunks); - - const concreteAssets = [...assetByPageNum.values()].sort( - (left, right) => left.pageNum - right.pageNum, - ); - if (concreteAssets.length > 0) { - yield* writeAssetIndexes(storage, context, concreteAssets).pipe( - Effect.match({ - onFailure: (error) => { - warnings.push(storageErrorToWarning(error, 'index_write_failed', context)); - return undefined; - }, - onSuccess: () => undefined, - }), - ); - } - - return { - result: input.result, - warnings, - }; - }, -); - -function assertPageCitationAssetDocumentId(input: PageCitationWorkflowInput): void { - if (!input.result.chunks.some(isPageChunk)) { - return; - } - - if (getPageCitationAssetDocumentId(input.result, input.fallbackDocumentId)) { - return; - } - - throw new ValidationError(MISSING_DOCUMENT_ID_MESSAGE); -} - -function getPageCitationAssetDocumentId( - result: ParseResult, - fallbackDocumentId: string | undefined, -): string | undefined { - return result.documentId ?? fallbackDocumentId; -} - -function isPageChunk(chunk: Chunk): chunk is Extract { - return chunk.type === 'page'; -} - -const resolveSourceFile = Effect.fn('Knowhere.resolvePageCitationSource')( - function*(documentId: string) { - const source = yield* PageCitationSourceService; - return yield* source.getSource(documentId); - }, -); - -const renderMissingAssets = Effect.fn('Knowhere.renderMissingPageCitationAssets')( - function*(input: { - readonly sourceFile: SourceFile; - readonly pageRequests: readonly PageRequest[]; - readonly context: PageAssetContext; - readonly options: NormalizedPageCitationOptions; - readonly warnings: PageCitationAssetWarning[]; - readonly assetByPageNum: Map; - }) { - const renderer = yield* PageCitationRendererService; - const storage = yield* PageCitationStorageService; - - for (const request of input.pageRequests) { - const rendered = yield* renderer - .renderPage({ - source: input.sourceFile.body, - pageNum: request.pageNum, - format: input.options.format, - scale: input.options.scale, - quality: input.options.quality, - }) - .pipe( - Effect.match({ - onFailure: (error) => { - input.warnings.push(renderErrorToWarning(error, request, input.context)); - return undefined; - }, - onSuccess: (page) => page, - }), - ); - - if (!rendered) { - continue; - } - - const asset = yield* writeRenderedPage(storage, request, rendered, input.context).pipe( - Effect.match({ - onFailure: (error) => { - input.warnings.push(storageErrorToWarning(error, 'storage_failed', input.context)); - return undefined; - }, - onSuccess: (writtenAsset) => writtenAsset, - }), - ); - - if (asset) { - input.assetByPageNum.set(request.pageNum, asset); - } - } - }, -); - -function createSourceLayer( - documents: DocumentPageCitationSourceClient, - sourceFetchTimeoutMs: number, -): Layer.Layer { - return Layer.succeed(PageCitationSourceService, { - getSource: Effect.fn('PageCitationSourceService.getSource')(function*(documentId: string) { - const source = yield* Effect.tryPromise({ - try: () => - runPromiseWithTimeout( - () => documents.getPageCitationSource(documentId), - sourceFetchTimeoutMs, - `Timed out resolving page citation source for document ${documentId}`, - ), - catch: (cause) => - new PageCitationSourceError({ - operation: 'resolve_source', - documentId, - cause, - }), - }); - - const body = yield* Effect.tryPromise({ - try: () => fetchSourceBytes(source.url, sourceFetchTimeoutMs), - catch: (cause) => - new PageCitationSourceError({ - operation: 'fetch_source', - documentId, - cause, - }), - }); - - return { source, body }; - }), - }); -} - -function createRendererLayer( - options: NormalizedPageCitationOptions, - rendererHandle: PageCitationRendererHandle, -): Layer.Layer { - return Layer.succeed(PageCitationRendererService, { - renderPage: Effect.fn('PageCitationRendererService.renderPage')( - function*(input: RenderPageInput) { - const renderer = yield* Effect.tryPromise({ - try: () => rendererHandle.getRenderer(), - catch: (cause) => new PageCitationRendererUnavailableError({ cause }), - }); - - return yield* Effect.tryPromise({ - try: () => - runPromiseWithTimeout( - () => renderer.renderPage(input), - options.pageRenderMs, - `Timed out rendering PDF page ${input.pageNum}`, - ), - catch: (cause) => - new PageCitationRenderError({ - pageNum: input.pageNum, - cause, - }), - }); - }, - ), - }); -} - -function createRendererHandle(renderer: PageRenderer | undefined): PageCitationRendererHandle { - let cachedRenderer: PageRenderer | undefined = renderer; - let ownedRenderer: CloseablePageRenderer | undefined; - - return { - async getRenderer(): Promise { - if (cachedRenderer) { - return cachedRenderer; - } - - const defaultRenderer: PageRenderer = await createDefaultPageRenderer(); - cachedRenderer = defaultRenderer; - ownedRenderer = isCloseablePageRenderer(defaultRenderer) ? defaultRenderer : undefined; - return defaultRenderer; - }, - - async closeOwnedRenderer(): Promise { - if (!ownedRenderer) { - return; - } - - const rendererToClose: CloseablePageRenderer = ownedRenderer; - ownedRenderer = undefined; - await rendererToClose.close(); - }, - }; -} - -function closeOwnedRenderer( - rendererHandle: PageCitationRendererHandle, -): Effect.Effect { - return Effect.tryPromise({ - try: () => rendererHandle.closeOwnedRenderer(), - catch: (cause) => cause, - }).pipe(Effect.orElseSucceed(() => undefined)); -} - -function createStorageLayer( - storage: KnowhereSdkStorage, - storageOperationTimeoutMs: number, -): Layer.Layer { - return Layer.succeed(PageCitationStorageService, { - headObject: Effect.fn('PageCitationStorageService.headObject')(function*(key: string) { - return yield* Effect.tryPromise({ - try: () => - runPromiseWithTimeout( - () => storage.headObject(key), - storageOperationTimeoutMs, - `Timed out reading storage metadata for ${key}`, - ), - catch: (cause) => - new PageCitationStorageError({ - operation: 'head', - key, - cause, - }), - }); - }), - writeObject: Effect.fn('PageCitationStorageService.writeObject')( - function*(input: KnowhereSdkStorageObject) { - return yield* Effect.tryPromise({ - try: () => - runPromiseWithTimeout( - () => storage.writeObject(input), - storageOperationTimeoutMs, - `Timed out writing storage object ${input.key}`, - ), - catch: (cause) => - new PageCitationStorageError({ - operation: 'write', - key: input.key, - cause, - }), - }); - }, - ), - getObjectUrl: Effect.fn('PageCitationStorageService.getObjectUrl')(function*(key: string) { - if (!storage.getObjectUrl) { - return undefined; - } - - return yield* Effect.tryPromise({ - try: () => - runPromiseWithTimeout( - () => storage.getObjectUrl?.(key) ?? Promise.resolve(null), - storageOperationTimeoutMs, - `Timed out resolving storage URL for ${key}`, - ), - catch: (cause) => - new PageCitationStorageError({ - operation: 'url', - key, - cause, - }), - }).pipe(Effect.map((url) => url ?? undefined)); - }), - }); -} - -function normalizeOptions(options: PageCitationAssetsOptions): NormalizedPageCitationOptions { - return { - storage: options.storage ?? createLocalKnowhereSdkStorage(), - renderer: options.renderer, - variant: options.variant ?? DEFAULT_VARIANT, - format: options.format ?? DEFAULT_FORMAT, - scale: options.scale ?? DEFAULT_SCALE, - quality: options.quality, - strict: options.strict ?? false, - maxPagesToRenderPerRun: - options.maxPagesToRenderPerRun ?? DEFAULT_PAGE_CITATION_ASSET_LIMITS.maxPagesToRenderPerRun, - totalMs: options.timeouts?.totalMs ?? DEFAULT_PAGE_CITATION_ASSET_LIMITS.totalMs, - sourceFetchMs: - options.timeouts?.sourceFetchMs ?? DEFAULT_PAGE_CITATION_ASSET_LIMITS.sourceFetchMs, - pageRenderMs: - options.timeouts?.pageRenderMs ?? DEFAULT_PAGE_CITATION_ASSET_LIMITS.pageRenderMs, - storageOperationMs: - options.timeouts?.storageOperationMs ?? - DEFAULT_PAGE_CITATION_ASSET_LIMITS.storageOperationMs, - }; -} - -function buildPageRequests( - chunks: readonly Chunk[], - context: PageAssetContext, - warnings: PageCitationAssetWarning[], -): PageRequest[] { - const chunkIdsByPageNum = new Map(); - - for (const chunk of chunks) { - if (chunk.type !== 'page') { - continue; - } - - const pageNumbers = getChunkPageNumbers(chunk.metadata); - if (pageNumbers.invalidCount > 0) { - warnings.push( - createWarning({ - code: 'invalid_page_number', - message: `${pageNumbers.invalidCount} invalid page number value(s) were ignored.`, - documentId: context.documentId, - jobId: context.jobId, - chunkId: chunk.chunkId, - }), - ); - } - - for (const pageNum of pageNumbers.validPageNumbers) { - const chunkIds = chunkIdsByPageNum.get(pageNum) ?? []; - chunkIds.push(chunk.chunkId); - chunkIdsByPageNum.set(pageNum, chunkIds); - } - } - - return [...chunkIdsByPageNum.entries()] - .sort(([left], [right]) => left - right) - .map(([pageNum, chunkIds]) => ({ - pageNum, - chunkIds, - key: createPageAssetKey(context, pageNum), - })); -} - -function getChunkPageNumbers(metadata: Record): { - validPageNumbers: readonly number[]; - invalidCount: number; -} { - const values = [metadata.pageNums, metadata.page_nums].filter(isReadonlyUnknownArray); - const validPageNumbers = new Set(); - let invalidCount = 0; - - for (const value of values) { - for (const pageNum of value) { - if (typeof pageNum === 'number' && Number.isInteger(pageNum) && pageNum > 0) { - validPageNumbers.add(pageNum); - } else { - invalidCount += 1; - } - } - } - - return { - validPageNumbers: [...validPageNumbers].sort((left, right) => left - right), - invalidCount, - }; -} - -function isReadonlyUnknownArray(value: unknown): value is readonly unknown[] { - return Array.isArray(value); -} - -const readCachedAsset = Effect.fn('Knowhere.readCachedPageCitationAsset')( - function*( - storage: PageCitationStorageServiceShape, - request: PageRequest, - context: PageAssetContext, - ) { - const head = yield* storage.headObject(request.key).pipe( - Effect.match({ - onFailure: () => null, - onSuccess: (metadata) => metadata, - }), - ); - - if (!head) { - return undefined; - } - - const metadata = head.metadata ?? {}; - const width = readPositiveInteger(metadata.width); - const height = readPositiveInteger(metadata.height); - const mimeType = normalizeMimeType(metadata.mimeType ?? head.contentType); - - if (!width || !height || !mimeType) { - return undefined; - } - - const assetUrl = yield* storage.getObjectUrl(request.key).pipe( - Effect.match({ - onFailure: () => undefined, - onSuccess: (url) => url, - }), - ); - - return { - pageNum: request.pageNum, - key: request.key, - assetUrl, - mimeType, - width, - height, - source: PAGE_ASSET_SOURCE, - variant: context.variant, - }; - }, -); - -const writeRenderedPage = Effect.fn('Knowhere.writeRenderedPageCitationAsset')( - function*( - storage: PageCitationStorageServiceShape, - request: PageRequest, - rendered: RenderedPage, - context: PageAssetContext, - ) { - const writeResult = yield* storage.writeObject({ - key: request.key, - body: rendered.body, - contentType: rendered.mimeType, - metadata: { - pageNum: String(request.pageNum), - width: String(rendered.width), - height: String(rendered.height), - mimeType: rendered.mimeType, - source: PAGE_ASSET_SOURCE, - variant: context.variant, - documentId: context.documentId, - jobId: context.jobId, - }, - }); - const assetUrl = - writeResult.url ?? - (yield* storage.getObjectUrl(request.key).pipe( - Effect.match({ - onFailure: () => undefined, - onSuccess: (url) => url, - }), - )); - - return { - pageNum: request.pageNum, - key: request.key, - assetUrl, - mimeType: rendered.mimeType, - width: rendered.width, - height: rendered.height, - source: PAGE_ASSET_SOURCE, - variant: context.variant, - }; - }, -); - -function writeAssetIndexes( - storage: PageCitationStorageServiceShape, - context: PageAssetContext, - assets: readonly PageCitationAsset[], -): Effect.Effect { - const generatedAt = new Date().toISOString(); - const indexKey = createPageAssetIndexKey(context); - const currentKey = createPageAssetCurrentIndexKey(context); - const index: PageCitationAssetIndex = { - version: 1, - documentId: context.documentId, - namespace: context.namespace, - jobId: context.jobId, - jobResultId: context.jobResultId, - variant: context.variant, - generatedAt, - assets, - }; - const current: PageCitationAssetCurrentIndex = { - version: 1, - documentId: context.documentId, - namespace: context.namespace, - jobId: context.jobId, - jobResultId: context.jobResultId, - variant: context.variant, - indexKey, - updatedAt: generatedAt, - }; - - return Effect.gen(function*() { - yield* storage.writeObject({ - key: indexKey, - body: Buffer.from(JSON.stringify(index, null, 2)), - contentType: 'application/json', - metadata: { - documentId: context.documentId, - jobId: context.jobId, - variant: context.variant, - }, - }); - yield* storage.writeObject({ - key: currentKey, - body: Buffer.from(JSON.stringify(current, null, 2)), - contentType: 'application/json', - metadata: { - documentId: context.documentId, - jobId: context.jobId, - variant: context.variant, - }, - }); - }); -} - -function attachAssetsToChunks( - chunks: readonly Chunk[], - assetByPageNum: ReadonlyMap, -): Chunk[] { - return chunks.map((chunk) => { - if (chunk.type !== 'page') { - return chunk; - } - - const assets = getChunkPageNumbers(chunk.metadata).validPageNumbers - .map((pageNum) => assetByPageNum.get(pageNum)) - .filter((asset): asset is PageCitationAsset => asset !== undefined); - - if (assets.length === 0) { - return chunk; - } - - return { - ...chunk, - pageAssets: assets, - }; - }); -} - -function createPageAssetKey(context: PageAssetContext, pageNum: number): string { - const extension = context.format === 'image/png' ? 'png' : 'jpg'; - return [ - 'page-citation-assets', - 'documents', - toSafeKeySegment(context.documentId), - 'jobs', - toSafeKeySegment(context.jobId), - 'variants', - toSafeKeySegment(context.variant), - `scale-${toSafeKeySegment(String(context.scale))}`, - `page-${String(pageNum).padStart(6, '0')}.${extension}`, - ].join('/'); -} - -function createPageAssetIndexKey(context: PageAssetContext): string { - return [ - 'page-citation-assets', - 'documents', - toSafeKeySegment(context.documentId), - 'jobs', - toSafeKeySegment(context.jobId), - 'variants', - toSafeKeySegment(context.variant), - 'index.json', - ].join('/'); -} - -function createPageAssetCurrentIndexKey(context: PageAssetContext): string { - return [ - 'page-citation-assets', - 'documents', - toSafeKeySegment(context.documentId), - 'current.json', - ].join('/'); -} - -function toSafeKeySegment(value: string): string { - const normalized = value.replace(/[^A-Za-z0-9._-]+/g, '_').replace(/^_+|_+$/g, ''); - if (normalized.length > 0 && normalized === value) { - return normalized; - } - - const hash = createHash('sha256').update(value).digest('hex').slice(0, 12); - return `${(normalized || 'value').slice(0, 48)}-${hash}`; -} - -async function createDefaultPageRenderer(): Promise { - const rendererModule = await import('../page-renderer-pdfjs.js'); - return rendererModule.createPdfJsPageRenderer(); -} - -function isCloseablePageRenderer(renderer: PageRenderer): renderer is CloseablePageRenderer { - return 'close' in renderer && typeof renderer.close === 'function'; -} - -async function fetchSourceBytes(url: string, timeoutMs: number): Promise { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), timeoutMs); - try { - const response = await fetch(url, { signal: controller.signal }); - if (!response.ok) { - throw new Error(`Source fetch failed with HTTP ${response.status}`); - } - return new Uint8Array(await response.arrayBuffer()); - } finally { - clearTimeout(timeout); - } -} - -function createWarning(params: { - code: PageCitationAssetWarningCode; - message: string; - documentId?: string; - jobId?: string; - chunkId?: string; - pageNum?: number; - key?: string; - cause?: string; -}): PageCitationAssetWarning { - return { - code: params.code, - message: params.message, - documentId: params.documentId, - jobId: params.jobId, - chunkId: params.chunkId, - pageNum: params.pageNum, - key: params.key, - cause: params.cause, - }; -} - -function sourceErrorToWarning( - error: PageCitationSourceError, - context: PageAssetContext, -): PageCitationAssetWarning { - return createWarning({ - code: 'source_fetch_failed', - message: `Unable to ${error.operation === 'resolve_source' ? 'resolve' : 'fetch'} page citation source.`, - documentId: context.documentId, - jobId: context.jobId, - cause: getErrorMessage(error.cause), - }); -} - -function renderErrorToWarning( - error: PageCitationRenderError | PageCitationRendererUnavailableError, - request: PageRequest, - context: PageAssetContext, -): PageCitationAssetWarning { - if (error._tag === 'PageCitationRendererUnavailableError') { - return createWarning({ - code: 'renderer_unavailable', - message: 'The page citation renderer could not be loaded.', - documentId: context.documentId, - jobId: context.jobId, - pageNum: request.pageNum, - key: request.key, - cause: getErrorMessage(error.cause), - }); - } - - return createWarning({ - code: 'render_failed', - message: `Unable to render PDF page ${request.pageNum}.`, - documentId: context.documentId, - jobId: context.jobId, - pageNum: request.pageNum, - key: request.key, - cause: getErrorMessage(error.cause), - }); -} - -function storageErrorToWarning( - error: PageCitationStorageError, - code: 'storage_failed' | 'index_write_failed', - context: PageAssetContext, -): PageCitationAssetWarning { - return createWarning({ - code, - message: - code === 'index_write_failed' - ? 'Unable to write the page citation asset index.' - : `Unable to ${error.operation} page citation asset storage object.`, - documentId: context.documentId, - jobId: context.jobId, - key: error.key, - cause: getErrorMessage(error.cause), - }); -} - -function readPositiveInteger(value: string | undefined): number | undefined { - if (value === undefined) { - return undefined; - } - - const parsed = Number(value); - return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; -} - -function normalizeMimeType(value: string | undefined): PageCitationAssetMimeType | undefined { - if (value === 'image/png' || value === 'image/jpeg') { - return value; - } - return undefined; -} - -function getErrorMessage(error: unknown): string { - if (error instanceof Error) { - return error.message; - } - return String(error); -} - -function runPromiseWithTimeout( - promiseFactory: () => Promise, - timeoutMs: number, - timeoutMessage: string, -): Promise { - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => reject(new Error(timeoutMessage)), timeoutMs); - void promiseFactory() - .then(resolve, reject) - .finally(() => clearTimeout(timeout)); - }); -} diff --git a/src/page-renderer-pdfjs-worker.ts b/src/page-renderer-pdfjs-worker.ts deleted file mode 100644 index fe47244..0000000 --- a/src/page-renderer-pdfjs-worker.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { Canvas, createCanvas, DOMMatrix, ImageData, Path2D } from '@napi-rs/canvas'; - -interface PdfJsPageRenderTaskInput { - source: Uint8Array; - pageNum: number; - format: 'image/png' | 'image/jpeg'; - scale: number; - background: string; - quality?: number; -} - -interface PdfJsPageRenderTaskOutput { - body: Uint8Array; - mimeType: 'image/png' | 'image/jpeg'; - width: number; - height: number; -} - -interface PdfJsModule { - getDocument(params: PdfDocumentInitParameters): PdfLoadingTask; -} - -interface PdfDocumentInitParameters { - data: Uint8Array; - useSystemFonts: boolean; -} - -interface PdfLoadingTask { - promise: Promise; - destroy(): Promise; -} - -interface PdfDocument { - getPage(pageNum: number): Promise; - cleanup(): Promise; -} - -interface PdfPage { - getViewport(params: { scale: number }): PdfViewport; - render(params: PdfRenderParameters): PdfRenderTask; - cleanup(): void; -} - -interface PdfViewport { - width: number; - height: number; -} - -interface PdfRenderParameters { - canvas: null; - canvasContext: object; - viewport: PdfViewport; - background: string; -} - -interface PdfRenderTask { - promise: Promise; -} - -export async function renderPage( - input: PdfJsPageRenderTaskInput, -): Promise { - const pdfjs: PdfJsModule = await loadPdfJsModule(); - const loadingTask: PdfLoadingTask = pdfjs.getDocument({ - data: new Uint8Array(input.source), - useSystemFonts: true, - }); - const document: PdfDocument = await loadingTask.promise; - - try { - const page: PdfPage = await document.getPage(input.pageNum); - const viewport: PdfViewport = page.getViewport({ scale: input.scale }); - const width: number = Math.ceil(viewport.width); - const height: number = Math.ceil(viewport.height); - const canvas: Canvas = createCanvas(width, height); - const context: object = canvas.getContext('2d'); - - await page.render({ - canvas: null, - canvasContext: context, - viewport, - background: input.background, - }).promise; - page.cleanup(); - - const body: Uint8Array = - input.format === 'image/png' - ? canvas.toBuffer('image/png') - : canvas.toBuffer('image/jpeg', input.quality); - - return { - body, - mimeType: input.format, - width, - height, - }; - } finally { - await document.cleanup(); - await loadingTask.destroy(); - } -} - -async function loadPdfJsModule(): Promise { - installPdfJsDomPolyfills(); - const importedModule: unknown = await import('pdfjs-dist/legacy/build/pdf.mjs'); - if (isPdfJsModule(importedModule)) { - return importedModule; - } - - throw new Error('pdfjs-dist/build/pdf.mjs did not expose getDocument'); -} - -function installPdfJsDomPolyfills(): void { - installGlobalValue('DOMMatrix', DOMMatrix); - installGlobalValue('ImageData', ImageData); - installGlobalValue('Path2D', Path2D); -} - -function installGlobalValue(name: string, value: unknown): void { - if (name in globalThis) { - return; - } - - Object.defineProperty(globalThis, name, { - value, - configurable: true, - writable: true, - }); -} - -function isPdfJsModule(value: unknown): value is PdfJsModule { - return ( - typeof value === 'object' && - value !== null && - 'getDocument' in value && - typeof value.getDocument === 'function' - ); -} diff --git a/src/page-renderer-pdfjs.ts b/src/page-renderer-pdfjs.ts deleted file mode 100644 index 1ad4466..0000000 --- a/src/page-renderer-pdfjs.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { existsSync } from 'fs'; -import { createRequire } from 'module'; -import path from 'path'; -import { pathToFileURL } from 'url'; - -import Piscina from 'piscina'; - -import type { RenderedPage, RenderPageInput } from './types/index.js'; - -const DEFAULT_BACKGROUND: string = 'rgb(255,255,255)'; -const DEFAULT_MAX_THREADS: number = 1; -const DEFAULT_IDLE_TIMEOUT_MS: number = 30_000; -const DEFAULT_CONCURRENT_TASKS_PER_WORKER: number = 1; -const PACKAGE_RENDERER_SUBPATH: string = '@ontos-ai/knowhere-sdk/page-renderer-pdfjs'; -const WORKER_BASENAME: string = 'page-renderer-pdfjs-worker'; -const WORKER_TASK_NAME: string = 'renderPage'; - -interface PdfJsPageRenderTaskInput { - source: Uint8Array; - pageNum: number; - format: 'image/png' | 'image/jpeg'; - scale: number; - background: string; - quality?: number; -} - -export interface PdfJsPageRenderer { - renderPage(input: RenderPageInput): Promise; - close(): Promise; - destroy(): Promise; -} - -export interface PdfJsPageRendererOptions { - background?: string; - maxThreads?: number; - idleTimeoutMs?: number; - workerFile?: string | URL; -} - -export function createPdfJsPageRenderer( - options: PdfJsPageRendererOptions = {}, -): PdfJsPageRenderer { - const background: string = options.background ?? DEFAULT_BACKGROUND; - const pool: Piscina = createWorkerPool(options); - - return { - async renderPage(input: RenderPageInput): Promise { - throwIfAborted(input.signal); - const taskInput: PdfJsPageRenderTaskInput = { - source: new Uint8Array(input.source), - pageNum: input.pageNum, - format: input.format, - scale: input.scale, - background, - quality: input.quality, - }; - - if (input.signal) { - return pool.run(taskInput, { signal: input.signal }); - } - - return pool.run(taskInput); - }, - - close(): Promise { - return pool.close(); - }, - - destroy(): Promise { - return pool.destroy(); - }, - }; -} - -function createWorkerPool( - options: PdfJsPageRendererOptions, -): Piscina { - return new Piscina({ - filename: resolveWorkerFilename(options.workerFile), - name: WORKER_TASK_NAME, - minThreads: 0, - maxThreads: normalizeMaxThreads(options.maxThreads), - idleTimeout: options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS, - concurrentTasksPerWorker: DEFAULT_CONCURRENT_TASKS_PER_WORKER, - }); -} - -function normalizeMaxThreads(maxThreads: number | undefined): number { - if (maxThreads === undefined) { - return DEFAULT_MAX_THREADS; - } - - return Math.max(1, Math.min(2, Math.floor(maxThreads))); -} - -function resolveWorkerFilename(workerFile: string | URL | undefined): string { - if (workerFile instanceof URL) { - return workerFile.href; - } - - if (workerFile) { - return workerFile; - } - - const bundledWorkerFilename: string | undefined = resolveBundledWorkerFilename(); - if (bundledWorkerFilename) { - return bundledWorkerFilename; - } - - const sourceWorkerPath: string = path.resolve(process.cwd(), 'src', `${WORKER_BASENAME}.ts`); - if (existsSync(sourceWorkerPath)) { - return pathToFileURL(sourceWorkerPath).href; - } - - throw new Error('Unable to resolve the PDF.js page renderer worker file.'); -} - -function resolveBundledWorkerFilename(): string | undefined { - try { - const projectRequire: NodeRequire = createRequire(path.join(process.cwd(), 'package.json')); - const rendererFilename: string = projectRequire.resolve(PACKAGE_RENDERER_SUBPATH); - const workerFilename: string = path.join( - path.dirname(rendererFilename), - `${WORKER_BASENAME}.js`, - ); - return existsSync(workerFilename) ? pathToFileURL(workerFilename).href : undefined; - } catch { - return undefined; - } -} - -function throwIfAborted(signal: AbortSignal | undefined): void { - if (signal?.aborted) { - throw new Error('PDF page render aborted'); - } -} diff --git a/src/resources/__tests__/documents.test.ts b/src/resources/__tests__/documents.test.ts index 3669ed0..761d183 100644 --- a/src/resources/__tests__/documents.test.ts +++ b/src/resources/__tests__/documents.test.ts @@ -120,7 +120,28 @@ describe('Documents Resource', () => { sourceChunkPath: 'Chapter 1/Pages 4-6', filePath: null, sortOrder: 0, - metadata: { summary: 'Revenue rose across the covered pages.', pageNums: [4, 5, 6] }, + metadata: { + summary: 'Revenue rose across the covered pages.', + pageNums: [4, 5, 6], + pageAssets: [ + { + pageNum: 4, + artifactRef: 'page_citation_assets/page-4.png', + assetUrl: 'https://assets.example/page-4.png', + contentType: 'image/png', + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + pageAssets: [ + { + pageNum: 4, + artifactRef: 'page_citation_assets/page-4.png', + assetUrl: 'https://assets.example/page-4.png', + contentType: 'image/png', + source: 'knowhere-rendered-page-citation-source', + }, + ], assetUrl: null, createdAt: new Date('2026-04-27T04:00:00Z'), }, @@ -151,6 +172,9 @@ describe('Documents Resource', () => { expect(response.chunks[0]?.id).toBe('dchk-123'); expect(response.chunks[0]?.chunkType).toBe('page'); expect(response.chunks[0]?.contentSource).toBe('summary'); + expect(response.chunks[0]?.pageAssets?.[0]?.assetUrl).toBe( + 'https://assets.example/page-4.png', + ); expect(response.pagination.totalPages).toBe(2); }); diff --git a/src/resources/__tests__/retrieval.test.ts b/src/resources/__tests__/retrieval.test.ts index 559b4c8..a9c9cc6 100644 --- a/src/resources/__tests__/retrieval.test.ts +++ b/src/resources/__tests__/retrieval.test.ts @@ -163,11 +163,31 @@ describe('Retrieval Resource', () => { { chunkId: 'chunk-1', documentId: 'doc-1', - chunkType: 'text', - sectionPath: 'Root', + chunkType: 'page', + sectionPath: 'Page 4', filePath: null, jobId: 'job-1', - assetUrl: 'https://example.com/1', + assetUrl: 'https://assets.example/page-4.png', + metadata: { + pageAssets: [ + { + pageNum: 4, + artifactRef: 'page_citation_assets/page-4.png', + assetUrl: 'https://assets.example/page-4.png', + contentType: 'image/png', + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + pageAssets: [ + { + pageNum: 4, + artifactRef: 'page_citation_assets/page-4.png', + assetUrl: 'https://assets.example/page-4.png', + contentType: 'image/png', + source: 'knowhere-rendered-page-citation-source', + }, + ], }, ], results: [], @@ -184,6 +204,9 @@ describe('Retrieval Resource', () => { expect(response.referencedChunks).toHaveLength(1); expect(referencedChunk?.chunkId).toBe('chunk-1'); expect(referencedChunk?.filePath).toBeNull(); + expect(referencedChunk?.pageAssets?.[0]?.assetUrl).toBe( + 'https://assets.example/page-4.png', + ); expect(response.decisionTrace).toHaveLength(2); expect(response.decisionTrace![0]).toHaveProperty('phase', 'discovery'); expect(response.decisionTrace![1]).toHaveProperty('phase', 'terminal'); diff --git a/src/resources/documents.ts b/src/resources/documents.ts index ffc8cc3..64263ff 100644 --- a/src/resources/documents.ts +++ b/src/resources/documents.ts @@ -67,7 +67,7 @@ export class Documents extends BaseResource { } /** - * Get a signed source file URL suitable for SDK-side page citation rendering. + * Get a signed normalized source-file URL for compatibility and debugging. */ async getPageCitationSource(documentId: string): Promise { return this.httpClient.get( diff --git a/src/storage/__tests__/local-storage.test.ts b/src/storage/__tests__/local-storage.test.ts deleted file mode 100644 index 65612b9..0000000 --- a/src/storage/__tests__/local-storage.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { mkdtemp, readFile, rm } from 'fs/promises'; -import { tmpdir } from 'os'; -import path from 'path'; -import { afterEach, describe, expect, it } from 'vitest'; - -import { LocalKnowhereSdkStorage } from '../local-storage.js'; - -describe('LocalKnowhereSdkStorage', () => { - const tempDirectories: string[] = []; - - afterEach(async () => { - await Promise.all( - tempDirectories.map((directory) => rm(directory, { recursive: true, force: true })), - ); - tempDirectories.length = 0; - }); - - it('writes objects with metadata sidecars and optional public URLs', async () => { - const rootDirectory = await createTempDirectory(tempDirectories); - const storage = new LocalKnowhereSdkStorage({ - rootDirectory, - publicBaseUrl: 'https://cdn.example/assets/', - }); - - const write = await storage.writeObject({ - key: 'documents/doc-1/page.png', - body: new TextEncoder().encode('image'), - contentType: 'image/png', - metadata: { width: '120', height: '240' }, - }); - const head = await storage.headObject('documents/doc-1/page.png'); - const read = await storage.readObject('documents/doc-1/page.png'); - const sidecar = await readFile( - path.join(rootDirectory, 'documents', 'doc-1', 'page.png.metadata.json'), - 'utf8', - ); - - expect(write.url).toBe('https://cdn.example/assets/documents/doc-1/page.png'); - expect(head).toMatchObject({ - contentType: 'image/png', - metadata: { width: '120', height: '240' }, - }); - expect(Buffer.from(read?.body ?? new Uint8Array()).toString('utf8')).toBe('image'); - expect(JSON.parse(sidecar)).toMatchObject({ - contentType: 'image/png', - metadata: { width: '120', height: '240' }, - }); - }); - - it('rejects unsafe keys', async () => { - const rootDirectory = await createTempDirectory(tempDirectories); - const storage = new LocalKnowhereSdkStorage({ rootDirectory }); - - await expect( - storage.writeObject({ - key: '../escape.txt', - body: new Uint8Array([1]), - }), - ).rejects.toThrow(/relative POSIX path/); - await expect(storage.headObject('/absolute.txt')).rejects.toThrow(/relative POSIX path/); - }); - - it('deletes prefixes inside the storage root', async () => { - const rootDirectory = await createTempDirectory(tempDirectories); - const storage = new LocalKnowhereSdkStorage({ rootDirectory }); - - await storage.writeObject({ - key: 'documents/doc-1/page.png', - body: new Uint8Array([1]), - }); - await storage.deletePrefix('documents/doc-1'); - - expect(await storage.headObject('documents/doc-1/page.png')).toBeNull(); - }); -}); - -async function createTempDirectory(tempDirectories: string[]): Promise { - const directory = await mkdtemp(path.join(tmpdir(), 'knowhere-storage-')); - tempDirectories.push(directory); - return directory; -} diff --git a/src/storage/local-storage.ts b/src/storage/local-storage.ts deleted file mode 100644 index dd79277..0000000 --- a/src/storage/local-storage.ts +++ /dev/null @@ -1,259 +0,0 @@ -import os from 'os'; -import path from 'path'; -import { promises as fs } from 'fs'; - -import type { - KnowhereSdkStorage, - KnowhereSdkStorageHead, - KnowhereSdkStorageObject, - KnowhereSdkStorageReadResult, - KnowhereSdkStorageWriteResult, -} from '../types/page-citation-assets.js'; - -const DEFAULT_STORAGE_DIRECTORY = path.join( - os.homedir(), - '.knowhere-node-sdk', - 'page-citation-assets', -); - -interface StoredObjectMetadata { - contentType?: string; - metadata?: Readonly>; -} - -export interface LocalKnowhereSdkStorageOptions { - rootDirectory?: string; - publicBaseUrl?: string; -} - -export class LocalKnowhereSdkStorage implements KnowhereSdkStorage { - private readonly rootDirectory: string; - private readonly publicBaseUrl?: string; - - constructor(options: LocalKnowhereSdkStorageOptions = {}) { - this.rootDirectory = path.resolve(options.rootDirectory ?? DEFAULT_STORAGE_DIRECTORY); - this.publicBaseUrl = options.publicBaseUrl?.replace(/\/+$/, ''); - } - - async headObject(key: string): Promise { - const objectPath = this.resolveObjectPath(key); - try { - const stat = await fs.stat(objectPath); - const storedMetadata = await this.readMetadata(objectPath); - return { - key, - contentType: storedMetadata.contentType, - contentLength: stat.size, - metadata: storedMetadata.metadata, - }; - } catch (error) { - if (isMissingFileError(error)) { - return null; - } - throw error; - } - } - - async writeObject(input: KnowhereSdkStorageObject): Promise { - const objectPath = this.resolveObjectPath(input.key); - await fs.mkdir(path.dirname(objectPath), { recursive: true }); - await fs.writeFile(objectPath, await readStorageBody(input.body)); - await this.writeMetadata(objectPath, { - contentType: input.contentType, - metadata: input.metadata, - }); - const url = await this.getObjectUrl(input.key); - return { - key: input.key, - url: url ?? undefined, - }; - } - - getObjectUrl(key: string): Promise { - this.resolveObjectPath(key); - if (!this.publicBaseUrl) { - return Promise.resolve(null); - } - - return Promise.resolve( - `${this.publicBaseUrl}/${key.split('/').map(encodeURIComponent).join('/')}`, - ); - } - - async readObject(key: string): Promise { - const objectPath = this.resolveObjectPath(key); - try { - const body = await fs.readFile(objectPath); - const storedMetadata = await this.readMetadata(objectPath); - return { - body, - contentType: storedMetadata.contentType, - metadata: storedMetadata.metadata, - }; - } catch (error) { - if (isMissingFileError(error)) { - return null; - } - throw error; - } - } - - async deleteObject(key: string): Promise { - const objectPath = this.resolveObjectPath(key); - await fs.rm(objectPath, { force: true }); - await fs.rm(this.getMetadataPath(objectPath), { force: true }); - } - - async deletePrefix(prefix: string): Promise { - const prefixPath = this.resolvePrefixPath(prefix); - await fs.rm(prefixPath, { recursive: true, force: true }); - } - - private resolveObjectPath(key: string): string { - validateStorageKey(key); - const objectPath = path.resolve(this.rootDirectory, key); - if (!isPathInsideDirectory(objectPath, this.rootDirectory)) { - throw new Error(`Storage key resolves outside the storage root: ${key}`); - } - return objectPath; - } - - private resolvePrefixPath(prefix: string): string { - validateStoragePrefix(prefix); - const prefixPath = path.resolve(this.rootDirectory, prefix); - if (!isPathInsideDirectory(prefixPath, this.rootDirectory)) { - throw new Error(`Storage prefix resolves outside the storage root: ${prefix}`); - } - return prefixPath; - } - - private getMetadataPath(objectPath: string): string { - return `${objectPath}.metadata.json`; - } - - private async readMetadata(objectPath: string): Promise { - try { - const raw = await fs.readFile(this.getMetadataPath(objectPath), 'utf8'); - return normalizeStoredMetadata(JSON.parse(raw)); - } catch (error) { - if (isMissingFileError(error)) { - return {}; - } - throw error; - } - } - - private async writeMetadata( - objectPath: string, - metadata: StoredObjectMetadata, - ): Promise { - if (!metadata.contentType && !metadata.metadata) { - await fs.rm(this.getMetadataPath(objectPath), { force: true }); - return; - } - - await fs.writeFile(this.getMetadataPath(objectPath), JSON.stringify(metadata, null, 2)); - } -} - -export function createLocalKnowhereSdkStorage( - options?: LocalKnowhereSdkStorageOptions, -): LocalKnowhereSdkStorage { - return new LocalKnowhereSdkStorage(options); -} - -function validateStorageKey(key: string): void { - if ( - key.length === 0 || - key.startsWith('/') || - key.includes('\\') || - key.includes('\0') || - path.posix.normalize(key) !== key || - key.split('/').some((segment) => segment.length === 0 || segment === '..' || segment === '.') - ) { - throw new Error( - 'Storage key must be a relative POSIX path without empty, traversal, or absolute segments', - ); - } -} - -function validateStoragePrefix(prefix: string): void { - if (prefix.length === 0) { - throw new Error('Storage prefix is required'); - } - validateStorageKey(prefix.endsWith('/') ? `${prefix}__prefix_marker__` : prefix); -} - -async function readStorageBody(body: Uint8Array | ReadableStream | Blob): Promise { - if (body instanceof Uint8Array) { - return Buffer.from(body); - } - - if (isBlob(body)) { - return Buffer.from(await body.arrayBuffer()); - } - - return readReadableStream(body); -} - -function isBlob(value: unknown): value is Blob { - return typeof Blob !== 'undefined' && value instanceof Blob; -} - -async function readReadableStream(stream: ReadableStream): Promise { - const reader = stream.getReader(); - const chunks: Buffer[] = []; - - while (true) { - const result = await reader.read(); - if (result.done) { - return Buffer.concat(chunks); - } - chunks.push(Buffer.from(result.value)); - } -} - -function normalizeStoredMetadata(value: unknown): StoredObjectMetadata { - if (!isPlainRecord(value)) { - return {}; - } - - return { - contentType: typeof value.contentType === 'string' ? value.contentType : undefined, - metadata: normalizeMetadataMap(value.metadata), - }; -} - -function normalizeMetadataMap(value: unknown): Readonly> | undefined { - if (!isPlainRecord(value)) { - return undefined; - } - - const metadata: Record = {}; - for (const [key, metadataValue] of Object.entries(value)) { - if (typeof metadataValue === 'string') { - metadata[key] = metadataValue; - } - } - - return Object.keys(metadata).length > 0 ? metadata : undefined; -} - -function isPlainRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && value.constructor === Object; -} - -function isPathInsideDirectory(targetPath: string, parentDirectory: string): boolean { - const relativePath = path.relative(parentDirectory, targetPath); - return ( - relativePath.length === 0 || (!relativePath.startsWith('..') && !path.isAbsolute(relativePath)) - ); -} - -function isMissingFileError(error: unknown): boolean { - return hasErrorCode(error) && error.code === 'ENOENT'; -} - -function hasErrorCode(error: unknown): error is { readonly code: unknown } { - return typeof error === 'object' && error !== null && 'code' in error; -} diff --git a/src/types/document.ts b/src/types/document.ts index 3934b77..0a5654d 100644 --- a/src/types/document.ts +++ b/src/types/document.ts @@ -91,7 +91,7 @@ export interface DocumentChunkListParams { pageSize?: number; /** Optional chunk type filter */ chunkType?: DocumentChunkType; - /** Set true to include 7-day asset URLs for media chunks */ + /** Set true to include 7-day asset URLs for media and page citation assets */ includeAssetUrls?: boolean; } @@ -99,7 +99,7 @@ export interface DocumentChunkListParams { * Query parameters for GET /v2/documents/{document_id}/chunks/{document_chunk_id}. */ export interface DocumentChunkGetParams { - /** Set true to include 7-day asset URLs for media chunks */ + /** Set true to include 7-day asset URLs for media and page citation assets */ includeAssetUrls?: boolean; } @@ -129,9 +129,9 @@ export interface DocumentChunk { sortOrder: number; /** Chunk metadata returned by the API */ metadata: Record; - /** 7-day asset URL for media chunks when available */ + /** 7-day asset URL for media or page citation assets when available */ assetUrl?: string | null; - /** Rendered source-page citation assets attached by SDK-side generation when available. */ + /** Server-provided rendered source-page citation assets when available. */ pageAssets?: readonly PageCitationAsset[]; /** Chunk creation timestamp */ createdAt?: Date; diff --git a/src/types/page-citation-assets.ts b/src/types/page-citation-assets.ts index 16dbc20..1379ec5 100644 --- a/src/types/page-citation-assets.ts +++ b/src/types/page-citation-assets.ts @@ -1,135 +1,15 @@ -export type PageCitationAssetMimeType = 'image/png' | 'image/jpeg'; +export type PageCitationAssetContentType = 'image/png' | 'image/jpeg'; -export type PageCitationAssetSource = 'client-rendered-pdf-page'; +export type PageCitationAssetSource = + | 'knowhere-rendered-page-citation-source' + | 'client-rendered-page-citation-source'; export interface PageCitationAsset { pageNum: number; - key: string; + artifactRef: string; assetUrl?: string; - mimeType: PageCitationAssetMimeType; - width: number; - height: number; + contentType: PageCitationAssetContentType; + width?: number; + height?: number; source: PageCitationAssetSource; - variant: string; -} - -export type PageCitationAssetWarningCode = - | 'missing_document_id' - | 'invalid_page_number' - | 'source_fetch_failed' - | 'renderer_unavailable' - | 'render_failed' - | 'storage_failed' - | 'render_limit_exceeded' - | 'index_write_failed'; - -export interface PageCitationAssetWarning { - code: PageCitationAssetWarningCode; - message: string; - documentId?: string; - jobId?: string; - chunkId?: string; - pageNum?: number; - key?: string; - cause?: string; -} - -export interface PageCitationAssetTimeoutOptions { - totalMs?: number; - sourceFetchMs?: number; - pageRenderMs?: number; - storageOperationMs?: number; -} - -export interface PageCitationAssetsOptions { - storage?: KnowhereSdkStorage; - renderer?: PageRenderer; - variant?: string; - format?: PageCitationAssetMimeType; - scale?: number; - quality?: number; - strict?: boolean; - maxPagesToRenderPerRun?: number; - timeouts?: PageCitationAssetTimeoutOptions; -} - -export type KnowhereSdkStorageBody = Uint8Array | ReadableStream | Blob; - -export interface KnowhereSdkStorageObject { - key: string; - contentType?: string; - body: KnowhereSdkStorageBody; - metadata?: Readonly>; -} - -export interface KnowhereSdkStorageHead { - key: string; - contentType?: string; - contentLength?: number; - metadata?: Readonly>; -} - -export interface KnowhereSdkStorageWriteResult { - key: string; - url?: string; -} - -export interface KnowhereSdkStorageReadResult { - body: Uint8Array; - contentType?: string; - metadata?: Readonly>; -} - -export interface KnowhereSdkStorage { - headObject(key: string): Promise; - writeObject(input: KnowhereSdkStorageObject): Promise; - getObjectUrl?( - key: string, - options?: { expiresInSeconds?: number }, - ): Promise; - readObject?(key: string): Promise; - deleteObject?(key: string): Promise; - deletePrefix?(prefix: string): Promise; -} - -export interface RenderPageInput { - source: Uint8Array; - pageNum: number; - format: PageCitationAssetMimeType; - scale: number; - quality?: number; - signal?: AbortSignal; -} - -export interface RenderedPage { - body: Uint8Array; - mimeType: PageCitationAssetMimeType; - width: number; - height: number; -} - -export interface PageRenderer { - renderPage(input: RenderPageInput): Promise; -} - -export interface PageCitationAssetIndex { - version: 1; - documentId: string; - namespace?: string; - jobId: string; - jobResultId?: string; - variant: string; - generatedAt: string; - assets: readonly PageCitationAsset[]; -} - -export interface PageCitationAssetCurrentIndex { - version: 1; - documentId: string; - namespace?: string; - jobId: string; - jobResultId?: string; - variant: string; - indexKey: string; - updatedAt: string; } diff --git a/src/types/params.ts b/src/types/params.ts index 960654c..1e4ed4c 100644 --- a/src/types/params.ts +++ b/src/types/params.ts @@ -1,5 +1,4 @@ import type { ReadStream } from 'fs'; -import type { PageCitationAssetsOptions } from './page-citation-assets.js'; /** * Parsing model options @@ -156,8 +155,12 @@ export interface ParseParams { onPollProgress?: (status: PollProgress) => void; /** Abort signal */ signal?: AbortSignal; - /** Generate rendered page citation assets for page chunks. */ - pageCitationAssets?: PageCitationAssetsOptions; + /** + * @deprecated Page citation assets are generated by Knowhere during parsing + * and returned on page chunks when available. This option is accepted for + * compatibility and no longer triggers SDK-side rendering. + */ + pageCitationAssets?: Record; } /** diff --git a/src/types/pdfjs-dist-build-pdf.d.ts b/src/types/pdfjs-dist-build-pdf.d.ts deleted file mode 100644 index 656837f..0000000 --- a/src/types/pdfjs-dist-build-pdf.d.ts +++ /dev/null @@ -1,7 +0,0 @@ -declare module 'pdfjs-dist/build/pdf.mjs' { - export const getDocument: unknown; -} - -declare module 'pdfjs-dist/legacy/build/pdf.mjs' { - export const getDocument: unknown; -} diff --git a/src/types/result.ts b/src/types/result.ts index 89e7d96..9dc715f 100644 --- a/src/types/result.ts +++ b/src/types/result.ts @@ -1,4 +1,4 @@ -import type { PageCitationAsset, PageCitationAssetWarning } from './page-citation-assets.js'; +import type { PageCitationAsset } from './page-citation-assets.js'; /** * Statistics about the parsed document @@ -222,7 +222,7 @@ export interface TableChunk extends BaseChunk { */ export interface PageChunk extends BaseChunk { type: 'page'; - /** Rendered source-page citation assets attached by the SDK when requested. */ + /** Server-provided rendered source-page citation assets when available. */ pageAssets?: readonly PageCitationAsset[]; } @@ -245,8 +245,6 @@ export interface ParseResult { fullMarkdown?: string; /** Raw ZIP buffer */ rawZip: Buffer; - /** Non-fatal page citation asset diagnostics emitted by SDK-side generation. */ - pageCitationAssetWarnings?: readonly PageCitationAssetWarning[]; // Legacy — the current worker no longer emits these files /** @deprecated Current worker no longer emits chunks_slim.json */ diff --git a/src/types/retrieval.ts b/src/types/retrieval.ts index 0b120d7..50bdfd9 100644 --- a/src/types/retrieval.ts +++ b/src/types/retrieval.ts @@ -1,3 +1,5 @@ +import type { PageCitationAsset } from './page-citation-assets.js'; + /** * Section exclusion for follow-up retrieval queries. */ @@ -96,6 +98,8 @@ export interface RetrievalResult { filePath?: string | null; /** Chunk metadata returned by the API */ metadata?: Record; + /** Server-provided rendered source-page citation assets when available. */ + pageAssets?: readonly PageCitationAsset[]; /** Source reference for this result */ source: RetrievalSource; } @@ -124,6 +128,8 @@ export interface RetrievalReferencedChunk { assetUrl?: string | null; /** Chunk metadata returned by the API */ metadata?: Record; + /** Server-provided rendered source-page citation assets when available. */ + pageAssets?: readonly PageCitationAsset[]; } /** From 5577c2de974a5425e048e36da0a2ca5885743c75 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Fri, 3 Jul 2026 21:14:10 +0800 Subject: [PATCH 05/10] Preserve page citation assets in metadata --- .changeset/page-citation-assets-metadata.md | 6 + README.md | 13 +- src/knowledge/__tests__/knowledge.test.ts | 69 +++++---- src/knowledge/knowledge.ts | 2 - src/knowledge/types.ts | 11 +- src/lib/__tests__/result-parser.test.ts | 150 ++++++++++++++++++-- src/lib/result-parser.ts | 74 +--------- src/resources/__tests__/documents.test.ts | 18 +-- src/resources/__tests__/retrieval.test.ts | 18 +-- src/types/document.ts | 4 - src/types/page-citation-assets.ts | 4 +- src/types/params.ts | 6 - src/types/result.ts | 4 - src/types/retrieval.ts | 6 - 14 files changed, 202 insertions(+), 183 deletions(-) create mode 100644 .changeset/page-citation-assets-metadata.md diff --git a/.changeset/page-citation-assets-metadata.md b/.changeset/page-citation-assets-metadata.md new file mode 100644 index 0000000..95aa25f --- /dev/null +++ b/.changeset/page-citation-assets-metadata.md @@ -0,0 +1,6 @@ +--- +"@ontos-ai/knowhere-sdk": major +"@ontos-ai/knowhere-mcp": major +--- + +Preserve page citation asset descriptors only in chunk metadata and remove the deprecated SDK-side page citation asset generation options. diff --git a/README.md b/README.md index 9a96123..0991ffb 100644 --- a/README.md +++ b/README.md @@ -161,11 +161,11 @@ const result = await client.parse({ }); for (const chunk of result.pageChunks) { - console.log(chunk.pageAssets); + console.log(chunk.metadata.pageAssets); } ``` -`pageAssets` entries point at concrete page images stored with the Knowhere +`metadata.pageAssets` entries point at concrete page images stored with the Knowhere result. `artifactRef` is the durable result artifact path; `assetUrl` is an optional server-generated access URL. @@ -177,9 +177,7 @@ type PageCitationAsset = { contentType: 'image/png' | 'image/jpeg'; width?: number; height?: number; - source: - | 'knowhere-rendered-page-citation-source' - | 'client-rendered-page-citation-source'; + source: 'knowhere-rendered-page-citation-source'; }; ``` @@ -191,12 +189,9 @@ const chunks = await client.documents.listChunks('doc_123', { includeAssetUrls: true, }); -console.log(chunks.chunks[0]?.pageAssets?.[0]?.assetUrl); +console.log(chunks.chunks[0]?.metadata.pageAssets?.[0]?.assetUrl); ``` -`parse(...)` and `knowledge.cacheJobResult(...)` still accept the deprecated -`pageCitationAssets` option for source compatibility, but it is ignored. - ### Low-Level API For granular control over the job lifecycle: diff --git a/src/knowledge/__tests__/knowledge.test.ts b/src/knowledge/__tests__/knowledge.test.ts index dad9aaa..98294ba 100644 --- a/src/knowledge/__tests__/knowledge.test.ts +++ b/src/knowledge/__tests__/knowledge.test.ts @@ -438,12 +438,15 @@ describe('Knowledge', () => { limit: 1, }); - expect(read.chunks[0]?.pageAssets?.[0]).toMatchObject({ - pageNum: 1, - artifactRef: 'page_citation_assets/page-1.png', - width: 120, - height: 240, - }); + expect(read.chunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + width: 120, + height: 240, + }), + ]); + expect(read.chunks[0]).not.toHaveProperty('pageAssets'); }); it('should cache server-provided page assets without SDK-side rendering', async () => { @@ -455,7 +458,6 @@ describe('Knowledge', () => { const cached = await knowledge.cacheJobResult({ jobId: 'job-1', localDocumentId: 'local-report', - pageCitationAssets: { deprecated: true }, }); const read = await knowledge.readChunks({ localDocumentId: 'local-report', @@ -465,16 +467,22 @@ describe('Knowledge', () => { expect(jobsLoad).toHaveBeenCalledWith('job-1', { verifyChecksum: undefined }); expect(documentsGetPageCitationSource).not.toHaveBeenCalled(); - expect(cached.result.pageChunks[0]?.pageAssets?.[0]).toMatchObject({ - pageNum: 1, - artifactRef: 'page_citation_assets/page-1.png', - }); - expect(read.chunks[0]?.pageAssets?.[0]).toMatchObject({ - pageNum: 1, - artifactRef: 'page_citation_assets/page-1.png', - width: 120, - height: 240, - }); + expect(cached.result.pageChunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + }), + ]); + expect(cached.result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(read.chunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + width: 120, + height: 240, + }), + ]); + expect(read.chunks[0]).not.toHaveProperty('pageAssets'); }); async function createKnowledgeWithCachedResult( @@ -796,18 +804,21 @@ function createPageParseResultWithAssets(): ParseResult { content: 'Page one summary.', contentSource: 'summary', path: 'report.pdf/Page 1', - metadata: { summary: 'Page one summary.', pageNums: [1] }, - pageAssets: [ - { - pageNum: 1, - artifactRef: 'page_citation_assets/page-1.png', - assetUrl: 'https://assets.example/page-1.png', - contentType: 'image/png', - width: 120, - height: 240, - source: 'knowhere-rendered-page-citation-source', - }, - ], + metadata: { + summary: 'Page one summary.', + pageNums: [1], + pageAssets: [ + { + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: 'https://assets.example/page-1.png', + contentType: 'image/png', + width: 120, + height: 240, + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, }, ]; diff --git a/src/knowledge/knowledge.ts b/src/knowledge/knowledge.ts index 93b4e7b..e501b51 100644 --- a/src/knowledge/knowledge.ts +++ b/src/knowledge/knowledge.ts @@ -432,7 +432,6 @@ function indexChunks(result: ParseResult): IndexedKnowledgeChunk[] { sourceChunkPath: chunk.path, filePath, pageNumbers: getChunkPageNumbers(chunk.metadata), - pageAssets: chunk.type === 'page' ? chunk.pageAssets : undefined, metadata: chunk.metadata, }; }); @@ -673,7 +672,6 @@ function toReadChunk(chunk: IndexedKnowledgeChunk): KnowledgeReadChunk { sourceChunkPath: chunk.sourceChunkPath, filePath: chunk.filePath, pageNumbers: chunk.pageNumbers, - pageAssets: chunk.pageAssets, metadata: chunk.metadata, }; } diff --git a/src/knowledge/types.ts b/src/knowledge/types.ts index e4443ea..607be57 100644 --- a/src/knowledge/types.ts +++ b/src/knowledge/types.ts @@ -1,11 +1,6 @@ import type { ParseParams } from '../types/params.js'; import type { Job, JobResult } from '../types/job.js'; -import type { - Chunk, - DocumentChunkType, - PageCitationAsset, - ParseResult, -} from '../types/index.js'; +import type { Chunk, DocumentChunkType, ParseResult } from '../types/index.js'; export type KnowledgeChunkType = DocumentChunkType; @@ -81,8 +76,6 @@ export interface KnowledgeCacheJobResultParams { jobId: string; localDocumentId?: string; verifyChecksum?: boolean; - /** @deprecated Ignored. Page citation assets are provided by the server. */ - pageCitationAssets?: ParseParams['pageCitationAssets']; } export interface KnowledgeCacheDocumentParams { @@ -143,7 +136,6 @@ export interface KnowledgeReadChunk { sourceChunkPath: string; filePath?: string; pageNumbers?: number[]; - pageAssets?: readonly PageCitationAsset[]; metadata: Record; } @@ -231,6 +223,5 @@ export interface IndexedKnowledgeChunk { sourceChunkPath: string; filePath?: string; pageNumbers?: number[]; - pageAssets?: readonly PageCitationAsset[]; metadata: Record; } diff --git a/src/lib/__tests__/result-parser.test.ts b/src/lib/__tests__/result-parser.test.ts index f888b91..7a65c54 100644 --- a/src/lib/__tests__/result-parser.test.ts +++ b/src/lib/__tests__/result-parser.test.ts @@ -471,6 +471,64 @@ describe('Result Parser', () => { }, ], }, + }, + ], + }), + ); + const zipBuffer = await zip.generateAsync({ type: 'nodebuffer' }); + mockHttpClient.download.mockResolvedValue(zipBuffer); + + const result = await parseResult(mockHttpClient, 'https://s3.example.com/result.zip'); + + expect(result.statistics.pageChunks).toBe(1); + expect(result.pageChunks).toHaveLength(1); + expect(result.pageChunks[0].type).toBe('page'); + expect(result.pageChunks[0].contentSource).toBe('summary'); + expect(result.pageChunks[0].metadata.pageNums).toEqual([4, 5, 6]); + expect(result.pageChunks[0].metadata.pageAssets).toEqual([ + { + pageNum: 4, + artifactRef: 'page_citation_assets/page-4.png', + assetUrl: 'https://assets.example/page-4.png', + contentType: 'image/png', + width: 1200, + height: 1800, + source: 'knowhere-rendered-page-citation-source', + }, + ]); + expect(result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(result.textChunks).toHaveLength(0); + }); + + it('should ignore unsupported top-level page assets on page chunks', async () => { + const zip = new JSZip(); + zip.file( + 'manifest.json', + JSON.stringify({ + job_id: 'job-page-123', + source_file_name: 'manual.pdf', + statistics: { + total_chunks: 1, + text_chunks: 0, + image_chunks: 0, + table_chunks: 0, + page_chunks: 1, + }, + }), + ); + zip.file( + 'chunks.json', + JSON.stringify({ + chunks: [ + { + chunk_id: 'page-4', + type: 'page', + content: 'Summary for page 4.', + path: 'manual.pdf/Page 4', + metadata: { + summary: 'Summary for page 4.', + page_nums: [4], + }, page_assets: [ { page_num: 4, @@ -491,21 +549,11 @@ describe('Result Parser', () => { const result = await parseResult(mockHttpClient, 'https://s3.example.com/result.zip'); - expect(result.statistics.pageChunks).toBe(1); - expect(result.pageChunks).toHaveLength(1); - expect(result.pageChunks[0].type).toBe('page'); - expect(result.pageChunks[0].contentSource).toBe('summary'); - expect(result.pageChunks[0].metadata.pageNums).toEqual([4, 5, 6]); - expect(result.pageChunks[0].pageAssets?.[0]).toEqual({ - pageNum: 4, - artifactRef: 'page_citation_assets/page-4.png', - assetUrl: 'https://assets.example/page-4.png', - contentType: 'image/png', - width: 1200, - height: 1800, - source: 'knowhere-rendered-page-citation-source', + expect(result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(result.pageChunks[0].metadata).toEqual({ + summary: 'Summary for page 4.', + pageNums: [4], }); - expect(result.textChunks).toHaveLength(0); }); it('should extract full markdown if present', async () => { @@ -834,6 +882,80 @@ describe('Result Parser', () => { expect(reloaded.rawZip.length).toBe(0); }); + it('should preserve server-provided page citation asset files when saving expanded results', async () => { + const zip = new JSZip(); + zip.file( + 'manifest.json', + JSON.stringify({ + job_id: 'job-page-assets', + source_file_name: 'manual.pdf', + statistics: { + total_chunks: 1, + text_chunks: 0, + image_chunks: 0, + table_chunks: 0, + page_chunks: 1, + }, + }), + ); + zip.file( + 'chunks.json', + JSON.stringify({ + chunks: [ + { + chunk_id: 'page-1', + type: 'page', + content: 'Page one summary.', + path: 'manual.pdf/Page 1', + metadata: { + page_nums: [1], + page_assets: [ + { + page_num: 1, + artifact_ref: 'page_citation_assets/page-1.png', + content_type: 'image/png', + width: 120, + height: 240, + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + }, + ], + }), + ); + zip.file('page_citation_assets/page-1.png', Buffer.from('server-page-png')); + const zipBuffer = await zip.generateAsync({ type: 'nodebuffer' }); + mockHttpClient.download.mockResolvedValue(zipBuffer); + + const result = await parseResult(mockHttpClient, 'https://s3.example.com/result.zip'); + + await saveExpandedParseResult(result, testOutputDir); + + const savedPageAsset = await fs.readFile( + join(testOutputDir, 'page_citation_assets/page-1.png'), + ); + const savedChunks = JSON.parse( + await fs.readFile(join(testOutputDir, 'chunks.json'), 'utf8'), + ) as { + chunks: [ + { + metadata: { + page_assets: Array>; + }; + }, + ]; + }; + + expect(savedPageAsset.toString()).toBe('server-page-png'); + expect(savedChunks.chunks[0].metadata.page_assets).toEqual([ + expect.objectContaining({ + artifact_ref: 'page_citation_assets/page-1.png', + }), + ]); + expect(savedChunks.chunks[0]).not.toHaveProperty('page_assets'); + }); + it('should expose jobId property', async () => { const mockZipBuffer = await createMockResultZip(); mockHttpClient.download.mockResolvedValue(mockZipBuffer); diff --git a/src/lib/result-parser.ts b/src/lib/result-parser.ts index 295dc1e..9c96bef 100644 --- a/src/lib/result-parser.ts +++ b/src/lib/result-parser.ts @@ -15,7 +15,6 @@ import type { SlimChunk, DocNav, } from '../types/result.js'; -import type { PageCitationAsset } from '../types/page-citation-assets.js'; import type { LoadOptions } from '../types/params.js'; import { ChecksumError, KnowhereError } from '../errors/index.js'; import { sanitizePath, getFileExtension, parseDates, keysToCamel } from './utils.js'; @@ -28,7 +27,6 @@ type RawChunk = { path?: string; filePath?: string; metadata?: Record; - pageAssets?: readonly PageCitationAsset[]; }; type ChunkPayload = RawChunk[] | { chunks?: RawChunk[] }; @@ -225,7 +223,7 @@ export async function saveExpandedParseResult( result: ParseResult, directory: string, ): Promise { - if (result.rawZip.length > 0 && !hasEnrichedPageAssets(result)) { + if (result.rawZip.length > 0) { const didExtractZip = await tryExtractRawZip(result.rawZip, directory); if (didExtractZip) { return directory; @@ -478,7 +476,6 @@ function buildPageChunk(chunkData: RawChunk): PageChunk { content: chunkData.content ?? '', path: chunkData.path ?? '', metadata, - pageAssets: normalizePageAssets(chunkData.pageAssets ?? metadata.pageAssets), }; } @@ -619,18 +616,13 @@ async function processDirectoryChunk(directory: string, chunkData: RawChunk): Pr function serializeChunks(chunks: Chunk[]): { chunks: RawChunk[] } { return { chunks: chunks.map((chunk): RawChunk => { - const pageAssets = chunk.type === 'page' ? chunk.pageAssets : undefined; const rawChunk: RawChunk = { chunkId: chunk.chunkId, type: chunk.type, contentSource: chunk.contentSource, content: chunk.content, path: chunk.path, - metadata: - chunk.type === 'page' && pageAssets - ? { ...chunk.metadata, pageAssets } - : chunk.metadata, - pageAssets, + metadata: chunk.metadata, }; if (chunk.type === 'image' || chunk.type === 'table') { @@ -642,68 +634,6 @@ function serializeChunks(chunks: Chunk[]): { chunks: RawChunk[] } { }; } -function hasEnrichedPageAssets(result: ParseResult): boolean { - return result.chunks.some((chunk) => chunk.type === 'page' && chunk.pageAssets !== undefined); -} - -function normalizePageAssets(value: unknown): readonly PageCitationAsset[] | undefined { - if (!Array.isArray(value)) { - return undefined; - } - const assets = value.flatMap((item): PageCitationAsset[] => { - if (!isRecord(item)) { - return []; - } - const pageNum = getPositiveInteger(item.pageNum); - const artifactRef = getString(item.artifactRef); - const contentType = getPageCitationAssetContentType(item.contentType); - const source = getPageCitationAssetSource(item.source); - if (!pageNum || !artifactRef || !contentType || !source) { - return []; - } - return [ - { - pageNum, - artifactRef, - assetUrl: getString(item.assetUrl), - contentType, - width: getPositiveInteger(item.width), - height: getPositiveInteger(item.height), - source, - }, - ]; - }); - return assets.length > 0 ? assets : undefined; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function getString(value: unknown): string | undefined { - return typeof value === 'string' && value.trim() ? value : undefined; -} - -function getPositiveInteger(value: unknown): number | undefined { - if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) { - return undefined; - } - return value; -} - -function getPageCitationAssetContentType( - value: unknown, -): PageCitationAsset['contentType'] | undefined { - return value === 'image/png' || value === 'image/jpeg' ? value : undefined; -} - -function getPageCitationAssetSource(value: unknown): PageCitationAsset['source'] | undefined { - return value === 'knowhere-rendered-page-citation-source' || - value === 'client-rendered-page-citation-source' - ? value - : undefined; -} - async function readRequiredTextFile(directory: string, fileName: string): Promise { try { return await fs.readFile(join(directory, fileName), 'utf8'); diff --git a/src/resources/__tests__/documents.test.ts b/src/resources/__tests__/documents.test.ts index 761d183..f135760 100644 --- a/src/resources/__tests__/documents.test.ts +++ b/src/resources/__tests__/documents.test.ts @@ -133,15 +133,6 @@ describe('Documents Resource', () => { }, ], }, - pageAssets: [ - { - pageNum: 4, - artifactRef: 'page_citation_assets/page-4.png', - assetUrl: 'https://assets.example/page-4.png', - contentType: 'image/png', - source: 'knowhere-rendered-page-citation-source', - }, - ], assetUrl: null, createdAt: new Date('2026-04-27T04:00:00Z'), }, @@ -172,9 +163,12 @@ describe('Documents Resource', () => { expect(response.chunks[0]?.id).toBe('dchk-123'); expect(response.chunks[0]?.chunkType).toBe('page'); expect(response.chunks[0]?.contentSource).toBe('summary'); - expect(response.chunks[0]?.pageAssets?.[0]?.assetUrl).toBe( - 'https://assets.example/page-4.png', - ); + expect(response.chunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + assetUrl: 'https://assets.example/page-4.png', + }), + ]); + expect(response.chunks[0]).not.toHaveProperty('pageAssets'); expect(response.pagination.totalPages).toBe(2); }); diff --git a/src/resources/__tests__/retrieval.test.ts b/src/resources/__tests__/retrieval.test.ts index a9c9cc6..e7fb787 100644 --- a/src/resources/__tests__/retrieval.test.ts +++ b/src/resources/__tests__/retrieval.test.ts @@ -179,15 +179,6 @@ describe('Retrieval Resource', () => { }, ], }, - pageAssets: [ - { - pageNum: 4, - artifactRef: 'page_citation_assets/page-4.png', - assetUrl: 'https://assets.example/page-4.png', - contentType: 'image/png', - source: 'knowhere-rendered-page-citation-source', - }, - ], }, ], results: [], @@ -204,9 +195,12 @@ describe('Retrieval Resource', () => { expect(response.referencedChunks).toHaveLength(1); expect(referencedChunk?.chunkId).toBe('chunk-1'); expect(referencedChunk?.filePath).toBeNull(); - expect(referencedChunk?.pageAssets?.[0]?.assetUrl).toBe( - 'https://assets.example/page-4.png', - ); + expect(referencedChunk?.metadata?.pageAssets).toEqual([ + expect.objectContaining({ + assetUrl: 'https://assets.example/page-4.png', + }), + ]); + expect(referencedChunk).not.toHaveProperty('pageAssets'); expect(response.decisionTrace).toHaveLength(2); expect(response.decisionTrace![0]).toHaveProperty('phase', 'discovery'); expect(response.decisionTrace![1]).toHaveProperty('phase', 'terminal'); diff --git a/src/types/document.ts b/src/types/document.ts index 0a5654d..86df509 100644 --- a/src/types/document.ts +++ b/src/types/document.ts @@ -1,5 +1,3 @@ -import type { PageCitationAsset } from './page-citation-assets.js'; - /** * Canonical document state returned by document lifecycle endpoints. */ @@ -131,8 +129,6 @@ export interface DocumentChunk { metadata: Record; /** 7-day asset URL for media or page citation assets when available */ assetUrl?: string | null; - /** Server-provided rendered source-page citation assets when available. */ - pageAssets?: readonly PageCitationAsset[]; /** Chunk creation timestamp */ createdAt?: Date; } diff --git a/src/types/page-citation-assets.ts b/src/types/page-citation-assets.ts index 1379ec5..27f60e5 100644 --- a/src/types/page-citation-assets.ts +++ b/src/types/page-citation-assets.ts @@ -1,8 +1,6 @@ export type PageCitationAssetContentType = 'image/png' | 'image/jpeg'; -export type PageCitationAssetSource = - | 'knowhere-rendered-page-citation-source' - | 'client-rendered-page-citation-source'; +export type PageCitationAssetSource = 'knowhere-rendered-page-citation-source'; export interface PageCitationAsset { pageNum: number; diff --git a/src/types/params.ts b/src/types/params.ts index 1e4ed4c..296fddb 100644 --- a/src/types/params.ts +++ b/src/types/params.ts @@ -155,12 +155,6 @@ export interface ParseParams { onPollProgress?: (status: PollProgress) => void; /** Abort signal */ signal?: AbortSignal; - /** - * @deprecated Page citation assets are generated by Knowhere during parsing - * and returned on page chunks when available. This option is accepted for - * compatibility and no longer triggers SDK-side rendering. - */ - pageCitationAssets?: Record; } /** diff --git a/src/types/result.ts b/src/types/result.ts index 9dc715f..af4ff06 100644 --- a/src/types/result.ts +++ b/src/types/result.ts @@ -1,5 +1,3 @@ -import type { PageCitationAsset } from './page-citation-assets.js'; - /** * Statistics about the parsed document */ @@ -222,8 +220,6 @@ export interface TableChunk extends BaseChunk { */ export interface PageChunk extends BaseChunk { type: 'page'; - /** Server-provided rendered source-page citation assets when available. */ - pageAssets?: readonly PageCitationAsset[]; } /** diff --git a/src/types/retrieval.ts b/src/types/retrieval.ts index 50bdfd9..0b120d7 100644 --- a/src/types/retrieval.ts +++ b/src/types/retrieval.ts @@ -1,5 +1,3 @@ -import type { PageCitationAsset } from './page-citation-assets.js'; - /** * Section exclusion for follow-up retrieval queries. */ @@ -98,8 +96,6 @@ export interface RetrievalResult { filePath?: string | null; /** Chunk metadata returned by the API */ metadata?: Record; - /** Server-provided rendered source-page citation assets when available. */ - pageAssets?: readonly PageCitationAsset[]; /** Source reference for this result */ source: RetrievalSource; } @@ -128,8 +124,6 @@ export interface RetrievalReferencedChunk { assetUrl?: string | null; /** Chunk metadata returned by the API */ metadata?: Record; - /** Server-provided rendered source-page citation assets when available. */ - pageAssets?: readonly PageCitationAsset[]; } /** From 9bda767c575bd641e8954fb9bfdccb7247147f35 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sat, 4 Jul 2026 00:08:24 +0800 Subject: [PATCH 06/10] Add parse asset storage adapter --- src/__tests__/client.test.ts | 104 +++++- src/client.ts | 6 +- src/index.ts | 9 + src/knowledge/__tests__/knowledge.test.ts | 66 +++- src/knowledge/knowledge.ts | 27 +- src/knowledge/types.ts | 5 + src/lib/__tests__/result-parser.test.ts | 76 ++++ src/lib/result-parser.ts | 36 +- src/storage/__tests__/asset-storage.test.ts | 264 ++++++++++++++ src/storage/asset-storage.ts | 377 ++++++++++++++++++++ src/types/index.ts | 1 + src/types/params.ts | 6 + src/types/result.ts | 4 + src/types/storage.ts | 40 +++ 14 files changed, 1004 insertions(+), 17 deletions(-) create mode 100644 src/storage/__tests__/asset-storage.test.ts create mode 100644 src/storage/asset-storage.ts create mode 100644 src/types/storage.ts diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index 0883637..fe58aab 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -5,7 +5,13 @@ import { Knowhere } from '../client.js'; import { ValidationError } from '../errors/base.js'; import { Agent as HttpAgent } from 'http'; import { Agent as HttpsAgent } from 'https'; -import type { ParseResult, TextChunk } from '../types/result.js'; +import JSZip from 'jszip'; +import type { ParseResult, PageChunk, TextChunk } from '../types/result.js'; +import type { + KnowhereAssetStorageAdapter, + KnowhereAssetStorageObject, + KnowhereAssetStorageWriteResult, +} from '../types/storage.js'; describe('Knowhere Client', () => { // Store original env @@ -202,6 +208,94 @@ describe('Knowhere Client', () => { expect(result.documentId).toBe('doc-123'); }); + it('should store parse result assets when a storage adapter is provided', async () => { + const pageChunk: PageChunk = { + chunkId: 'page-1', + type: 'page', + content: 'Page one summary', + contentSource: 'summary', + path: 'report.pdf/Page 1', + metadata: { + pageNums: [1], + pageAssets: [ + { + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + contentType: 'image/png', + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + }; + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => + Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }), + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + + vi.spyOn(client.jobs, 'load').mockResolvedValueOnce({ + manifest: { + version: '2.0', + jobId: 'job-123', + sourceFileName: 'report.pdf', + statistics: { + totalChunks: 1, + textChunks: 0, + imageChunks: 0, + tableChunks: 0, + pageChunks: 1, + }, + }, + chunks: [pageChunk], + textChunks: [], + imageChunks: [], + tableChunks: [], + pageChunks: [pageChunk], + rawZip: await createZipBuffer({ + 'page_citation_assets/page-1.png': 'page-one-image', + }), + jobId: 'job-123', + statistics: { + totalChunks: 1, + textChunks: 0, + imageChunks: 0, + tableChunks: 0, + pageChunks: 1, + }, + getChunk: vi.fn(), + save: vi.fn(), + }); + + const result = await client.parse({ + url: 'https://example.com/doc.pdf', + storageAdapter: { + adapter, + keyPrefix: 'workspaces/workspace-1/sources/source-1/parsed-result', + }, + }); + + expect(writeObject).toHaveBeenCalledWith( + expect.objectContaining({ + key: 'workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + contentType: 'image/png', + }), + ); + expect(result.pageChunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + expect(result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(result.documentId).toBe('doc-123'); + }); + it('should parse document from file path', async () => { const result = await client.parse({ file: './test.pdf', @@ -492,3 +586,11 @@ describe('Knowhere Client', () => { }); }); }); + +async function createZipBuffer(files: Readonly>): Promise { + const zip = new JSZip(); + for (const [filePath, content] of Object.entries(files)) { + zip.file(filePath, Buffer.from(content)); + } + return Buffer.from(await zip.generateAsync({ type: 'arraybuffer' })); +} diff --git a/src/client.ts b/src/client.ts index 6b284a3..00ae169 100644 --- a/src/client.ts +++ b/src/client.ts @@ -13,6 +13,7 @@ import { Knowledge } from './knowledge/index.js'; import { DEFAULT_BASE_URL, ENV } from './constants.js'; import { ValidationError } from './errors/index.js'; import { enrichParseResult } from './lib/utils.js'; +import { storeParseResultAssets } from './storage/asset-storage.js'; function inferFileName(file: ParseParams['file'], explicitFileName?: string): string | undefined { if (explicitFileName) { @@ -145,10 +146,13 @@ export class Knowhere { verifyChecksum: params.verifyChecksum, }); - return enrichParseResult(result, { + const enrichedResult = enrichParseResult(result, { namespace: jobResult.namespace, documentId: jobResult.documentId ?? params.documentId, }); + + const storedResult = await storeParseResultAssets(enrichedResult, params.storageAdapter); + return storedResult.result; } /** diff --git a/src/index.ts b/src/index.ts index 27ef222..4e93c45 100644 --- a/src/index.ts +++ b/src/index.ts @@ -50,6 +50,13 @@ export type { ParseParams, UploadProgress, PollProgress, + KnowhereAssetStorageAdapter, + KnowhereAssetStorageBody, + KnowhereAssetStorageHead, + KnowhereAssetStorageObject, + KnowhereAssetStorageOptions, + KnowhereAssetStorageResult, + KnowhereAssetStorageWriteResult, PageCitationAsset, PageCitationAssetContentType, PageCitationAssetSource, @@ -89,6 +96,8 @@ export type { KnowledgeStartupRecoveryResponse, } from './types/index.js'; +export { storeParseResultAssets } from './storage/asset-storage.js'; + // Errors export { // Base errors diff --git a/src/knowledge/__tests__/knowledge.test.ts b/src/knowledge/__tests__/knowledge.test.ts index 98294ba..94f9a6b 100644 --- a/src/knowledge/__tests__/knowledge.test.ts +++ b/src/knowledge/__tests__/knowledge.test.ts @@ -1,6 +1,7 @@ import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import path from 'path'; +import JSZip from 'jszip'; import { describe, it, expect, vi, afterEach } from 'vitest'; import { Knowledge } from '../knowledge.js'; @@ -8,6 +9,9 @@ import type { Knowhere } from '../../client.js'; import type { Chunk, DocumentChunkListResponse, + KnowhereAssetStorageAdapter, + KnowhereAssetStorageObject, + KnowhereAssetStorageWriteResult, ParseResult, TextChunk, TableChunk, @@ -451,7 +455,7 @@ describe('Knowledge', () => { it('should cache server-provided page assets without SDK-side rendering', async () => { const cacheDirectory = await createTempDirectory(); - const parseResult = createPageParseResultWithAssets(); + const parseResult = await createPageParseResultWithRawPageAsset(); const { client, jobsLoad, documentsGetPageCitationSource } = createClient(parseResult); const knowledge = new Knowledge(client, { cacheDirectory }); @@ -485,6 +489,58 @@ describe('Knowledge', () => { expect(read.chunks[0]).not.toHaveProperty('pageAssets'); }); + it('should cache job result assets through a storage adapter and preserve metadata shape', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = await createPageParseResultWithRawPageAsset(); + const { client } = createClient(parseResult); + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => + Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }), + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + const knowledge = new Knowledge(client, { cacheDirectory }); + + const cached = await knowledge.cacheJobResult({ + jobId: 'job-1', + localDocumentId: 'local-report', + storageAdapter: { + adapter, + keyPrefix: 'workspaces/workspace-1/sources/source-1/parsed-result', + }, + }); + const read = await knowledge.readChunks({ + localDocumentId: 'local-report', + chunkType: 'page', + limit: 1, + }); + + expect(cached.assetUrlsByFilePath).toEqual({ + 'page_citation_assets/page-1.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }); + expect(cached.result.pageChunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + expect(cached.result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(read.chunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + expect(read.chunks[0]).not.toHaveProperty('pageAssets'); + }); + async function createKnowledgeWithCachedResult( parseResult = createParseResult(), ): Promise { @@ -824,3 +880,11 @@ function createPageParseResultWithAssets(): ParseResult { return createPageParseResult(chunks); } + +async function createPageParseResultWithRawPageAsset(): Promise { + const result = createPageParseResultWithAssets(); + const zip = new JSZip(); + zip.file('page_citation_assets/page-1.png', Buffer.from('page-one-image')); + result.rawZip = Buffer.from(await zip.generateAsync({ type: 'arraybuffer' })); + return result; +} diff --git a/src/knowledge/knowledge.ts b/src/knowledge/knowledge.ts index e501b51..7934694 100644 --- a/src/knowledge/knowledge.ts +++ b/src/knowledge/knowledge.ts @@ -9,6 +9,7 @@ import type { RetrievalResult, } from '../types/index.js'; import { ValidationError } from '../errors/index.js'; +import { storeParseResultAssets } from '../storage/asset-storage.js'; import { LocalKnowledgeStore } from './local-store.js'; import type { IndexedKnowledgeChunk, @@ -60,13 +61,18 @@ export class Knowledge { } async parse(params: KnowledgeParseParams): Promise { - const result = await this.client.parse(params); + const loadedResult = params.storageAdapter + ? await this.parseWithoutClientStorageAdapter(params) + : await this.client.parse(params); + const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter); + const result = storedAssets.result; const document = await this.store.saveResult(result, { localDocumentId: params.localDocumentId, }); return { document, result, + assetUrlsByFilePath: storedAssets.assetUrlsByFilePath, }; } @@ -108,16 +114,27 @@ export class Knowledge { const loadedResult = await this.client.jobs.load(params.jobId, { verifyChecksum: params.verifyChecksum, }); - const result = loadedResult; + const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter); + const result = storedAssets.result; const document = await this.store.saveResult(result, { localDocumentId: params.localDocumentId, }); return { document, result, + assetUrlsByFilePath: storedAssets.assetUrlsByFilePath, }; } + private async parseWithoutClientStorageAdapter( + params: KnowledgeParseParams, + ): Promise { + const parseParams: KnowledgeParseParams = { ...params }; + delete parseParams.localDocumentId; + delete parseParams.storageAdapter; + return this.client.parse(parseParams); + } + async cacheDocument(params: KnowledgeCacheDocumentParams): Promise { if (!params.documentId) { throw new ValidationError('documentId is required'); @@ -431,6 +448,7 @@ function indexChunks(result: ParseResult): IndexedKnowledgeChunk[] { sectionPath: normalizeSectionPath(chunk.path, result.manifest.sourceFileName), sourceChunkPath: chunk.path, filePath, + assetUrl: getChunkAssetUrl(chunk), pageNumbers: getChunkPageNumbers(chunk.metadata), metadata: chunk.metadata, }; @@ -467,6 +485,10 @@ function getChunkFilePath(chunk: Chunk): string | undefined { return typeof filePath === 'string' ? filePath : undefined; } +function getChunkAssetUrl(chunk: Chunk): string | undefined { + return chunk.type === 'image' || chunk.type === 'table' ? chunk.assetUrl : undefined; +} + function normalizeSectionPath(path: string, sourceFileName?: string): string { if (!path) { return ''; @@ -671,6 +693,7 @@ function toReadChunk(chunk: IndexedKnowledgeChunk): KnowledgeReadChunk { sectionPath: chunk.sectionPath, sourceChunkPath: chunk.sourceChunkPath, filePath: chunk.filePath, + assetUrl: chunk.assetUrl, pageNumbers: chunk.pageNumbers, metadata: chunk.metadata, }; diff --git a/src/knowledge/types.ts b/src/knowledge/types.ts index 607be57..ae4fa92 100644 --- a/src/knowledge/types.ts +++ b/src/knowledge/types.ts @@ -1,6 +1,7 @@ import type { ParseParams } from '../types/params.js'; import type { Job, JobResult } from '../types/job.js'; import type { Chunk, DocumentChunkType, ParseResult } from '../types/index.js'; +import type { KnowhereAssetStorageOptions } from '../types/storage.js'; export type KnowledgeChunkType = DocumentChunkType; @@ -35,6 +36,7 @@ export interface LocalKnowledgeDocument { export interface LocalKnowledgeParseResponse { document: LocalKnowledgeDocument; result: ParseResult; + assetUrlsByFilePath?: Readonly>; } export interface KnowledgeAsyncParseParams extends ParseParams { @@ -76,6 +78,7 @@ export interface KnowledgeCacheJobResultParams { jobId: string; localDocumentId?: string; verifyChecksum?: boolean; + storageAdapter?: KnowhereAssetStorageOptions; } export interface KnowledgeCacheDocumentParams { @@ -135,6 +138,7 @@ export interface KnowledgeReadChunk { sectionPath: string; sourceChunkPath: string; filePath?: string; + assetUrl?: string; pageNumbers?: number[]; metadata: Record; } @@ -222,6 +226,7 @@ export interface IndexedKnowledgeChunk { sectionPath: string; sourceChunkPath: string; filePath?: string; + assetUrl?: string; pageNumbers?: number[]; metadata: Record; } diff --git a/src/lib/__tests__/result-parser.test.ts b/src/lib/__tests__/result-parser.test.ts index 7a65c54..06a6da9 100644 --- a/src/lib/__tests__/result-parser.test.ts +++ b/src/lib/__tests__/result-parser.test.ts @@ -956,6 +956,82 @@ describe('Result Parser', () => { expect(savedChunks.chunks[0]).not.toHaveProperty('page_assets'); }); + it('should save rewritten page citation asset metadata after raw ZIP extraction', async () => { + const zip = new JSZip(); + zip.file( + 'manifest.json', + JSON.stringify({ + job_id: 'job-page-assets', + source_file_name: 'manual.pdf', + statistics: { + total_chunks: 1, + text_chunks: 0, + image_chunks: 0, + table_chunks: 0, + page_chunks: 1, + }, + }), + ); + zip.file( + 'chunks.json', + JSON.stringify({ + chunks: [ + { + chunk_id: 'page-1', + type: 'page', + content: 'Page one summary.', + path: 'manual.pdf/Page 1', + metadata: { + page_assets: [ + { + page_num: 1, + artifact_ref: 'page_citation_assets/page-1.png', + content_type: 'image/png', + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + }, + ], + }), + ); + zip.file('page_citation_assets/page-1.png', Buffer.from('server-page-png')); + mockHttpClient.download.mockResolvedValue(await zip.generateAsync({ type: 'nodebuffer' })); + const result = await parseResult(mockHttpClient, 'https://s3.example.com/result.zip'); + const pageAssets = result.pageChunks[0]?.metadata.pageAssets; + const firstPageAsset: unknown = Array.isArray(pageAssets) ? pageAssets[0] : undefined; + if ( + Array.isArray(pageAssets) && + typeof firstPageAsset === 'object' && + firstPageAsset !== null + ) { + pageAssets[0] = { + ...firstPageAsset, + assetUrl: 'https://blob.example/page_citation_assets/page-1.png', + }; + } + + await saveExpandedParseResult(result, testOutputDir); + + const savedChunks = JSON.parse( + await fs.readFile(join(testOutputDir, 'chunks.json'), 'utf8'), + ) as { + chunks: [ + { + metadata: { + page_assets: Array>; + }; + }, + ]; + }; + + expect(savedChunks.chunks[0].metadata.page_assets).toEqual([ + expect.objectContaining({ + asset_url: 'https://blob.example/page_citation_assets/page-1.png', + }), + ]); + }); + it('should expose jobId property', async () => { const mockZipBuffer = await createMockResultZip(); mockHttpClient.download.mockResolvedValue(mockZipBuffer); diff --git a/src/lib/result-parser.ts b/src/lib/result-parser.ts index 9c96bef..2ad60aa 100644 --- a/src/lib/result-parser.ts +++ b/src/lib/result-parser.ts @@ -17,7 +17,7 @@ import type { } from '../types/result.js'; import type { LoadOptions } from '../types/params.js'; import { ChecksumError, KnowhereError } from '../errors/index.js'; -import { sanitizePath, getFileExtension, parseDates, keysToCamel } from './utils.js'; +import { sanitizePath, getFileExtension, parseDates, keysToCamel, keysToSnake } from './utils.js'; type RawChunk = { chunkId?: string; @@ -26,6 +26,7 @@ type RawChunk = { content?: string; path?: string; filePath?: string; + assetUrl?: string; metadata?: Record; }; @@ -226,12 +227,30 @@ export async function saveExpandedParseResult( if (result.rawZip.length > 0) { const didExtractZip = await tryExtractRawZip(result.rawZip, directory); if (didExtractZip) { + await writeExpandedParseResultMetadata(result, directory); return directory; } } await fs.mkdir(directory, { recursive: true }); + await writeExpandedParseResultMetadata(result, directory); + + for (const imageChunk of result.imageChunks) { + await writeBinaryAsset(directory, imageChunk.filePath, imageChunk.data); + } + + for (const tableChunk of result.tableChunks) { + await writeTextAsset(directory, tableChunk.filePath, tableChunk.html); + } + + return directory; +} + +async function writeExpandedParseResultMetadata( + result: ParseResult, + directory: string, +): Promise { await fs.writeFile(join(directory, 'manifest.json'), JSON.stringify(result.manifest, null, 2)); if (result.docNav) { @@ -240,7 +259,7 @@ export async function saveExpandedParseResult( await fs.writeFile( join(directory, 'chunks.json'), - JSON.stringify(serializeChunks(result.chunks), null, 2), + JSON.stringify(keysToSnake(serializeChunks(result.chunks)), null, 2), ); if (result.chunksSlim) { @@ -275,16 +294,6 @@ export async function saveExpandedParseResult( if (result.hierarchyViewHtml) { await fs.writeFile(join(directory, 'hierarchy_view.html'), result.hierarchyViewHtml); } - - for (const imageChunk of result.imageChunks) { - await writeBinaryAsset(directory, imageChunk.filePath, imageChunk.data); - } - - for (const tableChunk of result.tableChunks) { - await writeTextAsset(directory, tableChunk.filePath, tableChunk.html); - } - - return directory; } async function tryExtractRawZip(zipBuffer: Buffer, directory: string): Promise { @@ -487,6 +496,7 @@ function buildImageChunk(chunkData: RawChunk, filePath: string, imageBuffer: Buf content: chunkData.content ?? '', path: chunkData.path ?? '', filePath, + assetUrl: chunkData.assetUrl, data: imageBuffer, metadata: chunkData.metadata ?? {}, @@ -508,6 +518,7 @@ function buildTableChunk(chunkData: RawChunk, filePath: string, html: string): T content: chunkData.content ?? '', path: chunkData.path ?? '', filePath, + assetUrl: chunkData.assetUrl, html, metadata: chunkData.metadata ?? {}, @@ -627,6 +638,7 @@ function serializeChunks(chunks: Chunk[]): { chunks: RawChunk[] } { if (chunk.type === 'image' || chunk.type === 'table') { rawChunk.filePath = chunk.filePath; + rawChunk.assetUrl = chunk.assetUrl; } return rawChunk; diff --git a/src/storage/__tests__/asset-storage.test.ts b/src/storage/__tests__/asset-storage.test.ts new file mode 100644 index 0000000..0e60b38 --- /dev/null +++ b/src/storage/__tests__/asset-storage.test.ts @@ -0,0 +1,264 @@ +import JSZip from 'jszip'; +import { describe, expect, it, vi } from 'vitest'; + +import { storeParseResultAssets } from '../asset-storage.js'; +import type { + KnowhereAssetStorageAdapter, + KnowhereAssetStorageHead, + KnowhereAssetStorageObject, + KnowhereAssetStorageWriteResult, +} from '../../types/storage.js'; +import type { + Chunk, + ImageChunk, + PageChunk, + ParseResult, + TableChunk, + TextChunk, +} from '../../types/result.js'; + +describe('storeParseResultAssets', () => { + it('stores image, table, and page citation assets and rewrites display URLs', async () => { + const result = await createParseResultWithAssets(); + const writes: Array<{ + readonly key: string; + readonly bodyText: string; + readonly contentType: string; + }> = []; + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => { + writes.push({ + key: input.key, + bodyText: Buffer.from(input.body).toString('utf8'), + contentType: input.contentType, + }); + return Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }); + }, + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + + const stored = await storeParseResultAssets(result, { + adapter, + keyPrefix: 'workspaces/workspace-1/sources/source-1/parsed-result', + }); + + expect(stored.assetUrlsByFilePath).toEqual({ + 'images/chart.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', + 'page_citation_assets/page-1.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + 'tables/revenue.html': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/tables/revenue.html', + }); + expect(writes).toEqual([ + { + key: 'workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', + bodyText: 'chart-image', + contentType: 'image/png', + }, + { + key: 'workspaces/workspace-1/sources/source-1/parsed-result/tables/revenue.html', + bodyText: 'Revenue
', + contentType: 'text/html; charset=utf-8', + }, + { + key: 'workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + bodyText: 'page-one-image', + contentType: 'image/png', + }, + ]); + expect(stored.result.imageChunks[0]?.assetUrl).toBe( + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', + ); + expect(stored.result.tableChunks[0]?.assetUrl).toBe( + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/tables/revenue.html', + ); + expect(stored.result.pageChunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + expect(stored.result.pageChunks[0]).not.toHaveProperty('pageAssets'); + }); + + it('uses existing storage URLs when skipExisting is enabled', async () => { + const result = createParseResult([ + { + chunkId: 'image-1', + type: 'image', + content: 'Chart summary', + path: 'report/images/chart.png', + filePath: 'images/chart.png', + data: Buffer.from('chart-image'), + metadata: {}, + format: 'png', + save: vi.fn(), + }, + ]); + const headObject = vi.fn( + (key: string): Promise => + Promise.resolve({ + key, + url: `https://blob.example/existing/${key}`, + }), + ); + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => + Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }), + ); + const adapter: KnowhereAssetStorageAdapter = { + headObject, + writeObject, + }; + + const stored = await storeParseResultAssets(result, { + adapter, + keyPrefix: 'parsed-result', + }); + + expect(writeObject).not.toHaveBeenCalled(); + expect(stored.assetUrlsByFilePath).toEqual({ + 'images/chart.png': 'https://blob.example/existing/parsed-result/images/chart.png', + }); + expect(stored.result.imageChunks[0]?.assetUrl).toBe( + 'https://blob.example/existing/parsed-result/images/chart.png', + ); + }); + + it('rejects unsafe storage key prefixes and ignores unsafe asset refs', async () => { + const result = createParseResult([ + { + chunkId: 'image-1', + type: 'image', + content: 'Chart summary', + path: 'report/images/chart.png', + filePath: '../chart.png', + data: Buffer.from('chart-image'), + metadata: {}, + format: 'png', + save: vi.fn(), + }, + ]); + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => + Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }), + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + + await expect( + storeParseResultAssets(result, { + adapter, + keyPrefix: '../parsed-result', + }), + ).rejects.toThrow(/keyPrefix/); + + const stored = await storeParseResultAssets(result, { + adapter, + keyPrefix: 'parsed-result', + }); + + expect(writeObject).not.toHaveBeenCalled(); + expect(stored.assetUrlsByFilePath).toEqual({}); + }); +}); + +async function createParseResultWithAssets(): Promise { + const zip = new JSZip(); + zip.file('page_citation_assets/page-1.png', Buffer.from('page-one-image')); + + return createParseResult( + [ + { + chunkId: 'image-1', + type: 'image', + content: 'Chart summary', + path: 'report/images/chart.png', + filePath: 'images/chart.png', + data: Buffer.from('chart-image'), + metadata: {}, + format: 'png', + save: vi.fn(), + }, + { + chunkId: 'table-1', + type: 'table', + content: 'Revenue
', + path: 'report/tables/revenue.html', + filePath: 'tables/revenue.html', + html: 'Revenue
', + metadata: {}, + save: vi.fn(), + }, + { + chunkId: 'page-1', + type: 'page', + content: 'Page one summary', + contentSource: 'summary', + path: 'report/Page 1', + metadata: { + pageNums: [1], + pageAssets: [ + { + pageNum: 1, + artifactRef: 'page_citation_assets/page-1.png', + contentType: 'image/png', + width: 120, + height: 240, + source: 'knowhere-rendered-page-citation-source', + }, + ], + }, + }, + ], + Buffer.from(await zip.generateAsync({ type: 'arraybuffer' })), + ); +} + +function createParseResult(chunks: readonly Chunk[], rawZip: Buffer = Buffer.alloc(0)): ParseResult { + return { + manifest: { + version: '2.0', + jobId: 'job-1', + sourceFileName: 'report.pdf', + statistics: { + totalChunks: chunks.length, + textChunks: chunks.filter((chunk) => chunk.type === 'text').length, + imageChunks: chunks.filter((chunk) => chunk.type === 'image').length, + tableChunks: chunks.filter((chunk) => chunk.type === 'table').length, + pageChunks: chunks.filter((chunk) => chunk.type === 'page').length, + }, + }, + chunks: [...chunks], + rawZip, + textChunks: chunks.filter((chunk): chunk is TextChunk => chunk.type === 'text'), + imageChunks: chunks.filter((chunk): chunk is ImageChunk => chunk.type === 'image'), + tableChunks: chunks.filter((chunk): chunk is TableChunk => chunk.type === 'table'), + pageChunks: chunks.filter((chunk): chunk is PageChunk => chunk.type === 'page'), + jobId: 'job-1', + statistics: { + totalChunks: chunks.length, + textChunks: chunks.filter((chunk) => chunk.type === 'text').length, + imageChunks: chunks.filter((chunk) => chunk.type === 'image').length, + tableChunks: chunks.filter((chunk) => chunk.type === 'table').length, + pageChunks: chunks.filter((chunk) => chunk.type === 'page').length, + }, + getChunk: (chunkId: string) => chunks.find((chunk) => chunk.chunkId === chunkId), + save: vi.fn(), + }; +} diff --git a/src/storage/asset-storage.ts b/src/storage/asset-storage.ts new file mode 100644 index 0000000..2a90434 --- /dev/null +++ b/src/storage/asset-storage.ts @@ -0,0 +1,377 @@ +import JSZip from 'jszip'; + +import type { + Chunk, + ImageChunk, + PageChunk, + ParseResult, + TableChunk, + TextChunk, +} from '../types/result.js'; +import type { PageCitationAsset } from '../types/page-citation-assets.js'; +import type { + KnowhereAssetStorageAdapter, + KnowhereAssetStorageBody, + KnowhereAssetStorageOptions, + KnowhereAssetStorageResult, +} from '../types/storage.js'; + +interface StorageAsset { + readonly sourcePath: string; + readonly storageKey: string; + readonly body: KnowhereAssetStorageBody; + readonly contentType: string; + readonly metadata: Readonly>; +} + +const pageCitationAssetsMetadataKey = 'pageAssets'; +const defaultImageContentType = 'application/octet-stream'; +const tableContentType = 'text/html; charset=utf-8'; + +export async function storeParseResultAssets( + result: ParseResult, + options?: KnowhereAssetStorageOptions, +): Promise { + if (!options) { + return { + result, + assetUrlsByFilePath: {}, + }; + } + + const normalizedKeyPrefix = normalizeStorageKeyPrefix(options.keyPrefix); + const assets = dedupeStorageAssets( + await collectStorageAssets(result, normalizedKeyPrefix), + ); + const assetUrlsByFilePath: Record = {}; + + for (const asset of assets) { + const storedUrl = await storeAsset(asset, options.adapter, options.skipExisting ?? true); + if (storedUrl) { + assetUrlsByFilePath[asset.sourcePath] = storedUrl; + } + } + + if (Object.keys(assetUrlsByFilePath).length === 0) { + return { + result, + assetUrlsByFilePath, + }; + } + + return { + result: rewriteResultAssetUrls(result, assetUrlsByFilePath), + assetUrlsByFilePath, + }; +} + +async function collectStorageAssets(result: ParseResult, keyPrefix: string): Promise { + const resultZip = await loadResultZipForPageCitationAssets(result); + return [ + ...result.imageChunks.flatMap((chunk) => collectImageStorageAsset(chunk, keyPrefix)), + ...result.tableChunks.flatMap((chunk) => collectTableStorageAsset(chunk, keyPrefix)), + ...(resultZip + ? await collectPageCitationStorageAssets(result.pageChunks, keyPrefix, resultZip) + : []), + ]; +} + +function collectImageStorageAsset(chunk: ImageChunk, keyPrefix: string): StorageAsset[] { + const sourcePath = normalizeResultAssetPath(chunk.filePath); + if (!sourcePath) return []; + + return [ + { + sourcePath, + storageKey: `${keyPrefix}/${sourcePath}`, + body: chunk.data, + contentType: getImageContentType(chunk.filePath), + metadata: { + chunkId: chunk.chunkId, + chunkType: chunk.type, + sourcePath, + }, + }, + ]; +} + +function collectTableStorageAsset(chunk: TableChunk, keyPrefix: string): StorageAsset[] { + const sourcePath = normalizeResultAssetPath(chunk.filePath); + if (!sourcePath) return []; + + return [ + { + sourcePath, + storageKey: `${keyPrefix}/${sourcePath}`, + body: Buffer.from(chunk.html, 'utf8'), + contentType: tableContentType, + metadata: { + chunkId: chunk.chunkId, + chunkType: chunk.type, + sourcePath, + }, + }, + ]; +} + +async function collectPageCitationStorageAssets( + chunks: readonly PageChunk[], + keyPrefix: string, + resultZip: JSZip, +): Promise { + const assets: StorageAsset[] = []; + + for (const chunk of chunks) { + const pageAssets = parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]); + for (const pageAsset of pageAssets) { + const sourcePath = normalizeResultAssetPath(pageAsset.artifactRef); + if (!sourcePath) continue; + + const zipEntry = resultZip.file(sourcePath); + if (!zipEntry) continue; + + assets.push({ + sourcePath, + storageKey: `${keyPrefix}/${sourcePath}`, + body: await zipEntry.async('uint8array'), + contentType: pageAsset.contentType, + metadata: getPageCitationAssetMetadata(chunk, pageAsset, sourcePath), + }); + } + } + + return assets; +} + +function getPageCitationAssetMetadata( + chunk: PageChunk, + pageAsset: PageCitationAsset, + sourcePath: string, +): Readonly> { + return { + chunkId: chunk.chunkId, + chunkType: chunk.type, + pageNum: String(pageAsset.pageNum), + sourcePath, + }; +} + +async function loadResultZipForPageCitationAssets(result: ParseResult): Promise { + if (!hasPageCitationAssetReferences(result) || result.rawZip.length === 0) { + return null; + } + + try { + return await JSZip.loadAsync(result.rawZip); + } catch (cause) { + throw new Error('Unable to read page citation assets from the Knowhere result ZIP.', { + cause, + }); + } +} + +function hasPageCitationAssetReferences(result: ParseResult): boolean { + return result.pageChunks.some( + (chunk) => parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]).length > 0, + ); +} + +function dedupeStorageAssets(assets: readonly StorageAsset[]): StorageAsset[] { + const assetsBySourcePath = new Map(); + + for (const asset of assets) { + if (!assetsBySourcePath.has(asset.sourcePath)) { + assetsBySourcePath.set(asset.sourcePath, asset); + } + } + + return [...assetsBySourcePath.values()]; +} + +async function storeAsset( + asset: StorageAsset, + adapter: KnowhereAssetStorageAdapter, + skipExisting: boolean, +): Promise { + if (skipExisting && adapter.headObject) { + const existing = await adapter.headObject(asset.storageKey); + if (existing) { + const existingUrl = existing.url ?? (await adapter.getObjectUrl?.(asset.storageKey)); + if (existingUrl) return existingUrl; + } + } + + const writeResult = await adapter.writeObject({ + key: asset.storageKey, + body: asset.body, + contentType: asset.contentType, + metadata: asset.metadata, + }); + return writeResult.url ?? (await adapter.getObjectUrl?.(writeResult.key)) ?? null; +} + +function rewriteResultAssetUrls( + result: ParseResult, + assetUrlsByFilePath: Readonly>, +): ParseResult { + const chunks = result.chunks.map((chunk) => rewriteChunkAssetUrls(chunk, assetUrlsByFilePath)); + result.chunks.splice(0, result.chunks.length, ...chunks); + refreshStaticChunkCollections(result); + return result; +} + +function rewriteChunkAssetUrls( + chunk: Chunk, + assetUrlsByPath: Readonly>, +): Chunk { + if (chunk.type === 'image' || chunk.type === 'table') { + const sourcePath = normalizeResultAssetPath(chunk.filePath); + const assetUrl = sourcePath ? assetUrlsByPath[sourcePath] : undefined; + if (!assetUrl) return chunk; + return { + ...chunk, + assetUrl, + }; + } + + if (chunk.type === 'page') { + const pageAssets = parsePageCitationAssets(chunk.metadata[pageCitationAssetsMetadataKey]); + if (pageAssets.length === 0) return chunk; + + const rewrittenPageAssets = pageAssets.map((pageAsset) => { + const sourcePath = normalizeResultAssetPath(pageAsset.artifactRef); + const assetUrl = sourcePath ? assetUrlsByPath[sourcePath] : undefined; + return assetUrl ? { ...pageAsset, assetUrl } : pageAsset; + }); + + return { + ...chunk, + metadata: { + ...chunk.metadata, + [pageCitationAssetsMetadataKey]: rewrittenPageAssets, + }, + }; + } + + return chunk; +} + +function parsePageCitationAssets(value: unknown): PageCitationAsset[] { + if (!Array.isArray(value)) return []; + + return value.flatMap((item): PageCitationAsset[] => { + if (!isRecord(item)) return []; + const pageNum = item.pageNum; + const artifactRef = item.artifactRef; + const contentType = item.contentType; + const source = item.source; + + if (typeof pageNum !== 'number' || !Number.isSafeInteger(pageNum) || pageNum <= 0) return []; + if (typeof artifactRef !== 'string') return []; + if (contentType !== 'image/png' && contentType !== 'image/jpeg') return []; + if (source !== 'knowhere-rendered-page-citation-source') return []; + + return [ + { + pageNum, + artifactRef, + assetUrl: typeof item.assetUrl === 'string' ? item.assetUrl : undefined, + contentType, + width: typeof item.width === 'number' ? item.width : undefined, + height: typeof item.height === 'number' ? item.height : undefined, + source, + }, + ]; + }); +} + +function normalizeStorageKeyPrefix(value: string): string { + const normalized = normalizeStoragePath(value, { allowDotPrefix: false }); + if (!normalized) { + throw new Error('storageAdapter keyPrefix must be a non-empty relative POSIX path.'); + } + return normalized; +} + +function normalizeResultAssetPath(value: string | undefined): string | null { + return normalizeStoragePath(value, { allowDotPrefix: true }); +} + +function normalizeStoragePath( + value: string | undefined, + options: { readonly allowDotPrefix: boolean }, +): string | null { + if (!value) return null; + const trimmed = value.trim(); + if (trimmed.length === 0 || trimmed.includes('\0')) return null; + if (trimmed.startsWith('/') || /^[A-Za-z]:/.test(trimmed)) return null; + + const normalized = options.allowDotPrefix + ? trimmed.replaceAll('\\', '/').replace(/^\.\/+/, '') + : trimmed.replaceAll('\\', '/'); + + if (normalized.length === 0) return null; + + const parts = normalized.split('/'); + if (parts.some((part) => part.length === 0 || part === '.' || part === '..')) return null; + + return parts.join('/'); +} + +type StaticChunkCollections = { + textChunks: TextChunk[]; + imageChunks: ImageChunk[]; + tableChunks: TableChunk[]; + pageChunks: PageChunk[]; +}; + +function refreshStaticChunkCollections(result: ParseResult): void { + replaceStaticChunkCollection( + result, + 'textChunks', + result.chunks.filter((chunk): chunk is TextChunk => chunk.type === 'text'), + ); + replaceStaticChunkCollection( + result, + 'imageChunks', + result.chunks.filter((chunk): chunk is ImageChunk => chunk.type === 'image'), + ); + replaceStaticChunkCollection( + result, + 'tableChunks', + result.chunks.filter((chunk): chunk is TableChunk => chunk.type === 'table'), + ); + replaceStaticChunkCollection( + result, + 'pageChunks', + result.chunks.filter((chunk): chunk is PageChunk => chunk.type === 'page'), + ); +} + +function replaceStaticChunkCollection( + result: ParseResult, + property: Key, + value: StaticChunkCollections[Key], +): void { + const descriptor = Object.getOwnPropertyDescriptor(result, property); + if (!descriptor || descriptor.get || descriptor.writable !== true) return; + + Object.defineProperty(result, property, { + ...descriptor, + value, + }); +} + +function getImageContentType(filePath: string): string { + const extension = filePath.split('.').at(-1)?.toLowerCase(); + if (extension === 'jpg' || extension === 'jpeg') return 'image/jpeg'; + if (extension === 'png') return 'image/png'; + if (extension === 'webp') return 'image/webp'; + if (extension === 'gif') return 'image/gif'; + if (extension === 'svg') return 'image/svg+xml'; + return defaultImageContentType; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} diff --git a/src/types/index.ts b/src/types/index.ts index 81cb84c..fae5e68 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -6,4 +6,5 @@ export * from './result.js'; export * from './retrieval.js'; export * from './document.js'; export * from './page-citation-assets.js'; +export * from './storage.js'; export * from '../knowledge/types.js'; diff --git a/src/types/params.ts b/src/types/params.ts index 296fddb..278af74 100644 --- a/src/types/params.ts +++ b/src/types/params.ts @@ -1,4 +1,5 @@ import type { ReadStream } from 'fs'; +import type { KnowhereAssetStorageOptions } from './storage.js'; /** * Parsing model options @@ -147,6 +148,11 @@ export interface ParseParams { pollTimeout?: number; /** Whether to verify ZIP checksum (default: true) */ verifyChecksum?: boolean; + /** + * Optional asset storage adapter used to copy parsed media and page citation + * assets into application-owned storage before the result is returned. + */ + storageAdapter?: KnowhereAssetStorageOptions; /** Webhook configuration */ webhook?: WebhookConfig; /** Upload progress callback */ diff --git a/src/types/result.ts b/src/types/result.ts index af4ff06..123d1e6 100644 --- a/src/types/result.ts +++ b/src/types/result.ts @@ -194,6 +194,8 @@ export interface ImageChunk extends BaseChunk { type: 'image'; /** Relative file path in ZIP */ filePath: string; + /** Application-owned or server-provided display URL for the image asset. */ + assetUrl?: string; /** Image data buffer */ data: Buffer; /** Image format (derived from file extension) */ @@ -209,6 +211,8 @@ export interface TableChunk extends BaseChunk { type: 'table'; /** Relative file path in ZIP */ filePath: string; + /** Application-owned or server-provided display URL for the table asset. */ + assetUrl?: string; /** HTML representation */ html: string; /** Save table HTML to disk */ diff --git a/src/types/storage.ts b/src/types/storage.ts new file mode 100644 index 0000000..b534cb4 --- /dev/null +++ b/src/types/storage.ts @@ -0,0 +1,40 @@ +import type { ParseResult } from './result.js'; + +export type KnowhereAssetStorageBody = Uint8Array; + +export interface KnowhereAssetStorageObject { + readonly key: string; + readonly body: KnowhereAssetStorageBody; + readonly contentType: string; + readonly metadata?: Readonly>; +} + +export interface KnowhereAssetStorageHead { + readonly key: string; + readonly contentType?: string; + readonly contentLength?: number; + readonly metadata?: Readonly>; + readonly url?: string; +} + +export interface KnowhereAssetStorageWriteResult { + readonly key: string; + readonly url?: string; +} + +export interface KnowhereAssetStorageAdapter { + writeObject(input: KnowhereAssetStorageObject): Promise; + headObject?(key: string): Promise; + getObjectUrl?(key: string): Promise; +} + +export interface KnowhereAssetStorageOptions { + readonly adapter: KnowhereAssetStorageAdapter; + readonly keyPrefix: string; + readonly skipExisting?: boolean; +} + +export interface KnowhereAssetStorageResult { + readonly result: ParseResult; + readonly assetUrlsByFilePath: Readonly>; +} From cbd87535e7941af06499d0c487bf63292562b6bd Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sat, 4 Jul 2026 01:23:08 +0800 Subject: [PATCH 07/10] Add parsed result snapshot storage --- README.md | 5 + packages/mcp/src/__tests__/mcp.test.ts | 4 +- packages/mcp/src/index.ts | 4 +- src/knowledge/__tests__/knowledge.test.ts | 18 +++ src/knowledge/knowledge.ts | 2 + src/knowledge/types.ts | 6 +- src/storage/__tests__/asset-storage.test.ts | 129 ++++++++++++++- src/storage/asset-storage.ts | 165 +++++++++++++++++++- src/types/storage.ts | 60 +++++++ 9 files changed, 379 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0991ffb..9ea30e8 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,11 @@ console.log(serverSearch.references); Local grep and reads use the cached parse result, not server-side chunk scans. Search uses the Knowhere API retrieval query; local document IDs only help map returned server document IDs back to local cache IDs when available. +When `parse(...)` or `knowledge.cacheJobResult(...)` receives a +`storageAdapter`, the SDK also writes a parsed snapshot through that adapter: +media/table/page-citation assets, paged chunk JSON files, and +`manifest/current.json`. Local `readChunks`, `grepChunks`, and outline reads use +the cached parse result with any Blob asset URLs preserved in chunk metadata. If a search result only has a published `documentId`, or an async parse flow only has a completed `jobId`, read-oriented helpers can accept that remote identifier directly and will sync the result into the local cache before reading. For a diff --git a/packages/mcp/src/__tests__/mcp.test.ts b/packages/mcp/src/__tests__/mcp.test.ts index 497ce3b..a5259e4 100644 --- a/packages/mcp/src/__tests__/mcp.test.ts +++ b/packages/mcp/src/__tests__/mcp.test.ts @@ -30,8 +30,8 @@ describe('knowhere MCP wrapper', () => { expect(statusTool?.description).toContain('5s, 10s, 20s, 40s, 80s'); expect(statusTool?.description).toContain('Large PDFs or OCR-heavy files can take 10+ minutes'); expect(listTool?.description).toContain('remote API'); - expect(readTool?.description).toContain('resultDirectoryPath'); - expect(readTool?.description).toContain('chunks.json'); + expect(readTool?.description).toContain('cached parse result'); + expect(readTool?.description).toContain('Blob asset URLs'); await client.close(); await server.close(); }); diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 5a4bf32..30228c1 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -242,7 +242,7 @@ export async function createKnowhereMcpServer( 'knowhere_read_chunks', { description: - 'Read exact chunks from a parsed document. Pass localDocumentId, published documentId, or completed jobId. The response document includes resultDirectoryPath; expanded chunks are stored in chunks.json under that directory.', + 'Read exact chunks from a parsed document. Pass localDocumentId, published documentId, or completed jobId. The SDK reads its cached parse result and preserves storage-adapter Blob asset URLs in chunk metadata.', inputSchema: { localDocumentId: z.string().optional(), documentId: z.string().optional(), @@ -263,7 +263,7 @@ export async function createKnowhereMcpServer( 'knowhere_grep_chunks', { description: - 'Run grep-style literal or regex matching against parsed document chunks. Pass localDocumentId, published documentId, or completed jobId. The response document includes resultDirectoryPath; expanded chunks are stored in chunks.json under that directory.', + 'Run grep-style literal or regex matching against parsed document chunks. Pass localDocumentId, published documentId, or completed jobId. The SDK scans its cached parse result and preserves storage-adapter Blob asset URLs in chunk metadata.', inputSchema: { localDocumentId: z.string().optional(), documentId: z.string().optional(), diff --git a/src/knowledge/__tests__/knowledge.test.ts b/src/knowledge/__tests__/knowledge.test.ts index 94f9a6b..510d8f3 100644 --- a/src/knowledge/__tests__/knowledge.test.ts +++ b/src/knowledge/__tests__/knowledge.test.ts @@ -523,6 +523,24 @@ describe('Knowledge', () => { 'page_citation_assets/page-1.png': 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', }); + expect(cached.parsedSnapshot).toMatchObject({ + manifestKey: 'workspaces/workspace-1/sources/source-1/parsed-result/manifest/current.json', + manifestUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/manifest/current.json', + manifest: { + kind: 'knowhere-parsed-result-snapshot', + jobId: 'job-1', + documentId: 'doc-1', + totalChunks: 1, + chunkPages: [ + { + page: 1, + key: 'workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + url: 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + }, + ], + }, + }); expect(cached.result.pageChunks[0]?.metadata.pageAssets).toEqual([ expect.objectContaining({ artifactRef: 'page_citation_assets/page-1.png', diff --git a/src/knowledge/knowledge.ts b/src/knowledge/knowledge.ts index 7934694..bad5dac 100644 --- a/src/knowledge/knowledge.ts +++ b/src/knowledge/knowledge.ts @@ -73,6 +73,7 @@ export class Knowledge { document, result, assetUrlsByFilePath: storedAssets.assetUrlsByFilePath, + parsedSnapshot: storedAssets.snapshot, }; } @@ -123,6 +124,7 @@ export class Knowledge { document, result, assetUrlsByFilePath: storedAssets.assetUrlsByFilePath, + parsedSnapshot: storedAssets.snapshot, }; } diff --git a/src/knowledge/types.ts b/src/knowledge/types.ts index ae4fa92..7c62027 100644 --- a/src/knowledge/types.ts +++ b/src/knowledge/types.ts @@ -1,7 +1,10 @@ import type { ParseParams } from '../types/params.js'; import type { Job, JobResult } from '../types/job.js'; import type { Chunk, DocumentChunkType, ParseResult } from '../types/index.js'; -import type { KnowhereAssetStorageOptions } from '../types/storage.js'; +import type { + KnowhereAssetStorageOptions, + KnowhereParsedSnapshot, +} from '../types/storage.js'; export type KnowledgeChunkType = DocumentChunkType; @@ -37,6 +40,7 @@ export interface LocalKnowledgeParseResponse { document: LocalKnowledgeDocument; result: ParseResult; assetUrlsByFilePath?: Readonly>; + parsedSnapshot?: KnowhereParsedSnapshot; } export interface KnowledgeAsyncParseParams extends ParseParams { diff --git a/src/storage/__tests__/asset-storage.test.ts b/src/storage/__tests__/asset-storage.test.ts index 0e60b38..b84818a 100644 --- a/src/storage/__tests__/asset-storage.test.ts +++ b/src/storage/__tests__/asset-storage.test.ts @@ -55,7 +55,7 @@ describe('storeParseResultAssets', () => { 'tables/revenue.html': 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/tables/revenue.html', }); - expect(writes).toEqual([ + expect(writes.filter((write) => write.contentType !== 'application/json; charset=utf-8')).toEqual([ { key: 'workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', bodyText: 'chart-image', @@ -87,6 +87,123 @@ describe('storeParseResultAssets', () => { }), ]); expect(stored.result.pageChunks[0]).not.toHaveProperty('pageAssets'); + expect(stored.snapshot?.manifest).toMatchObject({ + version: 1, + kind: 'knowhere-parsed-result-snapshot', + jobId: 'job-1', + sourceFileName: 'report.pdf', + totalChunks: 3, + chunkPageSize: 200, + assetUrlsByFilePath: { + 'images/chart.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', + 'page_citation_assets/page-1.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + 'tables/revenue.html': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/tables/revenue.html', + }, + }); + expect(stored.snapshot?.manifestKey).toBe( + 'workspaces/workspace-1/sources/source-1/parsed-result/manifest/current.json', + ); + expect(stored.snapshot?.indexKey).toBe( + 'workspaces/workspace-1/sources/source-1/parsed-result/index.json', + ); + expect(stored.snapshot?.manifest.chunkPages).toEqual([ + expect.objectContaining({ + page: 1, + chunkCount: 3, + key: 'workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + url: 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + }), + ]); + const chunkPageWrite = writes.find((write) => write.key.endsWith('/chunks/page-1.json')); + expect(chunkPageWrite).toBeDefined(); + const chunkPage = JSON.parse(chunkPageWrite?.bodyText ?? '{}') as { + readonly chunks: readonly { + readonly chunkId: string; + readonly chunkType: string; + readonly assetUrl?: string; + readonly metadata: Record; + }[]; + }; + expect(chunkPage.chunks.map((chunk) => chunk.chunkId)).toEqual([ + 'image-1', + 'table-1', + 'page-1', + ]); + expect(chunkPage.chunks[0]).toMatchObject({ + chunkType: 'image', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', + }); + expect(chunkPage.chunks[2]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + }); + + it('stores paged chunk snapshots even when a result has no media assets', async () => { + const result = createParseResult([ + { + chunkId: 'text-1', + type: 'text', + content: 'Alpha', + path: 'report/Intro', + metadata: { summary: 'Intro' }, + }, + { + chunkId: 'text-2', + type: 'text', + content: 'Beta', + path: 'report/Body', + metadata: {}, + }, + ]); + const writes: Array<{ + readonly key: string; + readonly bodyText: string; + readonly contentType: string; + }> = []; + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => { + writes.push({ + key: input.key, + bodyText: Buffer.from(input.body).toString('utf8'), + contentType: input.contentType, + }); + return Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }); + }, + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + + const stored = await storeParseResultAssets(result, { + adapter, + keyPrefix: 'parsed-result', + chunkPageSize: 1, + }); + + expect(stored.assetUrlsByFilePath).toEqual({}); + expect(stored.snapshot?.manifest.chunkPages).toEqual([ + expect.objectContaining({ page: 1, chunkCount: 1, key: 'parsed-result/chunks/page-1.json' }), + expect.objectContaining({ page: 2, chunkCount: 1, key: 'parsed-result/chunks/page-2.json' }), + ]); + expect(writes.map((write) => write.key)).toEqual([ + 'parsed-result/chunks/page-1.json', + 'parsed-result/chunks/page-2.json', + 'parsed-result/manifest/current.json', + 'parsed-result/index.json', + ]); + expect(writes.every((write) => write.contentType === 'application/json; charset=utf-8')).toBe( + true, + ); }); it('uses existing storage URLs when skipExisting is enabled', async () => { @@ -127,13 +244,18 @@ describe('storeParseResultAssets', () => { keyPrefix: 'parsed-result', }); - expect(writeObject).not.toHaveBeenCalled(); + expect( + writeObject.mock.calls.filter( + ([input]) => input.contentType !== 'application/json; charset=utf-8', + ), + ).toEqual([]); expect(stored.assetUrlsByFilePath).toEqual({ 'images/chart.png': 'https://blob.example/existing/parsed-result/images/chart.png', }); expect(stored.result.imageChunks[0]?.assetUrl).toBe( 'https://blob.example/existing/parsed-result/images/chart.png', ); + expect(stored.snapshot?.manifestUrl).toBe('https://blob.example/parsed-result/manifest/current.json'); }); it('rejects unsafe storage key prefixes and ignores unsafe asset refs', async () => { @@ -173,8 +295,9 @@ describe('storeParseResultAssets', () => { keyPrefix: 'parsed-result', }); - expect(writeObject).not.toHaveBeenCalled(); + expect(writeObject).toHaveBeenCalledTimes(3); expect(stored.assetUrlsByFilePath).toEqual({}); + expect(stored.snapshot?.manifest.totalChunks).toBe(1); }); }); diff --git a/src/storage/asset-storage.ts b/src/storage/asset-storage.ts index 2a90434..afc1ed1 100644 --- a/src/storage/asset-storage.ts +++ b/src/storage/asset-storage.ts @@ -1,6 +1,7 @@ import JSZip from 'jszip'; import type { + ChunkMetadata, Chunk, ImageChunk, PageChunk, @@ -14,6 +15,11 @@ import type { KnowhereAssetStorageBody, KnowhereAssetStorageOptions, KnowhereAssetStorageResult, + KnowhereParsedSnapshot, + KnowhereParsedSnapshotChunk, + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotChunkPageReference, + KnowhereParsedSnapshotManifest, } from '../types/storage.js'; interface StorageAsset { @@ -27,6 +33,10 @@ interface StorageAsset { const pageCitationAssetsMetadataKey = 'pageAssets'; const defaultImageContentType = 'application/octet-stream'; const tableContentType = 'text/html; charset=utf-8'; +const snapshotManifestKey = 'manifest/current.json'; +const snapshotIndexKey = 'index.json'; +const defaultSnapshotChunkPageSize = 200; +const snapshotJsonContentType = 'application/json; charset=utf-8'; export async function storeParseResultAssets( result: ParseResult, @@ -52,19 +62,162 @@ export async function storeParseResultAssets( } } - if (Object.keys(assetUrlsByFilePath).length === 0) { - return { - result, - assetUrlsByFilePath, + const normalizedResult = + Object.keys(assetUrlsByFilePath).length === 0 + ? result + : rewriteResultAssetUrls(result, assetUrlsByFilePath); + const snapshot = await storeParsedSnapshot({ + result: normalizedResult, + assetUrlsByFilePath, + keyPrefix: normalizedKeyPrefix, + options, + }); + + return { + result: normalizedResult, + assetUrlsByFilePath, + snapshot, + }; +} + +async function storeParsedSnapshot(input: { + readonly result: ParseResult; + readonly assetUrlsByFilePath: Readonly>; + readonly keyPrefix: string; + readonly options: KnowhereAssetStorageOptions; +}): Promise { + const pageSize = normalizeSnapshotChunkPageSize(input.options.chunkPageSize); + const totalChunks = input.result.chunks.length; + const totalPages = Math.max(1, Math.ceil(totalChunks / pageSize)); + const chunkPageReferences: KnowhereParsedSnapshotChunkPageReference[] = []; + const chunkPageUrlsByPage: Record = {}; + + for (let page = 1; page <= totalPages; page += 1) { + const pageChunks = input.result.chunks.slice((page - 1) * pageSize, page * pageSize); + const pageKey = `${input.keyPrefix}/chunks/page-${page}.json`; + const chunkPage: KnowhereParsedSnapshotChunkPage = { + version: 1, + jobId: input.result.jobId, + documentId: input.result.documentId, + namespace: input.result.namespace, + sourceFileName: input.result.manifest.sourceFileName, + page, + pageSize, + total: totalChunks, + totalPages, + chunks: pageChunks.map((chunk, index) => + toSnapshotChunk(chunk, (page - 1) * pageSize + index + 1), + ), }; + const pageUrl = await writeJsonObject({ + key: pageKey, + value: chunkPage, + adapter: input.options.adapter, + }); + if (pageUrl) chunkPageUrlsByPage[page] = pageUrl; + chunkPageReferences.push({ + page, + pageSize, + chunkCount: pageChunks.length, + key: pageKey, + url: pageUrl, + }); } + const manifest: KnowhereParsedSnapshotManifest = { + version: 1, + kind: 'knowhere-parsed-result-snapshot', + jobId: input.result.jobId, + documentId: input.result.documentId, + namespace: input.result.namespace, + sourceFileName: input.result.manifest.sourceFileName, + totalChunks, + chunkPageSize: pageSize, + chunkPages: chunkPageReferences, + assetUrlsByFilePath: input.assetUrlsByFilePath, + createdAt: new Date().toISOString(), + }; + const manifestKey = `${input.keyPrefix}/${snapshotManifestKey}`; + const indexKey = `${input.keyPrefix}/${snapshotIndexKey}`; + const manifestUrl = await writeJsonObject({ + key: manifestKey, + value: manifest, + adapter: input.options.adapter, + }); + const indexUrl = await writeJsonObject({ + key: indexKey, + value: manifest, + adapter: input.options.adapter, + }); + return { - result: rewriteResultAssetUrls(result, assetUrlsByFilePath), - assetUrlsByFilePath, + manifest, + manifestKey, + manifestUrl, + indexKey, + indexUrl, + chunkPageUrlsByPage, }; } +async function writeJsonObject(input: { + readonly key: string; + readonly value: unknown; + readonly adapter: KnowhereAssetStorageAdapter; +}): Promise { + const result = await input.adapter.writeObject({ + key: input.key, + body: Buffer.from(JSON.stringify(input.value, null, 2), 'utf8'), + contentType: snapshotJsonContentType, + metadata: { + kind: 'knowhere-parsed-result-snapshot', + }, + }); + return result.url ?? (await input.adapter.getObjectUrl?.(result.key)) ?? undefined; +} + +function normalizeSnapshotChunkPageSize(value: number | undefined): number { + if (!Number.isInteger(value) || value === undefined) return defaultSnapshotChunkPageSize; + return Math.min(Math.max(value, 1), 1000); +} + +function toSnapshotChunk(chunk: Chunk, position: number): KnowhereParsedSnapshotChunk { + const filePath = getSnapshotChunkFilePath(chunk); + return { + id: chunk.chunkId, + chunkId: chunk.chunkId, + chunkType: chunk.type, + contentSource: chunk.contentSource, + content: chunk.content, + sectionPath: normalizeSnapshotSectionPath(chunk.path), + sourceChunkPath: chunk.path, + filePath, + sortOrder: position, + metadata: cloneChunkMetadata(chunk.metadata), + assetUrl: getSnapshotChunkAssetUrl(chunk), + }; +} + +function normalizeSnapshotSectionPath(path: string): string | undefined { + return path.length > 0 ? path : undefined; +} + +function getSnapshotChunkFilePath(chunk: Chunk): string | undefined { + if (chunk.type === 'image' || chunk.type === 'table') { + return chunk.filePath; + } + const filePath = chunk.metadata.filePath; + return typeof filePath === 'string' ? filePath : undefined; +} + +function getSnapshotChunkAssetUrl(chunk: Chunk): string | undefined { + return chunk.type === 'image' || chunk.type === 'table' ? chunk.assetUrl : undefined; +} + +function cloneChunkMetadata(metadata: ChunkMetadata): Record { + return { ...metadata }; +} + async function collectStorageAssets(result: ParseResult, keyPrefix: string): Promise { const resultZip = await loadResultZipForPageCitationAssets(result); return [ diff --git a/src/types/storage.ts b/src/types/storage.ts index b534cb4..b6add3b 100644 --- a/src/types/storage.ts +++ b/src/types/storage.ts @@ -32,9 +32,69 @@ export interface KnowhereAssetStorageOptions { readonly adapter: KnowhereAssetStorageAdapter; readonly keyPrefix: string; readonly skipExisting?: boolean; + readonly chunkPageSize?: number; +} + +export interface KnowhereParsedSnapshotChunk { + readonly id: string; + readonly chunkId: string; + readonly chunkType: string; + readonly contentSource?: string; + readonly content: string; + readonly sectionPath?: string; + readonly sourceChunkPath: string; + readonly filePath?: string; + readonly sortOrder: number; + readonly metadata: Record; + readonly assetUrl?: string; +} + +export interface KnowhereParsedSnapshotChunkPage { + readonly version: 1; + readonly jobId: string; + readonly documentId?: string; + readonly namespace?: string; + readonly sourceFileName: string; + readonly page: number; + readonly pageSize: number; + readonly total: number; + readonly totalPages: number; + readonly chunks: readonly KnowhereParsedSnapshotChunk[]; +} + +export interface KnowhereParsedSnapshotChunkPageReference { + readonly page: number; + readonly pageSize: number; + readonly chunkCount: number; + readonly key: string; + readonly url?: string; +} + +export interface KnowhereParsedSnapshotManifest { + readonly version: 1; + readonly kind: 'knowhere-parsed-result-snapshot'; + readonly jobId: string; + readonly documentId?: string; + readonly namespace?: string; + readonly sourceFileName: string; + readonly totalChunks: number; + readonly chunkPageSize: number; + readonly chunkPages: readonly KnowhereParsedSnapshotChunkPageReference[]; + readonly assetUrlsByFilePath: Readonly>; + readonly createdAt: string; +} + +export interface KnowhereParsedSnapshot { + readonly manifest: KnowhereParsedSnapshotManifest; + readonly manifestKey: string; + readonly manifestUrl?: string; + readonly indexKey: string; + readonly indexUrl?: string; + readonly chunkPageUrlsByPage: Readonly>; } export interface KnowhereAssetStorageResult { readonly result: ParseResult; readonly assetUrlsByFilePath: Readonly>; + readonly snapshot?: KnowhereParsedSnapshot; } From b5b4339958c4e2f66f747721c0c49a6380c41cf3 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sat, 4 Jul 2026 15:50:01 +0800 Subject: [PATCH 08/10] Clarify local knowledge result imports --- .changeset/page-citation-assets-metadata.md | 2 +- README.md | 31 ++++-- packages/mcp/src/__tests__/mcp.test.ts | 14 +-- packages/mcp/src/index.ts | 4 +- src/index.ts | 3 + src/knowledge/__tests__/knowledge.test.ts | 116 ++++++++++++++++++-- src/knowledge/index.ts | 3 + src/knowledge/knowledge.ts | 43 ++++++-- src/knowledge/types.ts | 16 ++- 9 files changed, 188 insertions(+), 44 deletions(-) diff --git a/.changeset/page-citation-assets-metadata.md b/.changeset/page-citation-assets-metadata.md index 95aa25f..d088ce6 100644 --- a/.changeset/page-citation-assets-metadata.md +++ b/.changeset/page-citation-assets-metadata.md @@ -3,4 +3,4 @@ "@ontos-ai/knowhere-mcp": major --- -Preserve page citation asset descriptors only in chunk metadata and remove the deprecated SDK-side page citation asset generation options. +Preserve page citation asset descriptors only in chunk metadata, remove the deprecated SDK-side page citation asset generation options, and split local-cache/server-safe result handling into explicit `knowledge.parseToLocalCache(...)`, `knowledge.importJobResult(...)`, and `knowledge.loadJobResult(...)` methods. diff --git a/README.md b/README.md index 9ea30e8..80857c2 100644 --- a/README.md +++ b/README.md @@ -347,11 +347,16 @@ console.log(archived.status); ### Local Knowledge Tools The SDK can also keep parsed results in a local cache and run exact inspection -tools over that cached copy. This is the implementation used by the separate +tools over that cached copy. These `client.knowledge.*` local import helpers +write under the SDK cache directory by design, so server workflows should prefer +top-level `client.parse(...)` for direct parsing or +`client.knowledge.loadJobResult(...)` with a provided storage adapter when they +need to mirror an existing completed job result without local cache state. This +local-cache flow is the implementation used by the separate `@ontos-ai/knowhere-mcp` package. ```typescript -const parsed = await client.knowledge.parse({ +const parsed = await client.knowledge.parseToLocalCache({ file: './manual.pdf', localDocumentId: 'manual', }); @@ -384,16 +389,17 @@ console.log(serverSearch.references); Local grep and reads use the cached parse result, not server-side chunk scans. Search uses the Knowhere API retrieval query; local document IDs only help map returned server document IDs back to local cache IDs when available. -When `parse(...)` or `knowledge.cacheJobResult(...)` receives a -`storageAdapter`, the SDK also writes a parsed snapshot through that adapter: -media/table/page-citation assets, paged chunk JSON files, and -`manifest/current.json`. Local `readChunks`, `grepChunks`, and outline reads use -the cached parse result with any Blob asset URLs preserved in chunk metadata. +When `knowledge.parseToLocalCache(...)`, `knowledge.importJobResult(...)`, or +`knowledge.loadJobResult(...)` receives a `storageAdapter`, the SDK also writes a +parsed snapshot through that adapter: media/table/page-citation assets, paged +chunk JSON files, and `manifest/current.json`. Local `readChunks`, +`grepChunks`, and outline reads use the cached parse result with any Blob asset +URLs preserved in chunk metadata. If a search result only has a published `documentId`, or an async parse flow only has a completed `jobId`, read-oriented helpers can accept that remote identifier directly and will sync the result into the local cache before reading. For a `documentId`, the SDK resolves the document's current published `jobId` and then -downloads the parser result ZIP through the same `cacheJobResult(...)` path: +downloads the parser result ZIP through the same `importJobResult(...)` path: ```typescript const remoteRead = await client.knowledge.readChunks({ @@ -435,9 +441,12 @@ if (status.job.isDone && status.cache.document) { When the job was started through `client.knowledge.startParse(...)`, `getJobStatus(...)` automatically caches the completed result locally the first -time it observes `status.job.isDone`. Use `cacheJobResult(...)` only to recover a -completed job that was not started through the local knowledge helper, or to -retry a cache step explicitly. +time it observes `status.job.isDone`. Use `importJobResult(...)` to recover a +completed job into the local cache when it was not started through the local +knowledge helper, or to retry a local import step explicitly. Use +`loadJobResult(...)` for server workflows that should load a completed result and +write a provided storage adapter snapshot without creating SDK local-disk cache +state. Follow-up queries can exclude documents or sections for one request: diff --git a/packages/mcp/src/__tests__/mcp.test.ts b/packages/mcp/src/__tests__/mcp.test.ts index a5259e4..703ec64 100644 --- a/packages/mcp/src/__tests__/mcp.test.ts +++ b/packages/mcp/src/__tests__/mcp.test.ts @@ -107,7 +107,7 @@ describe('knowhere MCP wrapper', () => { }, }); - expect(knowhereClient.knowledge.parse).toHaveBeenCalledWith({ + expect(knowhereClient.knowledge.parseToLocalCache).toHaveBeenCalledWith({ file: './report.md', fileName: undefined, namespace: undefined, @@ -414,7 +414,7 @@ function createClient(): Knowhere & { knowledge: KnowledgeWithMocks; } { const knowledge: KnowledgeWithMocks = { - parse: vi.fn().mockResolvedValue({ + parseToLocalCache: vi.fn().mockResolvedValue({ document: { localDocumentId: 'local-report' }, result: { jobId: 'job-1' }, }), @@ -430,7 +430,7 @@ function createClient(): Knowhere & { document: { localDocumentId: 'local-report' }, }, }), - cacheJobResult: vi.fn().mockResolvedValue({ + importJobResult: vi.fn().mockResolvedValue({ document: { localDocumentId: 'local-report' }, }), recoverPendingAsyncParseJobs: vi.fn().mockResolvedValue({ @@ -482,10 +482,10 @@ function createClient(): Knowhere & { type KnowledgeWithMocks = Pick< Knowledge, - | 'parse' + | 'parseToLocalCache' | 'startParse' | 'getJobStatus' - | 'cacheJobResult' + | 'importJobResult' | 'recoverPendingAsyncParseJobs' | 'listDocuments' | 'getDocumentOutline' @@ -494,10 +494,10 @@ type KnowledgeWithMocks = Pick< | 'search' | 'withCacheDirectory' > & { - parse: Mock; + parseToLocalCache: Mock; startParse: Mock; getJobStatus: Mock; - cacheJobResult: Mock; + importJobResult: Mock; recoverPendingAsyncParseJobs: Mock; listDocuments: Mock; getDocumentOutline: Mock; diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 30228c1..7070167 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -82,7 +82,7 @@ export async function createKnowhereMcpServer( }, async (input) => createToolResult( - await knowledge.parse({ + await knowledge.parseToLocalCache({ url: input.url, namespace: input.namespace, localDocumentId: input.localDocumentId, @@ -109,7 +109,7 @@ export async function createKnowhereMcpServer( }, async (input) => createToolResult( - await knowledge.parse({ + await knowledge.parseToLocalCache({ file: input.file, fileName: input.fileName, namespace: input.namespace, diff --git a/src/index.ts b/src/index.ts index 4e93c45..ea61bfd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -78,6 +78,9 @@ export type { KnowledgeCacheDocumentParams, KnowledgeCacheJobResultParams, KnowledgeDocumentReference, + KnowledgeImportJobResultParams, + KnowledgeJobResultResponse, + KnowledgeLoadJobResultParams, KnowledgeParseParams, LocalKnowledgeDocument, LocalKnowledgeParseResponse, diff --git a/src/knowledge/__tests__/knowledge.test.ts b/src/knowledge/__tests__/knowledge.test.ts index 510d8f3..c631a0b 100644 --- a/src/knowledge/__tests__/knowledge.test.ts +++ b/src/knowledge/__tests__/knowledge.test.ts @@ -35,7 +35,7 @@ describe('Knowledge', () => { const { client, parse } = createClient(parseResult); const knowledge = new Knowledge(client, { cacheDirectory }); - const response = await knowledge.parse({ + const response = await knowledge.parseToLocalCache({ url: 'https://example.com/report.md', namespace: 'support-center', localDocumentId: 'local-report', @@ -65,6 +65,24 @@ describe('Knowledge', () => { await expectFileMissing(path.join(cacheDirectory, 'documents', 'local-report', 'result.zip')); }); + it('should keep the deprecated parse alias for compatibility', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = createParseResult(); + const { client, parse } = createClient(parseResult); + const knowledge = new Knowledge(client, { cacheDirectory }); + + const response = await knowledge.parse({ + url: 'https://example.com/report.md', + localDocumentId: 'local-report', + }); + + expect(parse).toHaveBeenCalledWith({ + url: 'https://example.com/report.md', + localDocumentId: 'local-report', + }); + expect(response.document.localDocumentId).toBe('local-report'); + }); + it('should reject local document ids that resolve outside the cache', async () => { const cacheDirectory = await createTempDirectory(); const siblingDirectory = path.join(path.dirname(cacheDirectory), 'knowhere-victim'); @@ -77,7 +95,7 @@ describe('Knowledge', () => { await writeFile(path.join(siblingDirectory, 'sentinel.txt'), 'keep'); await expect( - knowledge.parse({ + knowledge.parseToLocalCache({ url: 'https://example.com/report.md', localDocumentId: `../../${path.basename(siblingDirectory)}`, }), @@ -144,12 +162,12 @@ describe('Knowledge', () => { expect(documents).toHaveLength(1); }); - it('should still allow manual completed job result caching', async () => { + it('should import a completed job result into the local cache', async () => { const cacheDirectory = await createTempDirectory(); const { client, jobsLoad } = createClient(createParseResult()); const knowledge = new Knowledge(client, { cacheDirectory }); - const cached = await knowledge.cacheJobResult({ + const cached = await knowledge.importJobResult({ jobId: 'job-1', localDocumentId: 'local-report', verifyChecksum: false, @@ -159,6 +177,20 @@ describe('Knowledge', () => { expect(cached.document.localDocumentId).toBe('local-report'); }); + it('should keep the deprecated job result cache alias for compatibility', async () => { + const cacheDirectory = await createTempDirectory(); + const { client, jobsLoad } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }); + + const cached = await knowledge.cacheJobResult({ + jobId: 'job-1', + localDocumentId: 'local-report', + }); + + expect(jobsLoad).toHaveBeenCalledWith('job-1', { verifyChecksum: undefined }); + expect(cached.document.localDocumentId).toBe('local-report'); + }); + it('should recover and cache pending async parse jobs on startup', async () => { const cacheDirectory = await createTempDirectory(); const { client, jobsGet, jobsLoad } = createClient(createParseResult()); @@ -361,7 +393,7 @@ describe('Knowledge', () => { const cacheDirectory = await createTempDirectory(); const { client, retrievalQuery } = createClient(createParseResult()); const knowledge = new Knowledge(client, { cacheDirectory }); - await knowledge.parse({ + await knowledge.parseToLocalCache({ url: 'https://example.com/report.md', localDocumentId: 'local-report', }); @@ -453,13 +485,13 @@ describe('Knowledge', () => { expect(read.chunks[0]).not.toHaveProperty('pageAssets'); }); - it('should cache server-provided page assets without SDK-side rendering', async () => { + it('should import server-provided page assets without SDK-side rendering', async () => { const cacheDirectory = await createTempDirectory(); const parseResult = await createPageParseResultWithRawPageAsset(); const { client, jobsLoad, documentsGetPageCitationSource } = createClient(parseResult); const knowledge = new Knowledge(client, { cacheDirectory }); - const cached = await knowledge.cacheJobResult({ + const cached = await knowledge.importJobResult({ jobId: 'job-1', localDocumentId: 'local-report', }); @@ -489,15 +521,75 @@ describe('Knowledge', () => { expect(read.chunks[0]).not.toHaveProperty('pageAssets'); }); - it('should cache job result assets through a storage adapter and preserve metadata shape', async () => { + it('should load job result assets through a storage adapter without saving locally', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = await createPageParseResultWithRawPageAsset(); + const { client, jobsLoad } = createClient(parseResult); + const writeObject = vi.fn( + (input: KnowhereAssetStorageObject): Promise => + Promise.resolve({ + key: input.key, + url: `https://blob.example/${input.key}`, + }), + ); + const adapter: KnowhereAssetStorageAdapter = { + writeObject, + }; + const knowledge = new Knowledge(client, { cacheDirectory }); + + const loaded = await knowledge.loadJobResult({ + jobId: 'job-1', + storageAdapter: { + adapter, + keyPrefix: 'workspaces/workspace-1/sources/source-1/parsed-result', + }, + }); + const documents = await knowledge.listDocuments(); + + expect(jobsLoad).toHaveBeenCalledWith('job-1', { verifyChecksum: undefined }); + expect(documents).toEqual([]); + expect(loaded.assetUrlsByFilePath).toEqual({ + 'page_citation_assets/page-1.png': + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }); + expect(loaded.parsedSnapshot).toMatchObject({ + manifestKey: 'workspaces/workspace-1/sources/source-1/parsed-result/manifest/current.json', + manifestUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/manifest/current.json', + manifest: { + kind: 'knowhere-parsed-result-snapshot', + jobId: 'job-1', + documentId: 'doc-1', + totalChunks: 1, + chunkPages: [ + { + page: 1, + key: 'workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + url: 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/chunks/page-1.json', + }, + ], + }, + }); + expect(loaded.result.pageChunks[0]?.metadata.pageAssets).toEqual([ + expect.objectContaining({ + artifactRef: 'page_citation_assets/page-1.png', + assetUrl: + 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/page_citation_assets/page-1.png', + }), + ]); + expect(loaded.result.pageChunks[0]).not.toHaveProperty('pageAssets'); + await expectFileMissing(path.join(cacheDirectory, 'documents')); + }); + + it('should import job result assets through a storage adapter and preserve metadata shape', async () => { const cacheDirectory = await createTempDirectory(); const parseResult = await createPageParseResultWithRawPageAsset(); const { client } = createClient(parseResult); const writeObject = vi.fn( (input: KnowhereAssetStorageObject): Promise => Promise.resolve({ - key: input.key, - url: `https://blob.example/${input.key}`, + key: input.key, + url: `https://blob.example/${input.key}`, }), ); const adapter: KnowhereAssetStorageAdapter = { @@ -505,7 +597,7 @@ describe('Knowledge', () => { }; const knowledge = new Knowledge(client, { cacheDirectory }); - const cached = await knowledge.cacheJobResult({ + const cached = await knowledge.importJobResult({ jobId: 'job-1', localDocumentId: 'local-report', storageAdapter: { @@ -565,7 +657,7 @@ describe('Knowledge', () => { const cacheDirectory = await createTempDirectory(); const { client } = createClient(parseResult); const knowledge = new Knowledge(client, { cacheDirectory }); - await knowledge.parse({ + await knowledge.parseToLocalCache({ url: 'https://example.com/report.md', localDocumentId: 'local-report', }); diff --git a/src/knowledge/index.ts b/src/knowledge/index.ts index 3a5c5c5..fc782db 100644 --- a/src/knowledge/index.ts +++ b/src/knowledge/index.ts @@ -10,6 +10,9 @@ export type { KnowledgeGrepMatch, KnowledgeGrepParams, KnowledgeGrepResponse, + KnowledgeImportJobResultParams, + KnowledgeJobResultResponse, + KnowledgeLoadJobResultParams, KnowledgeOutline, KnowledgeParseParams, KnowledgeReadChunk, diff --git a/src/knowledge/knowledge.ts b/src/knowledge/knowledge.ts index bad5dac..6c981ab 100644 --- a/src/knowledge/knowledge.ts +++ b/src/knowledge/knowledge.ts @@ -23,6 +23,9 @@ import type { KnowledgeGrepMatch, KnowledgeGrepParams, KnowledgeGrepResponse, + KnowledgeImportJobResultParams, + KnowledgeJobResultResponse, + KnowledgeLoadJobResultParams, KnowledgeOutline, KnowledgeParseParams, KnowledgeReadChunk, @@ -60,7 +63,7 @@ export class Knowledge { return new Knowledge(this.client, { cacheDirectory }); } - async parse(params: KnowledgeParseParams): Promise { + async parseToLocalCache(params: KnowledgeParseParams): Promise { const loadedResult = params.storageAdapter ? await this.parseWithoutClientStorageAdapter(params) : await this.client.parse(params); @@ -77,6 +80,11 @@ export class Knowledge { }; } + /** @deprecated Use parseToLocalCache for local cache imports or the top-level client.parse for server-safe parsing. */ + async parse(params: KnowledgeParseParams): Promise { + return this.parseToLocalCache(params); + } + async startParse(params: KnowledgeAsyncParseParams): Promise { const job = await this.client.startParse(params); await this.store.saveAsyncParseJob({ @@ -109,25 +117,42 @@ export class Knowledge { }; } - async cacheJobResult( - params: KnowledgeCacheJobResultParams, - ): Promise { + async loadJobResult(params: KnowledgeLoadJobResultParams): Promise { const loadedResult = await this.client.jobs.load(params.jobId, { verifyChecksum: params.verifyChecksum, }); const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter); const result = storedAssets.result; + return { + result, + assetUrlsByFilePath: storedAssets.assetUrlsByFilePath, + parsedSnapshot: storedAssets.snapshot, + }; + } + + async importJobResult( + params: KnowledgeImportJobResultParams, + ): Promise { + const stored = await this.loadJobResult(params); + const result = stored.result; const document = await this.store.saveResult(result, { localDocumentId: params.localDocumentId, }); return { document, result, - assetUrlsByFilePath: storedAssets.assetUrlsByFilePath, - parsedSnapshot: storedAssets.snapshot, + assetUrlsByFilePath: stored.assetUrlsByFilePath, + parsedSnapshot: stored.parsedSnapshot, }; } + /** @deprecated Use importJobResult for local cache imports or loadJobResult for server-safe loads. */ + async cacheJobResult( + params: KnowledgeCacheJobResultParams, + ): Promise { + return this.importJobResult(params); + } + private async parseWithoutClientStorageAdapter( params: KnowledgeParseParams, ): Promise { @@ -146,7 +171,7 @@ export class Knowledge { params.documentId, ); const jobId: string = await this.getPublishedDocumentJobId(params.documentId); - return this.cacheJobResult({ + return this.importJobResult({ jobId, localDocumentId: params.localDocumentId ?? @@ -195,7 +220,7 @@ export class Knowledge { } try { - const cached = await this.cacheJobResult({ + const cached = await this.importJobResult({ jobId, localDocumentId: trackedJob.localDocumentId, }); @@ -362,7 +387,7 @@ export class Knowledge { } if (normalized.jobId) { - return this.cacheJobResult({ + return this.importJobResult({ jobId: normalized.jobId, localDocumentId: normalized.localDocumentId, }); diff --git a/src/knowledge/types.ts b/src/knowledge/types.ts index 7c62027..46c6cf2 100644 --- a/src/knowledge/types.ts +++ b/src/knowledge/types.ts @@ -78,13 +78,25 @@ export interface KnowledgeStartupRecoveryResponse { results: KnowledgeAsyncJobStatusResponse[]; } -export interface KnowledgeCacheJobResultParams { +export interface KnowledgeLoadJobResultParams { jobId: string; - localDocumentId?: string; verifyChecksum?: boolean; storageAdapter?: KnowhereAssetStorageOptions; } +export interface KnowledgeImportJobResultParams extends KnowledgeLoadJobResultParams { + localDocumentId?: string; +} + +export interface KnowledgeJobResultResponse { + result: ParseResult; + assetUrlsByFilePath?: Readonly>; + parsedSnapshot?: KnowhereParsedSnapshot; +} + +/** @deprecated Use KnowledgeImportJobResultParams with importJobResult instead. */ +export type KnowledgeCacheJobResultParams = KnowledgeImportJobResultParams; + export interface KnowledgeCacheDocumentParams { /** Canonical Knowhere document identifier from published retrieval results. */ documentId: string; From 2d006ae8e21db6e1a27e88c91f1a250da7e1f541 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sat, 4 Jul 2026 19:04:45 +0800 Subject: [PATCH 09/10] Add parsed storage backed knowledge reads --- .changeset/parsed-storage-redesign.md | 11 + README.md | 74 +- packages/mcp/README.md | 29 +- packages/mcp/src/__tests__/mcp.test.ts | 23 +- packages/mcp/src/index.ts | 31 +- src/index.ts | 21 + src/knowledge/__tests__/knowledge.test.ts | 399 ++++- src/knowledge/index.ts | 3 + src/knowledge/knowledge.ts | 1628 ++++++++++++++++++- src/knowledge/types.ts | 27 + src/storage/asset-storage.ts | 24 +- src/storage/disk-parsed-document-storage.ts | 208 +++ src/storage/parsed-document-storage.ts | 173 ++ src/types/storage.ts | 90 + 14 files changed, 2655 insertions(+), 86 deletions(-) create mode 100644 .changeset/parsed-storage-redesign.md create mode 100644 src/storage/disk-parsed-document-storage.ts create mode 100644 src/storage/parsed-document-storage.ts diff --git a/.changeset/parsed-storage-redesign.md b/.changeset/parsed-storage-redesign.md new file mode 100644 index 0000000..348b686 --- /dev/null +++ b/.changeset/parsed-storage-redesign.md @@ -0,0 +1,11 @@ +--- +"@ontos-ai/knowhere-sdk": minor +"@ontos-ai/knowhere-mcp": minor +--- + +Move parsed document reads to a storage-first SDK model with remote chunk fallback. + +Adds parsed snapshot storage types, `knowledge.withParsedStorage(...)`, +`knowledge.syncParsedDocument(...)`, paged `readChunks(...)` display params, +durable asset URL hardening, bounded remote grep/outline fallback, and explicit +disk parsed storage for MCP cache-directory usage. diff --git a/README.md b/README.md index 80857c2..9dbbb1c 100644 --- a/README.md +++ b/README.md @@ -346,14 +346,13 @@ console.log(archived.status); ### Local Knowledge Tools -The SDK can also keep parsed results in a local cache and run exact inspection -tools over that cached copy. These `client.knowledge.*` local import helpers -write under the SDK cache directory by design, so server workflows should prefer -top-level `client.parse(...)` for direct parsing or -`client.knowledge.loadJobResult(...)` with a provided storage adapter when they -need to mirror an existing completed job result without local cache state. This -local-cache flow is the implementation used by the separate -`@ontos-ai/knowhere-mcp` package. +The SDK can also run exact inspection tools over parsed results. Local import +helpers still write under the SDK cache directory by design, while published +`documentId` reads use configured parsed storage first and fall back to +Knowhere's remote chunk API without importing the result ZIP into local disk. +Server workflows can configure parsed storage with +`client.knowledge.withParsedStorage(...)` so Notebook, MCP, and CLI surfaces +share the same snapshot model. ```typescript const parsed = await client.knowledge.parseToLocalCache({ @@ -386,33 +385,49 @@ console.log(grep.matches); console.log(serverSearch.references); ``` -Local grep and reads use the cached parse result, not server-side chunk scans. -Search uses the Knowhere API retrieval query; local document IDs only help map -returned server document IDs back to local cache IDs when available. +Local grep and reads use the cached parse result. Published `documentId` grep +streams `documents.listChunks(...)` page by page when parsed storage is missing +or stale. Search uses the Knowhere API retrieval query; local document IDs only +help map returned server document IDs back to local cache IDs when available. When `knowledge.parseToLocalCache(...)`, `knowledge.importJobResult(...)`, or -`knowledge.loadJobResult(...)` receives a `storageAdapter`, the SDK also writes a -parsed snapshot through that adapter: media/table/page-citation assets, paged -chunk JSON files, and `manifest/current.json`. Local `readChunks`, -`grepChunks`, and outline reads use the cached parse result with any Blob asset -URLs preserved in chunk metadata. -If a search result only has a published `documentId`, or an async parse flow only -has a completed `jobId`, read-oriented helpers can accept that remote identifier -directly and will sync the result into the local cache before reading. For a -`documentId`, the SDK resolves the document's current published `jobId` and then -downloads the parser result ZIP through the same `importJobResult(...)` path: +`knowledge.loadJobResult(...)` runs with configured parsed storage, the SDK +writes chunk pages, durable assets, sync progress, and a committed +`manifest/current.json` before returning. Partial snapshots are ignored until +the manifest is committed. ```typescript -const remoteRead = await client.knowledge.readChunks({ +const knowledge = client.knowledge.withParsedStorage({ + storage: myParsedDocumentStorage, + scheduler: myBackgroundScheduler, + limits: { remotePageSize: 100, maxPagesPerSync: 10 }, +}); +``` + +If a search result only has a published `documentId`, read-oriented helpers can +accept that remote identifier directly. `readChunks` supports display paging; +`assetUrlPolicy: 'durable'` hardens only returned assets through configured +storage and omits asset URLs if hardening fails: + +```typescript +const remoteRead = await knowledge.readChunks({ + documentId: 'doc_123', + page: 1, + pageSize: 20, + chunkType: 'page', + assetUrlPolicy: 'durable', +}); + +const remoteOutline = await knowledge.getDocumentOutline({ documentId: 'doc_123', - sectionPath: 'Overview', - limit: 5, }); -const remoteOutline = await client.knowledge.getDocumentOutline({ +const grep = await knowledge.grepChunks({ documentId: 'doc_123', + pattern: 'warranty', + maxResults: 10, }); -const jobRead = await client.knowledge.readChunks({ +const jobRead = await knowledge.readChunks({ jobId: jobResult.jobId, localDocumentId: 'manual', limit: 5, @@ -444,9 +459,10 @@ When the job was started through `client.knowledge.startParse(...)`, time it observes `status.job.isDone`. Use `importJobResult(...)` to recover a completed job into the local cache when it was not started through the local knowledge helper, or to retry a local import step explicitly. Use -`loadJobResult(...)` for server workflows that should load a completed result and -write a provided storage adapter snapshot without creating SDK local-disk cache -state. +`loadJobResult(...)` for server workflows that should load a completed result +without creating SDK local-disk cache state. Use `syncParsedDocument(...)` to +explicitly resume or retry parsed-storage sync for an existing `documentId`, +`jobId`, or local parsed result. Follow-up queries can exclude documents or sections for one request: diff --git a/packages/mcp/README.md b/packages/mcp/README.md index d0e434a..d71e7ed 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -19,9 +19,11 @@ npx -y @ontos-ai/knowhere-mcp login npx -y @ontos-ai/knowhere-mcp ``` -The server uses stdio transport and stores expanded Knowhere result files under -the SDK local knowledge cache by default. `knowhere-mcp login` opens the -Knowhere dashboard in your browser and stores a local MCP login at +The server uses stdio transport. Parse tools store expanded Knowhere result +files under the SDK local knowledge cache; published `documentId` reads use the +SDK's remote chunk fallback unless the host explicitly configures a cache +directory for parsed-storage snapshots. `knowhere-mcp login` opens the Knowhere +dashboard in your browser and stores a local MCP login at `~/.knowhere-node-sdk/mcp/auth.json`. During login, the dashboard asks for a Permission: @@ -196,18 +198,23 @@ When logged in with Read only permission, the MCP server exposes only passed to outline/read/grep tools. - `knowhere_delete_document`: archive, or soft-delete, a published Knowhere document through the Knowhere API. -- `knowhere_get_document_outline`: inspect a parsed document outline by passing - `localDocumentId`, published `documentId`, or completed `jobId`. -- `knowhere_read_chunks`: read exact chunks from a parsed document by passing - `localDocumentId`, published `documentId`, or completed `jobId`. -- `knowhere_grep_chunks`: run literal or regex grep over parsed document chunks - by passing `localDocumentId`, published `documentId`, or completed `jobId`. +- `knowhere_get_document_outline`: inspect one parsed document outline by + passing `localDocumentId`, published `documentId`, or completed `jobId`. +- `knowhere_read_chunks`: read exact chunks from one parsed document by passing + `localDocumentId`, published `documentId`, or completed `jobId`. Use + `page`/`pageSize` for display reads; `assetUrlPolicy: "durable"` returns only + storage-hardened asset URLs. +- `knowhere_grep_chunks`: run literal or regex grep over one parsed document by + passing `localDocumentId`, published `documentId`, or completed `jobId`. + Broad workspace search belongs to `knowhere_search`. - `knowhere_search`: search published documents through the Knowhere API retrieval query. -Read-related tool responses include `document.resultDirectoryPath`; expanded +Local-cache tool responses include `document.resultDirectoryPath`; expanded chunks are stored at `/chunks.json` for direct filesystem -reads when needed. +reads when a local cache entry exists. Remote fallback responses use +`remote:` or `parsed-storage:` markers instead of +claiming a local expanded result directory. ## Package Boundary diff --git a/packages/mcp/src/__tests__/mcp.test.ts b/packages/mcp/src/__tests__/mcp.test.ts index 703ec64..4a39656 100644 --- a/packages/mcp/src/__tests__/mcp.test.ts +++ b/packages/mcp/src/__tests__/mcp.test.ts @@ -30,8 +30,8 @@ describe('knowhere MCP wrapper', () => { expect(statusTool?.description).toContain('5s, 10s, 20s, 40s, 80s'); expect(statusTool?.description).toContain('Large PDFs or OCR-heavy files can take 10+ minutes'); expect(listTool?.description).toContain('remote API'); - expect(readTool?.description).toContain('cached parse result'); - expect(readTool?.description).toContain('Blob asset URLs'); + expect(readTool?.description).toContain('configured parsed storage first'); + expect(readTool?.description).toContain('assetUrlPolicy is durable'); await client.close(); await server.close(); }); @@ -160,16 +160,21 @@ describe('knowhere MCP wrapper', () => { name: 'knowhere_read_chunks', arguments: { documentId: 'doc_remote', - sectionPath: 'Overview', + revisionKey: 'jres_remote', + page: 1, + pageSize: 3, chunkType: 'page', limit: 3, + assetUrlPolicy: 'none', }, }); await client.callTool({ name: 'knowhere_grep_chunks', arguments: { documentId: 'doc_remote', + revisionKey: 'jres_remote', pattern: 'revenue', + continuationCursor: 'cursor-1', chunkType: 'page', }, }); @@ -179,13 +184,18 @@ describe('knowhere MCP wrapper', () => { }); expect(knowhereClient.knowledge.readChunks).toHaveBeenCalledWith({ documentId: 'doc_remote', - sectionPath: 'Overview', + revisionKey: 'jres_remote', + page: 1, + pageSize: 3, chunkType: 'page', limit: 3, + assetUrlPolicy: 'none', }); expect(knowhereClient.knowledge.grepChunks).toHaveBeenCalledWith({ documentId: 'doc_remote', + revisionKey: 'jres_remote', pattern: 'revenue', + continuationCursor: 'cursor-1', chunkType: 'page', }); await client.close(); @@ -318,6 +328,7 @@ describe('knowhere MCP wrapper', () => { expect(knowhereClient.knowledge.withCacheDirectory).toHaveBeenCalledWith( '/tmp/knowhere-mcp-cache', ); + expect(knowhereClient.knowledge.withParsedStorage).toHaveBeenCalledOnce(); expect(knowhereClient.knowledge.recoverPendingAsyncParseJobs).toHaveBeenCalledOnce(); expect(knowhereClient.knowledge.readChunks).toHaveBeenCalledWith({ localDocumentId: 'local-report', @@ -448,8 +459,10 @@ function createClient(): Knowhere & { grepChunks: vi.fn().mockResolvedValue({ matches: [] }), search: vi.fn().mockResolvedValue({ results: [] }), withCacheDirectory: vi.fn(), + withParsedStorage: vi.fn(), }; knowledge.withCacheDirectory.mockReturnValue(knowledge as unknown as Knowledge); + knowledge.withParsedStorage.mockReturnValue(knowledge as unknown as Knowledge); const archiveDocument = vi.fn().mockResolvedValue({ documentId: 'doc-1', status: 'archived', @@ -493,6 +506,7 @@ type KnowledgeWithMocks = Pick< | 'grepChunks' | 'search' | 'withCacheDirectory' + | 'withParsedStorage' > & { parseToLocalCache: Mock; startParse: Mock; @@ -505,4 +519,5 @@ type KnowledgeWithMocks = Pick< grepChunks: Mock; search: Mock; withCacheDirectory: Mock; + withParsedStorage: Mock; }; diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 7070167..da0e5e1 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -1,6 +1,9 @@ +import path from 'path'; + import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; import { + DiskParsedDocumentStorage, Knowhere, VERSION, ValidationError, @@ -51,10 +54,7 @@ export async function createKnowhereMcpServer( authTokenProvider: options?.authTokenProvider, baseURL: options?.baseURL, }); - const knowledge: Knowledge = - options?.cacheDirectory === undefined - ? client.knowledge - : client.knowledge.withCacheDirectory(options.cacheDirectory); + const knowledge: Knowledge = createKnowledgeClient(client, options?.cacheDirectory); if (options?.recoverPendingJobsOnStart !== false) { await knowledge.recoverPendingAsyncParseJobs(); } @@ -227,11 +227,12 @@ export async function createKnowhereMcpServer( 'knowhere_get_document_outline', { description: - 'Return the outline for a parsed document. Pass localDocumentId, published documentId, or completed jobId. The response document includes resultDirectoryPath; expanded chunks are stored in chunks.json under that directory.', + 'Return the outline for one parsed document. Pass localDocumentId, published documentId, or completed jobId. The SDK reads configured parsed storage first, falls back to remote document chunks for documentId reads, and schedules bounded sync only when parsed storage is explicitly configured.', inputSchema: { localDocumentId: z.string().optional(), documentId: z.string().optional(), jobId: z.string().optional(), + revisionKey: z.string().optional(), }, outputSchema: objectOutputSchema, }, @@ -242,17 +243,21 @@ export async function createKnowhereMcpServer( 'knowhere_read_chunks', { description: - 'Read exact chunks from a parsed document. Pass localDocumentId, published documentId, or completed jobId. The SDK reads its cached parse result and preserves storage-adapter Blob asset URLs in chunk metadata.', + 'Read exact chunks from one parsed document. Pass localDocumentId, published documentId, or completed jobId. page/pageSize are for display reads and cannot be combined with sectionPath/startChunk/endChunk/chunkId. The SDK reads configured parsed storage first, falls back to remote document chunks for documentId reads, and only returns durable asset URLs when assetUrlPolicy is durable and storage hardening succeeds.', inputSchema: { localDocumentId: z.string().optional(), documentId: z.string().optional(), jobId: z.string().optional(), + revisionKey: z.string().optional(), + page: z.number().int().positive().optional(), + pageSize: z.number().int().positive().optional(), sectionPath: z.string().optional(), startChunk: z.number().int().positive().optional(), endChunk: z.number().int().positive().optional(), chunkId: z.string().optional(), chunkType: z.enum(['text', 'image', 'table', 'page']).optional(), limit: z.number().int().positive().optional(), + assetUrlPolicy: z.enum(['none', 'durable']).optional(), }, outputSchema: objectOutputSchema, }, @@ -263,12 +268,14 @@ export async function createKnowhereMcpServer( 'knowhere_grep_chunks', { description: - 'Run grep-style literal or regex matching against parsed document chunks. Pass localDocumentId, published documentId, or completed jobId. The SDK scans its cached parse result and preserves storage-adapter Blob asset URLs in chunk metadata.', + 'Run grep-style literal or regex matching against one parsed document. Pass localDocumentId, published documentId, or completed jobId. Broad workspace search belongs to knowhere_search; documentId grep streams remote chunks without asset URLs when parsed storage is missing or stale.', inputSchema: { localDocumentId: z.string().optional(), documentId: z.string().optional(), jobId: z.string().optional(), + revisionKey: z.string().optional(), pattern: z.string(), + continuationCursor: z.string().optional(), isRegex: z.boolean().optional(), isCaseSensitive: z.boolean().optional(), maxResults: z.number().int().positive().optional(), @@ -307,6 +314,16 @@ export async function runKnowhereMcpServer(options?: KnowhereMcpServerOptions): await server.connect(transport); } +function createKnowledgeClient(client: Knowhere, cacheDirectory: string | undefined): Knowledge { + if (cacheDirectory === undefined) { + return client.knowledge; + } + + return client.knowledge.withCacheDirectory(cacheDirectory).withParsedStorage({ + storage: new DiskParsedDocumentStorage(path.join(cacheDirectory, 'parsed-documents')), + }); +} + function createToolResult(result: ToolResult): { content: { type: 'text'; text: string }[]; structuredContent: { result: ToolResult }; diff --git a/src/index.ts b/src/index.ts index ea61bfd..99cbda8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -57,6 +57,22 @@ export type { KnowhereAssetStorageOptions, KnowhereAssetStorageResult, KnowhereAssetStorageWriteResult, + KnowhereParsedSnapshot, + KnowhereParsedSnapshotChunk, + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotChunkPageReference, + KnowhereParsedSnapshotManifest, + ParsedDocumentAssetUrlPolicy, + ParsedDocumentStorage, + ParsedDocumentStorageAsset, + ParsedDocumentStorageAssetParams, + ParsedDocumentStorageChunkPageParams, + ParsedDocumentStorageConfig, + ParsedDocumentStorageDocument, + ParsedDocumentStorageLimits, + ParsedDocumentStorageManifestParams, + ParsedDocumentSyncProgress, + ParsedDocumentSyncScheduler, PageCitationAsset, PageCitationAssetContentType, PageCitationAssetSource, @@ -86,6 +102,7 @@ export type { LocalKnowledgeParseResponse, KnowledgeOutline, KnowledgeSection, + KnowledgeParsedStorageOptions, KnowledgeReadParams, KnowledgeReadChunk, KnowledgeReadResponse, @@ -97,9 +114,13 @@ export type { KnowledgeSearchResponse, KnowledgeSearchResult, KnowledgeStartupRecoveryResponse, + KnowledgeSyncParsedDocumentParams, + KnowledgeSyncParsedDocumentResponse, } from './types/index.js'; export { storeParseResultAssets } from './storage/asset-storage.js'; +export { DiskParsedDocumentStorage } from './storage/disk-parsed-document-storage.js'; +export { syncParseResultToParsedDocumentStorage } from './storage/parsed-document-storage.js'; // Errors export { diff --git a/src/knowledge/__tests__/knowledge.test.ts b/src/knowledge/__tests__/knowledge.test.ts index c631a0b..f928699 100644 --- a/src/knowledge/__tests__/knowledge.test.ts +++ b/src/knowledge/__tests__/knowledge.test.ts @@ -8,11 +8,17 @@ import { Knowledge } from '../knowledge.js'; import type { Knowhere } from '../../client.js'; import type { Chunk, + DocumentChunk, + DocumentChunkListParams, DocumentChunkListResponse, KnowhereAssetStorageAdapter, KnowhereAssetStorageObject, KnowhereAssetStorageWriteResult, + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, ParseResult, + ParsedDocumentStorage, + ParsedDocumentSyncProgress, TextChunk, TableChunk, } from '../../types/index.js'; @@ -241,7 +247,7 @@ describe('Knowledge', () => { expect(read.nextChunk).toBeUndefined(); }); - it('should sync a remote document id into the local cache before reads', async () => { + it('should read a remote document id without importing into the local cache', async () => { const cacheDirectory = await createTempDirectory(); const { client, documentsListChunks, jobsLoad } = createClient(createParseResult()); const knowledge = new Knowledge(client, { cacheDirectory }); @@ -261,15 +267,17 @@ describe('Knowledge', () => { expect(documentsListChunks).toHaveBeenCalledWith('doc_remote', { page: 1, - pageSize: 1, + pageSize: 100, + chunkType: undefined, + includeAssetUrls: false, }); - expect(jobsLoad).toHaveBeenCalledWith('job_remote', { verifyChecksum: undefined }); + expect(jobsLoad).not.toHaveBeenCalled(); expect(read.document).toMatchObject({ localDocumentId: 'doc_remote', documentId: 'doc_remote', namespace: 'support-center', jobId: 'job_remote', - sourceFileName: 'job_remote.md', + sourceFileName: 'doc_remote', chunkCount: 3, }); expect(read.chunks.map((chunk) => chunk.chunkId)).toEqual(['chunk-intro']); @@ -278,9 +286,140 @@ describe('Knowledge', () => { chunkId: 'chunk-intro', sectionPath: 'Intro', }); - expect(documents).toHaveLength(1); - await expectFileExists(path.join(cacheDirectory, 'documents', 'doc_remote', 'manifest.json')); - await expectFileExists(path.join(cacheDirectory, 'documents', 'doc_remote', 'chunks.json')); + expect(documents).toHaveLength(0); + await expectFileMissing(path.join(cacheDirectory, 'documents', 'doc_remote', 'manifest.json')); + await expectFileMissing(path.join(cacheDirectory, 'documents', 'doc_remote', 'chunks.json')); + }); + + it('should read configured parsed storage for an explicit revision without remote calls', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = createParseResult(); + const storage = createInMemoryParsedStorage(); + storage.seedSnapshot({ + documentId: 'doc_remote', + revisionKey: 'jres_remote', + result: parseResult, + }); + const { client, documentsListChunks } = createClient(parseResult); + const knowledge = new Knowledge(client, { cacheDirectory }).withParsedStorage({ storage }); + + const read = await knowledge.readChunks({ + documentId: 'doc_remote', + revisionKey: 'jres_remote', + page: 1, + pageSize: 2, + }); + + expect(documentsListChunks).not.toHaveBeenCalled(); + expect(read.document.resultDirectoryPath).toBe('parsed-storage:doc_remote'); + expect(read.chunks.map((chunk) => chunk.chunkId)).toEqual(['chunk-intro', 'chunk-table']); + expect(read.totalChunks).toBe(3); + }); + + it('should fall back to remote chunks and schedule parsed storage sync on storage misses', async () => { + const cacheDirectory = await createTempDirectory(); + const storage = createInMemoryParsedStorage(); + const scheduledTasks: Array<() => Promise> = []; + const scheduler = { + schedule: vi.fn((task: () => Promise): void => { + scheduledTasks.push(task); + }), + }; + const { client, documentsListChunks, jobsLoad } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }).withParsedStorage({ + storage, + scheduler, + limits: { + remotePageSize: 2, + maxPagesPerSync: 10, + }, + }); + + const read = await knowledge.readChunks({ + documentId: 'doc_remote', + page: 1, + pageSize: 2, + }); + + expect(read.chunks.map((chunk) => chunk.chunkId)).toEqual(['chunk-intro', 'chunk-table']); + expect(jobsLoad).not.toHaveBeenCalled(); + expect(scheduler.schedule).toHaveBeenCalledOnce(); + expect(storage.manifestCount()).toBe(0); + + await scheduledTasks[0]?.(); + + expect(documentsListChunks).toHaveBeenCalledWith('doc_remote', { + page: 1, + pageSize: 2, + includeAssetUrls: true, + }); + expect(storage.manifestCount()).toBe(1); + expect(storage.getManifest('doc_remote', 'jres_remote')?.totalChunks).toBe(3); + expect(storage.getProgress('doc_remote', 'jres_remote')?.status).toBe('completed'); + }); + + it('should sync configured parsed storage before parse returns', async () => { + const cacheDirectory = await createTempDirectory(); + const storage = createInMemoryParsedStorage(); + const { client } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }).withParsedStorage({ storage }); + + const parsed = await knowledge.parseToLocalCache({ + url: 'https://example.com/report.md', + localDocumentId: 'local-report', + }); + + expect(parsed.document.localDocumentId).toBe('local-report'); + expect(storage.getManifest('doc-1', 'job-1')).toMatchObject({ + documentId: 'doc-1', + revisionKey: 'job-1', + totalChunks: 3, + }); + expect(storage.getPage('doc-1', 'job-1', 1)?.chunks).toHaveLength(3); + }); + + it('should reject paged read params combined with scan-heavy filters', async () => { + const cacheDirectory = await createTempDirectory(); + const { client, documentsListChunks } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }); + + await expect( + knowledge.readChunks({ + documentId: 'doc_remote', + page: 1, + sectionPath: 'Intro', + }), + ).rejects.toThrow(/cannot be combined/); + expect(documentsListChunks).not.toHaveBeenCalled(); + }); + + it('should grep a remote document with truncation cursor without asset URLs', async () => { + const cacheDirectory = await createTempDirectory(); + const { client, documentsListChunks } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }); + + const grep = await knowledge.grepChunks({ + documentId: 'doc_remote', + pattern: 'revenue', + maxResults: 1, + }); + + expect(grep.matches).toHaveLength(1); + expect(grep.truncated).toBe(true); + expect(grep.continuationCursor).toEqual(expect.any(String)); + const resumed = await knowledge.grepChunks({ + documentId: 'doc_remote', + pattern: 'revenue', + maxResults: 1, + continuationCursor: grep.continuationCursor, + }); + expect(resumed.matches[0]?.chunkId).toBe('chunk-table'); + expect(documentsListChunks).toHaveBeenCalledWith('doc_remote', { + page: 1, + pageSize: 100, + chunkType: undefined, + includeAssetUrls: false, + }); }); it('should fail clearly when a remote document id has no published job id', async () => { @@ -307,7 +446,7 @@ describe('Knowledge', () => { documentId: 'doc_missing_job', limit: 5, }), - ).rejects.toThrow('Cannot sync server document doc_missing_job'); + ).rejects.toThrow('Cannot read server document doc_missing_job'); }); it('should sync a completed job id into the local cache before reads', async () => { @@ -716,22 +855,32 @@ function createClient(parseResult: ParseResult): { }, }), ); - const documentsListChunks = vi.fn().mockImplementation( - (_documentId: string): Promise => - Promise.resolve({ - documentId: 'doc_remote', - namespace: 'support-center', - jobId: 'job_remote', - jobResultId: 'jres_remote', - chunks: [], - pagination: { - page: 1, - pageSize: 1, - total: 0, - totalPages: 1, - }, - }), - ); + const documentsListChunks = vi + .fn() + .mockImplementation( + ( + documentId: string, + params?: DocumentChunkListParams, + ): Promise => { + const page = params?.page ?? 1; + const pageSize = params?.pageSize ?? 50; + const chunks = toRemoteDocumentChunks(parseResult, documentId, params); + const pageChunks = chunks.slice((page - 1) * pageSize, page * pageSize); + return Promise.resolve({ + documentId, + namespace: 'support-center', + jobId: documentId === 'doc_remote' ? 'job_remote' : parseResult.jobId, + jobResultId: documentId === 'doc_remote' ? 'jres_remote' : `jres_${parseResult.jobId}`, + chunks: pageChunks, + pagination: { + page, + pageSize, + total: chunks.length, + totalPages: Math.max(1, Math.ceil(chunks.length / pageSize)), + }, + }); + }, + ); const documentsGetPageCitationSource = vi.fn().mockResolvedValue({ documentId: 'doc-1', namespace: 'support-center', @@ -805,6 +954,204 @@ async function expectFileMissing(filePath: string): Promise { await expect(access(filePath)).rejects.toMatchObject({ code: 'ENOENT' }); } +function toRemoteDocumentChunks( + result: ParseResult, + documentId: string, + params?: DocumentChunkListParams, +): DocumentChunk[] { + return result.chunks + .map((chunk, index): DocumentChunk => { + const sectionPath = chunk.path.replace(`${result.manifest.sourceFileName}/`, ''); + const filePath = + chunk.type === 'image' || chunk.type === 'table' + ? chunk.filePath + : typeof chunk.metadata.filePath === 'string' + ? chunk.metadata.filePath + : undefined; + return { + id: `${documentId}-${chunk.chunkId}`, + chunkId: chunk.chunkId, + chunkType: chunk.type, + contentSource: chunk.contentSource, + content: chunk.content, + sectionPath, + sourceChunkPath: chunk.path, + filePath, + sortOrder: index, + metadata: chunk.metadata, + assetUrl: + params?.includeAssetUrls && (chunk.type === 'image' || chunk.type === 'table') + ? chunk.assetUrl + : undefined, + }; + }) + .filter((chunk) => !params?.chunkType || chunk.chunkType === params.chunkType); +} + +function createInMemoryParsedStorage(): ParsedDocumentStorage & { + seedSnapshot(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly result: ParseResult; + }): void; + getManifest(documentId: string, revisionKey: string): KnowhereParsedSnapshotManifest | undefined; + getPage( + documentId: string, + revisionKey: string, + page: number, + ): KnowhereParsedSnapshotChunkPage | undefined; + getProgress(documentId: string, revisionKey: string): ParsedDocumentSyncProgress | undefined; + manifestCount(): number; +} { + const manifests = new Map(); + const pages = new Map(); + const assetUrls = new Map(); + const progressByRevision = new Map(); + + function getRevisionKey(documentId: string, revisionKey: string): string { + return `${documentId}:${revisionKey}`; + } + + function getPageKey(documentId: string, revisionKey: string, page: number): string { + return `${getRevisionKey(documentId, revisionKey)}:${page}`; + } + + function seedSnapshot(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly result: ParseResult; + }): void { + const page = createSnapshotPage(params); + const manifest = createSnapshotManifest(params); + pages.set(getPageKey(params.documentId, params.revisionKey, 1), page); + manifests.set(getRevisionKey(params.documentId, params.revisionKey), manifest); + } + + return { + seedSnapshot, + getManifest: (documentId: string, revisionKey: string) => + manifests.get(getRevisionKey(documentId, revisionKey)), + getPage: (documentId: string, revisionKey: string, page: number) => + pages.get(getPageKey(documentId, revisionKey, page)), + getProgress: (documentId: string, revisionKey: string) => + progressByRevision.get(getRevisionKey(documentId, revisionKey)), + manifestCount: () => manifests.size, + readManifest: (params): Promise => + Promise.resolve(manifests.get(getRevisionKey(params.documentId, params.revisionKey)) ?? null), + writeManifest: (params): Promise => { + manifests.set(getRevisionKey(params.documentId, params.revisionKey), params.manifest); + return Promise.resolve(); + }, + readChunkPage: (params): Promise => + Promise.resolve( + pages.get(getPageKey(params.documentId, params.revisionKey, params.page)) ?? null, + ), + writeChunkPage: (params): Promise => { + pages.set(getPageKey(params.documentId, params.revisionKey, params.page.page), params.page); + return Promise.resolve(); + }, + writeAsset: (params): Promise<{ readonly sourcePath: string; readonly url?: string }> => { + const key = `${getRevisionKey(params.documentId, params.revisionKey)}:${params.sourcePath}`; + const url = `https://blob.example/${params.documentId}/${params.revisionKey}/${params.sourcePath}`; + assetUrls.set(key, url); + return Promise.resolve({ + sourcePath: params.sourcePath, + url, + }); + }, + getAssetUrl: (params): Promise => + Promise.resolve( + assetUrls.get( + `${getRevisionKey(params.documentId, params.revisionKey)}:${params.sourcePath}`, + ) ?? null, + ), + readSyncProgress: (params): Promise => + Promise.resolve( + progressByRevision.get(getRevisionKey(params.documentId, params.revisionKey)) ?? null, + ), + writeSyncProgress: (params): Promise => { + progressByRevision.set(getRevisionKey(params.documentId, params.revisionKey), params); + return Promise.resolve(); + }, + }; +} + +function createSnapshotManifest(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly result: ParseResult; +}): KnowhereParsedSnapshotManifest { + return { + version: 1, + kind: 'knowhere-parsed-result-snapshot', + jobId: params.result.jobId, + revisionKey: params.revisionKey, + documentId: params.documentId, + namespace: params.result.namespace, + sourceFileName: params.result.manifest.sourceFileName, + totalChunks: params.result.chunks.length, + typeCounts: countFixtureChunkTypes(params.result), + chunkPageSize: params.result.chunks.length, + chunkPages: [ + { + page: 1, + pageSize: params.result.chunks.length, + chunkCount: params.result.chunks.length, + key: 'parsed/chunks/page-1.json', + }, + ], + assetUrlsByFilePath: {}, + createdAt: '2026-01-01T00:00:00.000Z', + }; +} + +function createSnapshotPage(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly result: ParseResult; +}): KnowhereParsedSnapshotChunkPage { + return { + version: 1, + jobId: params.result.jobId, + revisionKey: params.revisionKey, + documentId: params.documentId, + namespace: params.result.namespace, + sourceFileName: params.result.manifest.sourceFileName, + page: 1, + pageSize: params.result.chunks.length, + total: params.result.chunks.length, + totalPages: 1, + chunks: params.result.chunks.map((chunk, index) => ({ + id: chunk.chunkId, + chunkId: chunk.chunkId, + chunkType: chunk.type, + contentSource: chunk.contentSource, + content: chunk.content, + sectionPath: chunk.path.replace(`${params.result.manifest.sourceFileName}/`, ''), + sourceChunkPath: chunk.path, + filePath: + chunk.type === 'image' || chunk.type === 'table' + ? chunk.filePath + : typeof chunk.metadata.filePath === 'string' + ? chunk.metadata.filePath + : undefined, + sortOrder: index + 1, + metadata: chunk.metadata, + assetUrl: chunk.type === 'image' || chunk.type === 'table' ? chunk.assetUrl : undefined, + })), + }; +} + +function countFixtureChunkTypes(result: ParseResult): Readonly> { + return result.chunks.reduce>( + (counts, chunk) => { + counts[chunk.type] += 1; + return counts; + }, + { text: 0, image: 0, table: 0, page: 0 }, + ); +} + function createParseResult(): ParseResult { const chunks: [TextChunk, TableChunk, TextChunk] = [ { @@ -915,7 +1262,9 @@ function createParseResultWithFullPaths(): ParseResult { } function createPageParseResult(chunks: Chunk[] = [createPageChunk()]): ParseResult { - const pageChunks = chunks.filter((chunk): chunk is Extract => chunk.type === 'page'); + const pageChunks = chunks.filter( + (chunk): chunk is Extract => chunk.type === 'page', + ); return { manifest: { diff --git a/src/knowledge/index.ts b/src/knowledge/index.ts index fc782db..4ed7014 100644 --- a/src/knowledge/index.ts +++ b/src/knowledge/index.ts @@ -15,6 +15,7 @@ export type { KnowledgeLoadJobResultParams, KnowledgeOutline, KnowledgeParseParams, + KnowledgeParsedStorageOptions, KnowledgeReadChunk, KnowledgeReadParams, KnowledgeReadResponse, @@ -24,6 +25,8 @@ export type { KnowledgeSearchResult, KnowledgeSection, KnowledgeStartupRecoveryResponse, + KnowledgeSyncParsedDocumentParams, + KnowledgeSyncParsedDocumentResponse, LocalKnowledgeDocument, LocalKnowledgeParseResponse, } from './types.js'; diff --git a/src/knowledge/knowledge.ts b/src/knowledge/knowledge.ts index 6c981ab..1993437 100644 --- a/src/knowledge/knowledge.ts +++ b/src/knowledge/knowledge.ts @@ -3,6 +3,7 @@ import { createHash } from 'crypto'; import type { Knowhere } from '../client.js'; import type { Chunk, + DocumentChunk, DocNavSection, DocumentChunkListResponse, ParseResult, @@ -10,6 +11,7 @@ import type { } from '../types/index.js'; import { ValidationError } from '../errors/index.js'; import { storeParseResultAssets } from '../storage/asset-storage.js'; +import { syncParseResultToParsedDocumentStorage } from '../storage/parsed-document-storage.js'; import { LocalKnowledgeStore } from './local-store.js'; import type { IndexedKnowledgeChunk, @@ -28,6 +30,7 @@ import type { KnowledgeLoadJobResultParams, KnowledgeOutline, KnowledgeParseParams, + KnowledgeParsedStorageOptions, KnowledgeReadChunk, KnowledgeReadParams, KnowledgeReadResponse, @@ -37,30 +40,70 @@ import type { KnowledgeSearchResult, KnowledgeSection, KnowledgeStartupRecoveryResponse, + KnowledgeSyncParsedDocumentParams, + KnowledgeSyncParsedDocumentResponse, KnowledgeChunkType, LocalKnowledgeDocument, LocalKnowledgeParseResponse, } from './types.js'; +import type { + KnowhereParsedSnapshotChunk, + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, + ParsedDocumentAssetUrlPolicy, + ParsedDocumentStorageConfig, + ParsedDocumentStorageLimits, +} from '../types/storage.js'; const DEFAULT_READ_LIMIT = 12; const MAX_READ_LIMIT = 40; +const DEFAULT_PAGE_SIZE = 20; +const MAX_PAGE_SIZE = 200; const DEFAULT_GREP_LIMIT = 20; const MAX_GREP_LIMIT = 50; const DEFAULT_CONTEXT_CHARS = 80; +const DEFAULT_REMOTE_SCAN_PAGE_SIZE = 100; +const DEFAULT_MAX_SYNC_PAGES = 10; +const DEFAULT_MAX_SYNC_ASSETS = 20; +const DEFAULT_SYNC_DEADLINE_MS = 8000; +const DEFAULT_GREP_MAX_PAGES = 50; +const DEFAULT_GREP_DEADLINE_MS = 8000; +const DEFAULT_OUTLINE_MAX_PAGES = 50; +const DEFAULT_OUTLINE_DEADLINE_MS = 8000; const SAFE_LOCAL_DOCUMENT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; const REMOTE_DOCUMENT_ID_PATTERN = /^doc[_-]/; export class Knowledge { private readonly client: Knowhere; private readonly store: LocalKnowledgeStore; - - constructor(client: Knowhere, options?: { cacheDirectory?: string }) { + private readonly cacheDirectory: string | undefined; + private readonly parsedStorageConfig: ParsedDocumentStorageConfig | undefined; + + constructor( + client: Knowhere, + options?: { + cacheDirectory?: string; + parsedStorage?: ParsedDocumentStorageConfig; + }, + ) { this.client = client; + this.cacheDirectory = options?.cacheDirectory; + this.parsedStorageConfig = options?.parsedStorage; this.store = new LocalKnowledgeStore(options?.cacheDirectory); } withCacheDirectory(cacheDirectory: string): Knowledge { - return new Knowledge(this.client, { cacheDirectory }); + return new Knowledge(this.client, { + cacheDirectory, + parsedStorage: this.parsedStorageConfig, + }); + } + + withParsedStorage(parsedStorage: KnowledgeParsedStorageOptions): Knowledge { + return new Knowledge(this.client, { + cacheDirectory: this.cacheDirectory, + parsedStorage, + }); } async parseToLocalCache(params: KnowledgeParseParams): Promise { @@ -68,7 +111,8 @@ export class Knowledge { ? await this.parseWithoutClientStorageAdapter(params) : await this.client.parse(params); const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter); - const result = storedAssets.result; + const syncedStorage = await this.syncParseResultToConfiguredStorage(storedAssets.result); + const result = syncedStorage.result; const document = await this.store.saveResult(result, { localDocumentId: params.localDocumentId, }); @@ -122,11 +166,12 @@ export class Knowledge { verifyChecksum: params.verifyChecksum, }); const storedAssets = await storeParseResultAssets(loadedResult, params.storageAdapter); - const result = storedAssets.result; + const syncedStorage = await this.syncParseResultToConfiguredStorage(storedAssets.result); + const result = syncedStorage.result; return { result, assetUrlsByFilePath: storedAssets.assetUrlsByFilePath, - parsedSnapshot: storedAssets.snapshot, + parsedSnapshot: syncedStorage.snapshot ?? storedAssets.snapshot, }; } @@ -153,6 +198,77 @@ export class Knowledge { return this.importJobResult(params); } + async syncParsedDocument( + params: KnowledgeSyncParsedDocumentParams, + ): Promise { + const normalized = normalizeDocumentReference(params); + const parsedStorageConfig = this.requireParsedStorageConfig(); + + if (normalized.documentId) { + return this.syncRemoteDocumentToParsedStorage({ + documentId: normalized.documentId, + revisionKey: params.revisionKey, + parsedStorageConfig, + }); + } + + if (normalized.jobId) { + const loadedResult = await this.client.jobs.load(normalized.jobId); + const synced = await syncParseResultToParsedDocumentStorage({ + result: loadedResult, + storage: parsedStorageConfig.storage, + documentId: loadedResult.documentId ?? normalized.localDocumentId ?? normalized.jobId, + revisionKey: params.revisionKey ?? normalized.jobId, + chunkPageSize: parsedStorageConfig.limits?.chunkPageSize, + }); + return { + documentId: synced.documentId, + revisionKey: synced.revisionKey, + completed: true, + manifest: synced.snapshot?.manifest, + }; + } + + if (!normalized.localDocumentId) { + throw new ValidationError('localDocumentId, documentId, or jobId is required'); + } + + const { result } = await this.store.loadResult(normalized.localDocumentId); + const synced = await syncParseResultToParsedDocumentStorage({ + result, + storage: parsedStorageConfig.storage, + documentId: result.documentId ?? normalized.localDocumentId, + revisionKey: params.revisionKey ?? result.jobId, + chunkPageSize: parsedStorageConfig.limits?.chunkPageSize, + }); + return { + documentId: synced.documentId, + revisionKey: synced.revisionKey, + completed: true, + manifest: synced.snapshot?.manifest, + }; + } + + private async syncParseResultToConfiguredStorage(result: ParseResult): Promise<{ + result: ParseResult; + snapshot?: Awaited>['snapshot']; + }> { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return { result }; + } + + const synced = await syncParseResultToParsedDocumentStorage({ + result, + storage: parsedStorageConfig.storage, + chunkPageSize: parsedStorageConfig.limits?.chunkPageSize, + }); + return { + result: synced.result, + snapshot: synced.snapshot, + }; + } + private async parseWithoutClientStorageAdapter( params: KnowledgeParseParams, ): Promise { @@ -249,6 +365,19 @@ export class Knowledge { async getDocumentOutline( reference: string | KnowledgeDocumentReference, ): Promise { + const normalized = normalizeDocumentReference(reference); + if (normalized.documentId) { + return this.getRemoteDocumentOutline(normalized.documentId, normalized.revisionKey); + } + + if ( + normalized.localDocumentId && + looksLikeRemoteDocumentId(normalized.localDocumentId) && + !(await this.store.getDocument(normalized.localDocumentId)) + ) { + return this.getRemoteDocumentOutline(normalized.localDocumentId, normalized.revisionKey); + } + const { document, result } = await this.loadReadableResult(reference); const chunks = indexChunks(result); const sections = buildFlatSections(result, chunks); @@ -269,6 +398,20 @@ export class Knowledge { } async readChunks(params: KnowledgeReadParams): Promise { + validateReadParams(params); + const normalized = normalizeDocumentReference(params); + if (normalized.documentId) { + return this.readRemoteDocumentChunks(normalized.documentId, params); + } + + if ( + normalized.localDocumentId && + looksLikeRemoteDocumentId(normalized.localDocumentId) && + !(await this.store.getDocument(normalized.localDocumentId)) + ) { + return this.readRemoteDocumentChunks(normalized.localDocumentId, params); + } + const { document, result } = await this.loadReadableResult(params); const limit = clampLimit(params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT); const chunks = indexChunks(result).filter((chunk) => matchesReadScope(chunk, params)); @@ -291,6 +434,19 @@ export class Knowledge { throw new ValidationError('pattern is required'); } + const normalized = normalizeDocumentReference(params); + if (normalized.documentId) { + return this.grepRemoteDocumentChunks(normalized.documentId, params); + } + + if ( + normalized.localDocumentId && + looksLikeRemoteDocumentId(normalized.localDocumentId) && + !(await this.store.getDocument(normalized.localDocumentId)) + ) { + return this.grepRemoteDocumentChunks(normalized.localDocumentId, params); + } + const { document, result } = await this.loadReadableResult(params); const maxResults = clampLimit(params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT); const contextChars = params.contextChars ?? DEFAULT_CONTEXT_CHARS; @@ -412,6 +568,784 @@ export class Knowledge { throw new Error(`Local Knowhere document not found: ${normalized.localDocumentId}`); } + private async readRemoteDocumentChunks( + documentId: string, + params: KnowledgeReadParams, + ): Promise { + const explicitRevisionKey = params.revisionKey ?? params.jobId; + if (explicitRevisionKey) { + const stored = await this.tryReadChunksFromParsedStorage({ + documentId, + revisionKey: explicitRevisionKey, + params, + }); + if (stored) { + return stored; + } + } + + if (hasScanReadFilters(params)) { + return this.scanRemoteDocumentChunks(documentId, params); + } + + const pageParams = getReadPageParams(params); + const remotePage = await this.client.documents.listChunks(documentId, { + page: pageParams.page, + pageSize: pageParams.pageSize, + chunkType: params.chunkType, + includeAssetUrls: shouldRequestRemoteAssetUrls( + params.assetUrlPolicy, + this.parsedStorageConfig, + ), + }); + const revisionKey = getRemoteRevisionKey(remotePage); + const stored = await this.tryReadChunksFromParsedStorage({ + documentId, + revisionKey, + params, + }); + if (stored) { + return stored; + } + + const chunks = await this.toRemoteReadChunks({ + documentId, + revisionKey, + chunks: remotePage.chunks, + assetUrlPolicy: params.assetUrlPolicy, + }); + this.scheduleParsedStorageSync(documentId, revisionKey); + return { + document: createRemoteKnowledgeDocument({ + documentId, + response: remotePage, + revisionKey, + }), + chunks, + page: remotePage.pagination.page, + pageSize: remotePage.pagination.pageSize, + totalChunks: remotePage.pagination.total, + totalPages: remotePage.pagination.totalPages, + }; + } + + private async scanRemoteDocumentChunks( + documentId: string, + params: KnowledgeReadParams, + ): Promise { + const explicitRevisionKey = params.revisionKey ?? params.jobId; + if (explicitRevisionKey) { + const stored = await this.tryReadChunksFromParsedStorage({ + documentId, + revisionKey: explicitRevisionKey, + params, + }); + if (stored) { + return stored; + } + } + + const limit = clampLimit(params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT); + const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits); + const firstPage = await this.client.documents.listChunks(documentId, { + page: 1, + pageSize: limits.remotePageSize, + chunkType: params.chunkType, + includeAssetUrls: shouldRequestRemoteAssetUrls( + params.assetUrlPolicy, + this.parsedStorageConfig, + ), + }); + const revisionKey = getRemoteRevisionKey(firstPage); + const stored = await this.tryReadChunksFromParsedStorage({ + documentId, + revisionKey, + params, + }); + if (stored) { + return stored; + } + + const selected: DocumentChunk[] = []; + let scannedPages = 0; + let page = 1; + let response = firstPage; + const startedAt = Date.now(); + + while (true) { + scannedPages += 1; + selected.push(...response.chunks.filter((chunk) => matchesRemoteReadScope(chunk, params))); + if ( + selected.length >= limit || + response.pagination.page >= response.pagination.totalPages || + scannedPages >= limits.outlineMaxPages || + Date.now() - startedAt >= limits.outlineDeadlineMs + ) { + break; + } + + page += 1; + response = await this.client.documents.listChunks(documentId, { + page, + pageSize: limits.remotePageSize, + chunkType: params.chunkType, + includeAssetUrls: shouldRequestRemoteAssetUrls( + params.assetUrlPolicy, + this.parsedStorageConfig, + ), + }); + } + + const chunks = await this.toRemoteReadChunks({ + documentId, + revisionKey, + chunks: selected.slice(0, limit), + assetUrlPolicy: params.assetUrlPolicy, + }); + this.scheduleParsedStorageSync(documentId, revisionKey); + return { + document: createRemoteKnowledgeDocument({ + documentId, + response: firstPage, + revisionKey, + }), + chunks, + nextChunk: selected.length > limit ? toChunkPosition(selected[limit]) : undefined, + totalChunks: firstPage.pagination.total, + totalPages: firstPage.pagination.totalPages, + }; + } + + private async grepRemoteDocumentChunks( + documentId: string, + params: KnowledgeGrepParams, + ): Promise { + const explicitRevisionKey = params.revisionKey ?? params.jobId; + if (explicitRevisionKey && !params.continuationCursor) { + const stored = await this.tryGrepChunksFromParsedStorage({ + documentId, + revisionKey: explicitRevisionKey, + params, + }); + if (stored) { + return stored; + } + } + + const maxResults = clampLimit(params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT); + const contextChars = params.contextChars ?? DEFAULT_CONTEXT_CHARS; + const matcher = createMatcher(params); + const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits); + const cursor = parseContinuationCursor(params.continuationCursor); + const startPage = cursor?.documentId === documentId ? cursor.nextPage : 1; + const firstPage = await this.client.documents.listChunks(documentId, { + page: startPage, + pageSize: limits.remotePageSize, + chunkType: params.chunkType, + includeAssetUrls: false, + }); + const revisionKey = getRemoteRevisionKey(firstPage); + const stored = + startPage === 1 + ? await this.tryGrepChunksFromParsedStorage({ + documentId, + revisionKey, + params, + }) + : null; + if (stored) { + return stored; + } + + const matches: KnowledgeGrepMatch[] = []; + let scannedChunks = 0; + let scannedPages = 0; + let response = firstPage; + let page = startPage; + const startedAt = Date.now(); + + while (true) { + scannedPages += 1; + for (const [chunkIndex, chunk] of response.chunks.entries()) { + if ( + response.pagination.page === startPage && + cursor?.nextChunkIndex !== undefined && + chunkIndex < cursor.nextChunkIndex + ) { + continue; + } + if (!matchesGrepScope(toIndexedRemoteChunk(chunk), params)) { + continue; + } + scannedChunks += 1; + const chunkMatches = matcher(chunk.content ?? ''); + for (const [matchIndex, match] of chunkMatches.entries()) { + if ( + response.pagination.page === startPage && + cursor?.nextChunkIndex === chunkIndex && + cursor.nextMatchIndex !== undefined && + matchIndex < cursor.nextMatchIndex + ) { + continue; + } + matches.push({ + position: toChunkPosition(chunk), + chunkId: chunk.chunkId, + chunkType: chunk.chunkType, + sectionPath: chunk.sectionPath ?? '', + sourceChunkPath: chunk.sourceChunkPath ?? chunk.sectionPath ?? '', + filePath: chunk.filePath ?? undefined, + startOffset: match.startOffset, + endOffset: match.endOffset, + snippet: buildSnippet( + chunk.content ?? '', + match.startOffset, + match.endOffset, + contextChars, + ), + }); + if (matches.length >= maxResults) { + const continuationCursor = createGrepContinuationCursor({ + documentId, + revisionKey, + response, + chunkIndex, + matchIndex, + chunkMatchCount: chunkMatches.length, + }); + this.scheduleParsedStorageSync(documentId, revisionKey); + return { + document: createRemoteKnowledgeDocument({ + documentId, + response: firstPage, + revisionKey, + }), + matches, + scannedChunks, + truncated: continuationCursor !== undefined, + continuationCursor, + }; + } + } + } + + const shouldStop = + response.pagination.page >= response.pagination.totalPages || + scannedPages >= limits.grepMaxPages || + Date.now() - startedAt >= limits.grepDeadlineMs; + if (shouldStop) { + const truncated = response.pagination.page < response.pagination.totalPages; + this.scheduleParsedStorageSync(documentId, revisionKey); + return { + document: createRemoteKnowledgeDocument({ + documentId, + response: firstPage, + revisionKey, + }), + matches, + scannedChunks, + truncated, + continuationCursor: truncated + ? createContinuationCursor({ + documentId, + revisionKey, + nextPage: response.pagination.page + 1, + }) + : undefined, + }; + } + + page += 1; + response = await this.client.documents.listChunks(documentId, { + page, + pageSize: limits.remotePageSize, + chunkType: params.chunkType, + includeAssetUrls: false, + }); + } + } + + private async getRemoteDocumentOutline( + documentId: string, + revisionKeyOverride?: string, + ): Promise { + if (revisionKeyOverride) { + const stored = await this.tryReadOutlineFromParsedStorage(documentId, revisionKeyOverride); + if (stored) { + return stored; + } + } + + const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits); + const firstPage = await this.client.documents.listChunks(documentId, { + page: 1, + pageSize: limits.remotePageSize, + includeAssetUrls: false, + }); + const revisionKey = getRemoteRevisionKey(firstPage); + const stored = await this.tryReadOutlineFromParsedStorage(documentId, revisionKey); + if (stored) { + return stored; + } + + const chunks: IndexedKnowledgeChunk[] = []; + let page = 1; + let response = firstPage; + let scannedPages = 0; + const startedAt = Date.now(); + + while (true) { + scannedPages += 1; + chunks.push(...response.chunks.map(toIndexedRemoteChunk)); + if ( + response.pagination.page >= response.pagination.totalPages || + scannedPages >= limits.outlineMaxPages || + Date.now() - startedAt >= limits.outlineDeadlineMs + ) { + break; + } + page += 1; + response = await this.client.documents.listChunks(documentId, { + page, + pageSize: limits.remotePageSize, + includeAssetUrls: false, + }); + } + + const truncated = response.pagination.page < response.pagination.totalPages; + const sections = buildFlatSectionsFromIndexedChunks(chunks); + this.scheduleParsedStorageSync(documentId, revisionKey); + return { + document: createRemoteKnowledgeDocument({ + documentId, + response: firstPage, + revisionKey, + typeCounts: countIndexedTypes(chunks), + }), + totalChunks: firstPage.pagination.total, + typeCounts: countIndexedTypes(chunks), + sections, + sectionTree: nestSections(sections), + truncated, + continuationCursor: truncated + ? createContinuationCursor({ + documentId, + revisionKey, + nextPage: response.pagination.page + 1, + }) + : undefined, + }; + } + + private async tryReadChunksFromParsedStorage(params: { + documentId: string; + revisionKey: string; + params: KnowledgeReadParams; + }): Promise { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return null; + } + + const manifest = await this.readFreshParsedStorageManifest( + params.documentId, + params.revisionKey, + ); + if (!manifest) { + return null; + } + + const pageParams = getReadPageParams(params.params); + const indexedChunks = await this.readStoredSnapshotChunks({ + documentId: params.documentId, + revisionKey: params.revisionKey, + manifest, + chunkType: params.params.chunkType, + }); + if (!indexedChunks) { + return null; + } + + const scopedChunks = indexedChunks.filter((chunk) => matchesReadScope(chunk, params.params)); + const selected = hasScanReadFilters(params.params) + ? selectReadWindow( + scopedChunks, + params.params, + clampLimit(params.params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT), + ) + : scopedChunks.slice( + (pageParams.page - 1) * pageParams.pageSize, + pageParams.page * pageParams.pageSize, + ); + const visibleChunks = applyAssetUrlPolicyToReadChunks( + selected.map(toReadChunk), + params.params.assetUrlPolicy, + ); + const totalChunks = scopedChunks.length; + return { + document: createStoredKnowledgeDocument({ + documentId: params.documentId, + manifest, + revisionKey: params.revisionKey, + typeCounts: countIndexedTypes(indexedChunks), + }), + chunks: visibleChunks, + page: hasScanReadFilters(params.params) ? undefined : pageParams.page, + pageSize: hasScanReadFilters(params.params) ? undefined : pageParams.pageSize, + totalChunks, + totalPages: Math.max(1, Math.ceil(totalChunks / pageParams.pageSize)), + nextChunk: getNextChunkPosition(scopedChunks, selected), + }; + } + + private async tryGrepChunksFromParsedStorage(params: { + documentId: string; + revisionKey: string; + params: KnowledgeGrepParams; + }): Promise { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return null; + } + + const manifest = await this.readFreshParsedStorageManifest( + params.documentId, + params.revisionKey, + ); + if (!manifest) { + return null; + } + + const indexedChunks = await this.readStoredSnapshotChunks({ + documentId: params.documentId, + revisionKey: params.revisionKey, + manifest, + chunkType: params.params.chunkType, + }); + if (!indexedChunks) { + return null; + } + + const maxResults = clampLimit(params.params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT); + const contextChars = params.params.contextChars ?? DEFAULT_CONTEXT_CHARS; + const matcher = createMatcher(params.params); + const scopedChunks = indexedChunks.filter((chunk) => matchesGrepScope(chunk, params.params)); + const matches: KnowledgeGrepMatch[] = []; + let scannedChunks = 0; + + for (const chunk of scopedChunks) { + scannedChunks += 1; + for (const match of matcher(chunk.content)) { + matches.push({ + position: chunk.position, + chunkId: chunk.chunkId, + chunkType: chunk.chunkType, + sectionPath: chunk.sectionPath, + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath, + startOffset: match.startOffset, + endOffset: match.endOffset, + snippet: buildSnippet(chunk.content, match.startOffset, match.endOffset, contextChars), + }); + if (matches.length >= maxResults) { + return { + document: createStoredKnowledgeDocument({ + documentId: params.documentId, + manifest, + revisionKey: params.revisionKey, + typeCounts: countIndexedTypes(indexedChunks), + }), + matches, + scannedChunks, + truncated: true, + }; + } + } + } + + return { + document: createStoredKnowledgeDocument({ + documentId: params.documentId, + manifest, + revisionKey: params.revisionKey, + typeCounts: countIndexedTypes(indexedChunks), + }), + matches, + scannedChunks, + truncated: false, + }; + } + + private async tryReadOutlineFromParsedStorage( + documentId: string, + revisionKey: string, + ): Promise { + const manifest = await this.readFreshParsedStorageManifest(documentId, revisionKey); + if (!manifest) { + return null; + } + + const indexedChunks = await this.readStoredSnapshotChunks({ + documentId, + revisionKey, + manifest, + }); + if (!indexedChunks) { + return null; + } + + const sections = buildFlatSectionsFromIndexedChunks(indexedChunks); + const typeCounts = countIndexedTypes(indexedChunks); + return { + document: createStoredKnowledgeDocument({ + documentId, + manifest, + revisionKey, + typeCounts, + }), + totalChunks: manifest.totalChunks, + typeCounts, + sections, + sectionTree: nestSections(sections), + truncated: false, + }; + } + + private async readFreshParsedStorageManifest( + documentId: string, + revisionKey: string, + ): Promise { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return null; + } + + const manifest = await parsedStorageConfig.storage.readManifest({ + documentId, + revisionKey, + }); + if (!manifest || !isFreshManifest(manifest, revisionKey)) { + return null; + } + return manifest; + } + + private async readStoredSnapshotChunks(params: { + documentId: string; + revisionKey: string; + manifest: KnowhereParsedSnapshotManifest; + chunkType?: KnowledgeChunkType; + }): Promise { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return null; + } + + const chunks: IndexedKnowledgeChunk[] = []; + for (const pageReference of params.manifest.chunkPages) { + const page = await parsedStorageConfig.storage.readChunkPage({ + documentId: params.documentId, + revisionKey: params.revisionKey, + page: pageReference.page, + chunkType: params.chunkType, + }); + if (!page) { + return null; + } + chunks.push( + ...page.chunks + .filter((chunk) => !params.chunkType || chunk.chunkType === params.chunkType) + .map(toIndexedSnapshotChunk), + ); + } + return chunks.sort((left, right) => left.position - right.position); + } + + private async toRemoteReadChunks(params: { + documentId: string; + revisionKey: string; + chunks: readonly DocumentChunk[]; + assetUrlPolicy: ParsedDocumentAssetUrlPolicy | undefined; + }): Promise { + const readChunks = params.chunks.map((chunk) => toRemoteReadChunk(chunk)); + if (params.assetUrlPolicy === 'durable') { + return this.hardenReadChunkAssetUrls({ + documentId: params.documentId, + revisionKey: params.revisionKey, + chunks: readChunks, + }); + } + return applyAssetUrlPolicyToReadChunks(readChunks, params.assetUrlPolicy); + } + + private async hardenReadChunkAssetUrls(params: { + documentId: string; + revisionKey: string; + chunks: readonly KnowledgeReadChunk[]; + }): Promise { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return applyAssetUrlPolicyToReadChunks(params.chunks, 'none'); + } + + const hardenedChunks: KnowledgeReadChunk[] = []; + for (const chunk of params.chunks) { + hardenedChunks.push( + await hardenReadChunkAssetUrls({ + storageConfig: parsedStorageConfig, + documentId: params.documentId, + revisionKey: params.revisionKey, + chunk, + }), + ); + } + return hardenedChunks; + } + + private scheduleParsedStorageSync(documentId: string, revisionKey: string): void { + const parsedStorageConfig = this.parsedStorageConfig; + if (!parsedStorageConfig) { + return; + } + + const scheduler = parsedStorageConfig.scheduler ?? defaultParsedStorageScheduler; + const task = async (): Promise => { + try { + await this.syncRemoteDocumentToParsedStorage({ + documentId, + revisionKey, + parsedStorageConfig, + }); + } catch (error) { + await parsedStorageConfig.storage.writeSyncProgress({ + documentId, + revisionKey, + nextChunkPage: 1, + nextAssetIndex: 0, + status: 'failed', + updatedAt: new Date().toISOString(), + error: error instanceof Error ? error.message : String(error), + }); + } + }; + + void Promise.resolve(scheduler.schedule(task)).catch(() => undefined); + } + + private async syncRemoteDocumentToParsedStorage(params: { + documentId: string; + revisionKey?: string; + parsedStorageConfig: ParsedDocumentStorageConfig; + }): Promise { + const limits = getParsedStorageLimits(params.parsedStorageConfig.limits); + const existingProgress = params.revisionKey + ? await params.parsedStorageConfig.storage.readSyncProgress({ + documentId: params.documentId, + revisionKey: params.revisionKey, + }) + : null; + let nextPage = existingProgress?.status === 'running' ? existingProgress.nextChunkPage : 1; + let response = await this.client.documents.listChunks(params.documentId, { + page: nextPage, + pageSize: limits.remotePageSize, + includeAssetUrls: true, + }); + const revisionKey = params.revisionKey ?? getRemoteRevisionKey(response); + if (params.revisionKey && getRemoteRevisionKey(response) !== params.revisionKey) { + response = await this.client.documents.listChunks(params.documentId, { + page: 1, + pageSize: limits.remotePageSize, + includeAssetUrls: true, + }); + nextPage = 1; + } + + const totalPages = response.pagination.totalPages; + const chunkPages: KnowhereParsedSnapshotChunkPage[] = []; + let syncedPages = 0; + const startedAt = Date.now(); + + while (nextPage <= totalPages) { + if (syncedPages > 0) { + response = await this.client.documents.listChunks(params.documentId, { + page: nextPage, + pageSize: limits.remotePageSize, + includeAssetUrls: true, + }); + } + + const readChunks = await this.hardenReadChunkAssetUrls({ + documentId: params.documentId, + revisionKey, + chunks: response.chunks.map(toRemoteReadChunk), + }); + const page = toSnapshotChunkPageFromReadChunks({ + response, + revisionKey, + chunks: readChunks, + }); + await params.parsedStorageConfig.storage.writeChunkPage({ + documentId: params.documentId, + revisionKey, + page, + }); + chunkPages.push(page); + nextPage += 1; + syncedPages += 1; + await params.parsedStorageConfig.storage.writeSyncProgress({ + documentId: params.documentId, + revisionKey, + nextChunkPage: nextPage, + nextAssetIndex: 0, + status: 'running', + updatedAt: new Date().toISOString(), + }); + + if ( + syncedPages >= limits.maxPagesPerSync || + Date.now() - startedAt >= limits.syncDeadlineMs + ) { + return { + documentId: params.documentId, + revisionKey, + completed: false, + }; + } + } + + const manifest = createRemoteSnapshotManifest({ + response, + revisionKey, + chunkPages, + }); + await params.parsedStorageConfig.storage.writeManifest({ + documentId: params.documentId, + revisionKey, + manifest, + }); + await params.parsedStorageConfig.storage.writeSyncProgress({ + documentId: params.documentId, + revisionKey, + nextChunkPage: totalPages + 1, + nextAssetIndex: 0, + status: 'completed', + updatedAt: new Date().toISOString(), + }); + return { + documentId: params.documentId, + revisionKey, + completed: true, + manifest, + }; + } + + private requireParsedStorageConfig(): ParsedDocumentStorageConfig { + if (!this.parsedStorageConfig) { + throw new ValidationError('parsed document storage is not configured'); + } + return this.parsedStorageConfig; + } + private async findLocalDocumentByRemoteId( documentId: string, ): Promise { @@ -460,6 +1394,688 @@ function createLocalDocumentIdForRemote(documentId: string): string { return `remote_${hash}`; } +interface NormalizedParsedStorageLimits { + readonly chunkPageSize: number; + readonly remotePageSize: number; + readonly maxPagesPerSync: number; + readonly maxAssetsPerSync: number; + readonly syncDeadlineMs: number; + readonly grepMaxPages: number; + readonly grepDeadlineMs: number; + readonly outlineMaxPages: number; + readonly outlineDeadlineMs: number; +} + +interface ContinuationCursor { + readonly documentId: string; + readonly revisionKey: string; + readonly nextPage: number; + readonly nextChunkIndex?: number; + readonly nextMatchIndex?: number; +} + +const defaultParsedStorageScheduler = { + schedule(task: () => Promise): void { + void task(); + }, +}; + +function getParsedStorageLimits( + limits: ParsedDocumentStorageLimits | undefined, +): NormalizedParsedStorageLimits { + return { + chunkPageSize: clampLimit(limits?.chunkPageSize, DEFAULT_REMOTE_SCAN_PAGE_SIZE, MAX_PAGE_SIZE), + remotePageSize: clampLimit( + limits?.remotePageSize, + DEFAULT_REMOTE_SCAN_PAGE_SIZE, + MAX_PAGE_SIZE, + ), + maxPagesPerSync: clampLimit(limits?.maxPagesPerSync, DEFAULT_MAX_SYNC_PAGES, 1000), + maxAssetsPerSync: clampLimit(limits?.maxAssetsPerSync, DEFAULT_MAX_SYNC_ASSETS, 1000), + syncDeadlineMs: clampLimit(limits?.syncDeadlineMs, DEFAULT_SYNC_DEADLINE_MS, 60000), + grepMaxPages: clampLimit(limits?.grepMaxPages, DEFAULT_GREP_MAX_PAGES, 1000), + grepDeadlineMs: clampLimit(limits?.grepDeadlineMs, DEFAULT_GREP_DEADLINE_MS, 60000), + outlineMaxPages: clampLimit(limits?.outlineMaxPages, DEFAULT_OUTLINE_MAX_PAGES, 1000), + outlineDeadlineMs: clampLimit(limits?.outlineDeadlineMs, DEFAULT_OUTLINE_DEADLINE_MS, 60000), + }; +} + +function validateReadParams(params: KnowledgeReadParams): void { + const usesPagedDisplay = + params.page !== undefined || + params.pageSize !== undefined || + params.assetUrlPolicy !== undefined; + if (!usesPagedDisplay) { + return; + } + + if ( + params.sectionPath !== undefined || + params.startChunk !== undefined || + params.endChunk !== undefined || + params.chunkId !== undefined + ) { + throw new ValidationError( + 'page, pageSize, and assetUrlPolicy cannot be combined with sectionPath, startChunk, endChunk, or chunkId', + ); + } +} + +function hasScanReadFilters(params: KnowledgeReadParams): boolean { + return ( + params.sectionPath !== undefined || + params.startChunk !== undefined || + params.endChunk !== undefined || + params.chunkId !== undefined + ); +} + +function getReadPageParams(params: KnowledgeReadParams): { + readonly page: number; + readonly pageSize: number; +} { + return { + page: clampLimit(params.page, 1, Number.MAX_SAFE_INTEGER), + pageSize: clampLimit(params.pageSize ?? params.limit, DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE), + }; +} + +function shouldRequestRemoteAssetUrls( + assetUrlPolicy: ParsedDocumentAssetUrlPolicy | undefined, + parsedStorageConfig: ParsedDocumentStorageConfig | undefined, +): boolean { + return assetUrlPolicy === 'durable' && parsedStorageConfig !== undefined; +} + +function getRemoteRevisionKey(response: DocumentChunkListResponse): string { + const revisionKey = response.jobResultId ?? response.jobId; + if (!revisionKey) { + throw new Error( + `Cannot read server document ${response.documentId}: current published revision was not returned.`, + ); + } + return revisionKey; +} + +function createRemoteKnowledgeDocument(params: { + readonly documentId: string; + readonly response: DocumentChunkListResponse; + readonly revisionKey: string; + readonly typeCounts?: Record; +}): LocalKnowledgeDocument { + return { + localDocumentId: params.documentId, + jobId: params.response.jobId ?? params.revisionKey, + documentId: params.documentId, + namespace: params.response.namespace, + sourceFileName: params.documentId, + chunkCount: params.response.pagination.total, + typeCounts: params.typeCounts ?? countRemoteChunkTypes(params.response.chunks), + resultDirectoryPath: `remote:${params.documentId}`, + createdAt: new Date(0), + updatedAt: new Date(0), + }; +} + +function createStoredKnowledgeDocument(params: { + readonly documentId: string; + readonly manifest: KnowhereParsedSnapshotManifest; + readonly revisionKey: string; + readonly typeCounts: Record; +}): LocalKnowledgeDocument { + return { + localDocumentId: params.documentId, + jobId: params.manifest.jobId || params.revisionKey, + documentId: params.manifest.documentId ?? params.documentId, + namespace: params.manifest.namespace, + sourceFileName: params.manifest.sourceFileName, + chunkCount: params.manifest.totalChunks, + typeCounts: params.typeCounts, + resultDirectoryPath: `parsed-storage:${params.documentId}`, + createdAt: new Date(params.manifest.createdAt), + updatedAt: new Date(params.manifest.createdAt), + }; +} + +function isFreshManifest(manifest: KnowhereParsedSnapshotManifest, revisionKey: string): boolean { + return (manifest.revisionKey ?? manifest.jobId) === revisionKey; +} + +function toRemoteReadChunk(chunk: DocumentChunk): KnowledgeReadChunk { + const metadata = cloneMetadata(chunk.metadata); + return { + position: toChunkPosition(chunk), + chunkId: chunk.chunkId, + chunkType: chunk.chunkType, + contentSource: chunk.contentSource ?? undefined, + content: chunk.content ?? '', + readableContent: getReadableRemoteChunkContent(chunk, metadata), + sectionPath: chunk.sectionPath ?? '', + sourceChunkPath: chunk.sourceChunkPath ?? chunk.sectionPath ?? '', + filePath: chunk.filePath ?? undefined, + assetUrl: chunk.assetUrl ?? undefined, + pageNumbers: getChunkPageNumbers(metadata), + metadata, + }; +} + +function getReadableRemoteChunkContent( + chunk: DocumentChunk, + metadata: Record, +): string { + if (chunk.chunkType !== 'page') { + return chunk.content ?? ''; + } + + const summary = metadata.summary; + return typeof summary === 'string' && summary.trim().length > 0 ? summary : (chunk.content ?? ''); +} + +function toIndexedRemoteChunk(chunk: DocumentChunk): IndexedKnowledgeChunk { + const readChunk = toRemoteReadChunk(chunk); + return toIndexedReadChunk(readChunk); +} + +function toIndexedSnapshotChunk(chunk: KnowhereParsedSnapshotChunk): IndexedKnowledgeChunk { + return toIndexedReadChunk({ + position: chunk.sortOrder, + chunkId: chunk.chunkId, + chunkType: chunk.chunkType as KnowledgeChunkType, + contentSource: chunk.contentSource, + content: chunk.content, + readableContent: + chunk.chunkType === 'page' && + typeof chunk.metadata.summary === 'string' && + chunk.metadata.summary.trim().length > 0 + ? chunk.metadata.summary + : chunk.content, + sectionPath: chunk.sectionPath ?? '', + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath, + assetUrl: chunk.assetUrl, + pageNumbers: getChunkPageNumbers(chunk.metadata), + metadata: cloneMetadata(chunk.metadata), + }); +} + +function toIndexedReadChunk(chunk: KnowledgeReadChunk): IndexedKnowledgeChunk { + return { + source: createSyntheticChunk(chunk), + position: chunk.position, + chunkId: chunk.chunkId, + chunkType: chunk.chunkType, + contentSource: chunk.contentSource, + content: chunk.content, + readableContent: chunk.readableContent, + sectionPath: chunk.sectionPath, + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath, + assetUrl: chunk.assetUrl, + pageNumbers: chunk.pageNumbers, + metadata: chunk.metadata, + }; +} + +function createSyntheticChunk(chunk: KnowledgeReadChunk): Chunk { + return { + chunkId: chunk.chunkId, + type: chunk.chunkType, + contentSource: chunk.contentSource, + content: chunk.content, + path: chunk.sourceChunkPath, + metadata: chunk.metadata, + } as Chunk; +} + +function toChunkPosition(chunk: DocumentChunk | undefined): number { + return chunk ? chunk.sortOrder + 1 : 1; +} + +function buildFlatSectionsFromIndexedChunks(chunks: IndexedKnowledgeChunk[]): KnowledgeSection[] { + const byPath = new Map(); + for (const chunk of chunks) { + const path = chunk.sectionPath || chunk.sourceChunkPath; + const existing = byPath.get(path); + if (existing) { + addChunkToSection(existing, chunk); + } else { + byPath.set(path, createSectionFromChunk(path, chunk)); + } + } + return [...byPath.values()].sort(compareSections); +} + +function matchesRemoteReadScope(chunk: DocumentChunk, params: KnowledgeReadParams): boolean { + return matchesReadScope(toIndexedRemoteChunk(chunk), params); +} + +function getNextChunkPosition( + chunks: readonly IndexedKnowledgeChunk[], + selected: readonly IndexedKnowledgeChunk[], +): number | undefined { + const lastSelected = selected[selected.length - 1]; + if (!lastSelected || selected.length >= chunks.length) { + return undefined; + } + return chunks[chunks.indexOf(lastSelected) + 1]?.position; +} + +function applyAssetUrlPolicyToReadChunks( + chunks: readonly KnowledgeReadChunk[], + assetUrlPolicy: ParsedDocumentAssetUrlPolicy | undefined, +): KnowledgeReadChunk[] { + if (assetUrlPolicy !== 'none') { + return chunks.map((chunk) => ({ + ...chunk, + metadata: cloneMetadata(chunk.metadata), + })); + } + + return chunks.map((chunk) => ({ + ...chunk, + assetUrl: undefined, + metadata: removeAssetUrlsFromMetadata(chunk.metadata), + })); +} + +async function hardenReadChunkAssetUrls(params: { + readonly storageConfig: ParsedDocumentStorageConfig; + readonly documentId: string; + readonly revisionKey: string; + readonly chunk: KnowledgeReadChunk; +}): Promise { + const metadata = cloneMetadata(params.chunk.metadata); + const assetUrl = await hardenPrimaryAssetUrl(params, metadata); + const pageAssets = await hardenPageAssetUrls(params, metadata); + if (pageAssets) { + metadata.pageAssets = pageAssets; + } + + return { + ...params.chunk, + assetUrl, + metadata, + }; +} + +async function hardenPrimaryAssetUrl( + params: { + readonly storageConfig: ParsedDocumentStorageConfig; + readonly documentId: string; + readonly revisionKey: string; + readonly chunk: KnowledgeReadChunk; + }, + metadata: Record, +): Promise { + if (!params.chunk.assetUrl || !params.chunk.filePath) { + return undefined; + } + + return writeDurableAssetUrl({ + storageConfig: params.storageConfig, + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: params.chunk.filePath, + sourceUrl: params.chunk.assetUrl, + fallbackContentType: inferContentTypeFromPath(params.chunk.filePath), + metadata: { + chunkId: params.chunk.chunkId, + chunkType: params.chunk.chunkType, + sourcePath: params.chunk.filePath, + ...stringifyFlatMetadata(metadata), + }, + }); +} + +async function hardenPageAssetUrls( + params: { + readonly storageConfig: ParsedDocumentStorageConfig; + readonly documentId: string; + readonly revisionKey: string; + readonly chunk: KnowledgeReadChunk; + }, + metadata: Record, +): Promise { + const pageAssets = toUnknownArray(metadata.pageAssets); + if (!pageAssets) { + return undefined; + } + + const hardenedAssets: unknown[] = []; + for (const pageAsset of pageAssets) { + if (!isRecord(pageAsset)) { + hardenedAssets.push(pageAsset); + continue; + } + + const artifactRef = pageAsset.artifactRef; + const sourceUrl = pageAsset.assetUrl; + if (typeof artifactRef !== 'string' || typeof sourceUrl !== 'string') { + hardenedAssets.push(removeAssetUrlField(pageAsset)); + continue; + } + + const durableUrl = await writeDurableAssetUrl({ + storageConfig: params.storageConfig, + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: artifactRef, + sourceUrl, + fallbackContentType: + typeof pageAsset.contentType === 'string' + ? pageAsset.contentType + : inferContentTypeFromPath(artifactRef), + metadata: { + chunkId: params.chunk.chunkId, + chunkType: params.chunk.chunkType, + sourcePath: artifactRef, + }, + }); + const rest = removeAssetUrlField(pageAsset); + hardenedAssets.push(durableUrl ? { ...rest, assetUrl: durableUrl } : rest); + } + + return hardenedAssets; +} + +async function writeDurableAssetUrl(params: { + readonly storageConfig: ParsedDocumentStorageConfig; + readonly documentId: string; + readonly revisionKey: string; + readonly sourcePath: string; + readonly sourceUrl: string; + readonly fallbackContentType: string; + readonly metadata: Readonly>; +}): Promise { + const existingUrl = await params.storageConfig.storage.getAssetUrl({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: params.sourcePath, + }); + if (existingUrl) { + return existingUrl; + } + + try { + const fetched = await fetchAsset(params.sourceUrl, params.fallbackContentType); + const written = await params.storageConfig.storage.writeAsset({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: params.sourcePath, + body: fetched.body, + contentType: fetched.contentType, + metadata: params.metadata, + }); + return ( + written.url ?? + (await params.storageConfig.storage.getAssetUrl({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: params.sourcePath, + })) ?? + undefined + ); + } catch { + return undefined; + } +} + +async function fetchAsset( + sourceUrl: string, + fallbackContentType: string, +): Promise<{ + readonly body: Uint8Array; + readonly contentType: string; +}> { + const response = await fetch(sourceUrl); + if (!response.ok) { + throw new Error(`Unable to fetch parsed asset: ${response.status}`); + } + + return { + body: new Uint8Array(await response.arrayBuffer()), + contentType: response.headers.get('content-type') ?? fallbackContentType, + }; +} + +function removeAssetUrlsFromMetadata(metadata: Record): Record { + const cloned = cloneMetadata(metadata); + const pageAssets = toUnknownArray(cloned.pageAssets); + if (!pageAssets) { + return cloned; + } + + cloned.pageAssets = pageAssets.map((pageAsset) => { + if (!isRecord(pageAsset)) { + return pageAsset; + } + return removeAssetUrlField(pageAsset); + }); + return cloned; +} + +function removeAssetUrlField(value: Record): Record { + const cloned = { ...value }; + delete cloned.assetUrl; + return cloned; +} + +function toSnapshotChunkPageFromReadChunks(params: { + readonly response: DocumentChunkListResponse; + readonly revisionKey: string; + readonly chunks: readonly KnowledgeReadChunk[]; +}): KnowhereParsedSnapshotChunkPage { + return { + version: 1, + jobId: params.response.jobId ?? params.revisionKey, + revisionKey: params.revisionKey, + documentId: params.response.documentId, + namespace: params.response.namespace, + sourceFileName: params.response.documentId, + page: params.response.pagination.page, + pageSize: params.response.pagination.pageSize, + total: params.response.pagination.total, + totalPages: params.response.pagination.totalPages, + chunks: params.chunks.map(toSnapshotChunkFromReadChunk), + }; +} + +function toSnapshotChunkFromReadChunk(chunk: KnowledgeReadChunk): KnowhereParsedSnapshotChunk { + return { + id: chunk.chunkId, + chunkId: chunk.chunkId, + chunkType: chunk.chunkType, + contentSource: chunk.contentSource, + content: chunk.content, + sectionPath: chunk.sectionPath, + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath, + sortOrder: chunk.position, + metadata: cloneMetadata(chunk.metadata), + assetUrl: chunk.assetUrl, + }; +} + +function createRemoteSnapshotManifest(params: { + readonly response: DocumentChunkListResponse; + readonly revisionKey: string; + readonly chunkPages: readonly KnowhereParsedSnapshotChunkPage[]; +}): KnowhereParsedSnapshotManifest { + return { + version: 1, + kind: 'knowhere-parsed-result-snapshot', + jobId: params.response.jobId ?? params.revisionKey, + revisionKey: params.revisionKey, + documentId: params.response.documentId, + namespace: params.response.namespace, + sourceFileName: params.response.documentId, + totalChunks: params.response.pagination.total, + typeCounts: countSnapshotPagesTypes(params.chunkPages), + chunkPageSize: params.response.pagination.pageSize, + chunkPages: Array.from({ length: params.response.pagination.totalPages }, (_, index) => ({ + page: index + 1, + pageSize: params.response.pagination.pageSize, + chunkCount: + index + 1 === params.response.pagination.totalPages + ? params.response.pagination.total - + params.response.pagination.pageSize * (params.response.pagination.totalPages - 1) + : params.response.pagination.pageSize, + key: `parsed/chunks/page-${index + 1}.json`, + })), + assetUrlsByFilePath: {}, + createdAt: new Date().toISOString(), + }; +} + +function countSnapshotPagesTypes( + pages: readonly KnowhereParsedSnapshotChunkPage[], +): Record { + return pages + .flatMap((page) => page.chunks) + .reduce>( + (counts, chunk) => { + const chunkType = chunk.chunkType as KnowledgeChunkType; + counts[chunkType] += 1; + return counts; + }, + { text: 0, image: 0, table: 0, page: 0 }, + ); +} + +function countRemoteChunkTypes( + chunks: readonly DocumentChunk[], +): Record { + return chunks.reduce>( + (counts, chunk) => { + counts[chunk.chunkType] += 1; + return counts; + }, + { text: 0, image: 0, table: 0, page: 0 }, + ); +} + +function createContinuationCursor(cursor: ContinuationCursor): string { + return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64url'); +} + +function createGrepContinuationCursor(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly response: DocumentChunkListResponse; + readonly chunkIndex: number; + readonly matchIndex: number; + readonly chunkMatchCount: number; +}): string | undefined { + if (params.matchIndex + 1 < params.chunkMatchCount) { + return createContinuationCursor({ + documentId: params.documentId, + revisionKey: params.revisionKey, + nextPage: params.response.pagination.page, + nextChunkIndex: params.chunkIndex, + nextMatchIndex: params.matchIndex + 1, + }); + } + + if (params.chunkIndex + 1 < params.response.chunks.length) { + return createContinuationCursor({ + documentId: params.documentId, + revisionKey: params.revisionKey, + nextPage: params.response.pagination.page, + nextChunkIndex: params.chunkIndex + 1, + nextMatchIndex: 0, + }); + } + + if (params.response.pagination.page < params.response.pagination.totalPages) { + return createContinuationCursor({ + documentId: params.documentId, + revisionKey: params.revisionKey, + nextPage: params.response.pagination.page + 1, + nextChunkIndex: 0, + nextMatchIndex: 0, + }); + } + + return undefined; +} + +function parseContinuationCursor(value: string | undefined): ContinuationCursor | null { + if (!value) { + return null; + } + + try { + const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')) as unknown; + if (!isRecord(parsed)) { + return null; + } + const documentId = parsed.documentId; + const revisionKey = parsed.revisionKey; + const nextPage = parsed.nextPage; + const nextChunkIndex = parsed.nextChunkIndex; + const nextMatchIndex = parsed.nextMatchIndex; + if ( + typeof documentId !== 'string' || + typeof revisionKey !== 'string' || + typeof nextPage !== 'number' || + !Number.isInteger(nextPage) || + nextPage < 1 || + !isOptionalNonNegativeInteger(nextChunkIndex) || + !isOptionalNonNegativeInteger(nextMatchIndex) + ) { + return null; + } + return { + documentId, + revisionKey, + nextPage, + nextChunkIndex, + nextMatchIndex, + }; + } catch { + return null; + } +} + +function isOptionalNonNegativeInteger(value: unknown): value is number | undefined { + return ( + value === undefined || (typeof value === 'number' && Number.isInteger(value) && value >= 0) + ); +} + +function cloneMetadata(metadata: Record): Record { + return { ...metadata }; +} + +function stringifyFlatMetadata(metadata: Record): Record { + const entries = Object.entries(metadata).flatMap(([key, value]): Array<[string, string]> => { + if (typeof value === 'string') { + return [[key, value]]; + } + if (typeof value === 'number' || typeof value === 'boolean') { + return [[key, String(value)]]; + } + return []; + }); + return Object.fromEntries(entries); +} + +function toUnknownArray(value: unknown): readonly unknown[] | null { + return Array.isArray(value) ? (value as readonly unknown[]) : null; +} + +function inferContentTypeFromPath(filePath: string): string { + const extension = filePath.split('.').at(-1)?.toLowerCase(); + if (extension === 'png') return 'image/png'; + if (extension === 'jpg' || extension === 'jpeg') return 'image/jpeg'; + if (extension === 'html' || extension === 'htm') return 'text/html; charset=utf-8'; + return 'application/octet-stream'; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + function indexChunks(result: ParseResult): IndexedKnowledgeChunk[] { return result.chunks.map((chunk, index) => { const filePath = getChunkFilePath(chunk); diff --git a/src/knowledge/types.ts b/src/knowledge/types.ts index 46c6cf2..198dc3b 100644 --- a/src/knowledge/types.ts +++ b/src/knowledge/types.ts @@ -3,7 +3,10 @@ import type { Job, JobResult } from '../types/job.js'; import type { Chunk, DocumentChunkType, ParseResult } from '../types/index.js'; import type { KnowhereAssetStorageOptions, + KnowhereParsedSnapshotManifest, KnowhereParsedSnapshot, + ParsedDocumentAssetUrlPolicy, + ParsedDocumentStorageConfig, } from '../types/storage.js'; export type KnowledgeChunkType = DocumentChunkType; @@ -111,6 +114,19 @@ export interface KnowledgeDocumentReference { documentId?: string; /** Server parse job identifier whose completed result should be cached before reading. */ jobId?: string; + /** Current parsed revision key when the caller already has it. */ + revisionKey?: string; +} + +export type KnowledgeParsedStorageOptions = ParsedDocumentStorageConfig; + +export type KnowledgeSyncParsedDocumentParams = KnowledgeDocumentReference; + +export interface KnowledgeSyncParsedDocumentResponse { + documentId: string; + revisionKey: string; + completed: boolean; + manifest?: KnowhereParsedSnapshotManifest; } export interface KnowledgeSection { @@ -131,15 +147,20 @@ export interface KnowledgeOutline { typeCounts: Record; sections: KnowledgeSection[]; sectionTree: KnowledgeSection[]; + truncated?: boolean; + continuationCursor?: string; } export interface KnowledgeReadParams extends KnowledgeDocumentReference { + page?: number; + pageSize?: number; sectionPath?: string; startChunk?: number; endChunk?: number; chunkId?: string; chunkType?: KnowledgeChunkType; limit?: number; + assetUrlPolicy?: ParsedDocumentAssetUrlPolicy; } export interface KnowledgeReadChunk { @@ -163,10 +184,15 @@ export interface KnowledgeReadResponse { document: LocalKnowledgeDocument; chunks: KnowledgeReadChunk[]; nextChunk?: number; + page?: number; + pageSize?: number; + totalChunks?: number; + totalPages?: number; } export interface KnowledgeGrepParams extends KnowledgeDocumentReference { pattern: string; + continuationCursor?: string; isRegex?: boolean; isCaseSensitive?: boolean; maxResults?: number; @@ -192,6 +218,7 @@ export interface KnowledgeGrepResponse { matches: KnowledgeGrepMatch[]; scannedChunks: number; truncated: boolean; + continuationCursor?: string; } export interface KnowledgeSearchParams { diff --git a/src/storage/asset-storage.ts b/src/storage/asset-storage.ts index afc1ed1..7f88b98 100644 --- a/src/storage/asset-storage.ts +++ b/src/storage/asset-storage.ts @@ -50,9 +50,7 @@ export async function storeParseResultAssets( } const normalizedKeyPrefix = normalizeStorageKeyPrefix(options.keyPrefix); - const assets = dedupeStorageAssets( - await collectStorageAssets(result, normalizedKeyPrefix), - ); + const assets = dedupeStorageAssets(await collectStorageAssets(result, normalizedKeyPrefix)); const assetUrlsByFilePath: Record = {}; for (const asset of assets) { @@ -98,6 +96,7 @@ async function storeParsedSnapshot(input: { const chunkPage: KnowhereParsedSnapshotChunkPage = { version: 1, jobId: input.result.jobId, + revisionKey: input.options.revisionKey ?? input.result.jobId, documentId: input.result.documentId, namespace: input.result.namespace, sourceFileName: input.result.manifest.sourceFileName, @@ -128,10 +127,12 @@ async function storeParsedSnapshot(input: { version: 1, kind: 'knowhere-parsed-result-snapshot', jobId: input.result.jobId, + revisionKey: input.options.revisionKey ?? input.result.jobId, documentId: input.result.documentId, namespace: input.result.namespace, sourceFileName: input.result.manifest.sourceFileName, totalChunks, + typeCounts: countSnapshotChunkTypes(input.result.chunks), chunkPageSize: pageSize, chunkPages: chunkPageReferences, assetUrlsByFilePath: input.assetUrlsByFilePath, @@ -160,6 +161,18 @@ async function storeParsedSnapshot(input: { }; } +function countSnapshotChunkTypes( + chunks: readonly Chunk[], +): Readonly> { + return chunks.reduce>( + (counts, chunk) => { + counts[chunk.type] += 1; + return counts; + }, + { text: 0, image: 0, table: 0, page: 0 }, + ); +} + async function writeJsonObject(input: { readonly key: string; readonly value: unknown; @@ -218,7 +231,10 @@ function cloneChunkMetadata(metadata: ChunkMetadata): Record { return { ...metadata }; } -async function collectStorageAssets(result: ParseResult, keyPrefix: string): Promise { +async function collectStorageAssets( + result: ParseResult, + keyPrefix: string, +): Promise { const resultZip = await loadResultZipForPageCitationAssets(result); return [ ...result.imageChunks.flatMap((chunk) => collectImageStorageAsset(chunk, keyPrefix)), diff --git a/src/storage/disk-parsed-document-storage.ts b/src/storage/disk-parsed-document-storage.ts new file mode 100644 index 0000000..58fc0c9 --- /dev/null +++ b/src/storage/disk-parsed-document-storage.ts @@ -0,0 +1,208 @@ +import { createHash } from 'crypto'; +import { promises as fs } from 'fs'; +import path from 'path'; + +import type { + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, + ParsedDocumentStorage, + ParsedDocumentSyncProgress, +} from '../types/storage.js'; + +export class DiskParsedDocumentStorage implements ParsedDocumentStorage { + private readonly rootDirectory: string; + + constructor(rootDirectory: string) { + this.rootDirectory = rootDirectory; + } + + async readManifest(params: { + readonly documentId: string; + readonly revisionKey: string; + }): Promise { + return this.readOptionalJson( + this.getManifestPath(params.documentId, params.revisionKey), + ); + } + + async writeManifest(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly manifest: KnowhereParsedSnapshotManifest; + }): Promise { + await this.writeJson( + this.getManifestPath(params.documentId, params.revisionKey), + params.manifest, + ); + } + + async readChunkPage(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly page: number; + }): Promise { + return this.readOptionalJson( + this.getChunkPagePath(params.documentId, params.revisionKey, params.page), + ); + } + + async writeChunkPage(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly page: KnowhereParsedSnapshotChunkPage; + }): Promise { + await this.writeJson( + this.getChunkPagePath(params.documentId, params.revisionKey, params.page.page), + params.page, + ); + } + + async writeAsset(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly sourcePath: string; + readonly body: Uint8Array; + readonly contentType: string; + readonly metadata?: Readonly>; + }): Promise<{ + readonly sourcePath: string; + readonly url?: string; + }> { + const assetPath = this.getAssetPath(params.documentId, params.revisionKey, params.sourcePath); + await fs.mkdir(path.dirname(assetPath), { recursive: true }); + await fs.writeFile(assetPath, params.body); + await this.writeJson(`${assetPath}.metadata.json`, { + contentType: params.contentType, + metadata: params.metadata ?? {}, + }); + return { + sourcePath: params.sourcePath, + url: toFileUrl(assetPath), + }; + } + + async getAssetUrl(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly sourcePath: string; + }): Promise { + const assetPath = this.getAssetPath(params.documentId, params.revisionKey, params.sourcePath); + try { + await fs.access(assetPath); + return toFileUrl(assetPath); + } catch (error) { + if (isMissingFileError(error)) { + return null; + } + throw error; + } + } + + async readSyncProgress(params: { + readonly documentId: string; + readonly revisionKey: string; + }): Promise { + return this.readOptionalJson( + this.getProgressPath(params.documentId, params.revisionKey), + ); + } + + async writeSyncProgress(params: ParsedDocumentSyncProgress): Promise { + await this.writeJson(this.getProgressPath(params.documentId, params.revisionKey), params); + } + + private getManifestPath(documentId: string, revisionKey: string): string { + return path.join( + this.getRevisionDirectory(documentId, revisionKey), + 'manifest', + 'current.json', + ); + } + + private getChunkPagePath(documentId: string, revisionKey: string, page: number): string { + return path.join( + this.getRevisionDirectory(documentId, revisionKey), + 'chunks', + `page-${page}.json`, + ); + } + + private getProgressPath(documentId: string, revisionKey: string): string { + return path.join(this.getRevisionDirectory(documentId, revisionKey), 'sync-progress.json'); + } + + private getAssetPath(documentId: string, revisionKey: string, sourcePath: string): string { + const normalizedSourcePath = normalizeRelativeStoragePath(sourcePath); + const assetPath = path.resolve( + this.getRevisionDirectory(documentId, revisionKey), + 'assets', + normalizedSourcePath, + ); + const assetsDirectory = path.resolve( + this.getRevisionDirectory(documentId, revisionKey), + 'assets', + ); + if (!isPathInsideDirectory(assetPath, assetsDirectory)) { + throw new Error(`Parsed asset path resolves outside storage: ${sourcePath}`); + } + return assetPath; + } + + private getRevisionDirectory(documentId: string, revisionKey: string): string { + return path.join(this.rootDirectory, hashPathPart(documentId), hashPathPart(revisionKey)); + } + + private async readOptionalJson(filePath: string): Promise { + try { + return JSON.parse(await fs.readFile(filePath, 'utf8')) as T; + } catch (error) { + if (isMissingFileError(error)) { + return null; + } + throw error; + } + } + + private async writeJson(filePath: string, value: unknown): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, JSON.stringify(value, null, 2), { + encoding: 'utf8', + flag: 'w', + }); + } +} + +function hashPathPart(value: string): string { + return createHash('sha256').update(value).digest('hex').slice(0, 24); +} + +function normalizeRelativeStoragePath(value: string): string { + const normalized = value.replaceAll('\\', '/').replace(/^\.\/+/, ''); + if ( + normalized.length === 0 || + normalized.includes('\0') || + normalized.startsWith('/') || + normalized.split('/').some((part) => part.length === 0 || part === '.' || part === '..') + ) { + throw new Error(`Invalid parsed storage path: ${value}`); + } + return normalized; +} + +function isPathInsideDirectory(filePath: string, directory: string): boolean { + const relative = path.relative(directory, filePath); + return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative)); +} + +function toFileUrl(filePath: string): string { + return `file://${filePath}`; +} + +function isMissingFileError(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code?: string }).code === 'ENOENT' + ); +} diff --git a/src/storage/parsed-document-storage.ts b/src/storage/parsed-document-storage.ts new file mode 100644 index 0000000..e9f6a55 --- /dev/null +++ b/src/storage/parsed-document-storage.ts @@ -0,0 +1,173 @@ +import { storeParseResultAssets } from './asset-storage.js'; +import type { ParseResult } from '../types/result.js'; +import type { + KnowhereAssetStorageAdapter, + KnowhereAssetStorageHead, + KnowhereAssetStorageObject, + KnowhereAssetStorageWriteResult, + KnowhereParsedSnapshot, + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, + ParsedDocumentStorage, +} from '../types/storage.js'; + +const parsedStorageKeyPrefix = 'parsed'; +const manifestStoragePath = 'manifest/current.json'; +const indexStoragePath = 'index.json'; +const chunkPagePathPattern = /^chunks\/page-(\d+)\.json$/; + +export async function syncParseResultToParsedDocumentStorage(params: { + readonly result: ParseResult; + readonly storage: ParsedDocumentStorage; + readonly documentId?: string; + readonly revisionKey?: string; + readonly chunkPageSize?: number; +}): Promise<{ + readonly result: ParseResult; + readonly snapshot?: KnowhereParsedSnapshot; + readonly documentId: string; + readonly revisionKey: string; +}> { + const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId; + const revisionKey = params.revisionKey ?? params.result.jobId; + const adapter = createParsedDocumentObjectStorageAdapter({ + storage: params.storage, + documentId, + revisionKey, + }); + const stored = await storeParseResultAssets(params.result, { + adapter, + keyPrefix: parsedStorageKeyPrefix, + chunkPageSize: params.chunkPageSize, + revisionKey, + }); + return { + result: stored.result, + snapshot: stored.snapshot, + documentId, + revisionKey, + }; +} + +function createParsedDocumentObjectStorageAdapter(params: { + readonly storage: ParsedDocumentStorage; + readonly documentId: string; + readonly revisionKey: string; +}): KnowhereAssetStorageAdapter { + return { + writeObject: (input) => writeParsedStorageObject(params, input), + headObject: (key) => headParsedStorageObject(params, key), + getObjectUrl: (key) => getParsedStorageObjectUrl(params, key), + }; +} + +async function writeParsedStorageObject( + params: { + readonly storage: ParsedDocumentStorage; + readonly documentId: string; + readonly revisionKey: string; + }, + input: KnowhereAssetStorageObject, +): Promise { + const storagePath = stripParsedStorageKeyPrefix(input.key); + if (storagePath === manifestStoragePath) { + const manifest = JSON.parse( + Buffer.from(input.body).toString('utf8'), + ) as KnowhereParsedSnapshotManifest; + await params.storage.writeManifest({ + documentId: params.documentId, + revisionKey: params.revisionKey, + manifest, + }); + return { key: input.key }; + } + + if (storagePath === indexStoragePath) { + return { key: input.key }; + } + + const chunkPageMatch = storagePath.match(chunkPagePathPattern); + if (chunkPageMatch) { + const page = JSON.parse( + Buffer.from(input.body).toString('utf8'), + ) as KnowhereParsedSnapshotChunkPage; + await params.storage.writeChunkPage({ + documentId: params.documentId, + revisionKey: params.revisionKey, + page, + }); + return { key: input.key }; + } + + const stored = await params.storage.writeAsset({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: storagePath, + body: input.body, + contentType: input.contentType, + metadata: input.metadata, + }); + return { + key: input.key, + url: stored.url, + }; +} + +async function headParsedStorageObject( + params: { + readonly storage: ParsedDocumentStorage; + readonly documentId: string; + readonly revisionKey: string; + }, + key: string, +): Promise { + const storagePath = stripParsedStorageKeyPrefix(key); + if ( + storagePath === manifestStoragePath || + storagePath === indexStoragePath || + chunkPagePathPattern.test(storagePath) + ) { + return null; + } + + const url = await params.storage.getAssetUrl({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: storagePath, + }); + return url + ? { + key, + url, + } + : null; +} + +async function getParsedStorageObjectUrl( + params: { + readonly storage: ParsedDocumentStorage; + readonly documentId: string; + readonly revisionKey: string; + }, + key: string, +): Promise { + const storagePath = stripParsedStorageKeyPrefix(key); + if ( + storagePath === manifestStoragePath || + storagePath === indexStoragePath || + chunkPagePathPattern.test(storagePath) + ) { + return null; + } + + return params.storage.getAssetUrl({ + documentId: params.documentId, + revisionKey: params.revisionKey, + sourcePath: storagePath, + }); +} + +function stripParsedStorageKeyPrefix(key: string): string { + const prefix = `${parsedStorageKeyPrefix}/`; + return key.startsWith(prefix) ? key.slice(prefix.length) : key; +} diff --git a/src/types/storage.ts b/src/types/storage.ts index b6add3b..39f4b63 100644 --- a/src/types/storage.ts +++ b/src/types/storage.ts @@ -1,4 +1,5 @@ import type { ParseResult } from './result.js'; +import type { DocumentChunkType } from './document.js'; export type KnowhereAssetStorageBody = Uint8Array; @@ -33,6 +34,7 @@ export interface KnowhereAssetStorageOptions { readonly keyPrefix: string; readonly skipExisting?: boolean; readonly chunkPageSize?: number; + readonly revisionKey?: string; } export interface KnowhereParsedSnapshotChunk { @@ -52,6 +54,7 @@ export interface KnowhereParsedSnapshotChunk { export interface KnowhereParsedSnapshotChunkPage { readonly version: 1; readonly jobId: string; + readonly revisionKey?: string; readonly documentId?: string; readonly namespace?: string; readonly sourceFileName: string; @@ -74,10 +77,12 @@ export interface KnowhereParsedSnapshotManifest { readonly version: 1; readonly kind: 'knowhere-parsed-result-snapshot'; readonly jobId: string; + readonly revisionKey?: string; readonly documentId?: string; readonly namespace?: string; readonly sourceFileName: string; readonly totalChunks: number; + readonly typeCounts?: Readonly>>; readonly chunkPageSize: number; readonly chunkPages: readonly KnowhereParsedSnapshotChunkPageReference[]; readonly assetUrlsByFilePath: Readonly>; @@ -98,3 +103,88 @@ export interface KnowhereAssetStorageResult { readonly assetUrlsByFilePath: Readonly>; readonly snapshot?: KnowhereParsedSnapshot; } + +export type ParsedDocumentAssetUrlPolicy = 'none' | 'durable'; + +export interface ParsedDocumentStorageDocument { + readonly documentId: string; + readonly revisionKey: string; +} + +export type ParsedDocumentStorageManifestParams = ParsedDocumentStorageDocument; + +export interface ParsedDocumentStorageChunkPageParams extends ParsedDocumentStorageDocument { + readonly page: number; + readonly chunkType?: DocumentChunkType; +} + +export interface ParsedDocumentStorageAsset { + readonly sourcePath: string; + readonly body: KnowhereAssetStorageBody; + readonly contentType: string; + readonly metadata?: Readonly>; +} + +export interface ParsedDocumentStorageAssetParams extends ParsedDocumentStorageDocument { + readonly sourcePath: string; +} + +export interface ParsedDocumentSyncProgress { + readonly documentId: string; + readonly revisionKey: string; + readonly nextChunkPage: number; + readonly nextAssetIndex: number; + readonly status: 'running' | 'completed' | 'failed'; + readonly updatedAt: string; + readonly error?: string; +} + +export interface ParsedDocumentStorage { + readManifest( + params: ParsedDocumentStorageManifestParams, + ): Promise; + writeManifest(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly manifest: KnowhereParsedSnapshotManifest; + }): Promise; + readChunkPage( + params: ParsedDocumentStorageChunkPageParams, + ): Promise; + writeChunkPage(params: { + readonly documentId: string; + readonly revisionKey: string; + readonly page: KnowhereParsedSnapshotChunkPage; + }): Promise; + writeAsset(params: ParsedDocumentStorageDocument & ParsedDocumentStorageAsset): Promise<{ + readonly sourcePath: string; + readonly url?: string; + }>; + getAssetUrl(params: ParsedDocumentStorageAssetParams): Promise; + readSyncProgress( + params: ParsedDocumentStorageDocument, + ): Promise; + writeSyncProgress(params: ParsedDocumentSyncProgress): Promise; +} + +export interface ParsedDocumentSyncScheduler { + schedule(task: () => Promise): void | Promise; +} + +export interface ParsedDocumentStorageLimits { + readonly chunkPageSize?: number; + readonly remotePageSize?: number; + readonly maxPagesPerSync?: number; + readonly maxAssetsPerSync?: number; + readonly syncDeadlineMs?: number; + readonly grepMaxPages?: number; + readonly grepDeadlineMs?: number; + readonly outlineMaxPages?: number; + readonly outlineDeadlineMs?: number; +} + +export interface ParsedDocumentStorageConfig { + readonly storage: ParsedDocumentStorage; + readonly scheduler?: ParsedDocumentSyncScheduler; + readonly limits?: ParsedDocumentStorageLimits; +} From 41162c57039ee1889799b7378189b8c2edcc7702 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sun, 5 Jul 2026 14:45:27 +0800 Subject: [PATCH 10/10] chore: format --- ...ommand-caveatcaveat-the-messages-below.txt | 1837 +++++++++++++++++ docs/new-design.md | 118 ++ src/__tests__/client.test.ts | 4 +- src/storage/__tests__/asset-storage.test.ts | 21 +- 4 files changed, 1971 insertions(+), 9 deletions(-) create mode 100644 2026-07-04-222626-local-command-caveatcaveat-the-messages-below.txt create mode 100644 docs/new-design.md diff --git a/2026-07-04-222626-local-command-caveatcaveat-the-messages-below.txt b/2026-07-04-222626-local-command-caveatcaveat-the-messages-below.txt new file mode 100644 index 0000000..02f8500 --- /dev/null +++ b/2026-07-04-222626-local-command-caveatcaveat-the-messages-below.txt @@ -0,0 +1,1837 @@ +╭─── Claude Code v2.1.201 ─────────────────────────────────────────────────────╮ +│ │ Tips for getting started │ +│ Welcome back! │ Run /init to create a CLAUDE.m… │ +│ │ ─────────────────────────────── │ +│ ▐▛███▜▌ │ What's new │ +│ ▝▜█████▛▘ │ Claude Sonnet 5 sessions no lo… │ +│ ▘▘ ▝▝ │ Changed `AskUserQuestion` dial… │ +│ │ Changed the "default" permissi… │ +│ Opus 4.8 · API Usage Billing │ /release-notes for more │ +│ ~/github.com/ontosAI/knowhere-node-sdk │ │ +╰──────────────────────────────────────────────────────────────────────────────╯ + + +❯ /clear + +❯ read the new design @docs/new-design.md, and lets plan on the notebookside + /home/suguan/github.com/ontosAI/knowhere-notebook + ⎿  Read docs/new-design.md (119 lines) + + Thought for 25s (ctrl+o to expand) + +● I've read the design doc. Now let me explore the notebook codebase to + understand the current implementation before planning. Let me launch parallel + Explore agents. + +● 3 background agents launched (↓ to manage) + ├ Explore parse/upload flow & source status + ├ Explore chunk read/display/chat & assets + └ Explore Blob, scheduler & SDK config + +● The three Explore agents are running in parallel. While they investigate, let + me also look at the SDK side to understand what's already built, since the + design references specific SDK methods. Let me read the current SDK state to + understand the contract the notebook will consume. + +● Bash(git log --oneline -15 && echo "---STRUCTURE---" && find src -type f -name + "*.ts" 2>/dev/null | head -60) + ⎿  2d006ae Add parsed storage backed knowledge reads + b5b4339 Clarify local knowledge result imports + cbd8753 Add parsed result snapshot storage + … +61 lines (ctrl+o to expand) + + Thought for 8s (ctrl+o to expand) + +● The SDK side is already substantially built. Let me read the key contract + files that the notebook will consume — the storage adapter interface, the + knowledge client public API, and relevant types. + + Thought for 19s, read 3 files (ctrl+o to expand) + +● Now let me read the storage types — this is the critical interface the + notebook must implement with Vercel Blob. + + Thought for 10s, read 1 file (ctrl+o to expand) + +● I now have a complete picture of the SDK contract. Let me read the disk + storage implementation as a reference model for the Vercel Blob adapter the + notebook will implement. + + Thought for 16s, read 1 file (ctrl+o to expand) + +● I've fully mapped the SDK contract. The key surfaces the notebook must + integrate with: + + - ParsedDocumentStorage interface (manifest, chunk pages, assets, sync + progress) — notebook implements a Vercel Blob version + - ParsedDocumentSyncScheduler — notebook implements an Upstash/Vercel-safe + version + - knowledge.withParsedStorage({ storage, scheduler, limits }) — config entry + point + - readChunks({ assetUrlPolicy: 'durable' }), grepChunks, getDocumentOutline — + reads + - syncParsedDocument — for parse-flow "sync before complete"; importJobResult + replaces cacheJobResult + + Now I'm waiting on the three Explore agents scanning the notebook repo + (Blob/scheduler/SDK config; chunk read/display/chat/assets; parse + flow/status/source_parse_results). I'll continue planning once they report + back. + +✻ Sautéed for 2m 3s + +● Agent "Explore Blob, scheduler & SDK config" finished · 4m 54s + +● First agent is back with a thorough map. Two more agents still running (chunk + read/display/chat/assets, and parse flow/status). Waiting for those before I + design the plan. + + Key takeaway from this report: the notebook currently uses the older + KnowhereAssetStorageOptions adapter + loadJobResult and reimplements + progress/batching/deadline logic itself. The redesign migrates it to the newer + ParsedDocumentStorage interface via withParsedStorage. The existing Vercel + Blob helpers, Upstash workflow self-continuation pattern, and + source-parse-result-repository are the seams to build on. + +✻ Baked for 9s + +● Agent "Explore chunk read/display/chat & assets" finished · 6m 15s + +● Second agent back. One more running (parse flow / source status / + source_parse_results schema / migrations). Critical new facts from this + report: + + - Notebook already has a rich chunk-read path with a Blob snapshot and asset + mirroring — but it reimplements the manifest/chunk-page/asset logic itself + (chunks/server.ts, remote-parsed-snapshot-sync.ts, + parse-result-storage-adapter.ts), all built on the old loadJobResult + + KnowhereAssetStorageOptions adapter, not readChunks/ParsedDocumentStorage. + - Chat uses Knowhere agentic retrieval.query — no + grepChunks/getDocumentOutline today. + - Asset hardening (copy-to-blob) already exists in two places + (media-asset-hardening.ts for chat, chunks/server.ts mirroring for the + viewer). + + Waiting on the parse-flow/status/schema agent before finalizing the plan. + +✻ Cooked for 11s + +● Agent "Explore parse/upload flow & source status" finished · 6m 30s + + Thought for 1m 26s (ctrl+o to expand) + +● All three Explore agents are back. I now have a complete two-sided picture: + + SDK side (already built, this branch): ParsedDocumentStorage interface, + withParsedStorage({storage, scheduler, limits}), + readChunks({assetUrlPolicy:'durable'}), grepChunks, getDocumentOutline, + syncParsedDocument, importJobResult, resumable background sync with progress + cursor + scheduler + deadline limits, and synchronous asset hardening. + + Notebook side (uses the OLD path): + - Reads via client.documents.listChunks + a hand-rolled Blob + snapshot/chunk-page cache + asset mirroring (chunks/server.ts, + remote-parsed-snapshot-sync.ts, parse-result-storage-adapter.ts, + parsed-result-assets.ts), all on loadJobResult + the old + KnowhereAssetStorageOptions adapter — not the new ParsedDocumentStorage. + - Parse lifecycle: uploading|parsing|ready|failed (text column, atomic + requiredStatus guards) driven by an Upstash workflow with a + deterministic-workflowRunId continuation pattern. Marks ready after the + snapshot is written. + - source_parse_results has resultBlobUrl, snapshotManifestUrl/Key, assetUrls — + no revision_key, sync_status, or failure_stage. Migrations in drizzle/, next + is 0011. + - Chat uses agentic retrieval.query (no grep/outline). Asset hardening + (copy-to-Blob) already exists twice: media-asset-hardening.ts (chat) and the + viewer mirror in chunks/server.ts. + + This is a substantial migration. A few forks materially change the plan, so + let me confirm scope before I design it. + +● User answered Claude's questions: + ⎿  · How aggressively should the notebook migrate to the SDK's new + ParsedDocumentStorage + readChunks path? → Full rip-and-replace + (Recommended) + · Beyond chunk read/display + parse lifecycle, should this effort also + unify chat citation asset hardening onto the SDK's durable policy? → + Include chat hardening (Recommended) + · How should the non-blocking background sync (for read-misses on existing + remote docs with no Blob snapshot) run on Vercel? → QStash-enqueued + resumable sync (Recommended) + + Thought for 9s (ctrl+o to expand) + +● All three recommended. Let me read the critical notebook files directly to + ground the plan before writing it — the current storage adapter, the reconcile + workflow, the chunk server path, the schema, and the SDK client wiring. + + Thought for 25s, read 3 files (ctrl+o to expand) + +● Now let me read the remaining critical files: the chunk read path, route + orchestration, schema, repository, and chat wiring. + + Thought for 22s, read 2 files (ctrl+o to expand) +● This is a large, well-understood migration. Let me read the last few grounding + files — schema, the source service (snapshot orchestration), and the workflow + runtime — then dispatch a Plan agent to synthesize the design. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && sed -n '60,200p' + src/infrastructure/db/schema.ts) + ⎿   * of a Knowhere parse + index job; the actual chunks / file bytes liv + e + * upstream in Knowhere, not here. + … +138 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 24s, read 2 files (ctrl+o to expand) + +● I have a thorough picture of both sides now. Let me dispatch a Plan agent to + synthesize the detailed implementation design, giving it the full context I've + gathered. It can read the chat-hardening files itself. + +● Plan(Design notebook migration plan) + ⎿  Done (43 tool uses · 257.7k tokens · 7m 36s) + (ctrl+o to expand) + +● The Plan agent produced a thorough, well-grounded design. I have everything I + need. Let me write the final plan file, then present it for approval. + +● Updated plan + ⎿  /plan to preview + +● User approved Claude's plan + ⎿  Plan saved to: ~/.claude/plans/read-the-new-design-precious-tower.md · + /plan to edit + Notebook Migration to SDK Parsed-Document Storage + + Context + + docs/new-design.md moves parsed chunk read/sync behavior into the Knowhere + SDK so Notebook, CLI, and MCP share one model. The SDK side is already + built on the current branch (ParsedDocumentStorage interface, + withParsedStorage, readChunks/grepChunks/getDocumentOutline, resumable + syncParsedDocument, synchronous assetUrlPolicy:'durable' hardening). This + plan covers the Notebook side + (/home/suguan/github.com/ontosAI/knowhere-notebook), which today still uses + the old path: loadJobResult + the KnowhereAssetStorageOptions adapter, + plus hand-rolled Blob snapshot readers, a chunk-page cache, and asset + mirroring. + + The intended outcome: Notebook provides one Vercel-Blob + ParsedDocumentStorage adapter + an Upstash/QStash-safe scheduler, routes + all chunk reads/display/chat asset hardening through the SDK, and marks + sources ready only after parsed storage is synced. Existing remote docs + remain readable immediately (SDK falls back to remote, backfills Blob in + the background). + + Decisions taken with the user: + 1. Full rip-and-replace of the hand-rolled snapshot machinery. + 2. Include chat: unify citation/media asset hardening onto the SDK durable + policy; keep agentic retrieval.query. + 3. QStash-enqueued resumable background sync (reuse the + deterministic-workflowRunId continuation pattern); parse flow syncs before + marking ready. + + SDK contract being consumed (no SDK changes) + + - knowledge.withParsedStorage({ storage, scheduler?, limits? }) → new + configured Knowledge (unwrapped copy). + - ParsedDocumentStorage (src/types/storage.ts:142): + readManifest/writeManifest, readChunkPage/writeChunkPage, + writeAsset/getAssetUrl, readSyncProgress/writeSyncProgress. Reference impl: + src/storage/disk-parsed-document-storage.ts. Storage is only ever called + with {documentId, revisionKey} — never workspace/source. + - Freshness: manifest is fresh iff manifest.revisionKey === revisionKey, + where revisionKey = jobResultId ?? jobId. Pass revisionKey into reads to + hit storage without a remote probe. + - readChunks(params) — paged (page/pageSize/assetUrlPolicy) cannot combine + with sectionPath/startChunk/endChunk/chunkId. Never returns zero for a + ready remote doc (falls back to remote listChunks, returns bounded page, + fire-and-forget scheduler.schedule to backfill). + - assetUrlPolicy:'durable' + configured storage → SDK synchronously hardens + primary assetUrl and metadata.pageAssets[].assetUrl via + writeAsset/getAssetUrl; on failure omits the URL (never leaks a presigned + Knowhere URL). + - syncParsedDocument({documentId, revisionKey}) — resumable via + sync-progress; bounded by limits; returns completed:false at page/deadline + bounds (re-invoke to continue); commits manifest only after all pages + present. + + Implementation + + 1. Vercel-Blob ParsedDocumentStorage adapter — + src/domains/sources/parsed-document-blob-storage.ts (new) + + class BlobParsedDocumentStorage implements ParsedDocumentStorage, + constructed workspace-scoped ({ workspaceId, blobStore? }) with an + injectable blobStore seam (default wraps @vercel/blob get/put/head/del), + mirroring the seam pattern in + chunks/server.ts/parse-result-storage-adapter.ts. + + Key layout (documentId-scoped, workspace-rooted — resolves the scoping + mismatch since one client serves many docs and the adapter only receives + documentId): + workspaces/{workspaceId}/parsed-documents/{documentId}/{revisionKey}/manife + st/current.json + workspaces/{workspaceId}/parsed-documents/{documentId}/{revisionKey}/chunks + /page-{N}.json + workspaces/{workspaceId}/parsed-documents/{documentId}/{revisionKey}/assets + /{normalizedSourcePath} + workspaces/{workspaceId}/parsed-documents/{documentId}/{revisionKey}/sync-p + rogress.json + - reads → blobStore.get + JSON parse, null on BlobNotFoundError/404 (reuse + readBlobText). readChunkPage ignores chunkType (SDK filters after read). + - JSON writes → put(..., {access:"public", allowOverwrite:true, + contentType:"application/json; charset=utf-8"}). + - writeAsset → put(Buffer.from(body), {access:"public", + allowOverwrite:true, contentType, multipart:true}) → {sourcePath, url: + blob.url}. getAssetUrl → head(key).url or null. + - Sanitize documentId/revisionKey/sourcePath with a traversal guard (mirror + DiskParsedDocumentStorage.normalizeRelativeStoragePath: reject .., + absolute, empty segments). Keying under {revisionKey} gives the freshness + contract for free. Chat's existing isNotebookOwnedAssetUrl already treats + *.blob.vercel-storage.com as owned, so these durable URLs pass through + un-re-hardened. + + 2. Scheduler + sync route + + - src/domains/sources/parsed-document-sync-scheduler.ts (new): + createQStashSyncScheduler({workspaceId, sourceId, documentId, revisionKey, + apiKey}). schedule(task) ignores the closure (can't serialize the client) + and triggers a QStash continuation, reusing background-reconcile.ts's + trigger shape: Client.trigger({ url: + ${NOTEBOOK_PUBLIC_URL}/api/sources/parsed-sync, body, workflowRunId: + ${documentId}-sync-${revisionKey}-${segmentIndex}, retries: 3 }). Add a + process-level cooldown guard keyed by documentId|revisionKey (like + lastTriggeredAtBySourceId) so read-path misses don't storm QStash. + - src/app/api/sources/parsed-sync/route.ts + + src/domains/sources/parsed-sync-route-workflow.ts (new), modeled on + source-reconcile-route-workflow.ts: rebuild a parsed-storage knowledge, + loop context.run("sync-N", () => knowledge.syncParsedDocument({documentId, + revisionKey})) until completed:true or budget (~45s / N segments); on + incomplete trigger continuation segmentIndex+1; on complete for the parse + flow mark ready (§5); failureFunction writes sync_status='failed' + + failure_stage='storage_sync' (does not fail an already-ready source — it + still serves via remote fallback). Partial-snapshot safety is SDK-side. + + 3. Client wiring — src/integrations/knowhere.ts + + Add makeKnowhereClientWithParsedStorage(apiKey, { workspaceId, scheduler?, + limits? }) → { client, knowledge }. Build the wrapped client via existing + makeKnowhereClient (keep for retrieval/jobs/documents), then knowledge = + client.knowledge.withParsedStorage({ storage: new + BlobParsedDocumentStorage({workspaceId}), scheduler, limits }). Invoking + withParsedStorage through the logging Proxy gives correct this and returns + a functional unwrapped Knowledge (its internal listChunks won't be logged — + acceptable). Vercel-safe limits: chunkPageSize:200, remotePageSize:100, + maxPagesPerSync:10, maxAssetsPerSync:20, syncDeadlineMs:8000, + outlineMaxPages:50, grepMaxPages:50. + + Scoping note: route-chunks reads one source at a time → build per-source + knowledge with a document-bound QStash scheduler. Chat's multi-doc asset + loop → build one workspace knowledge with a no-op/coalescing scheduler (its + reads use synchronous 'durable' hardening; full background sync isn't its + job). + + 4. Read path rewrite + + - Delete src/domains/chunks/server.ts (snapshot readers, + warmChunkPageCache, mirror machinery, chunk-page cache, scheduleWarm). + Reduce src/domains/chunks/index.ts to view mappers: keep toParsedChunkView, + resolveChunkConnectionTargets, getChunkPageParams, + ChunkPage/ChunkPageParams; add toParsedChunkViewFromReadChunk(chunk: + KnowledgeReadChunk, sourceTitle, documentId) (map chunkId/type/content/read + ableContent/sectionPath/filePath/assetUrl/metadata/pageNums/connections). + No assetUrlsByFilePath re-mapping — durable hardening already rewrote asset + URLs. + - src/domains/sources/route-chunks.ts for a ready workspace source: + - Paged display → knowledge.readChunks({documentId, revisionKey, page, + pageSize, assetUrlPolicy:'durable'}); pagination from + {page,pageSize,totalChunks,totalPages}. + - Load-all → loop readChunks({documentId, revisionKey, page:N, + pageSize:200, assetUrlPolicy:'durable'}) until page >= totalPages, then + resolveChunkConnectionTargets. + - Tree/structure (if any caller needs it) → + knowledge.getDocumentOutline({documentId, revisionKey}). + - Pass revisionKey from source_parse_results.revision_key (fallback + source.knowhereJobId); if unknown, omit → SDK probes remote once, returns + immediately, schedules background sync. + - New gating: source.status !== 'ready' → 202 "still preparing"; ready → + always serve. Delete ensureParsedSnapshotForRead/isCompleteSnapshot and the + snapshot/snapshotStore/onRevisionKey plumbing. + - Demo kind unchanged (HTTP demoApi.fetchChunkPage). Remote-library kind: + keep localizeRemoteDocument to get documentId, then readChunks; delete the + syncRemoteParsedSnapshot call (first read triggers background sync). + - Update counts.ts to use getDocumentOutline(...).totalChunks (or + readChunks(...).totalChunks) instead of readParsedResultSnapshotManifest. + + 5. Parse flow — source-reconcile-route-workflow.ts + + Replace the load-job-result-snapshot step (loadJobResult({storageAdapter})) + with a resumable sync loop: after ready-to-prepare {jobId, documentId}, + persist revision_key (= jobResultId ?? jobId, probe + listChunks(documentId,{page:1,pageSize:1}) if needed) + + sync_status='running'; build parsed-storage knowledge (§3); loop + context.run("sync-N", () => knowledge.syncParsedDocument({documentId, + revisionKey})). If completed:false at budget → trigger parsed-sync + continuation and return with source still parsing. On completed:true → + markSourceReadyAfterReconciliation + sync_status='completed'. On sync + failure after successful parse → markFailed with + failure_stage='storage_sync'. Use documentId-based (resumable, bounded) + sync, not jobId-based (one-shot zip load), to respect serverless deadlines. + markReadyEffect already hard-guards requiredStatus:'parsing', so a late + continuation can't double-ready. + + Retry (retry.ts / route-retry.ts): dispatch on failure_stage. 'parse'/null + → existing reparse from originalBlobUrl. 'storage_sync' → new + resumeParsedSync that re-enqueues the continuation from stored + documentId+revision_key (no reparse), resets sync_status='running'. + + 6. DB migration 0011 (new only; never edit existing) + + src/infrastructure/db/schema.ts: + - sourceParseResults: add revisionKey: text("revision_key"), syncStatus: + text("sync_status") (pending|running|completed|failed), syncError: + text("sync_error"). Resumable cursor stays in Blob sync-progress.json + (SDK-managed) — no cursor columns. Keep legacy + resultBlobUrl/snapshotManifest* for back-compat. + - sources: add failureStage: text("failure_stage") (nullable; + parse|storage_sync). + - pnpm db:generate → drizzle/0011_*.sql. + - Repos: source-parse-result-repository.ts (persist/read new cols + + updateSyncStatusEffect), workflow-runtime.ts facade (updateSyncStatus, + setFailureStage), source-row-repository.ts (markFailed accepts + failureStage; markReady clears it). + - Back-compat: old rows have revision_key = NULL → reads omit the fast-path + key, probe remote once, serve remote, schedule background sync that + populates the new layout. Old blob objects orphaned (later cleanup sweep, + out of scope). + + 7. Chat unification — route-answer.ts + media-asset-hardening.ts + + Keep agentic retrieval.query, citation selection, page-citation-assets.ts. + Delete copyAssetToBlob, resolveAssetFetchRequest, parseDemoAssetRoute, + getChatAssetBlobPathname, the chat-assets/ fetch-and-put path, and + HardeningContext.blobStore/fetchAsset. Keep isNotebookOwnedAssetUrl, + resolveParsedAssetUrl, enrichRetrievalResultsWithAssetUrls. + New loadSourceAssetUrls: build workspace parsed-storage knowledge once; per + source readChunks({documentId, revisionKey, page:N, pageSize:200, + assetUrlPolicy:'durable'}) (loop pages), build assetUrlsByFilePath from + chunk.filePath → chunk.assetUrl and metadata.pageAssets[].artifactRef → + assetUrl. This is the single hardening path (durable policy writes any + missing asset and returns durable URLs). hardenMediaAssetUrls shrinks to: + owned → keep; else resolve against the durable map; else keep raw (no + copy-to-blob). + + Critical files + + - New: src/domains/sources/parsed-document-blob-storage.ts, + parsed-document-sync-scheduler.ts, parsed-sync-route-workflow.ts, + src/app/api/sources/parsed-sync/route.ts, drizzle/0011_*.sql. + - Change: src/integrations/knowhere.ts, + src/domains/sources/route-chunks.ts, source-reconcile-route-workflow.ts, + retry.ts/route-retry.ts, src/infrastructure/db/schema.ts, + source-parse-result-repository.ts, workflow-runtime.ts, + source-row-repository.ts, src/domains/chunks/index.ts, counts.ts, + src/domains/chat/route-answer.ts, media-asset-hardening.ts. + - Delete: src/domains/chunks/server.ts, remote-parsed-snapshot-sync.ts, and + the snapshot-reader half of parse-result-storage-adapter.ts; strip + snapshot plumbing from service.ts, route-dependencies.ts, route-types.ts. + + Ordered task list + + 1. DB migration + repos — schema cols, pnpm db:generate → 0011, extend + repositories/facade (no behavior change). + 2. Blob storage adapter + tests (key layout, seam, traversal guard, + freshness). + 3. Client wiring — makeKnowhereClientWithParsedStorage + limits; verify + Proxy interaction. + 4. Scheduler + sync route — deterministic runIds + cooldown; continuation + loop. + 5. Parse flow — syncParsedDocument loop + continuation + ready-gating + + failure_stage. + 6. Read path — collapse route-chunks onto readChunks/getDocumentOutline; + add mapper; update counts.ts; delete chunks/server.ts. + 7. Chat unification — new loadSourceAssetUrls; shrink + media-asset-hardening.ts; update route-answer.ts. + 8. Retry — failure_stage dispatch (resume sync vs reparse). + 9. Delete legacy — parse-result-storage-adapter.ts snapshot readers, + remote-parsed-snapshot-sync.ts, chunks/server.ts; strip plumbing. + 10. Tests + verify. + + Risks + + - Logging Proxy vs withParsedStorage (unwrapped result; internal listChunks + unlogged — acceptable). + - Per-request storage scoping: scheduler only gets a closure → pre-bind + per-document (route-chunks) or no-op scheduler (chat asset reads). + - Serverless deadlines → documentId-based resumable sync + conservative + limits + QStash continuation. + - Demo (HTTP-only) and remote-library (localizeRemoteDocument first) kinds. + - Back-compat: old source_parse_results rows self-heal via remote fallback + + background re-sync; old blobs orphaned. + - Asset volume: synchronous durable hardening on a visible page is bounded + by page size but adds latency on image-heavy pages. + + Verification + + - Unit/integration (Vitest): storage adapter round-trip + guards; + scheduler/sync-route continuation + failure staging; route-chunks + paged/load-all/202/ready + demo + remote-library; reconcile parse flow + (syncParsedDocument, continuation, ready-only-on-complete, storage_sync + failure); repo persistence of revision_key/sync_status; retry failure_stage + dispatch; chat owned-URL passthrough + reference resolution + no + copy-to-blob. + - pnpm typecheck, pnpm test, pnpm db:generate (confirm only 0011 added). + - Smoke (local Knowhere + Notebook): upload a multi-page PDF with + images/tables → wait until ready → inspect Blob manifest/pages/assets under + parsed-documents/... → open chunks panel (durable Blob asset URLs render) + → chat with a citation (image/page citation renders durable Blob URL, never + a presigned Knowhere URL). Also verify an existing remote doc with no Blob + snapshot returns chunks immediately from remote and backfills Blob in the + background. + + Thought for 22s (ctrl+o to expand) + +● Plan approved. Let me clean up the stale task list (those are from a prior + SDK/CLI effort) and set up tasks for this notebook migration. First let me + check the notebook repo's git state, since all work happens there. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && git branch + --show-current && echo "---STATUS---" && git status --short && echo + "---SCRIPTS---" && node -…) + ⎿  feature/wangbinqi/notebook-page-citation-assets + ---STATUS--- + ---SCRIPTS--- + … +20 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 15s (ctrl+o to expand) + +● Clean tree, on a feature branch. Let me clear the stale tasks (prior SDK/CLI + work) and set up this migration's task list, then create a working branch on + the notebook repo. + + I'll replace the stale tasks with the plan's ordered steps. + Thought for 18s (ctrl+o to expand) + +● Let me set up the key dependencies to reflect the plan's order, then create a + working branch on the notebook repo. + +● Task graph is set. Now the branch. The notebook is on + feature/wangbinqi/notebook-page-citation-assets; per the git guidelines I + should branch for this new work rather than commit onto that one. Let me + create a branch in the correct xxx/wangbinqi/xxx format from the current + branch (which has the page-citation work this builds on). + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && git checkout -b + feature/wangbinqi/notebook-parsed-document-storage && git branch + --show-current) + ⎿  Switched to a new branch 'feature/wangbinqi/notebook-parsed-document-s + torage' + feature/wangbinqi/notebook-parsed-document-storage + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Branch created. Now Task #7 (DB migration + repos) and Task #8 (Blob storage + adapter) are both unblocked and independent — I'll start with them. Let me + begin with Task #8 (the Blob adapter), since it's the foundation the client + wiring and everything downstream depends on, and read the SDK's + DiskParsedDocumentStorage traversal guard details plus the notebook's existing + blob-seam conventions I need to mirror. + + Let me start Task #8. I already have the SDK's DiskParsedDocumentStorage and + parse-result-storage-adapter.ts in context. Let me check how @vercel/blob's + get/head are typed in the installed version and how the existing seam handles + get return shape. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && find + node_modules/@vercel/blob/dist -name "*.d.ts" | head && echo "---" && grep + -rn "export declare func…) + ⎿  node_modules/@vercel/blob/dist/index.d.ts + node_modules/@vercel/blob/dist/create-folder-vlS2Pu_G.d.ts + node_modules/@vercel/blob/dist/client.d.ts + … +16 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && sed -n '1,120p' + node_modules/@vercel/blob/dist/create-folder-vlS2Pu_G.d.ts | grep -n + "GetBlobResult\|Hea…) + ⎿  1:import { Readable } from 'stream'; + 39: * Defines the content type of the blob. By default, this value + is inferred from the pathname. Sent as the 'content-type' header when + … +5 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -n + "GetBlobResult\|HeadBlobResult\|GetCommandOptions\|interface .*Blob\|type + GetBlob\|type HeadBlob…) + ⎿  4:interface BlobCommandOptions { + 21:interface CommonCreateBlobOptions extends BlobCommandOptions { + 112:interface PutBlobResult { + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk +● The GetBlobResult/HeadBlobResult types live elsewhere. The existing + parse-result-storage-adapter.ts already models get returning {statusCode, + stream} and head returning {url, contentType, size}, so I have the shapes I + need from the code I've already read. Let me find where these types are + declared to get head's exact return. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -rn + "GetBlobResult\b\|HeadBlobResult\b" node_modules/@vercel/blob/dist/*.d.ts + | grep -v "export {"…) + ⎿  node_modules/@vercel/blob/dist/index.d.ts:76:interface HeadBlobResult + { + node_modules/@vercel/blob/dist/index.d.ts:121:declare function head(ur + … +13 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 12s, read 1 file (ctrl+o to expand) + +● I have all the types I need. Let me confirm the SDK's public type exports so I + can import the ParsedDocumentStorage interface and snapshot types by name. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -n + "ParsedDocumentStorage\|KnowhereParsedSnapshotManifest\|KnowhereParsedSnap + shotChunkPage\|ParsedD…) + ⎿  410:interface KnowhereParsedSnapshotChunkPageReference { + 417:interface KnowhereParsedSnapshotManifest { + 426: readonly chunkPages: readonly KnowhereParsedSnapshotChunkPageR + … +2 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 13s (ctrl+o to expand) + +● The dist typings are stale — they don't export the new + ParsedDocumentStorage/ParsedDocumentSyncProgress types the notebook needs to + import. Since the SDK is pnpm-linked via dist/, I need to confirm whether the + SDK dist is built from the current source. Let me check. + + Thought for 13s, listed 1 directory (ctrl+o to expand) + +● The SDK repo's own dist/index.d.ts has withParsedStorage/ParsedDocumentStorage + (22 hits), but the notebook's node_modules copy is stale (only 4 hits, + missing the new types). pnpm file: links can be copies. Let me check the link + type and whether the notebook's copy needs a rebuild/reinstall. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && ls -la + node_modules/@ontos-ai/ && echo "---is it symlink?---" && readlink -f + node_modules/@ontos-ai/know…) + ⎿  total 48 + drwxr-xr-x 2 suguan suguan 4096 Jul 3 23:40 . + drwxr-xr-x 68 suguan suguan 36864 Jul 4 15:36 .. + … +9 lines (ctrl+o to expand) + + Thought for 21s (ctrl+o to expand) + +● The notebook's pnpm copy of the SDK is a stale snapshot (0 hits for + withParsedStorage), even though the SDK repo's own dist/ is current. pnpm + copied the package at install time. I need to rebuild the SDK dist (to be + safe) and refresh the notebook's copy so the new ParsedDocumentStorage types + are importable. Let me check the SDK build script and rebuild. + +● Bash(node -e "const p=require('./package.json'); console.log('version', + p.version); console.log(JSON.stringify(p.scripts,null,2))") + ⎿  version 2.0.0 + { + "build": "tsup src/index.ts --format cjs,esm --dts --clean", + … +23 lines (ctrl+o to expand) + +● Let me rebuild the SDK dist and then refresh the notebook's pnpm copy so the + new types are available. First rebuild the SDK. + +● Bash(pnpm run build 2>&1 | tail -20 && echo "---verify types present---" && + grep -c "withParsedStorage" dist/index.d.ts) + ⎿  > @ontos-ai/knowhere-sdk@2.0.0 build /home/suguan/github.com/ontosAI/k + nowhere-node-sdk + > tsup src/index.ts --format cjs,esm --dts --clean + … +18 lines (ctrl+o to expand) + + Thought for 15s, searched for 1 pattern (ctrl+o to expand) + +● All the new types are now exported from the freshly built dist: + ParsedDocumentStorage, ParsedDocumentSyncProgress, + ParsedDocumentStorageConfig, ParsedDocumentStorageLimits, + KnowhereParsedSnapshotManifest, KnowhereParsedSnapshotChunkPage, etc. Now + refresh the notebook's pnpm copy so it picks up the rebuilt dist. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm install 2>&1 + | tail -15 && echo "---verify notebook copy now current---" && grep -c + "withParsedStor…) + ⎿  │ esbuild@0.27.7, msgpackr-extract@3.0.3, msw@2.14.3. + │ + │ Run "pnpm approve-builds" to pick which dependencies should be all + … +18 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -c + "interface ParsedDocumentStorage\b\|ParsedDocumentSyncProgress\|ParsedDocu + mentStorageConfig\|Par…) + ⎿  12 + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● The notebook now sees the new types. Now I'll write the Blob adapter (Task + #8). All work happens in the notebook repo. Let me create + parsed-document-blob-storage.ts. + +● Write(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/parsed-docume + nt-blob-storage.ts) + ⎿  Wrote 291 lines to ../knowhere-notebook/src/domains/sources/parsed-doc + ument-blob-storage.ts + 2 import "server-only" + 3 import { del, get, head, put, BlobNotFoundError } from "@vercel/ + blob" + 4 import type { + 5 KnowhereParsedSnapshotChunkPage, + 6 KnowhereParsedSnapshotManifest, + 7 ParsedDocumentStorage, + 8 ParsedDocumentStorageAsset, + 9 ParsedDocumentStorageAssetParams, + 10 ParsedDocumentStorageChunkPageParams, + … +314 lines (ctrl+o to expand) + +● Now let me check an existing test in the sources domain to match the test + conventions before writing the adapter's test. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && ls + src/domains/sources/*.test.ts 2>/dev/null | head && echo "---sample test + header---" && sed -n '1,40p'…) + ⎿  src/domains/sources/background-reconcile.test.ts + src/domains/sources/blob-upload.test.ts + src/domains/sources/counts.test.ts + … +48 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Now I'll write the adapter test with an in-memory blob store fake. + +● Write(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/parsed-docume + nt-blob-storage.test.ts) + ⎿  Wrote 219 lines to ../knowhere-notebook/src/domains/sources/parsed-doc + ument-blob-storage.test.ts + 1 import { describe, expect, it } from "vitest" + 2 import type { + 3 KnowhereParsedSnapshotChunkPage, + 4 KnowhereParsedSnapshotManifest, + 5 ParsedDocumentSyncProgress, + 6 } from "@ontos-ai/knowhere-sdk" + 7 + 8 import { + 9 BlobParsedDocumentStorage, + 10 type ParsedDocumentBlobStore, + … +229 lines (ctrl+o to expand) + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm vitest run + src/domains/sources/parsed-document-blob-storage.test.ts 2>&1 | tail -30) + ⎿   RUN v4.1.5 /home/suguan/github.com/ontosAI/knowhere-notebook + … +5 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● All 7 tests pass. Let me typecheck the new file, then mark Task #8 done and + move to Task #9 (client wiring). + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -20) + ⎿  > knowhere-notebook@0.1.0 typecheck /home/suguan/github.com/ontosAI/kn + owhere-notebook + > tsc --noEmit + … +9 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk +● A type-only issue in the test fake's new Response(body) when body is a Buffer. + Let me fix by normalizing to a Uint8Array in the fake. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/parsed-docum + ent-blob-storage.test.ts) + ⎿  Added 5 lines, removed 1 line + 27 get: async (pathname) => { + 28 const object = objects.get(pathname) + 29 if (!object) return null + 30 + const body = + 31 + typeof object.body === "string" + 32 + ? object.body + 33 + : new Uint8Array(object.body) + 34 return { + 35 statusCode: 200, + 32 - stream: new Response(object.body).body as ReadableStrea + -m, + 36 + stream: new Response(body).body as ReadableStream, + 37 } + 38 }, + 39 put: async (pathname, body, options) => { + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -8 && echo "===TEST===" && pnpm vitest run + src/domains/sources/parsed-documen…) + ⎿  > knowhere-notebook@0.1.0 typecheck /home/suguan/github.com/ontosAI/kn + owhere-notebook + > tsc --noEmit + … +8 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Typecheck clean, 7/7 tests pass. Task #8 done. Now Task #9 (client wiring) and + Task #7 (DB migration + repos) are both unblocked. Let me mark #8 complete + and start #9 (client wiring), which unblocks the most downstream work. + + Before wiring, I need to understand the SDK's + KnowledgeParsedStorageOptions/ParsedDocumentStorageConfig shape and how + withParsedStorage behaves through the logging Proxy. I have the SDK source. + Let me check route-dependencies.ts and request-context.ts to see how clients + are currently built and where a parsed-storage-configured knowledge should be + surfaced. + + Thought for 18s, read 1 file (ctrl+o to expand) + +● The Proxy interaction is clear from the SDK source: + client.knowledge.withParsedStorage(cfg) resolves through the namespace proxy + via Reflect.apply(fn, realKnowledge, [cfg]), and withParsedStorage returns new + Knowledge(this.client, ...) where this.client is the real unwrapped Knowhere + — so it works correctly. Let me check route-types.ts for the + SourceRouteKnowhereClient shape and how client.knowledge is typed/used. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -n + "SourceRouteKnowhereClient\|knowledge\|documents\|ChunkKnowhereClient" + src/domains/sources/route…) + ⎿  2: ChunkKnowhereClient, + 28:type SourceRouteKnowhereClient = UploadKnowhereClient & + 29: ChunkKnowhereClient & { + … +9 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 15s (ctrl+o to expand) + +● Now I'll add the client wiring helper. It returns { client, knowledge } where + knowledge is configured with the Blob storage + scheduler + limits. The + scheduler is injected by the caller (route-chunks binds per-document, chat + uses a no-op). + +● Update(~/github.com/ontosAI/knowhere-notebook/src/integrations/knowhere.ts) + ⎿  Added 53 lines + 1 import Knowhere from "@ontos-ai/knowhere-sdk" + 2 +import type { + 3 + Knowledge, + 4 + ParsedDocumentStorageLimits, + 5 + ParsedDocumentSyncScheduler, + 6 +} from "@ontos-ai/knowhere-sdk" + 7 import { logger } from "@/lib/logger" + 8 +import { BlobParsedDocumentStorage } from "@/domains/sources/pa + +rsed-document-blob-storage" + 9 + 10 /** + 11 + * Vercel-safe defaults for parsed-document reads and backgroun + +d sync. Each read + 12 + * miss and each background sync step is bounded by page count + +and a deadline so + 13 + * a single serverless invocation stays well under the platform + + ceiling; the + 14 + * SDK returns `completed:false` and the caller re-enqueues to + +continue. + 15 + */ + 16 +const defaultParsedStorageLimits: ParsedDocumentStorageLimits = + + { + 17 + chunkPageSize: 200, + 18 + remotePageSize: 100, + 19 + maxPagesPerSync: 10, + 20 + maxAssetsPerSync: 20, + 21 + syncDeadlineMs: 8000, + 22 + grepMaxPages: 50, + 23 + grepDeadlineMs: 8000, + 24 + outlineMaxPages: 50, + 25 + outlineDeadlineMs: 8000, + 26 +} + 27 + + 28 +type ParsedStorageOptions = { + 29 + readonly workspaceId: string + 30 + readonly scheduler?: ParsedDocumentSyncScheduler + 31 + readonly limits?: ParsedDocumentStorageLimits + 32 +} + 33 + + 34 +/** + 35 * Create a Knowhere client with the given API key. + 36 * Use for per-request clients created from Dashboard-issued JW + Ts. + 37 */ + ... + 44 return wrapKnowhereClient(client) + 45 } + 46 + 47 +/** + 48 + * Create a Knowhere client plus a `Knowledge` configured with + +a Vercel-Blob + 49 + * `ParsedDocumentStorage`. Reads through `knowledge` serve fro + +m Blob first and + 50 + * fall back to Knowhere remote transparently; the `scheduler` + +(when provided) + 51 + * backfills Blob in the background. Use `client` for retrieval + +/documents/jobs. + 52 + * + 53 + * `withParsedStorage` is invoked through the logging Proxy so + +`this` binds to + 54 + * the real Knowledge; the returned Knowledge wraps the unwrapp + +ed inner client + 55 + * (its internal `documents.listChunks` calls are not logged, w + +hich is fine). + 56 + */ + 57 +export function makeKnowhereClientWithParsedStorage( + 58 + apiKey: string, + 59 + options: ParsedStorageOptions, + 60 +): { readonly client: Knowhere; readonly knowledge: Knowledge } + + { + 61 + const client = makeKnowhereClient(apiKey) + 62 + const knowledge = client.knowledge.withParsedStorage({ + 63 + storage: new BlobParsedDocumentStorage({ workspaceId: optio + +ns.workspaceId }), + 64 + scheduler: options.scheduler, + 65 + limits: options.limits ?? defaultParsedStorageLimits, + 66 + }) + 67 + return { client, knowledge } + 68 +} + 69 + + 70 function wrapKnowhereClient(client: Knowhere): Knowhere { + 71 return new Proxy(client, { + 72 get(target, prop, receiver) { + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -15) + ⎿  > knowhere-notebook@0.1.0 typecheck /home/suguan/github.com/ontosAI/kn + owhere-notebook + > tsc --noEmit + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Typecheck clean. Task #9 done. Now Task #7 (DB migration + repos) is unblocked + and it, plus #10/#12, become available. Task #7 is a prerequisite for the + parse flow, read path, scheduler, and retry — let me do it next. + + Let me read the parse-result repository and source-row repository to see + exactly what to extend. + + Read 1 file (ctrl+o to expand) + +● Now let me read the source-row-repository's markFailedEffect and + markReadyEffect to add failureStage, and the updateInWorkspaceWithDb + signature. + + Read 1 file (ctrl+o to expand) + +● Now let me look at the SourceRowRepository type / interface signatures near + the top and SourceUpdate type. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && sed -n '1,120p' + src/domains/sources/source-row-repository.ts) + ⎿  import "server-only" + + import { and, desc, eq, isNull, sql } from "drizzle-orm" + … +117 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Now I have everything for Task #7. Let me make the schema changes. First, the + two schema edits: add failureStage to sources and + revisionKey/syncStatus/syncError to sourceParseResults. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/infrastructure/db/schema.ts) + ⎿  File must be read first + +● I read that region via sed, not the Read tool. Let me Read the two schema + regions properly, then edit. + + Read 1 file (ctrl+o to expand) + +● Update(~/github.com/ontosAI/knowhere-notebook/src/infrastructure/db/schema.ts) + ⎿  Added 1 line + 95 sizeBytes: bigint("size_bytes", { mode: "number" }).notNul + l(), + 96 status: text("status").notNull(), + 97 failureReason: text("failure_reason"), + 98 + failureStage: text("failure_stage"), + 99 knowhereJobId: text("knowhere_job_id"), + 100 knowhereDocumentId: text("knowhere_document_id"), + 101 stagedBlobPathname: text("staged_blob_pathname"), + +● Now update the doc comment for the sources table to document failure_stage, + and the source_parse_results table. Let me read those regions. + + Read 1 file (ctrl+o to expand) +● Update(~/github.com/ontosAI/knowhere-notebook/src/infrastructure/db/schema.ts) + ⎿  Added 3 lines + 177 resultBlobUrl: text("result_blob_url").notNull(), + 178 snapshotManifestUrl: text("snapshot_manifest_url"), + 179 snapshotManifestKey: text("snapshot_manifest_key"), + 180 + revisionKey: text("revision_key"), + 181 + syncStatus: text("sync_status"), + 182 + syncError: text("sync_error"), + 183 assetUrls: jsonb("asset_urls") + 184 .$type>>() + 185 .notNull(), + +● Now update the two doc comments to reflect the new columns. Let me read the + sources comment block and the source_parse_results comment block. + + Read 1 file (ctrl+o to expand) + +● Update(~/github.com/ontosAI/knowhere-notebook/src/infrastructure/db/schema.ts) + ⎿  Added 12 lines, removed 2 lines + 163 * Notebook-owned parse-result artifact index for one source. + 164 * + 165 * Blob is the Notebook-owned read model for parsed chunks aft + er source + 166 - * reconciliation completes. This row stores the current parse + -d snapshot + 167 - * manifest and the file-path-to-public-URL map for parsed med + -ia artifacts. + 166 + * reconciliation completes. The parsed snapshot itself (manif + +est, chunk pages, + 167 + * assets, and resumable sync progress) lives in Vercel Blob u + +nder + 168 + * `workspaces/{ws}/parsed-documents/{documentId}/{revisionKey + +}/...`, managed by + 169 + * the SDK `ParsedDocumentStorage`. This row records: + 170 + * - `revision_key` — current parsed revision (jobResultId + +?? jobId); the + 171 + * storage fast-path key passed into SDK + + reads + 172 + * - `sync_status` — pending | running | completed | faile + +d for the + 173 + * background/parse-time storage sync + 174 + * - `sync_error` — last storage-sync error detail when s + +ync_status=failed + 175 + * - `result_blob_url` / `snapshot_manifest_*` — legacy colu + +mns retained for + 176 + * rows written by the pre-migration man + +ifest format + 177 + * - `asset_urls` — legacy file-path-to-public-URL map fo + +r older rows + 178 */ + 179 export const sourceParseResults = pgTable( + 180 "source_parse_results", + +● Now update the sources doc comment to mention failure_stage. + + Read 1 file (ctrl+o to expand) + +● Update(~/github.com/ontosAI/knowhere-notebook/src/infrastructure/db/schema.ts) + ⎿  Added 2 lines + 66 * - `size_bytes` — original upload size (for display + quo + ta) + 67 * - `status` — lifecycle: uploading | parsing | ready + | failed + 68 * - `failure_reason` — human-readable error text when status + =failed + 69 + * - `failure_stage` — which stage failed: parse | storage_s + +ync; drives + 70 + * whether a retry reparses or only resu + +mes storage sync + 71 * - `knowhere_job_id` — set once the parse job is creat + ed + 72 * - `knowhere_document_id` — set when parsing completes; use + d to import + 73 * parsed snapshots and to exclude + a source from a + +● Now generate the migration. Let me run pnpm db:generate. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm db:generate + 2>&1 | tail -20 && echo "---new files---" && git status --short drizzle/) + ⎿  > knowhere-notebook@0.1.0 db:generate /home/suguan/github.com/ontosAI/ + knowhere-notebook + > drizzle-kit generate + … +17 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Migration 0011_brainy_the_leader.sql generated (only 0011, no edits to + existing). Let me verify its contents are exactly the four ADD COLUMNs. + + Read 1 file (ctrl+o to expand) + +● Clean — exactly the four additive columns, all nullable. Now extend the + repositories. First the parse-result repository: add + revisionKey/syncStatus/syncError to the save input and progress reads, and an + updateSyncStatusEffect. Let me update source-parse-result-repository.ts. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 21 lines + 14 readonly resultBlobUrl: string + 15 readonly snapshotManifestUrl?: string + 16 readonly snapshotManifestKey?: string + 17 + readonly revisionKey?: string + 18 + readonly syncStatus?: SourceParseSyncStatus + 19 + readonly syncError?: string | null + 20 readonly assetUrlsByFilePath: Readonly + > + 21 } + 22 + 23 +export type SourceParseSyncStatus = + 24 + | "pending" + 25 + | "running" + 26 + | "completed" + 27 + | "failed" + 28 + + 29 type SourceParseResultProgress = { + 30 readonly resultBlobUrl: string + 31 readonly snapshotManifestUrl?: string | null + 32 readonly snapshotManifestKey?: string | null + 33 + readonly revisionKey?: string | null + 34 + readonly syncStatus?: string | null + 35 + readonly syncError?: string | null + 36 readonly assetUrlsByFilePath: Readonly + > + 37 } + 38 + ... + 40 readonly resultBlobUrl: string + 41 readonly snapshotManifestUrl?: string | null + 42 readonly snapshotManifestKey?: string | null + 43 + readonly revisionKey?: string | null + 44 + readonly syncStatus?: string | null + 45 + readonly syncError?: string | null + 46 readonly assetUrlsByFilePath: Readonly + > + 47 } + 48 + 49 +type UpdateSyncStatusInput = { + 50 + readonly revisionKey?: string + 51 + readonly syncStatus: SourceParseSyncStatus + 52 + readonly syncError?: string | null + 53 +} + 54 + + 55 type SourceParseResultRepository = { + 56 readonly saveParseResultEffect: ( + 57 workspaceId: string, + +● Now add updateSyncStatusEffect to the repository interface and implementation, + and thread the new columns through save/merge/read. Let me update the + interface type. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 5 lines + 75 workspaceId: string, + 76 sourceId: string, + 77 ) => Effect.Effect>, never, D + bClient> + 78 + readonly updateSyncStatusEffect: ( + 79 + workspaceId: string, + 80 + sourceId: string, + 81 + input: UpdateSyncStatusInput, + 82 + ) => Effect.Effect + 83 } + 84 + 85 export function buildAtomicAssetUrlsMergeSql( + +● Now thread the new columns through saveParseResultEffect and + mergeParseAssetUrlsEffect insert+update sets. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 6 lines + 105 resultBlobUrl: input.resultBlobUrl, + 106 snapshotManifestUrl: input.snapshotManifestUrl, + 107 snapshotManifestKey: input.snapshotManifestKey, + 108 + revisionKey: input.revisionKey, + 109 + syncStatus: input.syncStatus, + 110 + syncError: input.syncError, + 111 assetUrls: input.assetUrlsByFilePath, + 112 }) + 113 .onConflictDoUpdate({ + ... + 116 resultBlobUrl: input.resultBlobUrl, + 117 snapshotManifestUrl: input.snapshotManifestUrl, + 118 snapshotManifestKey: input.snapshotManifestKey, + 119 + revisionKey: input.revisionKey, + 120 + syncStatus: input.syncStatus, + 121 + syncError: input.syncError, + 122 assetUrls: input.assetUrlsByFilePath, + 123 updatedAt: sql`now()`, + 124 }, + +● Now the merge effect: it also should persist revisionKey/syncStatus when + provided. Let me update its insert and update sets. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 6 lines + 146 resultBlobUrl: input.resultBlobUrl, + 147 snapshotManifestUrl: input.snapshotManifestUrl, + 148 snapshotManifestKey: input.snapshotManifestKey, + 149 + revisionKey: input.revisionKey, + 150 + syncStatus: input.syncStatus, + 151 + syncError: input.syncError, + 152 assetUrls: input.assetUrlsByFilePath, + 153 }) + 154 .onConflictDoUpdate({ + ... + 157 resultBlobUrl: input.resultBlobUrl, + 158 snapshotManifestUrl: input.snapshotManifestUrl, + 159 snapshotManifestKey: input.snapshotManifestKey, + 160 + revisionKey: input.revisionKey, + 161 + syncStatus: input.syncStatus, + 162 + syncError: input.syncError, + 163 assetUrls: buildAtomicAssetUrlsMergeSql( + 164 input.assetUrlsByFilePath, + 165 ), + +● Now update getParseResultProgressEffect to select and return the new columns, + and add updateSyncStatusEffect, plus register it in the export. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 33 lines + 187 resultBlobUrl: sourceParseResults.resultBlobUrl, + 188 snapshotManifestUrl: sourceParseResults.snapshotMa + nifestUrl, + 189 snapshotManifestKey: sourceParseResults.snapshotMa + nifestKey, + 190 + revisionKey: sourceParseResults.revisionKey, + 191 + syncStatus: sourceParseResults.syncStatus, + 192 + syncError: sourceParseResults.syncError, + 193 assetUrls: sourceParseResults.assetUrls, + 194 }) + 195 .from(sourceParseResults) + ... + 203 resultBlobUrl: progress.resultBlobUrl, + 204 snapshotManifestUrl: progress.snapshotManifestUrl, + 205 snapshotManifestKey: progress.snapshotManifestKey, + 206 + revisionKey: progress.revisionKey, + 207 + syncStatus: progress.syncStatus, + 208 + syncError: progress.syncError, + 209 assetUrlsByFilePath: progress.assetUrls, + 210 } + 211 }) + 212 + 213 +const updateSyncStatusEffect: SourceParseResultRepository["upd + +ateSyncStatusEffect"] = + 214 + (workspaceId: string, sourceId: string, input: UpdateSyncSta + +tusInput) => + 215 + Effect.gen(function* () { + 216 + const db = yield* DbClient + 217 + const source = yield* Effect.promise(() => + 218 + sourceRowRepository.findInWorkspaceWithDb(db, workspac + +eId, sourceId), + 219 + ) + 220 + if (!source) return null + 221 + + 222 + const [result] = yield* Effect.promise(() => + 223 + db + 224 + .update(sourceParseResults) + 225 + .set({ + 226 + ...(input.revisionKey !== undefined + 227 + ? { revisionKey: input.revisionKey } + 228 + : {}), + 229 + syncStatus: input.syncStatus, + 230 + syncError: input.syncError ?? null, + 231 + updatedAt: sql`now()`, + 232 + }) + 233 + .where(eq(sourceParseResults.sourceId, sourceId)) + 234 + .returning(), + 235 + ) + 236 + + 237 + return result ?? null + 238 + }) + 239 + + 240 const getParseAssetUrlsEffect: SourceParseResultRepository["ge + tParseAssetUrlsEffect"] = + 241 (workspaceId: string, sourceId: string) => + 242 Effect.gen(function* () { + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-parse + -result-repository.ts) + ⎿  Added 1 line + 270 getParseResultProgressEffect, + 271 getParseSnapshotMetadataEffect, + 272 getParseAssetUrlsEffect, + 273 + updateSyncStatusEffect, + 274 } + +● Now I need to check how repository.ts re-exports these (workflow-runtime + imports sourceRepository.saveParseResultEffect etc). Let me check + repository.ts. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -n + "ParseResult\|updateSyncStatus\|updateRevisionKey\|sourceParseResultReposi + tory\|\.\.\.\|export c…) + ⎿  4:import { sourceParseResultRepository } from "./source-parse-result-r + epository" + 17: readonly updateRevisionKeyEffect: typeof sourceRowRepository.upda + … +15 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Let me read repository.ts fully to add the new effects + (updateSyncStatusEffect, and markFailedEffect/markReadyEffect will need + failureStage support) and the interface. + + Read 1 file (ctrl+o to expand) + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/repository.t + s) + ⎿  Added 1 line + 23 readonly getParseResultProgressEffect: typeof sourceParseResu + ltRepository.getParseResultProgressEffect + 24 readonly getParseSnapshotMetadataEffect: typeof sourceParseRe + sultRepository.getParseSnapshotMetadataEffect + 25 readonly getParseAssetUrlsEffect: typeof sourceParseResultRep + ository.getParseAssetUrlsEffect + 26 + readonly updateSyncStatusEffect: typeof sourceParseResultRepo + +sitory.updateSyncStatusEffect + 27 } + 28 + 29 export const sourceRepository: SourceRepository = { + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/repository.t + s) ⎿  Added 1 line + 50 getParseSnapshotMetadataEffect: + 51 sourceParseResultRepository.getParseSnapshotMetadataEffect, + 52 getParseAssetUrlsEffect: sourceParseResultRepository.getParse + AssetUrlsEffect, + 53 + updateSyncStatusEffect: sourceParseResultRepository.updateSyn + +cStatusEffect, + 54 } + +● Now add failureStage support to markFailedEffect and clear it in + markReadyEffect in source-row-repository.ts. I need to add failureStage to the + SourceUpdate type and thread an optional param through markFailedEffect. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-row-r + epository.ts) + ⎿  Added 1 line + 26 | "sizeBytes" + 27 | "status" + 28 | "failureReason" + 29 + | "failureStage" + 30 | "knowhereJobId" + 31 | "knowhereDocumentId" + 32 | "stagedBlobPathname" + +● Now update the markFailedEffect signature in the interface and impl to accept + an optional failureStage, and clear failureStage in markReadyEffect and + markParsingEffect. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-row-r + epository.ts) + ⎿  Added 1 line + 84 sourceId: string, + 85 reason: string, + 86 requiredStatus?: string, + 87 + failureStage?: string, + 88 ) => Effect.Effect + 89 readonly clearStagedBlobEffect: ( + 90 workspaceId: string, + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-row-r + epository.ts) + ⎿  Added 2 lines + 195 knowhereJobId: jobId, + 196 knowhereDocumentId: documentId, + 197 failureReason: null, + 198 + failureStage: null, + 199 }, requiredStatus) + 200 + 201 const markReadyEffect: SourceRowRepository["markReadyEffect"] + = ( + ... + 207 status: "ready", + 208 knowhereDocumentId: documentId, + 209 failureReason: null, + 210 + failureStage: null, + 211 }, "parsing") + 212 + 213 const updateRevisionKeyEffect: SourceRowRepository["updateRevi + sionKeyEffect"] = ( + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/source-row-r + epository.ts) + ⎿  Added 2 lines + 224 sourceId: string, + 225 reason: string, + 226 requiredStatus?: string, + 227 + failureStage?: string, + 228 ) => + 229 updateInWorkspaceEffect(workspaceId, sourceId, { + 230 status: "failed", + 231 failureReason: reason, + 232 + failureStage: failureStage ?? null, + 233 }, requiredStatus) + 234 + 235 const clearStagedBlobEffect: SourceRowRepository["clearStagedB + lobEffect"] = ( + +● Now update the workflow-runtime.ts facade: markFailed should accept + failureStage, and add updateSyncStatus. Let me update the type and impls. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 1 line + 38 sourceId: string, + 39 reason: string, + 40 requiredStatus?: string, + 41 + failureStage?: string, + 42 ) => Promise + 43 } + 44 + +● Now add updateSyncStatus to the SourceWorkflowRuntime type and implement it. + Let me add to the type block after saveParseResult/mergeParseAssetUrls. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 5 lines + 106 sourceId: string, + 107 input: SaveSourceParseResultInput, + 108 ) => Promise + 109 + readonly updateSyncStatus: ( + 110 + workspaceId: string, + 111 + sourceId: string, + 112 + input: UpdateSyncStatusInput, + 113 + ) => Promise + 114 readonly softDelete: ( + 115 workspaceId: string, + 116 sourceId: string, + +● Now add the UpdateSyncStatusInput type alias near the other Parameters<...> + aliases, update markFailed impl to pass failureStage, add updateSyncStatus + impl, and register it in the export object. + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 4 lines + 17 typeof sourceRepository.saveParseResultEffect + 18 >[2] + 19 + 20 +type UpdateSyncStatusInput = Parameters< + 21 + typeof sourceRepository.updateSyncStatusEffect + 22 +>[2] + 23 + + 24 type UpsertMaterializedDemoSourceInput = Parameters< + 25 typeof sourceRepository.upsertMaterializedDemoSourceEffect + 26 >[1] + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 8 lines, removed 1 line + 210 sourceId: string, + 211 reason: string, + 212 requiredStatus?: string, + 213 + failureStage?: string, + 214 ) => + 215 databaseRuntime.runPromise( + 215 - sourceRepository.markFailedEffect(workspaceId, sourceId, r + -eason, requiredStatus), + 216 + sourceRepository.markFailedEffect( + 217 + workspaceId, + 218 + sourceId, + 219 + reason, + 220 + requiredStatus, + 221 + failureStage, + 222 + ), + 223 ) + 224 + 225 const clearStagedBlob: SourceWorkflowRuntime["clearStagedBlob" + ] = ( + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 9 lines + 262 sourceRepository.mergeParseAssetUrlsEffect(workspaceId, so + urceId, input), + 263 ) + 264 + 265 +const updateSyncStatus: SourceWorkflowRuntime["updateSyncStatu + +s"] = ( + 266 + workspaceId: string, + 267 + sourceId: string, + 268 + input: UpdateSyncStatusInput, + 269 +) => + 270 + databaseRuntime.runPromise( + 271 + sourceRepository.updateSyncStatusEffect(workspaceId, sourc + +eId, input), + 272 + ) + 273 + + 274 const getParseResultProgress: SourceWorkflowRuntime["getParseR + esultProgress"] = + 275 (workspaceId: string, sourceId: string) => + 276 databaseRuntime.runPromise( + +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/workflow-run + time.ts) + ⎿  Added 1 line + 342 mergeParseAssetUrls, + 343 saveParseResult, + 344 softDelete, + 345 + updateSyncStatus, + 346 upsertMaterializedDemoSource, + 347 } + +● Now typecheck to catch anything I missed in Task #7. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -20) + ⎿   Type 'undefined' is not assignable to type 'string | null'. + src/domains/sources/route-service.test.ts(18,7): error TS2741: Propert + y 'failureStage' is missing in type '{ id: string; workspaceId: string + … +61 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● The new non-null failureStage column on the Source type breaks test fixtures + that build full Source objects. Test fixtures likely use a shared builder. Let + me find how test Source objects are constructed — there's probably a factory + to update in one place. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -rln + "failureReason: null\|failureReason:" src --include=*.ts | grep -i + "test\|fixture\|factory\|mo…) + ⎿  (eval):1: no matches found: --include=*.ts + ---any shared source factory?--- + (eval):1: no matches found: --include=*.ts + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -rln + "buildSource\|makeSource\|createSource\|sourceFixture\|baseSource\|fakeSou + rce\|SourceFactory"…) + ⎿  src/app/api/sources/route.ts + src/app/api/sources/[sourceId]/chunks/route.ts + src/app/api/demo-sources/materialize/route.test.ts + … +53 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 21s (ctrl+o to expand) + +● The type is legitimately correct (real column), so the fixtures need updating. + A few already show failureStage? optional — meaning they spread from a shared + base. Let me find that base fixture and see the pattern. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== + source-reconcile-workflow.test.ts 1-40 ===" && sed -n '1,45p' + src/domains/sources/source-recon…) + ⎿  === source-reconcile-workflow.test.ts 1-40 === + import { describe, expect, it, vi } from "vitest" + import type { JobResult } from "@ontos-ai/knowhere-sdk" + … +43 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● These are per-file inline Source fixtures, each listing all columns. I need to + add failureStage: null to each. Rather than edit each manually, let me find + every occurrence of the fixture shape (the line failureReason: null, + immediately followed by knowhereJobId) in test/source files and insert + failureStage: null, after it. Let me first identify exactly which files + construct full Source objects (the failing set plus others), by searching for + the adjacent-line pattern. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -rlzP + "failureReason:[^\n]*\n\s*knowhereJobId:" src | tr '\0' '\n') ⎿  src/app/api/sources/[sourceId]/chunks/route.test.ts + src/app/api/sources/[sourceId]/route.test.ts + src/app/api/sources/route.test.ts + … +23 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Two of these are non-test source files (demo-source-repository.ts, + source-row-repository.ts) — those are DB inserts where the column is optional + (nullable, no default needed), so I should NOT add failureStage there. The + rest are test fixtures constructing full Source objects. Let me verify the two + source files don't need it (insert values, not full Source literals), then do + a targeted insert across the test files only. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== + demo-source-repository.ts ===" && grep -nP "failureReason:" + src/domains/sources/demo-source-re…) + ⎿  === demo-source-repository.ts === + 100: failureReason: null, + 113: failureReason: null, + … +7 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 30s (ctrl+o to expand) + +● These are full Source/NewSource literals (the two non-test source files are DB + insert/update values where the nullable column needs nothing). The Source + type now requires failureStage, so every full-source test fixture needs + failureStage: null. That's ~24 test files with the identical failureReason: … + ⏎ knowhereJobId: shape — a scripted insert is the right tool here rather than + dozens of manual edits. I'll insert failureStage: null, (matching indentation) + after each failureReason: line that's immediately followed by knowhereJobId:, + across test files only. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && files=$(grep + -rlzP "failureReason:[^\n]*\n\s*knowhereJobId:" src | tr '\0' '\n' | grep + "\.test\.ts$") &&…) + ⎿  src/app/api/sources/[sourceId]/chunks/route.test.ts + src/app/api/sources/[sourceId]/route.test.ts + src/app/api/sources/route.test.ts + … +48 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + + Thought for 27s (ctrl+o to expand) + +● The verify shows the insertion did NOT actually apply (line 26-27 unchanged), + and the bracketed [sourceId] paths errored due to shell globbing. Let me redo + this correctly — test on one file first to confirm the perl regex works. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && perl -0pi -e + 's/(failureReason:[^\n]*,\n)([ \t]*)(knowhereJobId:)/$1${2}failureStage: + null,\n$2$3/g' src…) + ⎿  26: failureReason: null, + 27- failureStage: null, + 28- knowhereJobId: "job_1", + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● The regex works when run per-file (the earlier for loop failed because the + file list from grep -rlz retained NUL/globbing issues). Now apply to the + remaining files using a null-safe loop. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && grep -rlzP + "failureReason:[^\n]*\n\s*knowhereJobId:" src | while IFS= read -r -d '' + f; do case "$f" in *…) + ⎿  DONE + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● source-reconcile-workflow.test.ts was already processed once — the loop would + double-insert. Let me check it didn't get a duplicate, then typecheck. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== dup + check (any file with 2 consecutive failureStage lines) ===" && grep -rzoP + "failureStage: n…) + ⎿  === dup check (any file with 2 consecutive failureStage lines) === + === reconcile-workflow fixture === + 26: failureReason: null, + … +64 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Progress but several remain. Two distinct cases: + 1. workflow-runtime.test.ts, route-service.test.ts — full Source literals + where knowhereJobId comes BEFORE failureReason (different field order), so my + regex missed them. + 2. retry.test.ts:118, upload.test.ts:18, view.test.ts:7, + initial-state.test.ts:642 — these show failureStage?: undefined meaning they + spread a base fixture that has failureStage optional, OR use Partial overrides + producing undefined. Let me look at each failing spot. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== + workflow-runtime.test.ts 20-45 ===" && sed -n '20,45p' + src/domains/sources/workflow-runtime.te…) + ⎿  === workflow-runtime.test.ts 20-45 === + repository.markSourceFailed("workspace_1", "source_1", "failed") + , + … +54 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● These two have a conditional failureReason: status === ... ? ... : null, + (workflow-runtime) — the [^\n]*,\n matched but the ternary contains no issue; + actually route-service.test.ts line 25 has plain failureReason: null, then + knowhereJobId: — my regex should have caught it. It seems these files weren't + in my processed list or the earlier run's file list differed. Let me just add + failureStage: null, after the failureReason line in these two specific files. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && for f in + src/domains/sources/workflow-runtime.test.ts + src/domains/sources/route-service.test.ts; do perl…) + ⎿  === workflow-runtime === + 33: failureReason: status === "failed" ? "failed" : null, + 34- failureStage: null, + … +4 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Now the other four (retry.test.ts:118, upload.test.ts:18, view.test.ts:7, + initial-state.test.ts:642) show failureStage?: undefined, meaning they spread + a base object that itself now has failureStage but the override object passes + undefined, OR they build via a helper with Partial. Let me look at these. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== + retry.test.ts 100-125 ===" && sed -n '100,125p' + src/domains/sources/retry.test.ts && echo "===…) + ⎿  === retry.test.ts 100-125 === + } + … +55 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● These makeSource helpers use ...overrides — the failureStage?: undefined in + the error is because the fixture object still lacks failureStage before the + spread. My regex requires failureReason:...,\nknowhereJobId:. In + view.test.ts the pattern matches (line 14→15), so why did it fail? The earlier + bulk loop reprocessed the already-done file and possibly the file list from + grep -rlz didn't include these because of how I piped. Let me just re-run the + per-file perl on the four remaining files directly. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && for f in + src/domains/sources/retry.test.ts src/domains/sources/upload.test.ts + src/domains/sources/view.t…) + ⎿  verify: + -- retry -- + 2 + … +5 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -20) + ⎿   Types of property 'failureStage' are incompatible. + Type 'string | null | undefined' is not assignable to type 'string + | null'. + … +47 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Fewer errors. counts.test.ts and reconcile.test.ts still need failureStage (my + regex likely missed them — perhaps failureReason uses a different value with + a trailing comment or the following line isn't knowhereJobId). And now a + SourceParseResult fixture in remote-parsed-snapshot-sync.test.ts:302 needs + revisionKey/syncStatus/syncError. Let me inspect these three spots. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && echo "=== + counts.test.ts 5-15 ===" && sed -n '5,15p' + src/domains/sources/counts.test.ts && echo "=== rec…) + ⎿  === counts.test.ts 5-15 === + + import type { Source } from "@/infrastructure/db/schema" + … +45 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● counts.test.ts and reconcile.test.ts have the standard shape, so my regex + should have caught them — they must not have been in the grep list when I ran + the loop (glob issue). Let me apply per-file, and add the three + SourceParseResult columns to the parse-result fixture. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && for f in + src/domains/sources/counts.test.ts src/domains/sources/reconcile.test.ts; + do perl -0pi -e 's/(f…) + ⎿  src/domains/sources/reconcile.test.ts:1 + src/domains/sources/counts.test.ts:1 + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Now the SourceParseResult fixture. Let me add revisionKey/syncStatus/syncError + to it after snapshotManifestKey. + + Read 1 file (ctrl+o to expand) +● Update(~/github.com/ontosAI/knowhere-notebook/src/domains/sources/remote-parse + d-snapshot-sync.test.ts) + ⎿  Added 3 lines + 308 "https://blob.example/workspaces/workspace_1/sources/000 + 00000-0000-0000-0000-000000000009/parsed-result/manifest/curre + nt.json", + 309 snapshotManifestKey: + 310 "workspaces/workspace_1/sources/00000000-0000-0000-0000- + 000000000009/parsed-result/manifest/current.json", + 311 + revisionKey: null, + 312 + syncStatus: null, + 313 + syncError: null, + 314 assetUrls: {}, + 315 createdAt: new Date("2026-05-10T00:00:00.000Z"), + 316 updatedAt: new Date("2026-05-10T00:00:00.000Z"), + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -20) + ⎿   Types of property 'failureStage' are incompatible. + Type 'string | null | undefined' is not assignable to type 'string + | null'. + … +46 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Still a handful: chat/service.test.ts, chunks/index.test.ts, + chunks/server.test.ts, and remote-parsed-snapshot-sync.test.ts:277. My bulk + loop clearly didn't touch several files consistently. Let me just check which + test files with a full Source literal still lack failureStage and fix them all + in one reliable pass. The safest approach: for every .test.ts under src that + has the failureReason:...\n...knowhereJobId: pattern but NOT already + failureStage, apply the perl. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && changed=""; while + IFS= read -r -d '' f; do if ! grep -qzP + "failureReason:[^\n]*,\n\s*failureStage:" "$f"…) + ⎿  patched: + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● The while-loop input got mangled. Let me take a simpler, deterministic path: + iterate a plain explicit list of the four remaining failing files. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && for f in + "src/domains/chat/service.test.ts" "src/domains/chunks/index.test.ts" + "src/domains/chunks/serve…) + ⎿  src/domains/chat/service.test.ts -> 1 + src/domains/chunks/index.test.ts -> 1 + src/domains/chunks/server.test.ts -> 1 + src/domains/sources/remote-parsed-snapshot-sync.test.ts -> 1 + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -20) + ⎿   Types of property 'failureStage' are incompatible. + Type 'string | null | undefined' is not assignable to type 'string + | null'. + … +46 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● More chat test files. The bulk loop is clearly unreliable in this shell. Let + me get the definitive list of ALL remaining offenders from tsc directly and + fix exactly those. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | grep -oP "^src/[^(]+\.test\.ts" | sort -u) + ⎿  src/app/api/demo-sources/materialize/route.test.ts + src/app/api/sources/route.test.ts + src/domains/chat/index.test.ts + … +4 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Seven files. Let me apply the perl to each explicitly (avoiding the loop-input + issue), then check whether any have a different field ordering. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && perl -0pi -e + 's/(failureReason:[^\n]*,\n)([ \t]*)(knowhereJobId:)/$1${2}failureStage: + null,\n$2$3/g' "sr…) + ⎿  src/app/api/demo-sources/materialize/route.test.ts -> 1 + src/app/api/sources/route.test.ts -> 1 + src/domains/chat/index.test.ts -> 1 + … +4 lines (ctrl+o to expand) + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● All patched. Some of these files (e.g. route.test.ts had failureStage reported + missing at multiple lines) may have more than one Source literal — the count + of 1 suggests only one matched. Let me typecheck to see if any remain. + +● Bash(cd /home/suguan/github.com/ontosAI/knowhere-notebook && pnpm typecheck + 2>&1 | tail -25) + ⎿  > knowhere-notebook@0.1.0 typecheck /home/suguan/github.com/ontosAI/kn + owhere-notebook + > tsc --noEmit + ⎿  Shell cwd was reset to + /home/suguan/github.com/ontosAI/knowhere-node-sdk + +● Please run /login · API Error: 403 预扣费额度失败, 用户剩余额度: + ¥0.155160, 需要预扣费额度: ¥0.495542 (request id: + 202607041425149789930498268d9d69rP7bjXn) + +✻ Crunched for 1h 3m 4s \ No newline at end of file diff --git a/docs/new-design.md b/docs/new-design.md new file mode 100644 index 0000000..63bbf1e --- /dev/null +++ b/docs/new-design.md @@ -0,0 +1,118 @@ + # SDK Parsed Storage Redesign + + ## Summary + Move parsed chunk read/sync behavior into the SDK so Notebook, CLI, and MCP share one model. Callers provide a parsed-document storage adapter; SDK reads storage first, falls back to Knowhere remote storage + transparently, and schedules non-blocking storage sync on read misses. Parse flow is different: after parsing succeeds, SDK must sync to provided storage before the caller treats the document as complete/ + ready. + + ## Public SDK Shape + - Keep `knowledge.readChunks`, `knowledge.grepChunks`, and `knowledge.getDocumentOutline` as the only public read methods; do not add public `readChunkPage`. + - Extend `readChunks` with paged display params: + - `page`, `pageSize`, `chunkType`, `assetUrlPolicy?: "none" | "durable"`. + - Paged mode must not combine with scan-heavy filters like `sectionPath`, `startChunk`, `endChunk`, or `chunkId`. + - Add storage configuration on the SDK/knowledge client, e.g. `knowledge.withParsedStorage({ storage, scheduler, limits })`. + - Replace new usage of `cacheJobResult` with clearer methods: + - `importJobResult` means explicit local import. + - `syncParsedDocument` means write a parsed snapshot/assets to the configured storage. + - Keep `cacheJobResult` only as deprecated compatibility alias. + + ## SDK Implementation + - Define a `ParsedDocumentStorage` adapter for parsed snapshots, not just assets: + - read/write manifest + - read/write chunk pages + - read/write durable assets + - get asset URL + - read/write sync progress metadata + - Use `revisionKey = jobResultId ?? jobId` for freshness. Storage is valid only when its manifest revision matches the current remote revision. + - Read flow: + - resolve one document reference: `documentId`, `localDocumentId`, or `jobId` + - try configured storage first + - if missing/stale, read remote via `documents.listChunks` + - return the requested bounded result immediately + - schedule background sync into configured storage + - Parse flow: + - parse completes on Knowhere + - SDK writes chunks/assets/manifest to configured storage + - only then returns completion to the caller + - Background sync must be resumable and Vercel-safe: + - cursor/progress includes `documentId`, `revisionKey`, next chunk page, next asset index + - bounded by page count, asset count, and deadline per step + - final manifest is written only after all required pages/assets are present + - partial snapshots are ignored by reads until manifest commit + + ## Grep And Outline + - `grepChunks` supports only one specific document, never broad workspace/search scope. + - Broad search continues to use Knowhere `retrieval.query`. + - Remote grep streams `documents.listChunks(..., includeAssetUrls: false)` page by page, applies literal/regex matching SDK-side, and stops at `maxResults`, deadline, max pages, or end of document. + - Grep response should include `truncated` and a continuation cursor when it stops early. + - `getDocumentOutline` reads full outline from storage when available. On remote fallback, build a lightweight section outline from chunk `sectionPath` within configured page/deadline limits and mark it + truncated if incomplete. + + ## Notebook Changes + - Notebook uses SDK directly for chunk read/display/chat tooling; do not route this through MCP. + - Notebook provides a Vercel Blob `ParsedDocumentStorage` adapter and an Upstash/Vercel-safe scheduler. + - New Notebook parse/upload flow remains hidden behind `parsing`: + - Knowhere parse success plus Blob sync success -> mark source `ready` + - Knowhere parse failure -> mark source `failed` + - Blob sync failure after parse success -> mark source `failed` with `failureStage: "storage_sync"` + - Retry behavior: + - parse failure retries parse/upload + - `storage_sync` failure resumes sync from existing `documentId/jobId/revisionKey`, no reparse + - Existing remote Knowhere documents are readable immediately: + - SDK reads remote if Blob snapshot is missing/stale + - Blob sync is non-blocking + - missing Blob data must not produce zero chunks or non-ready behavior + - User-visible asset URLs must be durable Notebook Blob URLs: + - `readChunks` display calls use `assetUrlPolicy: "durable"` + - only visible/cited assets are hardened synchronously + - grep uses no asset URLs + - if hardening fails, return text and omit the asset URL rather than exposing a presigned Knowhere URL + + ## CLI And MCP + - CLI/MCP configure disk `ParsedDocumentStorage` explicitly. + - CLI removes duplicate `documentId -> listChunks -> importJobResult` logic and delegates reads to SDK. + - MCP continues delegating to SDK `readChunks`, `grepChunks`, and `getDocumentOutline`; update descriptions/tests for storage-first remote-fallback behavior. + - Remote reads must not silently write local disk unless disk storage was explicitly configured. + + ## Database And Compatibility + - Add a new Notebook migration only; do not edit existing migrations. + - Store snapshot/cache metadata in `source_parse_results`, including at least: + - `revision_key` + - `sync_status` or equivalent + - `failure_stage` / sync failure detail where appropriate + - progress cursor for resumable sync if not stored elsewhere + - Keep existing snapshot manifest shape compatible where possible, but treat committed manifest as the only readable snapshot marker. + + ## Test Plan + - SDK: + - storage hit reads without remote calls + - stale/missing storage falls back to remote + - remote reads schedule non-blocking sync + - parse writes configured storage before completion + - paged `readChunks` behavior and invalid param combinations + - single-document bounded remote grep with truncation cursor + - remote outline fallback and storage outline path + - no implicit disk writes without configured disk storage + - Notebook: + - parse remains `parsing` until Blob sync completes + - parse success plus Blob sync failure marks `failed` with `storage_sync` + - retry of `storage_sync` resumes sync without reparsing + - existing remote document with no Blob snapshot returns chunks from remote + - missing/stale Blob never returns zero chunks for ready remote docs + - displayed/chat citation assets are Blob URLs or omitted, never Knowhere presigned URLs + - CLI/MCP: + - remote document read/grep/outline works through SDK + - disk storage is explicit + - duplicate remote-to-local import path is removed + - Verification: + - run targeted SDK storage/read tests + - run Notebook parse/sync/chunk/chat tests + - run CLI read/inspect tests and MCP tests + - run feasible typecheck/lint/test suites per repo + - smoke test local Knowhere + Notebook: parse PDF, wait until ready, inspect Blob manifest/pages/assets, open chunks, chat with citation, verify durable Blob URLs + + ## Assumptions + - `documents.listChunks` is the authoritative remote parsed read API for published parsed v2 documents. + - `jobResultId ?? jobId` is sufficient as the revision key. + - Full job-result ZIP loading is allowed for explicit import/sync workflows, but not for Notebook request-time reads. + - Notebook request-time work is bounded to the requested page/window plus visible/cited asset hardening. diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index fe58aab..fa78008 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -230,8 +230,8 @@ describe('Knowhere Client', () => { const writeObject = vi.fn( (input: KnowhereAssetStorageObject): Promise => Promise.resolve({ - key: input.key, - url: `https://blob.example/${input.key}`, + key: input.key, + url: `https://blob.example/${input.key}`, }), ); const adapter: KnowhereAssetStorageAdapter = { diff --git a/src/storage/__tests__/asset-storage.test.ts b/src/storage/__tests__/asset-storage.test.ts index b84818a..0eca3e5 100644 --- a/src/storage/__tests__/asset-storage.test.ts +++ b/src/storage/__tests__/asset-storage.test.ts @@ -55,7 +55,9 @@ describe('storeParseResultAssets', () => { 'tables/revenue.html': 'https://blob.example/workspaces/workspace-1/sources/source-1/parsed-result/tables/revenue.html', }); - expect(writes.filter((write) => write.contentType !== 'application/json; charset=utf-8')).toEqual([ + expect( + writes.filter((write) => write.contentType !== 'application/json; charset=utf-8'), + ).toEqual([ { key: 'workspaces/workspace-1/sources/source-1/parsed-result/images/chart.png', bodyText: 'chart-image', @@ -230,8 +232,8 @@ describe('storeParseResultAssets', () => { const writeObject = vi.fn( (input: KnowhereAssetStorageObject): Promise => Promise.resolve({ - key: input.key, - url: `https://blob.example/${input.key}`, + key: input.key, + url: `https://blob.example/${input.key}`, }), ); const adapter: KnowhereAssetStorageAdapter = { @@ -255,7 +257,9 @@ describe('storeParseResultAssets', () => { expect(stored.result.imageChunks[0]?.assetUrl).toBe( 'https://blob.example/existing/parsed-result/images/chart.png', ); - expect(stored.snapshot?.manifestUrl).toBe('https://blob.example/parsed-result/manifest/current.json'); + expect(stored.snapshot?.manifestUrl).toBe( + 'https://blob.example/parsed-result/manifest/current.json', + ); }); it('rejects unsafe storage key prefixes and ignores unsafe asset refs', async () => { @@ -275,8 +279,8 @@ describe('storeParseResultAssets', () => { const writeObject = vi.fn( (input: KnowhereAssetStorageObject): Promise => Promise.resolve({ - key: input.key, - url: `https://blob.example/${input.key}`, + key: input.key, + url: `https://blob.example/${input.key}`, }), ); const adapter: KnowhereAssetStorageAdapter = { @@ -353,7 +357,10 @@ async function createParseResultWithAssets(): Promise { ); } -function createParseResult(chunks: readonly Chunk[], rawZip: Buffer = Buffer.alloc(0)): ParseResult { +function createParseResult( + chunks: readonly Chunk[], + rawZip: Buffer = Buffer.alloc(0), +): ParseResult { return { manifest: { version: '2.0',