From 10ee98181c53c8f8de24b6b03960d445f624ec7a Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Tue, 16 Jun 2026 14:58:25 -0400 Subject: [PATCH 1/9] feat: support google discovery backends --- .changeset/google-discovery-media.md | 8 + CONTEXT.md | 13 + apps/docs/src/content/docs/capabilities.mdx | 14 +- .../content/docs/reference/caplet-files.mdx | 39 +- .../src/content/docs/reference/config.mdx | 29 +- .../docs/src/content/docs/troubleshooting.mdx | 26 + .../public/caplet-frontmatter.schema.json | 259 ++ apps/landing/public/config.schema.json | 431 ++++ ...-media-artifacts-for-non-inline-results.md | 20 + docs/architecture.md | 7 +- ...le-discovery-api-backend-implementation.md | 2151 +++++++++++++++++ ...2026-06-16-google-discovery-api-backend.md | 229 ++ packages/core/src/auth.ts | 54 +- packages/core/src/caplet-files-bundle.ts | 87 +- packages/core/src/caplet-sets.ts | 5 + packages/core/src/caplet-source/parse.ts | 4 + packages/core/src/cli.ts | 47 + packages/core/src/cli/add.ts | 36 +- packages/core/src/cli/auth.ts | 45 +- packages/core/src/cli/commands.ts | 2 +- packages/core/src/cli/completion-discovery.ts | 12 + packages/core/src/cli/doctor.ts | 2 + packages/core/src/cli/inspection.ts | 1 + packages/core/src/cli/setup-caplet.ts | 1 + packages/core/src/cli/setup.ts | 4 + packages/core/src/cloud/runtime-adapter.ts | 1 + packages/core/src/config-runtime.ts | 62 + packages/core/src/config.ts | 239 +- packages/core/src/config/paths.ts | 10 + packages/core/src/engine.ts | 75 +- packages/core/src/google-discovery/index.ts | 5 + packages/core/src/google-discovery/manager.ts | 607 +++++ .../core/src/google-discovery/operations.ts | 279 +++ packages/core/src/google-discovery/request.ts | 169 ++ packages/core/src/google-discovery/schema.ts | 77 + packages/core/src/google-discovery/types.ts | 61 + packages/core/src/http-actions.ts | 45 +- packages/core/src/http/response.ts | 188 ++ packages/core/src/media/artifacts.ts | 319 +++ packages/core/src/media/index.ts | 2 + packages/core/src/media/input.ts | 131 + packages/core/src/native/service.ts | 17 +- packages/core/src/openapi.ts | 26 +- packages/core/src/registry.ts | 24 + packages/core/src/remote-control/dispatch.ts | 32 +- packages/core/src/runtime-plan/planner.ts | 7 +- packages/core/src/runtime.ts | 4 + packages/core/src/tools.ts | 67 +- packages/core/test/auth.test.ts | 109 + packages/core/test/caplet-sets.test.ts | 42 + packages/core/test/caplet-source.test.ts | 47 +- packages/core/test/cli-completion.test.ts | 1 + packages/core/test/cli-remote.test.ts | 52 + packages/core/test/cli.test.ts | 183 +- packages/core/test/config.test.ts | 168 ++ packages/core/test/engine.test.ts | 18 + packages/core/test/exposure-discovery.test.ts | 1 + .../google-discovery/drive.discovery.json | 132 + packages/core/test/google-discovery.test.ts | 624 +++++ packages/core/test/http-actions.test.ts | 164 ++ packages/core/test/media-artifacts.test.ts | 249 ++ packages/core/test/native.test.ts | 70 + packages/core/test/openapi.test.ts | 79 +- packages/core/test/tools.test.ts | 27 + schemas/caplet.schema.json | 259 ++ schemas/caplets-config.schema.json | 431 ++++ 66 files changed, 8472 insertions(+), 157 deletions(-) create mode 100644 .changeset/google-discovery-media.md create mode 100644 CONTEXT.md create mode 100644 docs/adr/0002-media-artifacts-for-non-inline-results.md create mode 100644 docs/plans/2026-06-16-google-discovery-api-backend-implementation.md create mode 100644 docs/specs/2026-06-16-google-discovery-api-backend.md create mode 100644 packages/core/src/google-discovery/index.ts create mode 100644 packages/core/src/google-discovery/manager.ts create mode 100644 packages/core/src/google-discovery/operations.ts create mode 100644 packages/core/src/google-discovery/request.ts create mode 100644 packages/core/src/google-discovery/schema.ts create mode 100644 packages/core/src/google-discovery/types.ts create mode 100644 packages/core/src/http/response.ts create mode 100644 packages/core/src/media/artifacts.ts create mode 100644 packages/core/src/media/index.ts create mode 100644 packages/core/src/media/input.ts create mode 100644 packages/core/test/fixtures/google-discovery/drive.discovery.json create mode 100644 packages/core/test/google-discovery.test.ts create mode 100644 packages/core/test/media-artifacts.test.ts diff --git a/.changeset/google-discovery-media.md b/.changeset/google-discovery-media.md new file mode 100644 index 00000000..77c59d31 --- /dev/null +++ b/.changeset/google-discovery-media.md @@ -0,0 +1,8 @@ +--- +"@caplets/core": minor +"caplets": minor +"@caplets/opencode": minor +"@caplets/pi": minor +--- + +Add Google Discovery API Caplets with inferred OAuth scopes, operation filters, media upload/download handling, and shared HTTP-like media artifacts. diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 00000000..f88b2966 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,13 @@ +# Caplets + +Caplets is a capability gateway for coding agents. This glossary names the product concepts used when describing Caplet configuration and runtime behavior. + +## Language + +**Google Discovery API backend**: +A Caplets backend family for Google APIs whose machine-readable contract is a Google Discovery document rather than an OpenAPI document. +_Avoid_: Discovery backend, discovery document backend, OpenAPI-backed Google API + +**Media artifact**: +A file-backed Caplets result for response content that should not be returned inline, such as binary media or oversized textual content. +_Avoid_: Inline blob, base64 result, download blob diff --git a/apps/docs/src/content/docs/capabilities.mdx b/apps/docs/src/content/docs/capabilities.mdx index 92ac61a6..d68d60bf 100644 --- a/apps/docs/src/content/docs/capabilities.mdx +++ b/apps/docs/src/content/docs/capabilities.mdx @@ -3,8 +3,9 @@ title: Capabilities description: Add capability sources for agents. --- -Caplets can wrap MCP servers, OpenAPI specs, GraphQL endpoints, simple HTTP APIs, curated -CLI commands, and shared Markdown Caplet files such as `CAPLET.md`. +Caplets can wrap MCP servers, OpenAPI specs, Google Discovery APIs, GraphQL endpoints, +simple HTTP APIs, curated CLI commands, and shared Markdown Caplet files such as +`CAPLET.md`. ## Start with OSV @@ -30,11 +31,16 @@ Examples: ```sh caplets add mcp docs --command npx --arg -y --arg @upstash/context7-mcp caplets add openapi users --spec ./openapi.json --base-url https://api.example.com +caplets add google-discovery drive --discovery-url https://www.googleapis.com/discovery/v1/apis/drive/v3/rest caplets add graphql catalog --endpoint-url https://api.example.com/graphql --schema ./schema.graphql caplets add http status-api --base-url https://api.example.com --action get_status:GET:/status/{service} caplets add cli repo-tools --repo . --include git,gh,package ``` +Google Discovery Caplets use Google's Discovery documents directly. They can infer request +base URLs and OAuth scopes from the final exposed operation set, so narrow broad APIs with +`includeOperations` and `excludeOperations` when a capability only needs a subset. + Inspect from the CLI: ```sh @@ -50,5 +56,9 @@ caplets call-tool osv query_package_version --args '{"ecosystem":"npm","name":"r Good Caplets are narrow and named by the job they help an agent do. Prefer a few focused capabilities over one broad catch-all backend. +HTTP-like Caplets return small JSON and text inline. Binary downloads, Google media +downloads, and oversized responses are returned as media artifact metadata with local paths +or artifact references that agents can pass into later upload calls. + Use [Configuration](/configuration/) when a capability should be shared through `.caplets/config.json`. diff --git a/apps/docs/src/content/docs/reference/caplet-files.mdx b/apps/docs/src/content/docs/reference/caplet-files.mdx index 8fcdf537..64bd09b3 100644 --- a/apps/docs/src/content/docs/reference/caplet-files.mdx +++ b/apps/docs/src/content/docs/reference/caplet-files.mdx @@ -58,25 +58,26 @@ Use this Caplet when an agent needs the current repository's local test signal. ## Top-level fields -| Field | Status | Type | Description | -| ----------------- | -------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | -| `$schema` | Optional | string | Optional JSON Schema URL for editor validation. | -| `name` | Required | string | Human-readable Caplet display name. | -| `description` | Required | string | Compact capability description shown before the full Caplet card is disclosed. | -| `tags` | Optional | array | Optional tags for grouping or searching Caplets. | -| `exposure` | Optional | "direct" \| "progressive" \| "code_mode" \| "direct_and_code_mode" \| "progressive_and_code_mode" | How this Caplet is exposed to agents. | -| `shadowing` | Optional | "forbid" \| "allow" | Whether attached local Caplets may shadow this remote Caplet ID. | -| `useWhen` | Optional | string | When agents should prefer this Caplet or configured action. | -| `avoidWhen` | Optional | string | When agents should avoid this Caplet or configured action. | -| `setup` | Optional | object | Optional explicit setup and verification metadata for this Caplet. | -| `projectBinding` | Optional | object | Project Binding requirements for Caplets that need an attached project. | -| `runtime` | Optional | object | Runtime feature and resource requirements for hosted execution. | -| `mcpServer` | Optional | object | MCP server backend configuration for this Caplet. | -| `openapiEndpoint` | Optional | object | OpenAPI endpoint backend configuration for this Caplet. | -| `graphqlEndpoint` | Optional | object | GraphQL endpoint backend configuration for this Caplet. | -| `httpApi` | Optional | object | HTTP API backend configuration for this Caplet. | -| `cliTools` | Optional | object | CLI tools backend configuration for this Caplet. | -| `capletSet` | Optional | object | Nested Caplet collection backend configuration for this Caplet. | +| Field | Status | Type | Description | +| -------------------- | -------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| `$schema` | Optional | string | Optional JSON Schema URL for editor validation. | +| `name` | Required | string | Human-readable Caplet display name. | +| `description` | Required | string | Compact capability description shown before the full Caplet card is disclosed. | +| `tags` | Optional | array | Optional tags for grouping or searching Caplets. | +| `exposure` | Optional | "direct" \| "progressive" \| "code_mode" \| "direct_and_code_mode" \| "progressive_and_code_mode" | How this Caplet is exposed to agents. | +| `shadowing` | Optional | "forbid" \| "allow" | Whether attached local Caplets may shadow this remote Caplet ID. | +| `useWhen` | Optional | string | When agents should prefer this Caplet or configured action. | +| `avoidWhen` | Optional | string | When agents should avoid this Caplet or configured action. | +| `setup` | Optional | object | Optional explicit setup and verification metadata for this Caplet. | +| `projectBinding` | Optional | object | Project Binding requirements for Caplets that need an attached project. | +| `runtime` | Optional | object | Runtime feature and resource requirements for hosted execution. | +| `mcpServer` | Optional | object | MCP server backend configuration for this Caplet. | +| `openapiEndpoint` | Optional | object | OpenAPI endpoint backend configuration for this Caplet. | +| `googleDiscoveryApi` | Optional | object | Google Discovery API backend configuration for this Caplet. | +| `graphqlEndpoint` | Optional | object | GraphQL endpoint backend configuration for this Caplet. | +| `httpApi` | Optional | object | HTTP API backend configuration for this Caplet. | +| `cliTools` | Optional | object | CLI tools backend configuration for this Caplet. | +| `capletSet` | Optional | object | Nested Caplet collection backend configuration for this Caplet. | ## Major sections diff --git a/apps/docs/src/content/docs/reference/config.mdx b/apps/docs/src/content/docs/reference/config.mdx index afa40845..c486a94c 100644 --- a/apps/docs/src/content/docs/reference/config.mdx +++ b/apps/docs/src/content/docs/reference/config.mdx @@ -62,20 +62,21 @@ Public OpenAPI endpoint: ## Top-level fields -| Field | Status | Type | Description | -| -------------------- | -------- | ------- | ------------------------------------------------------ | -| `$schema` | Optional | string | Optional JSON Schema URL for editor validation. | -| `version` | Optional | number | Caplets config schema version. | -| `defaultSearchLimit` | Optional | integer | Default maximum number of same-server search results. | -| `maxSearchLimit` | Optional | integer | Maximum accepted search_tools limit. | -| `completion` | Optional | object | Shell completion discovery timeout and cache settings. | -| `options` | Optional | object | Global Caplets runtime options. | -| `mcpServers` | Optional | object | Downstream MCP servers keyed by stable server ID. | -| `openapiEndpoints` | Optional | object | OpenAPI endpoints keyed by stable Caplet ID. | -| `graphqlEndpoints` | Optional | object | GraphQL endpoints keyed by stable Caplet ID. | -| `httpApis` | Optional | object | HTTP APIs keyed by stable Caplet ID. | -| `cliTools` | Optional | object | CLI tools keyed by stable Caplet ID. | -| `capletSets` | Optional | object | Nested Caplet collections keyed by stable Caplet ID. | +| Field | Status | Type | Description | +| --------------------- | -------- | ------- | ------------------------------------------------------ | +| `$schema` | Optional | string | Optional JSON Schema URL for editor validation. | +| `version` | Optional | number | Caplets config schema version. | +| `defaultSearchLimit` | Optional | integer | Default maximum number of same-server search results. | +| `maxSearchLimit` | Optional | integer | Maximum accepted search_tools limit. | +| `completion` | Optional | object | Shell completion discovery timeout and cache settings. | +| `options` | Optional | object | Global Caplets runtime options. | +| `mcpServers` | Optional | object | Downstream MCP servers keyed by stable server ID. | +| `openapiEndpoints` | Optional | object | OpenAPI endpoints keyed by stable Caplet ID. | +| `googleDiscoveryApis` | Optional | object | Google Discovery APIs keyed by stable Caplet ID. | +| `graphqlEndpoints` | Optional | object | GraphQL endpoints keyed by stable Caplet ID. | +| `httpApis` | Optional | object | HTTP APIs keyed by stable Caplet ID. | +| `cliTools` | Optional | object | CLI tools keyed by stable Caplet ID. | +| `capletSets` | Optional | object | Nested Caplet collections keyed by stable Caplet ID. | ## Major sections diff --git a/apps/docs/src/content/docs/troubleshooting.mdx b/apps/docs/src/content/docs/troubleshooting.mdx index 1752c6ab..de3eb8dd 100644 --- a/apps/docs/src/content/docs/troubleshooting.mdx +++ b/apps/docs/src/content/docs/troubleshooting.mdx @@ -90,6 +90,32 @@ return await h.callTool("query_package_version", { Use the returned `error.code`, `error.message`, and Caplet ID to repair the call. Do not paste a full raw downstream payload back to the agent unless the error depends on it. +### Google auth asks for login again + +Expected symptom: a Google Discovery Caplet worked before, but `auth login` or a tool call +now reports missing scopes after you changed operation filters or the Discovery document. + +Google Discovery Caplets infer OAuth scopes from the exposed operation set unless +`auth.scopes` is configured. Re-run login for that Caplet so the stored token metadata +matches the current scope set: + +```sh +caplets auth login +caplets doctor +``` + +If the Caplet should never ask for newly inferred scopes, configure `auth.scopes` +explicitly and keep `includeOperations` narrow. + +### Download returned an artifact + +Expected symptom: a tool result contains `body.artifact` instead of inline bytes or text. + +Binary downloads, Google media downloads, and oversized HTTP-like responses are written as +Caplets media artifacts. Use the artifact `path` locally, or pass the artifact URI back to +another media-capable tool as `media.artifact`. If you need a specific destination for a +download, retry with `filename` or `outputPath` when the tool schema exposes those fields. + ### Remote attach fails Expected symptom: remote mode starts, but attach cannot reach the runtime or authenticate. diff --git a/apps/landing/public/caplet-frontmatter.schema.json b/apps/landing/public/caplet-frontmatter.schema.json index 7200d110..8baeb063 100644 --- a/apps/landing/public/caplet-frontmatter.schema.json +++ b/apps/landing/public/caplet-frontmatter.schema.json @@ -717,6 +717,265 @@ "additionalProperties": false, "description": "OpenAPI endpoint backend configuration for this Caplet." }, + "googleDiscoveryApi": { + "type": "object", + "properties": { + "discoveryPath": { + "description": "Local Google Discovery document path.", + "type": "string", + "minLength": 1 + }, + "discoveryUrl": { + "description": "Remote Google Discovery document URL.", + "type": "string", + "minLength": 1 + }, + "baseUrl": { + "description": "Override base URL for Google API requests.", + "type": "string", + "minLength": 1 + }, + "auth": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "none" + } + }, + "required": ["type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "bearer" + }, + "token": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "token"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "headers" + }, + "headers": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + } + }, + "required": ["type", "headers"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "oauth2" + }, + "authorizationUrl": { + "type": "string", + "minLength": 1 + }, + "tokenUrl": { + "type": "string", + "minLength": 1 + }, + "issuer": { + "type": "string", + "minLength": 1 + }, + "resourceMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "authorizationServerMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "openidConfigurationUrl": { + "type": "string", + "minLength": 1 + }, + "clientMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "clientId": { + "type": "string", + "minLength": 1 + }, + "clientSecret": { + "type": "string", + "minLength": 1 + }, + "scopes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "redirectUri": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "oidc" + }, + "authorizationUrl": { + "type": "string", + "minLength": 1 + }, + "tokenUrl": { + "type": "string", + "minLength": 1 + }, + "issuer": { + "type": "string", + "minLength": 1 + }, + "resourceMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "authorizationServerMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "openidConfigurationUrl": { + "type": "string", + "minLength": 1 + }, + "clientMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "clientId": { + "type": "string", + "minLength": 1 + }, + "clientSecret": { + "type": "string", + "minLength": 1 + }, + "scopes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "redirectUri": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type"], + "additionalProperties": false + } + ], + "description": "Explicit Google API request auth config. Use {\"type\":\"none\"} for public APIs." + }, + "requestTimeoutMs": { + "description": "Timeout in milliseconds for Google API HTTP requests.", + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "operationCacheTtlMs": { + "description": "Milliseconds Google Discovery operation metadata stays fresh. Set 0 to refresh every time.", + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "includeOperations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 160 + } + }, + "excludeOperations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 160 + } + }, + "disabled": { + "description": "When true, omit this Caplet from discovery.", + "type": "boolean" + }, + "projectBinding": { + "type": "object", + "properties": { + "required": { + "type": "boolean", + "const": true + } + }, + "required": ["required"], + "additionalProperties": false, + "description": "Project Binding requirements for Caplets that need an attached project." + }, + "runtime": { + "type": "object", + "properties": { + "features": { + "type": "array", + "items": { + "type": "string", + "enum": ["docker", "browser"] + } + }, + "resources": { + "type": "object", + "properties": { + "class": { + "type": "string", + "enum": ["standard", "large", "heavy"] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "description": "Runtime feature and resource requirements for hosted execution." + } + }, + "required": ["auth"], + "additionalProperties": false, + "description": "Google Discovery API backend configuration for this Caplet." + }, "graphqlEndpoint": { "type": "object", "properties": { diff --git a/apps/landing/public/config.schema.json b/apps/landing/public/config.schema.json index 3ec7d1ce..06d0eaff 100644 --- a/apps/landing/public/config.schema.json +++ b/apps/landing/public/config.schema.json @@ -962,6 +962,437 @@ "additionalProperties": false } }, + "googleDiscoveryApis": { + "default": {}, + "description": "Google Discovery APIs keyed by stable Caplet ID.", + "type": "object", + "propertyNames": { + "type": "string", + "pattern": "^[a-zA-Z0-9_-]{1,64}$" + }, + "additionalProperties": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 80, + "description": "Human-readable Google Discovery API display name." + }, + "description": { + "type": "string", + "description": "Capability description shown to agents before Google Discovery operations are disclosed." + }, + "discoveryPath": { + "description": "Local Google Discovery document path.", + "type": "string", + "minLength": 1 + }, + "discoveryUrl": { + "description": "Remote Google Discovery document URL.", + "type": "string", + "format": "uri" + }, + "baseUrl": { + "description": "Override base URL for Google API requests.", + "type": "string", + "format": "uri" + }, + "includeOperations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 160 + } + }, + "excludeOperations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 160 + } + }, + "auth": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "none" + } + }, + "required": ["type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "bearer" + }, + "token": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "token"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "headers" + }, + "headers": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + } + }, + "required": ["type", "headers"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "oauth2" + }, + "authorizationUrl": { + "type": "string", + "format": "uri" + }, + "tokenUrl": { + "type": "string", + "format": "uri" + }, + "issuer": { + "type": "string", + "format": "uri" + }, + "resourceMetadataUrl": { + "type": "string", + "format": "uri" + }, + "authorizationServerMetadataUrl": { + "type": "string", + "format": "uri" + }, + "openidConfigurationUrl": { + "type": "string", + "format": "uri" + }, + "clientMetadataUrl": { + "type": "string", + "format": "uri" + }, + "clientId": { + "type": "string", + "minLength": 1 + }, + "clientSecret": { + "type": "string", + "minLength": 1 + }, + "scopes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "redirectUri": { + "type": "string", + "format": "uri" + } + }, + "required": ["type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "oidc" + }, + "authorizationUrl": { + "type": "string", + "format": "uri" + }, + "tokenUrl": { + "type": "string", + "format": "uri" + }, + "issuer": { + "type": "string", + "format": "uri" + }, + "resourceMetadataUrl": { + "type": "string", + "format": "uri" + }, + "authorizationServerMetadataUrl": { + "type": "string", + "format": "uri" + }, + "openidConfigurationUrl": { + "type": "string", + "format": "uri" + }, + "clientMetadataUrl": { + "type": "string", + "format": "uri" + }, + "clientId": { + "type": "string", + "minLength": 1 + }, + "clientSecret": { + "type": "string", + "minLength": 1 + }, + "scopes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "redirectUri": { + "type": "string", + "format": "uri" + } + }, + "required": ["type"], + "additionalProperties": false + } + ], + "description": "Explicit Google API request auth config. Use {\"type\":\"none\"} for public APIs." + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 80 + } + }, + "exposure": { + "type": "string", + "enum": [ + "direct", + "progressive", + "code_mode", + "direct_and_code_mode", + "progressive_and_code_mode" + ], + "description": "How this Caplet is exposed to agents." + }, + "shadowing": { + "default": "forbid", + "description": "Whether attached local Caplets may shadow this remote Caplet ID.", + "type": "string", + "enum": ["forbid", "allow"] + }, + "useWhen": { + "description": "When agents should prefer this Caplet or configured action.", + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "avoidWhen": { + "description": "When agents should avoid this Caplet or configured action.", + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "setup": { + "type": "object", + "properties": { + "commands": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string", + "minLength": 1, + "description": "Human-readable setup or verification step label." + }, + "command": { + "type": "string", + "minLength": 1, + "description": "Executable command to spawn without a shell." + }, + "args": { + "description": "Arguments passed to the command.", + "type": "array", + "items": { + "type": "string" + } + }, + "env": { + "description": "Additional environment variables.", + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + } + }, + "cwd": { + "description": "Working directory for this command.", + "type": "string", + "minLength": 1 + }, + "timeoutMs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "maxOutputBytes": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": ["label", "command"], + "additionalProperties": false + } + }, + "verify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string", + "minLength": 1, + "description": "Human-readable setup or verification step label." + }, + "command": { + "type": "string", + "minLength": 1, + "description": "Executable command to spawn without a shell." + }, + "args": { + "description": "Arguments passed to the command.", + "type": "array", + "items": { + "type": "string" + } + }, + "env": { + "description": "Additional environment variables.", + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + } + }, + "cwd": { + "description": "Working directory for this command.", + "type": "string", + "minLength": 1 + }, + "timeoutMs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "maxOutputBytes": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": ["label", "command"], + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "projectBinding": { + "type": "object", + "properties": { + "required": { + "type": "boolean", + "const": true, + "description": "Requires Project Binding before this Caplet can run." + } + }, + "required": ["required"], + "additionalProperties": false, + "description": "Project Binding requirements for Caplets that need an attached project." + }, + "runtime": { + "type": "object", + "properties": { + "features": { + "type": "array", + "items": { + "type": "string", + "enum": ["docker", "browser"] + }, + "description": "Runtime features required by this Caplet." + }, + "resources": { + "description": "Hosted sandbox resource requirements.", + "type": "object", + "properties": { + "class": { + "description": "Requested hosted sandbox resource class.", + "type": "string", + "enum": ["standard", "large", "heavy"] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "description": "Runtime feature and resource requirements for hosted execution." + }, + "requestTimeoutMs": { + "default": 60000, + "description": "Timeout in milliseconds for Google Discovery HTTP requests.", + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "operationCacheTtlMs": { + "default": 30000, + "description": "Milliseconds Google Discovery operation metadata stays fresh. Set 0 to refresh every time.", + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "disabled": { + "default": false, + "description": "When true, omit this Google Discovery Caplet from discovery.", + "type": "boolean" + } + }, + "required": ["name", "description", "auth"], + "additionalProperties": false + } + }, "graphqlEndpoints": { "default": {}, "description": "GraphQL endpoints keyed by stable Caplet ID.", diff --git a/docs/adr/0002-media-artifacts-for-non-inline-results.md b/docs/adr/0002-media-artifacts-for-non-inline-results.md new file mode 100644 index 00000000..08f3506f --- /dev/null +++ b/docs/adr/0002-media-artifacts-for-non-inline-results.md @@ -0,0 +1,20 @@ +# ADR 0002: Use Media Artifacts For Non-Inline Results + +## Status + +Accepted + +## Context + +Caplets backends can return binary media, downloads, exports, and textual responses too large to fit comfortably in structured tool output. Inline base64 is hard for coding agents to inspect, expensive in tokens, fragile when truncated, and unsuitable as a common contract across local, remote, and hosted execution. + +## Decision + +Caplets will represent binary media and oversized response content as Media artifacts by default. Local execution may return absolute paths under Caplets-managed artifact storage, while remote and hosted execution must return artifact references or links rather than fake local paths. Small JSON and text responses remain inline. + +## Consequences + +- Media-capable backends share one result contract instead of inventing backend-specific blob behavior. +- Agents should prefer file paths or artifact references for media input and output. +- Data URLs are allowed only as small-input fallback values and must not be echoed in logs, errors, or result previews. +- Backends that currently read all HTTP responses as bounded text need to route binary and oversized responses through shared artifact writing. diff --git a/docs/architecture.md b/docs/architecture.md index 5a19ec17..94e2f2b3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,6 +14,7 @@ Supported backend families are: - `mcpServers` - `openapiEndpoints` +- `googleDiscoveryApis` - `graphqlEndpoints` - `httpApis` - `cliTools` @@ -89,9 +90,11 @@ Project Binding under `packages/core/src/project-binding/` connects a local proj MCP-backed Caplets preserve downstream tool results and expose resources, templates, prompts, and completion when the downstream server supports them. Direct exposure can register those downstream surfaces directly. -### OpenAPI, GraphQL, And HTTP +### OpenAPI, Google Discovery, GraphQL, And HTTP -OpenAPI, GraphQL, and HTTP backends expose explicit operation/action tools. They do not synthesize MCP resources or prompts. HTTP-like backends enforce safe URL handling, bounded response bodies, timeouts, and redacted errors. +OpenAPI, Google Discovery, GraphQL, and HTTP backends expose explicit operation/action tools. They do not synthesize MCP resources or prompts. HTTP-like backends enforce safe URL handling, bounded response bodies, timeouts, and redacted errors. + +Google Discovery backends load local or remote Google Discovery documents, infer request base URLs from the document unless overridden, expose filtered Discovery methods as tools, and infer OAuth scopes from the exposed operation set. Google media downloads and oversized or binary HTTP-like responses are written as Caplets media artifacts under the configured artifact root instead of being forced inline. ### CLI Tools diff --git a/docs/plans/2026-06-16-google-discovery-api-backend-implementation.md b/docs/plans/2026-06-16-google-discovery-api-backend-implementation.md new file mode 100644 index 00000000..4b6e900b --- /dev/null +++ b/docs/plans/2026-06-16-google-discovery-api-backend-implementation.md @@ -0,0 +1,2151 @@ +# Google Discovery API Backend Implementation Plan + +> **For agentic workers:** REQUIRED SKILL: Use `subagent-driven-development` (recommended) or `executing-plans` to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a first-class Google Discovery API backend with inferred scopes, operation filters, comprehensive media upload/download, and shared media artifact handling for HTTP-like backends. + +**Architecture:** Add `googleDiscoveryApis` as a new top-level Caplets backend family with its own manager and native Google Discovery parser. Build shared media artifact infrastructure first, wire HTTP/OpenAPI responses through it, then implement Google Discovery on top of shared auth, HTTP request, and artifact primitives. Keep Google Discovery distinct from OpenAPI while returning the same Caplets `Tool` and `CompatibilityCallToolResult` shapes used by other backends. + +**Tech Stack:** TypeScript, Zod, Vitest, MCP SDK `Tool`/`CompatibilityCallToolResult`, existing Caplets auth/config/engine/tool surfaces, Node filesystem and fetch APIs, pnpm 11.5.0, Node >=24. + +--- + +## Source Documents + +- Design spec: `docs/specs/2026-06-16-google-discovery-api-backend.md` +- Media ADR: `docs/adr/0002-media-artifacts-for-non-inline-results.md` +- Glossary: `CONTEXT.md` +- Existing backend references: `packages/core/src/openapi.ts`, `packages/core/src/http-actions.ts`, `packages/core/src/graphql.ts` +- Existing config references: `packages/core/src/config.ts`, `packages/core/src/caplet-files-bundle.ts`, `packages/core/src/registry.ts` + +## File Structure Map + +Create: + +- `packages/core/src/media/artifacts.ts` — writes Caplets-managed media artifacts, resolves artifact references, computes hashes, and enforces output path safety. +- `packages/core/src/media/input.ts` — reads media input from `path`, `artifact`, or `dataUrl` for upload-capable backends. +- `packages/core/src/http/response.ts` — shared HTTP response reader that returns inline JSON/text or a Media artifact. +- `packages/core/src/google-discovery/types.ts` — narrow TypeScript types for the Google Discovery document shapes Caplets consumes. +- `packages/core/src/google-discovery/schema.ts` — converts Google Discovery schemas to JSON Schema-like tool schemas. +- `packages/core/src/google-discovery/operations.ts` — walks `resources.*.methods.*`, applies operation filters, resolves scopes, and builds operation descriptors. +- `packages/core/src/google-discovery/request.ts` — builds normal JSON requests and media upload/download requests. +- `packages/core/src/google-discovery/manager.ts` — `GoogleDiscoveryManager` implementation. +- `packages/core/src/google-discovery/index.ts` — public exports for the manager and helper types. +- `packages/core/test/google-discovery.test.ts` — parser, manager, auth, filtering, media, and tool-surface tests. +- `packages/core/test/media-artifacts.test.ts` — shared artifact and media input tests. +- `packages/core/test/fixtures/google-discovery/drive.discovery.json` — small Drive-like fixture with JSON, download, simple upload, multipart upload, resumable upload, scopes, and destructive operations. + +Modify: + +- `packages/core/src/config/paths.ts` — add default artifact directory export. +- `packages/core/src/config.ts` — add `GoogleDiscoveryApiConfig`, schema, normalization, merge/source/reject logic, and config JSON schema support. +- `packages/core/src/caplet-files-bundle.ts` — add `googleDiscoveryApi` frontmatter support and schema generation. +- `packages/core/src/caplet-source/parse.ts` — include Google Discovery Caplets in parsed source output. +- `packages/core/src/registry.ts` — include backend detail for `googleDiscovery`. +- `packages/core/src/engine.ts` — instantiate/update/invalidate/dispatch `GoogleDiscoveryManager`. +- `packages/core/src/tools.ts` — accept `GoogleDiscoveryManager` in `handleServerTool` and `backendFor`. +- `packages/core/src/native/service.ts` and `packages/core/src/native/tools.ts` — include Google Discovery in native service/tool guidance. +- `packages/core/src/cli/auth.ts` — include Google Discovery auth targets and resolve inferred scopes before login/refresh. +- `packages/core/src/auth.ts` and `packages/core/src/auth/store.ts` — support backend-resolved OAuth scopes and compare requested scope metadata. +- `packages/core/src/cli/add.ts` and `packages/core/src/cli.ts` — add `caplets add google-discovery`. +- `packages/core/src/cli/inspection.ts`, `packages/core/src/cli/completion-discovery.ts`, `packages/core/src/cli/setup-caplet.ts`, `packages/core/src/cli/doctor.ts` — include `googleDiscoveryApis` anywhere all Caplets are enumerated. +- `packages/core/src/remote-control/dispatch.ts` and remote add types if remote add kinds are enumerated there. +- `packages/core/src/openapi.ts` and `packages/core/src/http-actions.ts` — use shared response/artifact reader. +- `packages/core/src/code-mode/runtime-api.d.ts` and generated output only if media artifact types are added to Code Mode declarations. +- `schemas/caplets-config.schema.json`, `schemas/caplet.schema.json`, and docs generated from config/caplet schemas. +- `apps/docs/src/content/docs/reference/config.mdx`, `apps/docs/src/content/docs/reference/caplet-files.mdx`, `apps/docs/src/content/docs/capabilities.mdx`, `apps/docs/src/content/docs/troubleshooting.mdx`, `apps/docs/src/content/docs/changelog.mdx`. +- `docs/architecture.md` — add Google Discovery to backend families and HTTP-like backend contract. + +## Implementation Tasks + +### Task 1: Shared Media Artifact Infrastructure + +**Files:** + +- Create: `packages/core/src/media/artifacts.ts` +- Create: `packages/core/src/media/input.ts` +- Modify: `packages/core/src/config/paths.ts` +- Test: `packages/core/test/media-artifacts.test.ts` + +- [ ] **Step 1: Write failing artifact tests** + +Create `packages/core/test/media-artifacts.test.ts` with these cases: + +```ts +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { readMediaInput, resolveMediaArtifact, writeMediaArtifact } from "../src/media"; + +describe("media artifacts", () => { + const dirs: string[] = []; + afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + + function tempDir(prefix: string) { + const dir = mkdtempSync(join(tmpdir(), prefix)); + dirs.push(dir); + return dir; + } + + it("writes artifact files with stable metadata", async () => { + const root = tempDir("caplets-artifacts-"); + const artifact = await writeMediaArtifact({ + rootDir: root, + capletId: "google-drive", + suggestedFilename: "report.pdf", + mimeType: "application/pdf", + bytes: Buffer.from("pdf-bytes"), + }); + + expect(artifact).toMatchObject({ + mimeType: "application/pdf", + byteLength: 9, + filename: "report.pdf", + }); + expect(artifact.path).toContain(join(root, "google-drive")); + expect(artifact.sha256).toHaveLength(64); + expect(readFileSync(artifact.path, "utf8")).toBe("pdf-bytes"); + }); + + it("rejects output paths outside an allowed root", async () => { + const root = tempDir("caplets-artifacts-"); + await expect( + writeMediaArtifact({ + rootDir: root, + capletId: "drive", + outputPath: join(root, "..", "escape.bin"), + bytes: Buffer.from("x"), + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); + + it("reads media input from path, artifact reference, and data URL", async () => { + const root = tempDir("caplets-artifacts-"); + const file = join(root, "image.png"); + writeFileSync(file, Buffer.from("png")); + const artifact = await writeMediaArtifact({ + rootDir: root, + capletId: "drive", + suggestedFilename: "existing.png", + mimeType: "image/png", + bytes: Buffer.from("artifact"), + }); + + await expect(readMediaInput({ path: file }, { artifactRoot: root })).resolves.toMatchObject({ + bytes: Buffer.from("png"), + filename: "image.png", + }); + await expect( + readMediaInput({ artifact: artifact.uri }, { artifactRoot: root }), + ).resolves.toMatchObject({ + bytes: Buffer.from("artifact"), + filename: "existing.png", + mimeType: "image/png", + }); + await expect( + readMediaInput( + { dataUrl: "data:text/plain;base64,aGVsbG8=", filename: "hello.txt" }, + { artifactRoot: root }, + ), + ).resolves.toMatchObject({ + bytes: Buffer.from("hello"), + filename: "hello.txt", + mimeType: "text/plain", + }); + }); + + it("rejects multiple media input sources and non-base64 data URLs", async () => { + const root = tempDir("caplets-artifacts-"); + await expect( + readMediaInput( + { path: "/tmp/a", dataUrl: "data:text/plain;base64,eA==" }, + { + artifactRoot: root, + }, + ), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + await expect( + readMediaInput( + { dataUrl: "data:text/plain,hello" }, + { + artifactRoot: root, + }, + ), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); +}); +``` + +- [ ] **Step 2: Run the failing artifact tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/media-artifacts.test.ts +``` + +Expected: fails because `../src/media` does not exist. + +- [ ] **Step 3: Add default artifact path** + +In `packages/core/src/config/paths.ts`, export a default artifact directory next to existing state/cache directory exports: + +```ts +export const DEFAULT_ARTIFACT_DIR = join(defaultStateBaseDir(), "artifacts"); +``` + +Use the existing local naming style in the file. If the file exposes helper functions rather than constants for nearby paths, match that shape with `defaultArtifactDir()`. + +- [ ] **Step 4: Implement artifact writing and lookup** + +Create `packages/core/src/media/artifacts.ts`: + +```ts +import { createHash, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { DEFAULT_ARTIFACT_DIR } from "../config/paths"; +import { CapletsError } from "../errors"; + +export type MediaArtifact = { + uri: string; + path: string; + filename: string; + mimeType?: string; + byteLength: number; + sha256: string; +}; + +export type WriteMediaArtifactInput = { + rootDir?: string; + capletId: string; + callId?: string; + suggestedFilename?: string; + outputPath?: string; + mimeType?: string; + bytes: Uint8Array | Buffer; +}; + +export function artifactUri(capletId: string, callId: string, filename: string): string { + return `caplets://artifacts/${encodeURIComponent(capletId)}/${encodeURIComponent(callId)}/${encodeURIComponent(filename)}`; +} + +export async function writeMediaArtifact(input: WriteMediaArtifactInput): Promise { + const rootDir = resolve(input.rootDir ?? DEFAULT_ARTIFACT_DIR); + const callId = + input.callId ?? `${new Date().toISOString().replace(/[:.]/gu, "-")}-${randomUUID()}`; + const filename = safeFilename(input.suggestedFilename ?? "response.bin"); + const target = input.outputPath + ? assertInsideRoot(rootDir, input.outputPath) + : resolve(rootDir, input.capletId, callId, filename); + mkdirSync(dirname(target), { recursive: true, mode: 0o700 }); + const bytes = Buffer.from(input.bytes); + writeFileSync(target, bytes, { mode: 0o600 }); + return { + uri: artifactUri(input.capletId, callId, basename(target)), + path: target, + filename: basename(target), + ...(input.mimeType ? { mimeType: input.mimeType } : {}), + byteLength: bytes.byteLength, + sha256: createHash("sha256").update(bytes).digest("hex"), + }; +} + +export function resolveMediaArtifact( + uri: string, + options: { artifactRoot?: string } = {}, +): MediaArtifact { + const parsed = parseArtifactUri(uri); + const rootDir = resolve(options.artifactRoot ?? DEFAULT_ARTIFACT_DIR); + const path = resolve(rootDir, parsed.capletId, parsed.callId, parsed.filename); + assertInsideRoot(rootDir, path); + if (!existsSync(path)) { + throw new CapletsError("REQUEST_INVALID", `Media artifact ${uri} was not found`); + } + const bytes = readFileSync(path); + return { + uri, + path, + filename: basename(path), + byteLength: bytes.byteLength, + sha256: createHash("sha256").update(bytes).digest("hex"), + }; +} + +function parseArtifactUri(uri: string): { capletId: string; callId: string; filename: string } { + const url = new URL(uri); + if (url.protocol !== "caplets:" || url.hostname !== "artifacts") { + throw new CapletsError( + "REQUEST_INVALID", + "Media artifact URI must start with caplets://artifacts/", + ); + } + const [capletId, callId, filename] = url.pathname + .split("/") + .filter(Boolean) + .map((part) => decodeURIComponent(part)); + if (!capletId || !callId || !filename) { + throw new CapletsError("REQUEST_INVALID", "Media artifact URI is missing required parts"); + } + return { capletId, callId, filename: safeFilename(filename) }; +} + +function assertInsideRoot(rootDir: string, candidate: string): string { + if (!isAbsolute(candidate)) { + throw new CapletsError("REQUEST_INVALID", "Media artifact outputPath must be absolute"); + } + const resolved = resolve(candidate); + const rel = relative(rootDir, resolved); + if (rel.startsWith("..") || isAbsolute(rel)) { + throw new CapletsError( + "REQUEST_INVALID", + "Media artifact outputPath must stay inside the artifact root", + ); + } + return resolved; +} + +function safeFilename(value: string): string { + const name = basename(value) + .replace(/[^\w.\- ]/gu, "_") + .trim(); + return name || "response.bin"; +} +``` + +- [ ] **Step 5: Implement media input reading** + +Create `packages/core/src/media/input.ts`: + +```ts +import { readFileSync, statSync } from "node:fs"; +import { basename } from "node:path"; +import { CapletsError } from "../errors"; +import { resolveMediaArtifact } from "./artifacts"; + +export type MediaInput = + | { path: string; artifact?: never; dataUrl?: never; filename?: string; mimeType?: string } + | { artifact: string; path?: never; dataUrl?: never; filename?: string; mimeType?: string } + | { dataUrl: string; path?: never; artifact?: never; filename?: string; mimeType?: string }; + +export type ResolvedMediaInput = { + bytes: Buffer; + filename: string; + mimeType?: string; +}; + +export async function readMediaInput( + input: unknown, + options: { artifactRoot?: string; maxBytes?: number } = {}, +): Promise { + if (!input || typeof input !== "object" || Array.isArray(input)) { + throw new CapletsError("REQUEST_INVALID", "media must be an object"); + } + const media = input as Record; + const sources = ["path", "artifact", "dataUrl"].filter((key) => typeof media[key] === "string"); + if (sources.length !== 1) { + throw new CapletsError( + "REQUEST_INVALID", + "media must define exactly one of path, artifact, or dataUrl", + ); + } + const filename = typeof media.filename === "string" ? media.filename : undefined; + const mimeType = typeof media.mimeType === "string" ? media.mimeType : undefined; + if (typeof media.path === "string") { + const stat = statSync(media.path); + enforceSize(stat.size, options.maxBytes); + const bytes = readFileSync(media.path); + return { bytes, filename: filename ?? basename(media.path), ...(mimeType ? { mimeType } : {}) }; + } + if (typeof media.artifact === "string") { + const artifact = resolveMediaArtifact(media.artifact, { artifactRoot: options.artifactRoot }); + enforceSize(artifact.byteLength, options.maxBytes); + const bytes = readFileSync(artifact.path); + return { + bytes, + filename: filename ?? artifact.filename, + ...((mimeType ?? artifact.mimeType) ? { mimeType: mimeType ?? artifact.mimeType } : {}), + }; + } + return readDataUrl(String(media.dataUrl), { filename, mimeType, maxBytes: options.maxBytes }); +} + +function readDataUrl( + dataUrl: string, + options: { filename?: string; mimeType?: string; maxBytes?: number }, +): ResolvedMediaInput { + const match = /^data:([^;,]+);base64,([A-Za-z0-9+/=]+)$/u.exec(dataUrl); + if (!match) { + throw new CapletsError("REQUEST_INVALID", "media.dataUrl must be a base64 data URL"); + } + const bytes = Buffer.from(match[2]!, "base64"); + enforceSize(bytes.byteLength, options.maxBytes); + return { + bytes, + filename: options.filename ?? "media.bin", + mimeType: options.mimeType ?? match[1], + }; +} + +function enforceSize(size: number, maxBytes = 100 * 1024 * 1024): void { + if (size > maxBytes) { + throw new CapletsError("REQUEST_INVALID", `media exceeds byte limit ${maxBytes}`); + } +} +``` + +- [ ] **Step 6: Add media barrel export** + +Create `packages/core/src/media/index.ts`: + +```ts +export * from "./artifacts"; +export * from "./input"; +``` + +- [ ] **Step 7: Run artifact tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/media-artifacts.test.ts +``` + +Expected: pass. + +### Task 2: Shared HTTP Response Reader And Existing Backend Media Artifacts + +**Files:** + +- Create: `packages/core/src/http/response.ts` +- Modify: `packages/core/src/openapi.ts` +- Modify: `packages/core/src/http-actions.ts` +- Test: `packages/core/test/openapi.test.ts` +- Test: `packages/core/test/http-actions.test.ts` + +- [ ] **Step 1: Add failing HTTP/OpenAPI artifact tests** + +In `packages/core/test/http-actions.test.ts`, extend the local server with: + +```ts +if (request.url === "/pdf") { + response.setHeader("content-type", "application/pdf"); + response.end(Buffer.from("%PDF-1.7 test")); + return; +} +``` + +Add this test: + +```ts +it("writes binary HTTP responses as media artifacts", async () => { + const artifactDir = mkdtempSync(join(tmpdir(), "caplets-http-artifacts-")); + try { + const manager = new HttpActionManager(registry(), { artifactDir }); + const api = httpApi({ actions: { pdf: { method: "GET", path: "/pdf" } } }); + + const result = await manager.callTool(api, "pdf", {}); + + expect(result.structuredContent).toMatchObject({ + status: 200, + headers: { "content-type": "application/pdf" }, + body: { + artifact: { + mimeType: "application/pdf", + byteLength: 13, + }, + }, + }); + const path = (result.structuredContent as any).body.artifact.path; + expect(readFileSync(path, "utf8")).toBe("%PDF-1.7 test"); + } finally { + rmSync(artifactDir, { recursive: true, force: true }); + } +}); +``` + +Mirror the same assertion in `packages/core/test/openapi.test.ts` using an OpenAPI operation whose response has `application/pdf`. + +- [ ] **Step 2: Run failing existing-backend media tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/http-actions.test.ts test/openapi.test.ts +``` + +Expected: fail because responses are still read as bounded text. + +- [ ] **Step 3: Implement shared response reader** + +Create `packages/core/src/http/response.ts`: + +```ts +import type { MediaArtifact } from "../media"; +import { writeMediaArtifact } from "../media"; +import { parseHttpBody, readLimitedText } from "./utils"; + +export type HttpLikeResponseBody = + | { body?: unknown; artifact?: never } + | { body: { artifact: MediaArtifact } }; + +export type ReadHttpLikeResponseOptions = { + capletId: string; + artifactDir?: string; + outputPath?: string; + filename?: string; + maxInlineBytes?: number; +}; + +export async function readHttpLikeResponse( + response: Response, + options: ReadHttpLikeResponseOptions, +): Promise> { + const contentType = response.headers.get("content-type") ?? ""; + const mime = contentType.split(";")[0]?.toLowerCase().trim() ?? ""; + const inlineText = isInlineTextMime(mime); + if (inlineText) { + try { + const text = await readLimitedText(response, { + maxBytes: options.maxInlineBytes, + errorMessage: "HTTP response exceeded inline byte limit", + }); + const body = parseHttpBody(contentType, text); + return baseResponse(response, contentType, body); + } catch (error) { + if (!response.body) throw error; + } + } + + const bytes = Buffer.from(await response.arrayBuffer()); + const artifact = await writeMediaArtifact({ + rootDir: options.artifactDir, + capletId: options.capletId, + outputPath: options.outputPath, + suggestedFilename: options.filename ?? filenameFromHeaders(response) ?? "response.bin", + mimeType: mime || undefined, + bytes, + }); + return baseResponse(response, contentType, { artifact }); +} + +function baseResponse( + response: Response, + contentType: string, + body: unknown, +): Record { + return { + status: response.status, + statusText: response.statusText, + headers: { "content-type": contentType }, + ...(body === undefined ? {} : { body }), + }; +} + +function isInlineTextMime(mime: string): boolean { + return ( + mime === "" || + mime === "application/json" || + mime.endsWith("+json") || + mime.endsWith("/json") || + mime.startsWith("text/") + ); +} + +function filenameFromHeaders(response: Response): string | undefined { + const disposition = response.headers.get("content-disposition") ?? ""; + return /filename="?([^";]+)"?/iu.exec(disposition)?.[1]; +} +``` + +- [ ] **Step 4: Pass artifact options through managers** + +Update constructors in `packages/core/src/http-actions.ts` and `packages/core/src/openapi.ts`: + +```ts +constructor( + private registry: ServerRegistry, + private readonly options: { authDir?: string; artifactDir?: string } = {}, +) {} +``` + +Replace local `readResponse` functions with calls to `readHttpLikeResponse(response, { capletId, artifactDir })`. + +For OpenAPI and HTTP actions, reserve `args.outputPath` and `args.filename` for artifact output by removing them from request query/body/path mappings only when the operation schema explicitly models them as media output controls. If this conflicts with existing actions, keep output controls under `_caplets: { outputPath, filename }` and document that path in the plan implementation notes. + +- [ ] **Step 5: Pass artifact directory from engine** + +Add `artifactDir?: string` to `CapletsEngineOptions`. Instantiate managers with: + +```ts +const sharedManagerOptions = { authDir: options.authDir, artifactDir: options.artifactDir }; +this.openapi = new OpenApiManager(this.registry, sharedManagerOptions); +this.graphql = new GraphQLManager(this.registry, selectAuthOptions(options.authDir)); +this.http = new HttpActionManager(this.registry, sharedManagerOptions); +``` + +GraphQL can remain text/JSON-only unless the implementation chooses to route it through the shared reader. + +- [ ] **Step 6: Run focused tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/media-artifacts.test.ts test/http-actions.test.ts test/openapi.test.ts +``` + +Expected: pass. + +### Task 3: Config Schema And Core Types For `googleDiscoveryApis` + +**Files:** + +- Modify: `packages/core/src/config.ts` +- Modify: `packages/core/src/registry.ts` +- Test: `packages/core/test/config.test.ts` + +- [ ] **Step 1: Add failing config tests** + +Add to `packages/core/test/config.test.ts`: + +```ts +it("loads Google Discovery APIs with defaults and safe registry details", () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files and permissions.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "oidc", issuer: "https://accounts.google.com", clientId: "client" }, + includeOperations: ["drive.files.*"], + excludeOperations: ["drive.files.delete"], + }, + }, + }); + expect(config.googleDiscoveryApis.drive).toMatchObject({ + server: "drive", + backend: "googleDiscovery", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + requestTimeoutMs: 60000, + operationCacheTtlMs: 30000, + disabled: false, + }); + expect(JSON.stringify(configJsonSchema())).toContain("googleDiscoveryApis"); +}); + +it("rejects invalid Google Discovery sources and duplicate Caplet IDs", () => { + expect(() => + parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Drive", + description: "Access Google Drive files.", + discoveryUrl: "ftp://example.com/discovery.json", + auth: { type: "none" }, + }, + }, + }), + ).toThrow(CapletsError); + expect(() => + parseConfig({ + openapiEndpoints: { + drive: { + name: "Drive OpenAPI", + description: "OpenAPI Drive wrapper.", + specUrl: "https://example.com/openapi.json", + auth: { type: "none" }, + }, + }, + googleDiscoveryApis: { + drive: { + name: "Drive", + description: "Access Google Drive files.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "none" }, + }, + }, + }), + ).toThrow(/already used/); +}); +``` + +- [ ] **Step 2: Run failing config tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/config.test.ts +``` + +Expected: fail because config does not know `googleDiscoveryApis`. + +- [ ] **Step 3: Add config types** + +In `packages/core/src/config.ts`, add: + +```ts +export type GoogleDiscoveryApiConfig = AgentSelectionHintsConfig & { + server: string; + backend: "googleDiscovery"; + name: string; + description: string; + exposure?: CapletExposure | undefined; + shadowing?: CapletShadowingPolicy | undefined; + tags?: string[] | undefined; + body?: string | undefined; + discoveryPath?: string | undefined; + discoveryUrl?: string | undefined; + baseUrl?: string | undefined; + includeOperations?: string[] | undefined; + excludeOperations?: string[] | undefined; + auth: OpenApiAuthConfig; + requestTimeoutMs: number; + operationCacheTtlMs: number; + disabled: boolean; + setup?: CapletSetupConfig | undefined; + projectBinding?: ProjectBindingConfig | undefined; + runtime?: RuntimeRequirementsConfig | undefined; +}; +``` + +Extend: + +```ts +export type CapletConfig = + | CapletServerConfig + | OpenApiEndpointConfig + | GoogleDiscoveryApiConfig + | GraphQlEndpointConfig + | HttpApiConfig + | CliToolsConfig + | CapletSetConfig; + +export type CapletsConfig = { + ... + googleDiscoveryApis: Record; + ... +}; +``` + +- [ ] **Step 4: Add Zod schema** + +Add `publicGoogleDiscoveryApiSchema` modeled after `publicOpenApiEndpointSchema`: + +```ts +const operationFilterSchema = z.array(z.string().trim().min(1).max(160)); + +const publicGoogleDiscoveryApiSchema = z + .object({ + name: z + .string() + .trim() + .min(1) + .max(80) + .describe("Human-readable Google Discovery API display name."), + description: z + .string() + .describe( + "Capability description shown to agents before Google Discovery operations are disclosed.", + ) + .refine( + (value) => value.trim().length >= 10, + "description must contain at least 10 non-whitespace characters", + ) + .refine((value) => value.length <= 1500, "description must be at most 1500 characters"), + discoveryPath: z.string().min(1).optional().describe("Local Google Discovery document path."), + discoveryUrl: z.string().url().optional().describe("Remote Google Discovery document URL."), + baseUrl: z.string().url().optional().describe("Override base URL for Google API requests."), + includeOperations: operationFilterSchema.optional(), + excludeOperations: operationFilterSchema.optional(), + auth: openApiAuthSchema.describe( + 'Explicit Google API request auth config. Use {"type":"none"} for public APIs.', + ), + tags: z.array(z.string().trim().min(1).max(80)).optional(), + exposure: exposureSchema.optional(), + shadowing: shadowingSchema, + ...agentSelectionHintsSchema, + setup: setupSchema.optional(), + projectBinding: projectBindingSchema.optional(), + runtime: runtimeRequirementsSchema.optional(), + requestTimeoutMs: z.number().int().positive().default(60_000), + operationCacheTtlMs: z.number().int().nonnegative().default(30_000), + disabled: z.boolean().default(false), + }) + .strict(); +``` + +Normalize parsed entries with `server` and `backend: "googleDiscovery"` the same way OpenAPI entries are normalized. + +- [ ] **Step 5: Add validation, merge, source, and project rejection support** + +Update every backend-map list in `packages/core/src/config.ts`: + +- `ConfigInput` +- `configSchemaFor(...)` +- duplicate ID checks +- URL safety checks for `discoveryUrl` and `baseUrl` +- exact-one-source check for `discoveryPath` xor `discoveryUrl` +- `normalizeLocalPaths` +- `rejectProjectConfigExecutableBackendMaps` +- `mergeConfigInputs` +- `removeCapletId` +- `capletIds` +- empty-config checks + +Use this duplicate ordering when checking `googleDiscoveryApis`: `mcpServers`, `openapiEndpoints`, then `googleDiscoveryApis`, then the remaining backend maps. + +- [ ] **Step 6: Update registry detail** + +In `packages/core/src/registry.ts`, add a backend detail variant: + +```ts +| { + type: "googleDiscovery"; + disabled: boolean; + requestTimeoutMs: number; + operationCacheTtlMs: number; + source: "discoveryPath" | "discoveryUrl"; + } +``` + +Add `googleDiscoveryApis` to `get()`, `allCaplets()`, and `backendDetail()`. + +- [ ] **Step 7: Run config tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/config.test.ts +``` + +Expected: pass. + +### Task 4: Caplet Files, Source Parsing, Inspection, And CLI Enumeration + +**Files:** + +- Modify: `packages/core/src/caplet-files-bundle.ts` +- Modify: `packages/core/src/caplet-source/parse.ts` +- Modify: `packages/core/src/cli/inspection.ts` +- Modify: `packages/core/src/cli/completion-discovery.ts` +- Modify: `packages/core/src/cli/setup-caplet.ts` +- Modify: `packages/core/src/cli/doctor.ts` +- Modify: `packages/core/src/cli.ts` +- Test: `packages/core/test/caplet-files.test.ts` +- Test: `packages/core/test/config.test.ts` +- Test: `packages/core/test/cli.test.ts` + +- [ ] **Step 1: Add failing Caplet file and list tests** + +In `packages/core/test/caplet-files.test.ts`: + +```ts +it("loads Google Discovery API backend Caplet files", () => { + const result = loadCapletFilesFromMap({ + files: [ + { + path: "drive/CAPLET.md", + content: `--- +name: Google Drive +description: Access Google Drive files. +googleDiscoveryApi: + discoveryPath: ./drive.discovery.json + includeOperations: + - drive.files.* + auth: + type: none +--- + +# Drive +`, + }, + ], + }); + + expect(result?.config.googleDiscoveryApis?.drive).toEqual( + expect.objectContaining({ + name: "Google Drive", + description: "Access Google Drive files.", + discoveryPath: "drive/drive.discovery.json", + includeOperations: ["drive.files.*"], + body: "\n# Drive\n", + }), + ); +}); +``` + +In `packages/core/test/cli.test.ts`, add a list/inspect assertion using `googleDiscoveryApis` and expect backend `googleDiscovery`. + +- [ ] **Step 2: Run failing tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/caplet-files.test.ts test/cli.test.ts +``` + +Expected: fail because file loading and listing do not include the new backend. + +- [ ] **Step 3: Add frontmatter schema** + +In `packages/core/src/caplet-files-bundle.ts`, add `capletGoogleDiscoveryApiSchema` with the same fields as config minus top-level display fields, add `googleDiscoveryApi` to `capletFileSchema`, and update the backend-count error text: + +```ts +"Caplet file must define exactly one backend: mcpServer, openapiEndpoint, googleDiscoveryApi, graphqlEndpoint, httpApi, cliTools, or capletSet"; +``` + +- [ ] **Step 4: Add Google Discovery file mapping** + +In `buildCapletFileLoadResultFromEntries`, add `googleDiscoveryApis` to duplicate detection and output config. In the frontmatter normalization function, add: + +```ts +if (frontmatter.googleDiscoveryApi) { + return { + ...frontmatter.googleDiscoveryApi, + discoveryPath: normalizePath(frontmatter.googleDiscoveryApi.discoveryPath, baseDir), + backend: "googleDiscovery", + name: frontmatter.name, + description: frontmatter.description, + ...sharedCapletFields(frontmatter), + body, + }; +} +``` + +- [ ] **Step 5: Add enumeration support** + +Add `googleDiscoveryApis` to all `allCaplets` or object-spread collections in: + +- `packages/core/src/caplet-source/parse.ts` +- `packages/core/src/cli/inspection.ts` +- `packages/core/src/cli/completion-discovery.ts` +- `packages/core/src/cli/setup-caplet.ts` +- `packages/core/src/cli/doctor.ts` +- `packages/core/src/cli.ts` `capletConfigKinds`, `hasEnabledCaplet`, and any local overlay removal helpers + +- [ ] **Step 6: Run file/list tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/caplet-files.test.ts test/config.test.ts test/cli.test.ts +``` + +Expected: pass. + +### Task 5: Google Discovery Parser, Schema Conversion, Filtering, And Scope Resolution + +**Files:** + +- Create: `packages/core/src/google-discovery/types.ts` +- Create: `packages/core/src/google-discovery/schema.ts` +- Create: `packages/core/src/google-discovery/operations.ts` +- Create: `packages/core/src/google-discovery/index.ts` +- Create: `packages/core/test/fixtures/google-discovery/drive.discovery.json` +- Test: `packages/core/test/google-discovery.test.ts` + +- [ ] **Step 1: Add a compact Drive-like fixture** + +Create `packages/core/test/fixtures/google-discovery/drive.discovery.json` with this shape: + +```json +{ + "kind": "discovery#restDescription", + "id": "drive:v3", + "name": "drive", + "version": "v3", + "title": "Drive API", + "rootUrl": "https://www.googleapis.com/", + "servicePath": "drive/v3/", + "baseUrl": "https://www.googleapis.com/drive/v3/", + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/drive": { "description": "Full Drive access." }, + "https://www.googleapis.com/auth/drive.readonly": { "description": "Read Drive files." } + } + } + }, + "parameters": { + "fields": { + "type": "string", + "location": "query", + "description": "Partial response selector." + }, + "prettyPrint": { "type": "boolean", "location": "query", "default": "true" } + }, + "schemas": { + "File": { + "id": "File", + "type": "object", + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "parents": { "type": "array", "items": { "type": "string" } } + } + }, + "FileList": { + "id": "FileList", + "type": "object", + "properties": { + "files": { "type": "array", "items": { "$ref": "File" } }, + "nextPageToken": { "type": "string" } + } + } + }, + "resources": { + "files": { + "methods": { + "list": { + "id": "drive.files.list", + "path": "files", + "httpMethod": "GET", + "description": "Lists files.", + "scopes": ["https://www.googleapis.com/auth/drive.readonly"], + "parameters": { + "pageSize": { + "type": "integer", + "format": "int32", + "location": "query", + "default": "100" + } + }, + "response": { "$ref": "FileList" } + }, + "delete": { + "id": "drive.files.delete", + "path": "files/{fileId}", + "httpMethod": "DELETE", + "description": "Permanently deletes a file.", + "parameters": { + "fileId": { "type": "string", "location": "path", "required": true } + }, + "scopes": ["https://www.googleapis.com/auth/drive"] + }, + "create": { + "id": "drive.files.create", + "path": "files", + "httpMethod": "POST", + "description": "Creates a file.", + "request": { "$ref": "File" }, + "response": { "$ref": "File" }, + "scopes": ["https://www.googleapis.com/auth/drive"], + "supportsMediaUpload": true, + "mediaUpload": { + "protocols": { + "simple": { "path": "/upload/drive/v3/files", "multipart": false }, + "multipart": { "path": "/upload/drive/v3/files", "multipart": true }, + "resumable": { "path": "/upload/drive/v3/files", "multipart": true } + } + } + }, + "download": { + "id": "drive.files.download", + "path": "files/{fileId}/download", + "httpMethod": "GET", + "description": "Downloads file media.", + "supportsMediaDownload": true, + "parameters": { + "fileId": { "type": "string", "location": "path", "required": true } + }, + "scopes": ["https://www.googleapis.com/auth/drive.readonly"] + } + } + } + } +} +``` + +- [ ] **Step 2: Write failing parser tests** + +In `packages/core/test/google-discovery.test.ts`: + +```ts +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { discoveryOperations, googleDiscoveryScopesForOperations } from "../src/google-discovery"; + +const fixture = JSON.parse( + readFileSync(join(__dirname, "fixtures/google-discovery/drive.discovery.json"), "utf8"), +); + +describe("Google Discovery parser", () => { + it("maps resources and methods to Caplets operations", () => { + const operations = discoveryOperations({ + server: "drive", + document: fixture, + includeOperations: ["drive.files.*"], + excludeOperations: ["drive.files.delete"], + }); + + expect(operations.map((operation) => operation.name)).toEqual([ + "drive.files.create", + "drive.files.download", + "drive.files.list", + ]); + expect(operations.find((operation) => operation.name === "drive.files.list")).toMatchObject({ + method: "get", + path: "files", + readOnlyHint: true, + destructiveHint: false, + inputSchema: { + properties: { + query: { + properties: { + fields: { type: "string" }, + pageSize: { type: "integer", default: 100 }, + }, + }, + }, + }, + }); + }); + + it("marks destructive operations and resolves filtered scopes", () => { + const operations = discoveryOperations({ server: "drive", document: fixture }); + expect(operations.find((operation) => operation.name === "drive.files.delete")).toMatchObject({ + destructiveHint: true, + }); + expect(googleDiscoveryScopesForOperations(operations)).toEqual([ + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/drive.readonly", + ]); + }); +}); +``` + +- [ ] **Step 3: Run failing parser tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/google-discovery.test.ts +``` + +Expected: fail because parser files do not exist. + +- [ ] **Step 4: Implement types** + +Create `packages/core/src/google-discovery/types.ts`: + +```ts +export type GoogleDiscoveryDocument = { + kind?: string; + id?: string; + name?: string; + version?: string; + title?: string; + rootUrl?: string; + servicePath?: string; + baseUrl?: string; + auth?: { oauth2?: { scopes?: Record } }; + parameters?: Record; + schemas?: Record; + resources?: Record; +}; + +export type GoogleDiscoveryResource = { + methods?: Record; + resources?: Record; +}; + +export type GoogleDiscoveryMethod = { + id?: string; + path?: string; + flatPath?: string; + httpMethod?: string; + description?: string; + parameters?: Record; + parameterOrder?: string[]; + request?: { $ref?: string }; + response?: { $ref?: string }; + scopes?: string[]; + supportsMediaUpload?: boolean; + supportsMediaDownload?: boolean; + mediaUpload?: { + accept?: string[]; + maxSize?: string; + protocols?: Record; + }; +}; + +export type GoogleDiscoveryParameter = GoogleDiscoverySchema & { + location?: "path" | "query" | "header"; + required?: boolean; + repeated?: boolean; + deprecated?: boolean; +}; + +export type GoogleDiscoverySchema = { + id?: string; + $ref?: string; + type?: string; + format?: string; + description?: string; + default?: unknown; + enum?: string[]; + repeated?: boolean; + properties?: Record; + items?: GoogleDiscoverySchema; + additionalProperties?: GoogleDiscoverySchema; +}; +``` + +- [ ] **Step 5: Implement schema conversion** + +Create `packages/core/src/google-discovery/schema.ts` with: + +```ts +import type { GoogleDiscoverySchema } from "./types"; + +export function googleDiscoverySchemaToJsonSchema( + value: GoogleDiscoverySchema | undefined, + schemas: Record = {}, + seen = new Set(), +): Record { + if (!value) return {}; + if (value.$ref) { + const target = schemas[value.$ref]; + if (!target || seen.has(value.$ref)) return { type: "object", additionalProperties: true }; + return googleDiscoverySchemaToJsonSchema(target, schemas, new Set([...seen, value.$ref])); + } + const type = value.type === "any" ? "object" : value.type; + const converted: Record = {}; + if (value.description) converted.description = collapseWhitespace(value.description); + if (type) converted.type = type; + if (value.format) converted.format = value.format; + if (value.enum) converted.enum = value.enum; + const defaultValue = convertedDefault(value.default, type); + if (defaultValue !== undefined) converted.default = defaultValue; + if (value.repeated) { + return { + ...(converted.description ? { description: converted.description } : {}), + type: "array", + items: omit(converted, ["description", "default"]), + }; + } + if (value.items) converted.items = googleDiscoverySchemaToJsonSchema(value.items, schemas, seen); + if (value.properties) { + converted.type = converted.type ?? "object"; + converted.properties = Object.fromEntries( + Object.entries(value.properties).map(([key, schema]) => [ + key, + googleDiscoverySchemaToJsonSchema(schema, schemas, seen), + ]), + ); + converted.additionalProperties = false; + } + if (value.additionalProperties) { + converted.additionalProperties = googleDiscoverySchemaToJsonSchema( + value.additionalProperties, + schemas, + seen, + ); + } + return converted; +} + +function convertedDefault(value: unknown, type: string | undefined): unknown { + if (value === undefined) return undefined; + if (type === "boolean" && typeof value === "string") return value === "true"; + if ((type === "integer" || type === "number") && typeof value === "string") { + const number = Number(value); + return Number.isFinite(number) ? number : value; + } + return value; +} + +function collapseWhitespace(value: string): string { + return value.replace(/\s+/gu, " ").trim(); +} + +function omit(value: Record, keys: string[]): Record { + return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key))); +} +``` + +- [ ] **Step 6: Implement operation mapping** + +Create `packages/core/src/google-discovery/operations.ts` with exported `GoogleDiscoveryOperation`, `discoveryOperations`, `googleDiscoveryScopesForOperations`, and glob matching. The operation type must carry: + +```ts +export type GoogleDiscoveryOperation = { + name: string; + method: "get" | "put" | "post" | "delete" | "patch"; + path: string; + description?: string; + inputSchema: Record; + outputSchema?: Record; + readOnlyHint: boolean; + destructiveHint: boolean; + scopes: string[]; + supportsMediaUpload: boolean; + supportsMediaDownload: boolean; + mediaUploadProtocols: Record; +}; +``` + +Use `method.id` as the operation name. Sort operations by name. Use path parameters as required. Apply include/exclude patterns against operation names before returning. + +- [ ] **Step 7: Export parser helpers and run tests** + +Create `packages/core/src/google-discovery/index.ts`: + +```ts +export * from "./types"; +export * from "./schema"; +export * from "./operations"; +``` + +Run: + +```bash +pnpm --filter @caplets/core test -- test/google-discovery.test.ts +``` + +Expected: parser tests pass. + +### Task 6: `GoogleDiscoveryManager` For Discovery, Descriptors, Search, And JSON Calls + +**Files:** + +- Create: `packages/core/src/google-discovery/request.ts` +- Create: `packages/core/src/google-discovery/manager.ts` +- Modify: `packages/core/src/google-discovery/index.ts` +- Test: `packages/core/test/google-discovery.test.ts` + +- [ ] **Step 1: Extend fixture server tests** + +In `packages/core/test/google-discovery.test.ts`, add a local HTTP server similar to `openapi.test.ts` that serves `/drive.discovery.json`, `/drive/v3/files`, `/drive/v3/files/{fileId}`, and JSON responses. Add a test: + +```ts +it("lists, describes, searches, and calls Google Discovery operations", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + includeOperations: ["drive.files.*"], + excludeOperations: ["drive.files.delete"], + }, + }, + }); + const registry = new ServerRegistry(config); + const manager = new GoogleDiscoveryManager(registry); + const caplet = config.googleDiscoveryApis.drive!; + + await expect(manager.listTools(caplet)).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "drive.files.list" }), + expect.objectContaining({ name: "drive.files.create" }), + ]), + ); + await expect(manager.getTool(caplet, "drive.files.list")).resolves.toMatchObject({ + inputSchema: { properties: { query: { properties: { pageSize: { type: "integer" } } } } }, + annotations: { readOnlyHint: true, destructiveHint: false }, + }); + await expect( + manager.callTool(caplet, "drive.files.list", { query: { pageSize: 2 } }), + ).resolves.toMatchObject({ + structuredContent: { status: 200, body: { files: [{ id: "1", name: "Report" }] } }, + isError: false, + }); +}); +``` + +- [ ] **Step 2: Run failing manager tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/google-discovery.test.ts +``` + +Expected: fail because `GoogleDiscoveryManager` is not implemented. + +- [ ] **Step 3: Implement request builder** + +Create `packages/core/src/google-discovery/request.ts` with functions: + +```ts +export function buildGoogleDiscoveryUrl( + api: GoogleDiscoveryApiConfig, + operation: GoogleDiscoveryOperation, + args: Record, +): URL; + +export function buildJsonRequestInit( + operation: GoogleDiscoveryOperation, + args: Record, + headers: Headers, +): RequestInit; +``` + +Rules: + +- Preserve base URL path like OpenAPI `buildOperationUrl`. +- Substitute `{path}` params from `args.path`. +- Append query params from `args.query`. +- Reject object/array query and path values. +- Set `content-type: application/json` only when `args.body` is present. +- Reject attempts to supply forbidden headers through args. + +- [ ] **Step 4: Implement manager skeleton** + +Create `packages/core/src/google-discovery/manager.ts` with the same public method shape as `OpenApiManager`: + +```ts +export class GoogleDiscoveryManager { + constructor( + private registry: ServerRegistry, + private readonly options: { authDir?: string; artifactDir?: string } = {}, + ) {} + + updateRegistry(registry: ServerRegistry): void; + invalidate(serverId: string): void; + checkApi(api: GoogleDiscoveryApiConfig): Promise<{ + id: string; + status: string; + toolCount?: number; + elapsedMs: number; + error?: unknown; + }>; + listTools(api: GoogleDiscoveryApiConfig): Promise; + getTool(api: GoogleDiscoveryApiConfig, toolName: string): Promise; + callTool( + api: GoogleDiscoveryApiConfig, + toolName: string, + args: Record, + ): Promise; + compact(api: GoogleDiscoveryApiConfig, tool: Tool): CompactTool; + search(api: GoogleDiscoveryApiConfig, tools: Tool[], query: string, limit: number): CompactTool[]; + resolveAuthScopes(api: GoogleDiscoveryApiConfig): Promise; +} +``` + +Cache parsed operations by server and source cache key. Load source from `discoveryPath` or `discoveryUrl`, reject redirects, enforce request timeout, parse JSON, and validate `kind === "discovery#restDescription"` or presence of `resources` plus `schemas`. + +- [ ] **Step 5: Implement JSON call path** + +In `callTool`, for operations without media upload/download: + +1. Build URL. +2. Apply auth via `genericOAuthHeaders(api as GenericAuthTarget, authDir)` or static auth helpers. +3. Fetch with `redirect: "manual"`. +4. Reject redirects. +5. Use `readHttpLikeResponse(response, { capletId: api.server, artifactDir: this.options.artifactDir })`. +6. Return `content`, `structuredContent`, and `isError: !response.ok`. + +- [ ] **Step 6: Run manager tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/google-discovery.test.ts +``` + +Expected: pass for parser and JSON manager tests. + +### Task 7: Engine, Tool, Native, Direct Exposure, And Code Mode Integration + +**Files:** + +- Modify: `packages/core/src/engine.ts` +- Modify: `packages/core/src/tools.ts` +- Modify: `packages/core/src/native/service.ts` +- Modify: `packages/core/src/native/tools.ts` +- Modify: `packages/core/src/exposure/discovery.ts` only if type narrowing requires it +- Test: `packages/core/test/google-discovery.test.ts` +- Test: `packages/core/test/native.test.ts` +- Test: `packages/core/test/code-mode-api.test.ts` + +- [ ] **Step 1: Add failing end-to-end tool surface test** + +In `packages/core/test/google-discovery.test.ts`, add: + +```ts +it("executes Google Discovery operations through handleServerTool", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const registry = new ServerRegistry(config); + const manager = new GoogleDiscoveryManager(registry); + const downstream = new DownstreamManager(registry); + const caplet = config.googleDiscoveryApis.drive!; + + const list = (await handleServerTool( + caplet, + { operation: "tools" }, + registry, + downstream, + undefined, + undefined, + undefined, + undefined, + undefined, + manager, + )) as any; + + expect(list.structuredContent.result.items.map((tool: { name: string }) => tool.name)).toContain( + "drive.files.list", + ); +}); +``` + +The exact argument ordering must match the final `handleServerTool` signature. + +- [ ] **Step 2: Run failing integration test** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/google-discovery.test.ts +``` + +Expected: fail because engine/tools do not accept the manager. + +- [ ] **Step 3: Add manager to engine** + +In `packages/core/src/engine.ts`: + +- import `GoogleDiscoveryManager` +- add private `googleDiscovery` +- instantiate with `{ authDir, artifactDir }` +- update registry on reload +- invalidate on backend changes +- include in `listCompletionTools`, `listTools`, and `callTool` +- include in `allCaplets` +- pass to `handleServerTool` + +- [ ] **Step 4: Add manager to `tools.ts`** + +In `packages/core/src/tools.ts`: + +- import `GoogleDiscoveryManager` +- add optional param to `handleServerTool` +- add optional param to `backendFor` +- add branch before OpenAPI fallback: + +```ts +if (server.backend === "googleDiscovery") { + if (!googleDiscovery) { + throw new CapletsError("INTERNAL_ERROR", "Google Discovery manager is not configured"); + } + return { + check: (...args: Parameters) => + googleDiscovery.checkApi(...args), + listTools: (...args: Parameters) => + googleDiscovery.listTools(...args), + getTool: (...args: Parameters) => + googleDiscovery.getTool(...args), + callTool: (...args: Parameters) => + googleDiscovery.callTool(...args), + compact: (...args: Parameters) => + googleDiscovery.compact(...args), + search: (...args: Parameters) => + googleDiscovery.search(...args), + }; +} +``` + +- [ ] **Step 5: Add native guidance** + +In `packages/core/src/native/tools.ts`, keep guidance HTTP-like: + +```ts +if (caplet.backend === "googleDiscovery") { + return [ + `${toolName} exposes Google API operations from a Google Discovery document.`, + "Use tools/searchTools to find exact operation IDs and describeTool before calling media or write operations.", + ]; +} +``` + +In `packages/core/src/native/service.ts`, any direct per-backend behavior that currently checks `http`, `cli`, or `mcp` should treat `googleDiscovery` as a tool-only HTTP-like backend. + +- [ ] **Step 6: Run tool/native/code-mode tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/google-discovery.test.ts test/native.test.ts test/code-mode-api.test.ts +``` + +Expected: pass. + +### Task 8: OAuth Scope Inference And Auth Login Integration + +**Files:** + +- Modify: `packages/core/src/auth.ts` +- Modify: `packages/core/src/auth/store.ts` +- Modify: `packages/core/src/cli/auth.ts` +- Test: `packages/core/test/auth.test.ts` +- Test: `packages/core/test/google-discovery.test.ts` + +- [ ] **Step 1: Add failing auth tests** + +In `packages/core/test/auth.test.ts`, add a test that creates a Google Discovery config fixture and starts `auth login --no-open` with a fake manual code against a local OAuth server. Assert the printed auth URL scope includes `openid profile email` plus filtered Discovery scopes and excludes scopes from filtered-out operations. + +Use the existing generic OIDC authorization-code flow tests around `runGenericOAuthFlow` as the local OAuth fixture template. + +- [ ] **Step 2: Run failing auth tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/auth.test.ts test/google-discovery.test.ts +``` + +Expected: fail because auth target discovery does not resolve Google Discovery scopes. + +- [ ] **Step 3: Add resolved-scope support to generic OAuth** + +In `packages/core/src/auth.ts`, extend `GenericAuthTarget`: + +```ts +resolvedScopes?: string[] | undefined; +``` + +Update `scopesFor`: + +```ts +function scopesFor(authConfig: OAuthLikeAuthConfig, resolvedScopes?: string[]): string | undefined { + if (authConfig.scopes?.length) return authConfig.scopes.join(" "); + if (resolvedScopes?.length) { + const scopes = + authConfig.type === "oidc" + ? ["openid", "profile", "email", ...resolvedScopes] + : resolvedScopes; + return [...new Set(scopes)].sort(scopeSort).join(" "); + } + return authConfig.type === "oidc" ? "openid profile email" : undefined; +} +``` + +Preserve OIDC ordering for the three identity scopes if tests depend on exact URL text; otherwise sort only API scopes and prepend identity scopes. + +- [ ] **Step 4: Store requested scopes metadata** + +In `writeTokenBundle` call sites inside `runGenericOAuthFlow` and `startGenericOAuthFlow.complete`, add: + +```ts +metadata: redactSecrets({ + protectedResource: target.url ?? target.baseUrl ?? target.specUrl, + authorizationServer: metadata, + requestedScopes: scope?.split(/\s+/u).filter(Boolean), + dynamicClient: client.dynamic ? { client_id: client.clientId } : undefined, +}) as Record, +``` + +Do not remove existing metadata fields. + +- [ ] **Step 5: Validate requested scopes on token reuse** + +In `assertTokenBundleMatchesTarget`, compare required resolved scopes to `bundle.metadata.requestedScopes` when present. If metadata is absent, fall back to checking `bundle.scope` contains every required API scope. Return `AUTH_REQUIRED` if required scopes are missing. + +Do not require exact equality on provider-returned `bundle.scope` because Google may canonicalize identity scopes such as `email` to `https://www.googleapis.com/auth/userinfo.email`. + +- [ ] **Step 6: Resolve Google Discovery auth targets** + +In `packages/core/src/cli/auth.ts`: + +- include `config.googleDiscoveryApis` in `authTargets` +- for Google Discovery entries, load operations with `GoogleDiscoveryManager` and attach `resolvedScopes` +- set protected resource origin from `baseUrl` or inferred document base URL + +Use a helper: + +```ts +async function googleDiscoveryAuthTarget( + api: GoogleDiscoveryApiConfig, + authDir?: string, +): Promise { + const manager = new GoogleDiscoveryManager( + new ServerRegistry(parseConfig({ googleDiscoveryApis: { [api.server]: api } })), + { authDir }, + ); + return { + ...api, + baseUrl: await manager.resolveBaseUrl(api), + resolvedScopes: await manager.resolveAuthScopes(api), + }; +} +``` + +Adapt the helper to avoid recursive config parsing if a cleaner constructor path is available. + +- [ ] **Step 7: Run auth tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/auth.test.ts test/google-discovery.test.ts +``` + +Expected: pass. + +### Task 9: Google Media Download And Upload Protocols + +**Files:** + +- Modify: `packages/core/src/google-discovery/request.ts` +- Modify: `packages/core/src/google-discovery/manager.ts` +- Modify: `packages/core/src/google-discovery/operations.ts` +- Test: `packages/core/test/google-discovery.test.ts` + +- [ ] **Step 1: Add failing media protocol tests** + +In the Google Discovery fixture server, add endpoints: + +- `GET /drive/v3/files/1/download` returns `application/pdf` bytes. +- `POST /upload/drive/v3/files?uploadType=media` records raw body. +- `POST /upload/drive/v3/files?uploadType=multipart` records multipart body and returns JSON. +- `POST /upload/drive/v3/files?uploadType=resumable` returns `Location: /upload/session/abc`. +- `PUT /upload/session/abc` accepts chunks and returns final JSON after the last chunk. + +Add tests: + +```ts +it("writes Google media downloads as artifacts", async () => { + const result = await manager.callTool(caplet, "drive.files.download", { + path: { fileId: "1" }, + filename: "report.pdf", + }); + expect(result.structuredContent).toMatchObject({ + status: 200, + body: { artifact: { filename: "report.pdf", mimeType: "application/pdf" } }, + }); +}); + +it("uploads media from path using multipart when metadata body is present", async () => { + const mediaPath = join(tempDir, "report.pdf"); + writeFileSync(mediaPath, "pdf"); + const result = await manager.callTool(caplet, "drive.files.create", { + body: { name: "report.pdf" }, + media: { path: mediaPath, mimeType: "application/pdf" }, + }); + expect(result.structuredContent).toMatchObject({ status: 200, body: { id: "uploaded" } }); + expect(lastUploadRequest.headers["content-type"]).toContain("multipart/related"); +}); + +it("uploads small media from dataUrl and never echoes the data URL", async () => { + const result = await manager.callTool(caplet, "drive.files.create", { + media: { dataUrl: "data:text/plain;base64,aGVsbG8=", filename: "hello.txt" }, + }); + expect(JSON.stringify(result)).not.toContain("aGVsbG8="); +}); +``` + +- [ ] **Step 2: Run failing media tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/google-discovery.test.ts +``` + +Expected: fail because media protocols are not implemented. + +- [ ] **Step 3: Add media args to tool input schemas** + +For operations with `supportsMediaUpload`, add: + +```ts +media: { + type: "object", + additionalProperties: false, + properties: { + path: { type: "string" }, + artifact: { type: "string" }, + dataUrl: { type: "string", description: "Small base64 data URL. Prefer media.path or media.artifact." }, + filename: { type: "string" }, + mimeType: { type: "string" } + } +} +``` + +For `supportsMediaDownload`, add optional top-level `outputPath` and `filename` unless the implementation uses `_caplets.outputPath`; keep the final input shape documented in test assertions. + +- [ ] **Step 4: Implement download path** + +For `supportsMediaDownload`, call the normal request URL, fetch the response, and route through `readHttpLikeResponse` with `filename` and `outputPath`. + +- [ ] **Step 5: Implement upload protocol selection** + +Protocol selection: + +- If `media` is absent, use normal JSON request. +- If media is present and `body` is present and `multipart` protocol exists, use multipart. +- If media is present and body is absent and simple protocol exists, use simple. +- If file size exceeds `resumableThresholdBytes` or neither simple nor multipart is available, use resumable when available. + +Add defaults: + +```ts +const DEFAULT_RESUMABLE_THRESHOLD_BYTES = 8 * 1024 * 1024; +const DEFAULT_RESUMABLE_CHUNK_BYTES = 8 * 1024 * 1024; +``` + +- [ ] **Step 6: Implement simple upload** + +Build upload URL from `mediaUpload.protocols.simple.path` and append `uploadType=media`. Send raw bytes with content type from resolved media input. + +- [ ] **Step 7: Implement multipart upload** + +Build `multipart/related` body with JSON metadata part and media part: + +```ts +const boundary = `caplets_${randomUUID().replace(/-/gu, "")}`; +const body = Buffer.concat([ + Buffer.from( + `--${boundary}\r\ncontent-type: application/json; charset=UTF-8\r\n\r\n${JSON.stringify(args.body ?? {})}\r\n`, + ), + Buffer.from( + `--${boundary}\r\ncontent-type: ${media.mimeType ?? "application/octet-stream"}\r\n\r\n`, + ), + media.bytes, + Buffer.from(`\r\n--${boundary}--\r\n`), +]); +``` + +- [ ] **Step 8: Implement single-call resumable upload** + +Start session with `uploadType=resumable`, `X-Upload-Content-Type`, and `X-Upload-Content-Length`. Read `Location`. Upload chunks with `Content-Range`. Retry 5xx and 429 chunk failures up to 3 attempts with short exponential backoff. Return the final JSON/media response through the shared response reader. + +- [ ] **Step 9: Run media tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/google-discovery.test.ts test/media-artifacts.test.ts +``` + +Expected: pass. + +### Task 10: CLI Add Command And Remote Add Shape + +**Files:** + +- Modify: `packages/core/src/cli/add.ts` +- Modify: `packages/core/src/cli.ts` +- Modify: `packages/core/src/remote-control/dispatch.ts` if remote add kind validation exists +- Test: `packages/core/test/cli.test.ts` +- Test: `packages/core/test/remote-control-dispatch.test.ts` if remote add kinds are tested + +- [ ] **Step 1: Add failing CLI add tests** + +In `packages/core/test/cli.test.ts`, add: + +```ts +it("prints added Google Discovery backend Caplets", async () => { + const out: string[] = []; + await runCli( + [ + "add", + "google-discovery", + "google-drive", + "--discovery-url", + "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + "--print", + ], + { writeOut: (value) => out.push(value) }, + ); + expect(out.join("\n")).toContain("googleDiscoveryApi:"); + expect(out.join("\n")).toContain("discoveryUrl:"); +}); + +it("writes Google Discovery local discovery paths that load from the original project file", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-add-google-discovery-path-")); + const projectRoot = join(dir, "project"); + const cwd = process.cwd(); + try { + mkdirSync(projectRoot, { recursive: true }); + writeFileSync( + join(projectRoot, "drive.discovery.json"), + JSON.stringify({ kind: "discovery#restDescription", resources: {} }), + ); + process.chdir(projectRoot); + + await runCli(["add", "google-discovery", "drive", "--discovery", "./drive.discovery.json"], { + writeOut: () => {}, + }); + + const config = loadConfig( + join(dir, "user", "config.json"), + join(projectRoot, ".caplets", "config.json"), + ); + expect(config.googleDiscoveryApis.drive?.discoveryPath).toBe( + join(projectRoot, "drive.discovery.json"), + ); + } finally { + process.chdir(cwd); + rmSync(dir, { recursive: true, force: true }); + } +}); +``` + +- [ ] **Step 2: Run failing CLI tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/cli.test.ts +``` + +Expected: fail because command does not exist. + +- [ ] **Step 3: Add `addGoogleDiscoveryCaplet`** + +In `packages/core/src/cli/add.ts`, add options and implementation: + +```ts +type AddGoogleDiscoveryOptions = AddDestinationOptions & { + discovery?: string; + discoveryUrl?: string; + baseUrl?: string; + tokenEnv?: string; +}; + +export function addGoogleDiscoveryCaplet( + id: string, + options: AddGoogleDiscoveryOptions, +): { path?: string; text: string } { + const source = options.discovery ?? options.discoveryUrl; + if (!source) { + throw new CapletsError( + "REQUEST_INVALID", + "Google Discovery Caplet requires --discovery or --discovery-url", + ); + } + return writeGeneratedCaplet( + id, + "Google Discovery", + "googleDiscoveryApi", + [ + [isUrlLike(source) ? "discoveryUrl" : "discoveryPath", source], + ["baseUrl", options.baseUrl], + ["auth", authFromTokenEnv(options.tokenEnv) ?? { type: "none" }], + ], + options, + ); +} +``` + +- [ ] **Step 4: Add CLI command** + +In `packages/core/src/cli.ts`, add: + +```ts +add + .command("google-discovery") + .description("Add a Google Discovery API backend Caplet.") + .argument("", "Caplet ID/display seed") + .option("--discovery ", "Google Discovery document path or URL") + .option("--discovery-url ", "remote Google Discovery document URL") + .option("--base-url ", "request base URL override") + .option("--token-env ", "bearer token environment variable reference") + .option("--project", "write to the project Caplets root") + .option("-g, --global", "write to the user Caplets root") + .option("--remote", "add through remote control") + .option("--print", "print generated Caplet text without writing a file") + .option("--output ", "output path") + .option("--force", "overwrite an existing destination file"); +``` + +Use kind `"googleDiscovery"` for remote add payloads. + +- [ ] **Step 5: Run CLI tests** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/cli.test.ts +``` + +Expected: pass. + +### Task 11: Docs, Schemas, And Generated References + +**Files:** + +- Modify: `docs/architecture.md` +- Modify: `apps/docs/src/content/docs/reference/config.mdx` +- Modify: `apps/docs/src/content/docs/reference/caplet-files.mdx` +- Modify: `apps/docs/src/content/docs/capabilities.mdx` +- Modify: `apps/docs/src/content/docs/troubleshooting.mdx` +- Modify: `apps/docs/src/content/docs/changelog.mdx` +- Generate: `schemas/caplets-config.schema.json` +- Generate: `schemas/caplet.schema.json` + +- [ ] **Step 1: Generate schemas** + +Run: + +```bash +pnpm schema:generate +``` + +Expected: `schemas/caplets-config.schema.json` and `schemas/caplet.schema.json` include `googleDiscoveryApis` and `googleDiscoveryApi`. + +- [ ] **Step 2: Update architecture docs** + +In `docs/architecture.md`, add `googleDiscoveryApis` to supported backend families and update the HTTP-like backend section to mention Google Discovery API backends expose operation tools and share media artifact behavior. + +- [ ] **Step 3: Update public docs** + +In `apps/docs/src/content/docs/capabilities.mdx`, add an example: + +```sh +caplets add google-discovery google-drive --discovery-url https://www.googleapis.com/discovery/v1/apis/drive/v3/rest +``` + +In reference docs, add tables for: + +- `googleDiscoveryApis` +- `googleDiscoveryApi` +- `discoveryUrl` +- `discoveryPath` +- `baseUrl` +- `includeOperations` +- `excludeOperations` +- media upload input fields + +- [ ] **Step 4: Update changelog** + +Add a concise entry to `apps/docs/src/content/docs/changelog.mdx` describing: + +- Google Discovery API backend +- inferred scopes +- media artifacts + +- [ ] **Step 5: Run doc/schema checks** + +Run: + +```bash +pnpm schema:check +pnpm format:check +``` + +Expected: pass. + +### Task 12: End-To-End Regression And Full Verification + +**Files:** + +- Test files touched by prior tasks +- Generated schemas and docs + +- [ ] **Step 1: Run focused backend test set** + +Run: + +```bash +pnpm --filter @caplets/core test -- test/google-discovery.test.ts test/media-artifacts.test.ts test/http-actions.test.ts test/openapi.test.ts test/auth.test.ts test/config.test.ts test/caplet-files.test.ts test/cli.test.ts +``` + +Expected: all tests pass. + +- [ ] **Step 2: Run typecheck** + +Run: + +```bash +pnpm typecheck +``` + +Expected: pass. + +- [ ] **Step 3: Run generated API checks** + +Run: + +```bash +pnpm code-mode:check-api +pnpm schema:check +``` + +Expected: pass. If Code Mode declaration changes are intentional, run `pnpm code-mode:generate-api` and re-run `pnpm code-mode:check-api`. + +- [ ] **Step 4: Run full verification** + +Run: + +```bash +pnpm verify +``` + +Expected: pass through `format:check`, `lint`, `code-mode:check-api`, `typecheck`, `schema:check`, `test`, `benchmark:check`, and `build`. + +- [ ] **Step 5: Manual local smoke** + +Use a local config with Google Drive: + +```json +{ + "googleDiscoveryApis": { + "google-drive": { + "name": "Google Drive", + "description": "Access and manipulate Google Drive files and folders.", + "discoveryUrl": "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + "auth": { + "type": "oidc", + "issuer": "https://accounts.google.com", + "clientId": "$env:GOOGLE_CLIENT_ID", + "clientSecret": "$env:GOOGLE_CLIENT_SECRET" + }, + "includeOperations": [ + "drive.files.list", + "drive.files.get", + "drive.files.create", + "drive.files.download" + ] + } + } +} +``` + +Run: + +```bash +caplets auth login google-drive +caplets list-tools google-drive +caplets call-tool google-drive.drive.files.list --args '{"query":{"pageSize":5,"fields":"files(id,name,mimeType),nextPageToken"}}' --format json +``` + +Expected: + +- auth URL includes inferred Drive scopes +- tool list includes Drive file operations +- `drive.files.list` returns HTTP `200` + +## Self-Review Checklist + +- Spec coverage: + - First-class backend: Tasks 3, 4, 6, 7, 10, 11. + - Native Discovery parser: Task 5. + - Inferred scopes: Task 8. + - Operation filters: Tasks 3 and 5. + - Comprehensive media: Tasks 1, 2, 9. + - Shared Media artifacts: Tasks 1 and 2. + - Existing surfaces preserved: Tasks 7, 10, 11, 12. +- No third-party converter dependency is introduced. +- No persisted resumable sessions are introduced. +- Data URLs are fallback media inputs only. +- Implementation proceeds TDD-first with focused tests before `pnpm verify`. diff --git a/docs/specs/2026-06-16-google-discovery-api-backend.md b/docs/specs/2026-06-16-google-discovery-api-backend.md new file mode 100644 index 00000000..f91f0746 --- /dev/null +++ b/docs/specs/2026-06-16-google-discovery-api-backend.md @@ -0,0 +1,229 @@ +# Google Discovery API Backend + +## Summary + +Add a first-class Google Discovery API backend for Google APIs whose machine-readable contract is a Google Discovery document rather than OpenAPI. The backend should be comprehensive: operation discovery, inferred OAuth scopes, operation filtering, JSON calls, media downloads, media uploads, and Caplet file support all ship as part of the design. + +This backend must not live under `openapiEndpoints`. It is a separate backend family with its own source format and manager, while sharing lower-level HTTP, auth, and media artifact infrastructure with other HTTP-like backends. + +## Goals + +- Add top-level `googleDiscoveryApis` config and `googleDiscoveryApi` Caplet file support. +- Parse Google Discovery documents natively instead of converting them to OpenAPI as the primary abstraction. +- Infer OAuth scopes from exposed Discovery operations unless `auth.scopes` overrides inference. +- Support operation include/exclude filters and compute inferred scopes after filtering. +- Support comprehensive Google media download and upload behavior. +- Add a shared Media artifact contract usable by Google Discovery, OpenAPI, HTTP, and future media-capable backends. +- Preserve existing Caplets exposure modes, Code Mode handles, progressive wrappers, native service behavior, and CLI inspection/call surfaces. + +## Non-Goals + +- Do not depend on hosted APIs.guru specs or a third-party Discovery-to-OpenAPI converter. +- Do not persist resumable upload sessions across separate Caplets calls in the first version. +- Do not auto-confirm destructive operations inside Caplets. Caplets exposes safety hints; clients and agents decide how to handle them. +- Do not treat inline data URLs as the primary agent media path. + +## Public Config + +Top-level config uses `googleDiscoveryApis`: + +```json +{ + "googleDiscoveryApis": { + "google-drive": { + "name": "Google Drive", + "description": "Access and manipulate Google Drive files and folders.", + "discoveryUrl": "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + "auth": { + "type": "oidc", + "issuer": "https://accounts.google.com", + "clientId": "$env:GOOGLE_CLIENT_ID", + "clientSecret": "$env:GOOGLE_CLIENT_SECRET" + } + } + } +} +``` + +Supported source fields: + +- `discoveryUrl`: remote Google Discovery document URL. +- `discoveryPath`: local Google Discovery document path. +- `baseUrl`: optional request base URL override; default is inferred from the document. + +Common Caplet fields match other backend families: `name`, `description`, `auth`, `tags`, `exposure`, `shadowing`, `useWhen`, `avoidWhen`, `setup`, `projectBinding`, `runtime`, `requestTimeoutMs`, `operationCacheTtlMs`, and `disabled`. + +Caplet files use `googleDiscoveryApi`: + +```yaml +googleDiscoveryApi: + discoveryUrl: https://www.googleapis.com/discovery/v1/apis/drive/v3/rest + auth: + type: oidc + issuer: https://accounts.google.com +``` + +The normalized backend discriminator is `backend: "googleDiscovery"`. + +## Discovery Mapping + +`GoogleDiscoveryManager` loads `discoveryUrl` or `discoveryPath`, validates that the document is a Google Discovery document, and recursively walks `resources.*.methods.*`. + +Each Discovery method becomes a Caplets tool: + +- tool name comes from `method.id`, such as `drive.files.list` +- HTTP method comes from `method.httpMethod` +- request path comes from `method.path` +- base URL is inferred from `baseUrl`, or `rootUrl + servicePath`, with config override support +- path/query parameters come from method parameters plus global parameters +- request body schema comes from `method.request.$ref` +- response body schema comes from `method.response.$ref` +- schemas come from top-level Discovery `schemas`, converted into the JSON Schema-like shape Caplets tools expose + +The manager preserves Google-specific metadata internally, including `supportsMediaUpload`, `mediaUpload.protocols`, `supportsMediaDownload`, `parameterOrder`, and operation scopes. + +## Operation Filters + +Google Discovery APIs support operation filters: + +```json +{ + "includeOperations": ["drive.files.*", "drive.permissions.list"], + "excludeOperations": ["*.delete", "drive.files.emptyTrash"] +} +``` + +Rules: + +- Operation IDs are Discovery `method.id` values. +- If `includeOperations` is absent, all operations are included. +- `excludeOperations` applies after include. +- Filtering controls tool discovery, tool execution, and inferred scopes. +- Glob matching is simple and documented; it is not regular-expression matching. + +## Auth And Scope Inference + +Google Discovery APIs infer OAuth scopes from the Discovery document unless `auth.scopes` overrides inference. + +Rules: + +- If `auth.scopes` is configured, use it exactly. +- Otherwise infer scopes from the final exposed operation set after operation filters. +- For `oidc`, include `openid profile email` plus inferred Google API scopes. +- De-duplicate and sort inferred API scopes for stable authorization URLs. +- Store the requested or granted scope string in the OAuth token bundle. +- Tool descriptors expose operation-level possible scopes as guidance. +- If config changes cause the inferred scope set to differ from the stored bundle, require re-login rather than silently calling with insufficient scopes. + +The generic OAuth flow needs a way for backends to supply resolved scopes at login time; today it only reads static `auth.scopes`. + +## Safety Annotations + +Safety annotations mirror existing HTTP-like behavior: + +- `GET` and `HEAD` operations are `readOnlyHint: true`. +- `DELETE` operations are `destructiveHint: true`. +- Known destructive non-DELETE methods also receive destructive hints using method ID and description patterns, such as `emptyTrash`. +- Caplets does not add confirmation prompts. + +Broad Google APIs are still exposed by default. Users can narrow them with operation filters. + +## Shared Media Artifact Contract + +Media-capable backends use a shared Media artifact contract. + +Small JSON and text responses remain inline. Binary responses, media downloads, and oversized text responses are written to Caplets-managed artifact storage and returned as metadata: + +```json +{ + "status": 200, + "headers": { "content-type": "application/pdf" }, + "body": { + "artifact": { + "path": "/Users/ianpascoe/.local/state/caplets/artifacts/google-drive/...", + "mimeType": "application/pdf", + "byteLength": 12345, + "sha256": "..." + } + } +} +``` + +Default artifact storage: + +```text +~/.local/state/caplets/artifacts/// +``` + +Rules: + +- Caller-provided `outputPath` is allowed for explicit download destinations. +- Local execution returns absolute local paths. +- Remote and hosted execution return artifact references or links, not pretend-local paths. +- Artifact metadata should also appear in `_meta.caplets.artifacts` when useful for clients. +- HTTP, OpenAPI, Google Discovery, and future media-capable backends share the infrastructure. + +## Google Media Download And Upload + +Downloads and exports: + +- `supportsMediaDownload` methods can return inline JSON/text when appropriate or Media artifacts for binary/large content. +- Download/export tools accept optional `outputPath` and `filename`. +- Response metadata includes status, content type, byte count, hash, and artifact information. + +Uploads use an agent-first media input contract: + +```json +{ + "body": { "name": "report.pdf" }, + "media": { "path": "/abs/path/report.pdf", "mimeType": "application/pdf" } +} +``` + +Supported media sources, in priority order: + +- `media.path`: primary for local coding agents. +- `media.artifact`: primary for chaining prior Caplets media outputs into uploads. +- `media.dataUrl`: supported only as a small-input fallback. + +Rules: + +- Exactly one of `path`, `artifact`, or `dataUrl` is accepted. +- Raw `dataUrl` content must not be echoed in output, logs, errors, or descriptors. +- Max decoded or input size is enforced. +- MIME type is inferred from file, artifact, or data URL when possible; explicit `mimeType` can override inference. + +Google upload protocols: + +- simple upload for media-only requests +- multipart upload for metadata plus media +- resumable upload for large files or when selected by protocol/defaults + +Resumable uploads are internal to one Caplets call. The backend may retry transient chunk failures within that call, but v1 does not expose persisted resumable session resume or cancel commands. + +## CLI And Docs + +CLI additions: + +- `caplets add google-discovery --discovery-url ` +- optional parity form: `--discovery ` +- existing `list`, `inspect`, `list-tools`, `get-tool`, `call-tool`, `auth login`, and Code Mode surfaces work with the new backend + +Docs updates: + +- configuration reference for `googleDiscoveryApis` +- Caplet file reference for `googleDiscoveryApi` +- capabilities docs for Google Discovery API backends +- media artifact documentation shared across HTTP-like backends +- ADR for path/artifact-based media result handling + +## Verification + +Implementation should include: + +- unit tests for Discovery parsing, schema conversion, operation filtering, scope inference, and request construction +- tests for media artifact writing and inline-vs-artifact response selection +- tests for simple, multipart, and single-call resumable upload behavior using local fixtures +- CLI tests for add/list/get/call/auth URL behavior +- schema generation and schema checks after config changes +- focused tests first, then `pnpm verify` once implementation is complete diff --git a/packages/core/src/auth.ts b/packages/core/src/auth.ts index b66388f4..486ed679 100644 --- a/packages/core/src/auth.ts +++ b/packages/core/src/auth.ts @@ -47,10 +47,11 @@ type OAuthLikeAuthConfig = { export type GenericAuthTarget = { server: string; - backend: "openapi" | "graphql" | "http"; + backend: "openapi" | "googleDiscovery" | "graphql" | "http"; url?: string | undefined; baseUrl?: string | undefined; specUrl?: string | undefined; + resolvedScopes?: string[] | undefined; auth?: OAuthLikeAuthConfig | { type: string } | undefined; requestTimeoutMs?: number | undefined; }; @@ -557,7 +558,7 @@ export async function startGenericOAuthFlow( redirectUri, allowLoopbackHttp, ); - const scope = scopesFor(authConfig); + const scope = scopesFor(authConfig, target.resolvedScopes); const authorizationUrl = new URL(authorizationEndpoint); authorizationUrl.searchParams.set("response_type", "code"); authorizationUrl.searchParams.set("client_id", client.clientId); @@ -621,6 +622,7 @@ export async function startGenericOAuthFlow( metadata: redactSecrets({ protectedResource: target.url ?? target.baseUrl ?? target.specUrl, authorizationServer: metadata, + requestedScopes: scope?.split(/\s+/u).filter(Boolean), dynamicClient: client.dynamic ? { client_id: client.clientId } : undefined, }) as Record, }), @@ -680,7 +682,7 @@ export async function runGenericOAuthFlow( redirectUri, allowLoopbackHttp, ); - const scope = scopesFor(authConfig); + const scope = scopesFor(authConfig, target.resolvedScopes); const authorizationUrl = new URL(authorizationEndpoint); authorizationUrl.searchParams.set("response_type", "code"); authorizationUrl.searchParams.set("client_id", client.clientId); @@ -755,6 +757,7 @@ export async function runGenericOAuthFlow( metadata: redactSecrets({ protectedResource: target.url ?? target.baseUrl ?? target.specUrl, authorizationServer: metadata, + requestedScopes: scope?.split(/\s+/u).filter(Boolean), dynamicClient: client.dynamic ? { client_id: client.clientId } : undefined, }) as Record, }); @@ -1098,7 +1101,8 @@ async function refreshGenericOAuthBundle( refreshToken: asString(tokenResponse.refresh_token) ?? bundle.refreshToken, tokenType: asString(tokenResponse.token_type) ?? bundle.tokenType, expiresAt: refreshedExpiresAt(tokenResponse.expires_in, bundle.expiresAt), - scope: asString(tokenResponse.scope) ?? bundle.scope ?? scopesFor(authConfig), + scope: + asString(tokenResponse.scope) ?? bundle.scope ?? scopesFor(authConfig, target.resolvedScopes), idToken: idToken ?? bundle.idToken, issuer: asString(idClaims?.iss) ?? bundle.issuer ?? metadata.issuer ?? authConfig.issuer, subject: asString(idClaims?.sub) ?? bundle.subject, @@ -1201,7 +1205,8 @@ function assertTokenBundleMatchesTarget( bundle.authType !== authConfig.type || (expectedOrigin && bundle.protectedResourceOrigin !== expectedOrigin) || (configuredClientId && bundle.clientId !== configuredClientId) || - (authConfig.issuer && bundle.issuer !== authConfig.issuer); + (authConfig.issuer && bundle.issuer !== authConfig.issuer) || + tokenBundleMissingScopes(bundle, authConfig, target.resolvedScopes); if (mismatch) { throw new CapletsError( "AUTH_REQUIRED", @@ -1242,10 +1247,47 @@ function isLoopbackHttpUrl(value: string): boolean { ); } -function scopesFor(authConfig: OAuthLikeAuthConfig): string | undefined { +function tokenBundleMissingScopes( + bundle: StoredOAuthTokenBundle, + authConfig: OAuthLikeAuthConfig, + resolvedScopes: string[] | undefined, +): boolean { + const required = requiredStoredScopes(authConfig, resolvedScopes); + if (required.length === 0) return false; + const metadataScopes = requestedScopesFromMetadata(bundle.metadata); + const actual = new Set(metadataScopes ?? bundle.scope?.split(/\s+/u).filter(Boolean) ?? []); + return required.some((scope) => !actual.has(scope)); +} + +function requiredStoredScopes( + authConfig: OAuthLikeAuthConfig, + resolvedScopes: string[] | undefined, +): string[] { + if (authConfig.scopes?.length) return authConfig.scopes; + return resolvedScopes?.length ? [...new Set(resolvedScopes)].sort() : []; +} + +function requestedScopesFromMetadata(metadata: unknown): string[] | undefined { + if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) return undefined; + const value = (metadata as Record).requestedScopes; + return Array.isArray(value) && value.every((entry) => typeof entry === "string") + ? value + : undefined; +} + +function scopesFor( + authConfig: OAuthLikeAuthConfig, + resolvedScopes?: string[] | undefined, +): string | undefined { if (authConfig.scopes?.length) { return authConfig.scopes.join(" "); } + if (resolvedScopes?.length) { + const apiScopes = [...new Set(resolvedScopes)].sort(); + return authConfig.type === "oidc" + ? ["openid", "profile", "email", ...apiScopes].join(" ") + : apiScopes.join(" "); + } return authConfig.type === "oidc" ? "openid profile email" : undefined; } diff --git a/packages/core/src/caplet-files-bundle.ts b/packages/core/src/caplet-files-bundle.ts index a86f15cd..df44d554 100644 --- a/packages/core/src/caplet-files-bundle.ts +++ b/packages/core/src/caplet-files-bundle.ts @@ -342,6 +342,67 @@ const capletOpenApiEndpointSchema = z validateEndpointAuthHeaders(endpoint.auth, ctx); }); +const capletGoogleDiscoveryOperationFilterSchema = z.array(z.string().trim().min(1).max(160)); + +const capletGoogleDiscoveryApiSchema = z + .object({ + discoveryPath: z.string().min(1).optional().describe("Local Google Discovery document path."), + discoveryUrl: z.string().min(1).optional().describe("Remote Google Discovery document URL."), + baseUrl: z.string().min(1).optional().describe("Override base URL for Google API requests."), + auth: capletEndpointAuthSchema.describe( + 'Explicit Google API request auth config. Use {"type":"none"} for public APIs.', + ), + requestTimeoutMs: z + .number() + .int() + .positive() + .optional() + .describe("Timeout in milliseconds for Google API HTTP requests."), + operationCacheTtlMs: z + .number() + .int() + .nonnegative() + .optional() + .describe( + "Milliseconds Google Discovery operation metadata stays fresh. Set 0 to refresh every time.", + ), + includeOperations: capletGoogleDiscoveryOperationFilterSchema.optional(), + excludeOperations: capletGoogleDiscoveryOperationFilterSchema.optional(), + disabled: z.boolean().optional().describe("When true, omit this Caplet from discovery."), + projectBinding: capletProjectBindingSchema.optional(), + runtime: capletRuntimeRequirementsSchema.optional(), + }) + .strict() + .superRefine((api, ctx) => { + if (Boolean(api.discoveryPath) === Boolean(api.discoveryUrl)) { + ctx.addIssue({ + code: "custom", + message: + "googleDiscoveryApi must define exactly one discovery source: discoveryPath or discoveryUrl", + }); + } + if ( + api.discoveryUrl && + !hasEnvReference(api.discoveryUrl) && + !isAllowedRemoteUrl(api.discoveryUrl) + ) { + ctx.addIssue({ + code: "custom", + path: ["discoveryUrl"], + message: "Google Discovery discoveryUrl must use https except loopback development urls", + }); + } + if (api.baseUrl && !hasEnvReference(api.baseUrl) && !isAllowedHttpBaseUrl(api.baseUrl)) { + ctx.addIssue({ + code: "custom", + path: ["baseUrl"], + message: + "Google Discovery baseUrl must use https except loopback development urls and must not include credentials, query, or fragment", + }); + } + validateEndpointAuthHeaders(api.auth, ctx); + }); + const capletGraphQlOperationSchema = z .object({ document: z.string().min(1).optional().describe("Inline GraphQL operation document."), @@ -653,6 +714,9 @@ export const capletFileSchema = z openapiEndpoint: capletOpenApiEndpointSchema .describe("OpenAPI endpoint backend configuration for this Caplet.") .optional(), + googleDiscoveryApi: capletGoogleDiscoveryApiSchema + .describe("Google Discovery API backend configuration for this Caplet.") + .optional(), graphqlEndpoint: capletGraphQlEndpointSchema .describe("GraphQL endpoint backend configuration for this Caplet.") .optional(), @@ -671,6 +735,7 @@ export const capletFileSchema = z const backendCount = Number(Boolean(frontmatter.mcpServer)) + Number(Boolean(frontmatter.openapiEndpoint)) + + Number(Boolean(frontmatter.googleDiscoveryApi)) + Number(Boolean(frontmatter.graphqlEndpoint)) + Number(Boolean(frontmatter.httpApi)) + Number(Boolean(frontmatter.cliTools)) + @@ -679,7 +744,7 @@ export const capletFileSchema = z ctx.addIssue({ code: "custom", message: - "Caplet file must define exactly one backend: mcpServer, openapiEndpoint, graphqlEndpoint, httpApi, cliTools, or capletSet", + "Caplet file must define exactly one backend: mcpServer, openapiEndpoint, googleDiscoveryApi, graphqlEndpoint, httpApi, cliTools, or capletSet", }); } }); @@ -699,6 +764,7 @@ export function capletJsonSchema(): unknown { export type CapletFileConfig = { mcpServers?: Record; openapiEndpoints?: Record; + googleDiscoveryApis?: Record; graphqlEndpoints?: Record; httpApis?: Record; cliTools?: Record; @@ -756,6 +822,7 @@ export function buildCapletFileLoadResultFromEntries( ): BestEffortCapletFileLoadResult | undefined { const servers: Record = {}; const openapiEndpoints: Record = {}; + const googleDiscoveryApis: Record = {}; const graphqlEndpoints: Record = {}; const httpApis: Record = {}; const cliTools: Record = {}; @@ -766,6 +833,7 @@ export function buildCapletFileLoadResultFromEntries( return Boolean( servers[id] || openapiEndpoints[id] || + googleDiscoveryApis[id] || graphqlEndpoints[id] || httpApis[id] || cliTools[id] || @@ -804,6 +872,9 @@ export function buildCapletFileLoadResultFromEntries( if (isPlainObject(config) && config.backend === "openapi") { const { backend: _backend, ...endpoint } = config; openapiEndpoints[candidate.id] = endpoint; + } else if (isPlainObject(config) && config.backend === "googleDiscovery") { + const { backend: _backend, ...api } = config; + googleDiscoveryApis[candidate.id] = api; } else if (isPlainObject(config) && config.backend === "graphql") { const { backend: _backend, ...endpoint } = config; graphqlEndpoints[candidate.id] = endpoint; @@ -823,6 +894,7 @@ export function buildCapletFileLoadResultFromEntries( const hasServers = Object.keys(servers).length > 0; const hasOpenApi = Object.keys(openapiEndpoints).length > 0; + const hasGoogleDiscovery = Object.keys(googleDiscoveryApis).length > 0; const hasGraphQl = Object.keys(graphqlEndpoints).length > 0; const hasHttpApis = Object.keys(httpApis).length > 0; const hasCliTools = Object.keys(cliTools).length > 0; @@ -830,6 +902,7 @@ export function buildCapletFileLoadResultFromEntries( const config = { ...(hasServers ? { mcpServers: servers } : {}), ...(hasOpenApi ? { openapiEndpoints } : {}), + ...(hasGoogleDiscovery ? { googleDiscoveryApis } : {}), ...(hasGraphQl ? { graphqlEndpoints } : {}), ...(hasHttpApis ? { httpApis } : {}), ...(hasCliTools ? { cliTools } : {}), @@ -920,6 +993,18 @@ function capletToServerConfig( }; } + if (frontmatter.googleDiscoveryApi) { + return { + ...frontmatter.googleDiscoveryApi, + discoveryPath: normalizePath(frontmatter.googleDiscoveryApi.discoveryPath, baseDir), + backend: "googleDiscovery", + name: frontmatter.name, + description: frontmatter.description, + ...sharedCapletFields(frontmatter), + body, + }; + } + if (frontmatter.graphqlEndpoint) { return { ...frontmatter.graphqlEndpoint, diff --git a/packages/core/src/caplet-sets.ts b/packages/core/src/caplet-sets.ts index 1b64c4ce..34242473 100644 --- a/packages/core/src/caplet-sets.ts +++ b/packages/core/src/caplet-sets.ts @@ -10,6 +10,7 @@ import { type CompactTool, } from "./downstream"; import { CapletsError, errorResult, toSafeError } from "./errors"; +import { GoogleDiscoveryManager } from "./google-discovery"; import { GraphQLManager } from "./graphql"; import { HttpActionManager } from "./http-actions"; import { OpenApiManager } from "./openapi"; @@ -25,6 +26,7 @@ type ChildRuntime = { graphql: GraphQLManager; http: HttpActionManager; cli: CliToolsManager; + googleDiscovery: GoogleDiscoveryManager; capletSets: CapletSetManager; cacheKey: string; configFingerprint: string; @@ -144,6 +146,8 @@ export class CapletSetManager { child.http, child.cli, child.capletSets, + {}, + child.googleDiscovery, )) as CompatibilityCallToolResult; } catch (error) { return errorResult(error) as CompatibilityCallToolResult; @@ -220,6 +224,7 @@ export class CapletSetManager { graphql: new GraphQLManager(registry, authOptions), http: new HttpActionManager(registry, authOptions), cli: new CliToolsManager(registry), + googleDiscovery: new GoogleDiscoveryManager(registry, authOptions), capletSets: new CapletSetManager(registry, { ...authOptions, ancestry: childAncestry, diff --git a/packages/core/src/caplet-source/parse.ts b/packages/core/src/caplet-source/parse.ts index 37bca581..1e9fe8a4 100644 --- a/packages/core/src/caplet-source/parse.ts +++ b/packages/core/src/caplet-source/parse.ts @@ -135,6 +135,7 @@ function capletsFromConfig(config: CapletsConfig): CapletConfig[] { return [ ...Object.values(config.mcpServers), ...Object.values(config.openapiEndpoints), + ...Object.values(config.googleDiscoveryApis ?? {}), ...Object.values(config.graphqlEndpoints), ...Object.values(config.httpApis), ...Object.values(config.cliTools), @@ -146,6 +147,9 @@ function localReferencePaths(caplet: CapletConfig): string[] { if (caplet.backend === "openapi") { return filterLocalReferences([caplet.specPath]); } + if (caplet.backend === "googleDiscovery") { + return filterLocalReferences([caplet.discoveryPath]); + } if (caplet.backend === "graphql") { return filterLocalReferences([ caplet.schemaPath, diff --git a/packages/core/src/cli.ts b/packages/core/src/cli.ts index ea5f2c6b..e2e287c9 100644 --- a/packages/core/src/cli.ts +++ b/packages/core/src/cli.ts @@ -4,6 +4,7 @@ import { createInterface } from "node:readline/promises"; import { version as packageJsonVersion } from "../package.json"; import { addCliCaplet, + addGoogleDiscoveryCaplet, addGraphqlCaplet, addHttpCaplet, addMcpCaplet, @@ -93,6 +94,7 @@ export { initConfig, starterConfig } from "./cli/init"; export { installCaplets, normalizeGitRepo } from "./cli/install"; export { addCliCaplet, + addGoogleDiscoveryCaplet, addGraphqlCaplet, addHttpCaplet, addMcpCaplet, @@ -1197,6 +1199,49 @@ export function createProgram(io: CliIO = {}): Command { }, ); + add + .command("google-discovery") + .description("Add a Google Discovery API backend Caplet.") + .argument("", "Caplet ID/display seed") + .option("--discovery ", "Google Discovery document path or URL") + .option("--discovery-url ", "remote Google Discovery document URL") + .option("--base-url ", "request base URL override") + .option("--token-env ", "bearer token environment variable reference") + .option("--project", "write to the project Caplets root") + .option("-g, --global", "write to the user Caplets root") + .option("--remote", "add through remote control") + .option("--print", "print generated Caplet text without writing a file") + .option("--output ", "output path") + .option("--force", "overwrite an existing destination file") + .action( + async ( + id: string, + options: AddBackendCliOptions & { + discovery?: string; + discoveryUrl?: string; + baseUrl?: string; + tokenEnv?: string; + }, + ) => { + const target = parseMutationTarget(options); + if (target === "remote") { + const remote = requireRemoteClientForTarget(io); + const result = await remote.request("add", { + kind: "googleDiscovery", + id, + options: remoteAddOptions(options), + }); + writeAddResult(writeOut, "Google Discovery", result as AddCliResult); + return; + } + const result = addGoogleDiscoveryCaplet(id, { + ...options, + destinationRoot: addDestinationRoot(target, currentConfigPath(), env), + }); + writeAddResult(writeOut, `${localMutationTargetLabel(target, io)}Google Discovery`, result); + }, + ); + add .command("graphql") .description("Add a GraphQL backend Caplet.") @@ -2368,6 +2413,7 @@ function mergePartialLocalOverlays( const capletConfigKinds = [ "mcpServers", "openapiEndpoints", + "googleDiscoveryApis", "graphqlEndpoints", "httpApis", "cliTools", @@ -2445,6 +2491,7 @@ function hasEnabledCaplet(config: CapletsConfig, id: string): boolean { const caplet = config.mcpServers[id] ?? config.openapiEndpoints[id] ?? + config.googleDiscoveryApis[id] ?? config.graphqlEndpoints[id] ?? config.httpApis[id] ?? config.cliTools[id] ?? diff --git a/packages/core/src/cli/add.ts b/packages/core/src/cli/add.ts index 2a8463c7..8d9c7b75 100644 --- a/packages/core/src/cli/add.ts +++ b/packages/core/src/cli/add.ts @@ -50,6 +50,13 @@ type AddOpenApiOptions = AddDestinationOptions & { tokenEnv?: string; }; +type AddGoogleDiscoveryOptions = AddDestinationOptions & { + discovery?: string; + discoveryUrl?: string; + baseUrl?: string; + tokenEnv?: string; +}; + type AddGraphqlOptions = AddDestinationOptions & { endpointUrl?: string; schema?: string; @@ -142,6 +149,30 @@ export function addOpenApiCaplet( ); } +export function addGoogleDiscoveryCaplet( + id: string, + options: AddGoogleDiscoveryOptions, +): { path?: string; text: string } { + const discovery = options.discovery ?? options.discoveryUrl; + if (!discovery) { + throw new CapletsError( + "REQUEST_INVALID", + "Google Discovery Caplet requires --discovery or --discovery-url", + ); + } + return writeGeneratedCaplet( + id, + "Google Discovery", + "googleDiscoveryApi", + [ + [isUrlLike(discovery) ? "discoveryUrl" : "discoveryPath", discovery], + ["baseUrl", options.baseUrl], + ["auth", authFromTokenEnv(options.tokenEnv) ?? { type: "none" }], + ], + options, + ); +} + export function addGraphqlCaplet( id: string, options: AddGraphqlOptions, @@ -366,7 +397,10 @@ function resolvePrintOutputPath(id: string, options: AddDestinationOptions): str function renderLocalPaths(fields: YamlField[], outputDir: string): YamlField[] { return fields.map(([key, value]) => { - if ((key !== "specPath" && key !== "schemaPath") || typeof value !== "string") { + if ( + (key !== "specPath" && key !== "schemaPath" && key !== "discoveryPath") || + typeof value !== "string" + ) { return [key, value]; } return [key, localPathRelativeToOutput(value, outputDir)]; diff --git a/packages/core/src/cli/auth.ts b/packages/core/src/cli/auth.ts index ce09f2cf..ceef9213 100644 --- a/packages/core/src/cli/auth.ts +++ b/packages/core/src/cli/auth.ts @@ -14,10 +14,13 @@ import { loadGlobalConfig, loadProjectConfig, type CapletsConfig, + type GoogleDiscoveryApiConfig, type GraphQlEndpointConfig, type HttpApiConfig, } from "../config"; import { CapletsError, toSafeError } from "../errors"; +import { GoogleDiscoveryManager } from "../google-discovery"; +import { ServerRegistry } from "../registry"; type AuthTarget = ReturnType[number]; type AuthListFormat = "plain" | "markdown" | "json"; @@ -43,7 +46,7 @@ export async function loginAuth( }, ): Promise { const config = options.config ?? loadConfig(options.configPath); - const server = findAuthTarget(serverId, config); + const server = await resolveAuthTarget(serverId, config, options.authDir); assertLoginTarget(server, serverId); try { @@ -108,7 +111,11 @@ export async function refreshAuthResult( serverId: string, options: { authDir?: string; configPath?: string; config?: CapletsConfig }, ): Promise<{ server: string }> { - const target = findAuthTarget(serverId, options.config ?? loadConfig(options.configPath)); + const target = await resolveAuthTarget( + serverId, + options.config ?? loadConfig(options.configPath), + options.authDir, + ); assertLoginTarget(target, serverId); await refreshOAuthTokenBundle(target, options.authDir); return { server: serverId }; @@ -260,6 +267,27 @@ export function findAuthTarget(serverId: string, config = loadConfig()): AuthTar return authTargets(config).find((server) => server.server === serverId); } +async function resolveAuthTarget( + serverId: string, + config: CapletsConfig, + authDir?: string, +): Promise { + const target = findAuthTarget(serverId, config); + if (target?.backend !== "googleDiscovery") return target; + const api = config.googleDiscoveryApis[serverId]; + if (!api || (api.auth.type !== "oauth2" && api.auth.type !== "oidc")) return target; + const manager = new GoogleDiscoveryManager( + new ServerRegistry(config), + authDir ? { authDir } : {}, + ); + const baseUrl = api.baseUrl ?? api.discoveryUrl; + return { + ...target, + ...(baseUrl ? { baseUrl } : {}), + ...(api.auth.scopes?.length ? {} : { resolvedScopes: await manager.resolveAuthScopes(api) }), + }; +} + function authTargets(config: ReturnType) { return [ ...Object.values(config.mcpServers).filter( @@ -270,6 +298,9 @@ function authTargets(config: ReturnType) { ...Object.values(config.openapiEndpoints).filter( (endpoint) => endpoint.auth?.type === "oauth2" || endpoint.auth?.type === "oidc", ), + ...Object.values(config.googleDiscoveryApis) + .filter((api) => api.auth?.type === "oauth2" || api.auth?.type === "oidc") + .map(googleDiscoveryAuthTarget), ...Object.values(config.graphqlEndpoints) .filter((endpoint) => endpoint.auth?.type === "oauth2" || endpoint.auth?.type === "oidc") .map(graphQlAuthTarget), @@ -279,6 +310,16 @@ function authTargets(config: ReturnType) { ]; } +function googleDiscoveryAuthTarget( + api: GoogleDiscoveryApiConfig, +): GoogleDiscoveryApiConfig & GenericAuthTarget { + const baseUrl = api.baseUrl ?? api.discoveryUrl; + return { + ...api, + ...(baseUrl ? { baseUrl } : {}), + }; +} + function graphQlAuthTarget( endpoint: GraphQlEndpointConfig, ): GraphQlEndpointConfig & GenericAuthTarget { diff --git a/packages/core/src/cli/commands.ts b/packages/core/src/cli/commands.ts index 2e398ff5..0b23b7da 100644 --- a/packages/core/src/cli/commands.ts +++ b/packages/core/src/cli/commands.ts @@ -63,7 +63,7 @@ export const topLevelCommandNames = [ ] as const; export const cliSubcommands = { - [cliCommands.add]: ["cli", "mcp", "openapi", "graphql", "http"], + [cliCommands.add]: ["cli", "mcp", "openapi", "google-discovery", "graphql", "http"], [cliCommands.auth]: ["login", "logout", "list", "refresh"], [cliCommands.cloud]: ["auth"], [cliCommands.codeMode]: ["types"], diff --git a/packages/core/src/cli/completion-discovery.ts b/packages/core/src/cli/completion-discovery.ts index 69777a3e..43efddcc 100644 --- a/packages/core/src/cli/completion-discovery.ts +++ b/packages/core/src/cli/completion-discovery.ts @@ -183,6 +183,17 @@ function secretFreeServerShape(server: CapletConfig): Record { authType: server.auth.type, requestTimeoutMs: server.requestTimeoutMs, }; + case "googleDiscovery": + return { + ...base, + discoveryPath: server.discoveryPath, + discoveryUrl: server.discoveryUrl, + baseUrl: server.baseUrl, + includeOperations: server.includeOperations, + excludeOperations: server.excludeOperations, + authType: server.auth.type, + requestTimeoutMs: server.requestTimeoutMs, + }; case "graphql": return { ...base, @@ -271,6 +282,7 @@ function enabledServer(serverId: string, config: CapletsConfig): CapletConfig | const server = config.mcpServers[serverId] ?? config.openapiEndpoints[serverId] ?? + config.googleDiscoveryApis[serverId] ?? config.graphqlEndpoints[serverId] ?? config.httpApis[serverId] ?? config.cliTools[serverId] ?? diff --git a/packages/core/src/cli/doctor.ts b/packages/core/src/cli/doctor.ts index 4b870758..5bf655d9 100644 --- a/packages/core/src/cli/doctor.ts +++ b/packages/core/src/cli/doctor.ts @@ -391,6 +391,7 @@ function allCaplets(config: { [key: string]: unknown }): CapletConfig[] { const typed = config as { mcpServers?: Record; openapiEndpoints?: Record; + googleDiscoveryApis?: Record; graphqlEndpoints?: Record; httpApis?: Record; cliTools?: Record; @@ -399,6 +400,7 @@ function allCaplets(config: { [key: string]: unknown }): CapletConfig[] { return [ ...Object.values(typed.mcpServers ?? {}), ...Object.values(typed.openapiEndpoints ?? {}), + ...Object.values(typed.googleDiscoveryApis ?? {}), ...Object.values(typed.graphqlEndpoints ?? {}), ...Object.values(typed.httpApis ?? {}), ...Object.values(typed.cliTools ?? {}), diff --git a/packages/core/src/cli/inspection.ts b/packages/core/src/cli/inspection.ts index c68a3da3..55c53954 100644 --- a/packages/core/src/cli/inspection.ts +++ b/packages/core/src/cli/inspection.ts @@ -62,6 +62,7 @@ function allCaplets(config: CapletsConfig): CapletConfig[] { return [ ...Object.values(config.mcpServers), ...Object.values(config.openapiEndpoints), + ...Object.values(config.googleDiscoveryApis ?? {}), ...Object.values(config.graphqlEndpoints), ...Object.values(config.httpApis), ...Object.values(config.cliTools), diff --git a/packages/core/src/cli/setup-caplet.ts b/packages/core/src/cli/setup-caplet.ts index e4c0c4eb..1b2bf0fe 100644 --- a/packages/core/src/cli/setup-caplet.ts +++ b/packages/core/src/cli/setup-caplet.ts @@ -34,6 +34,7 @@ export async function runCapletSetupCli( const caplet = Object.values({ ...config.mcpServers, ...config.openapiEndpoints, + ...config.googleDiscoveryApis, ...config.graphqlEndpoints, ...config.httpApis, ...config.cliTools, diff --git a/packages/core/src/cli/setup.ts b/packages/core/src/cli/setup.ts index c95af732..200fb858 100644 --- a/packages/core/src/cli/setup.ts +++ b/packages/core/src/cli/setup.ts @@ -142,6 +142,10 @@ export async function runSetup(integration: string, options: SetupOptions = {}): return await runCapletSetupCli(integration, { ...(options.yes === undefined ? {} : { yes: options.yes }), target: resolveSetupTargetKind(options), + ...(options.env?.CAPLETS_CONFIG ? { configPath: options.env.CAPLETS_CONFIG } : {}), + ...(options.env?.CAPLETS_PROJECT_CONFIG + ? { projectConfigPath: options.env.CAPLETS_PROJECT_CONFIG } + : {}), ...(options.remote === undefined && !isRemoteSetup(options) ? {} : { remote: isRemoteSetup(options) }), diff --git a/packages/core/src/cloud/runtime-adapter.ts b/packages/core/src/cloud/runtime-adapter.ts index 34eaa10e..d8bd82e8 100644 --- a/packages/core/src/cloud/runtime-adapter.ts +++ b/packages/core/src/cloud/runtime-adapter.ts @@ -134,6 +134,7 @@ class DefaultCloudRuntimeAdapter implements CloudRuntimeAdapter { return Object.values({ ...this.engine.currentConfig().mcpServers, ...this.engine.currentConfig().openapiEndpoints, + ...this.engine.currentConfig().googleDiscoveryApis, ...this.engine.currentConfig().graphqlEndpoints, ...this.engine.currentConfig().httpApis, ...this.engine.currentConfig().cliTools, diff --git a/packages/core/src/config-runtime.ts b/packages/core/src/config-runtime.ts index 70bbfcff..6ef34d42 100644 --- a/packages/core/src/config-runtime.ts +++ b/packages/core/src/config-runtime.ts @@ -92,6 +92,18 @@ export type OpenApiEndpointConfig = CommonCapletConfig & { operationCacheTtlMs: number; }; +export type GoogleDiscoveryApiConfig = CommonCapletConfig & { + backend: "googleDiscovery"; + discoveryPath?: string | undefined; + discoveryUrl?: string | undefined; + baseUrl?: string | undefined; + includeOperations?: string[] | undefined; + excludeOperations?: string[] | undefined; + auth: OpenApiAuthConfig; + requestTimeoutMs: number; + operationCacheTtlMs: number; +}; + export type GraphQlOperationConfig = AgentSelectionHintsConfig & { document?: string | undefined; documentPath?: string | undefined; @@ -174,6 +186,7 @@ export type CapletSetConfig = CommonCapletConfig & { export type CapletConfig = | CapletServerConfig | OpenApiEndpointConfig + | GoogleDiscoveryApiConfig | GraphQlEndpointConfig | HttpApiConfig | CliToolsConfig @@ -196,6 +209,7 @@ export type CapletsConfig = { }; mcpServers: Record; openapiEndpoints: Record; + googleDiscoveryApis: Record; graphqlEndpoints: Record; httpApis: Record; cliTools: Record; @@ -321,6 +335,20 @@ const openApiEndpointSchema = z operationCacheTtlMs: z.number().int().nonnegative().default(30_000), }) .strict(); +const operationFilterSchema = z.array(z.string().trim().min(1).max(160)); +const googleDiscoveryApiSchema = z + .object({ + ...commonSchema, + discoveryPath: z.string().min(1).optional(), + discoveryUrl: z.string().min(1).optional(), + baseUrl: z.string().min(1).optional(), + includeOperations: operationFilterSchema.optional(), + excludeOperations: operationFilterSchema.optional(), + auth: authSchema, + requestTimeoutMs: z.number().int().positive().default(60_000), + operationCacheTtlMs: z.number().int().nonnegative().default(30_000), + }) + .strict(); const graphQlOperationSchema = z .object({ document: z.string().min(1).optional(), @@ -479,6 +507,9 @@ const configSchema = z openapiEndpoints: z .record(z.string().regex(SERVER_ID_PATTERN), openApiEndpointSchema) .default({}), + googleDiscoveryApis: z + .record(z.string().regex(SERVER_ID_PATTERN), googleDiscoveryApiSchema) + .default({}), graphqlEndpoints: z .record(z.string().regex(SERVER_ID_PATTERN), graphQlEndpointSchema) .default({}), @@ -523,6 +554,7 @@ export function parseConfig(input: unknown): CapletsConfig { }; }), openapiEndpoints: mapBackend(config.openapiEndpoints, "openapi"), + googleDiscoveryApis: mapBackend(config.googleDiscoveryApis, "googleDiscovery"), graphqlEndpoints: mapBackend(config.graphqlEndpoints, "graphql"), httpApis: mapBackend(config.httpApis, "http"), cliTools: mapBackend(config.cliTools, "cli"), @@ -608,6 +640,36 @@ function validateBackends(config: z.infer, ctx: z.Refinemen } validateAuthHeaders(raw.auth, ctx, ["openapiEndpoints", server, "auth"]); } + for (const [server, raw] of Object.entries(config.googleDiscoveryApis)) { + if (Boolean(raw.discoveryPath) === Boolean(raw.discoveryUrl)) { + ctx.addIssue({ + code: "custom", + path: ["googleDiscoveryApis", server], + message: "Google Discovery API must define exactly one discovery source", + }); + } + if ( + raw.discoveryUrl && + !hasEnvReference(raw.discoveryUrl) && + !isAllowedRemoteUrl(raw.discoveryUrl) + ) { + ctx.addIssue({ + code: "custom", + path: ["googleDiscoveryApis", server, "discoveryUrl"], + message: + "Google Discovery API discoveryUrl must use https except loopback development urls", + }); + } + if (raw.baseUrl && !hasEnvReference(raw.baseUrl) && !isAllowedHttpBaseUrl(raw.baseUrl)) { + ctx.addIssue({ + code: "custom", + path: ["googleDiscoveryApis", server, "baseUrl"], + message: + "Google Discovery API baseUrl must use https except loopback development urls and must not include credentials, query, or fragment", + }); + } + validateAuthHeaders(raw.auth, ctx, ["googleDiscoveryApis", server, "auth"]); + } for (const [server, raw] of Object.entries(config.graphqlEndpoints)) { const sourceCount = Number(Boolean(raw.schemaPath)) + diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index 74f2babb..2c99f70c 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -217,6 +217,29 @@ export type HttpApiConfig = AgentSelectionHintsConfig & { runtime?: RuntimeRequirementsConfig | undefined; }; +export type GoogleDiscoveryApiConfig = AgentSelectionHintsConfig & { + server: string; + backend: "googleDiscovery"; + name: string; + description: string; + exposure?: CapletExposure | undefined; + shadowing?: CapletShadowingPolicy | undefined; + tags?: string[] | undefined; + body?: string | undefined; + discoveryPath?: string | undefined; + discoveryUrl?: string | undefined; + baseUrl?: string | undefined; + includeOperations?: string[] | undefined; + excludeOperations?: string[] | undefined; + auth: OpenApiAuthConfig; + requestTimeoutMs: number; + operationCacheTtlMs: number; + disabled: boolean; + setup?: CapletSetupConfig | undefined; + projectBinding?: ProjectBindingConfig | undefined; + runtime?: RuntimeRequirementsConfig | undefined; +}; + export type CliToolOutputConfig = { type: "text" | "json"; }; @@ -285,6 +308,7 @@ export type CapletSetConfig = AgentSelectionHintsConfig & { export type CapletConfig = | CapletServerConfig | OpenApiEndpointConfig + | GoogleDiscoveryApiConfig | GraphQlEndpointConfig | HttpApiConfig | CliToolsConfig @@ -311,6 +335,7 @@ export type CapletsConfig = { options: CapletsOptions; mcpServers: Record; openapiEndpoints: Record; + googleDiscoveryApis: Record; graphqlEndpoints: Record; httpApis: Record; cliTools: Record; @@ -619,6 +644,66 @@ const normalizedOpenApiEndpointSchema = publicOpenApiEndpointSchema.extend({ body: z.string().optional(), }); +const operationFilterSchema = z.array(z.string().trim().min(1).max(160)); + +const publicGoogleDiscoveryApiSchema = z + .object({ + name: z + .string() + .trim() + .min(1) + .max(80) + .describe("Human-readable Google Discovery API display name."), + description: z + .string() + .describe( + "Capability description shown to agents before Google Discovery operations are disclosed.", + ) + .refine( + (value) => value.trim().length >= 10, + "description must contain at least 10 non-whitespace characters", + ) + .refine((value) => value.length <= 1500, "description must be at most 1500 characters"), + discoveryPath: z.string().min(1).optional().describe("Local Google Discovery document path."), + discoveryUrl: z.string().url().optional().describe("Remote Google Discovery document URL."), + baseUrl: z.string().url().optional().describe("Override base URL for Google API requests."), + includeOperations: operationFilterSchema.optional(), + excludeOperations: operationFilterSchema.optional(), + auth: openApiAuthSchema.describe( + 'Explicit Google API request auth config. Use {"type":"none"} for public APIs.', + ), + tags: z.array(z.string().trim().min(1).max(80)).optional(), + exposure: exposureSchema.optional(), + shadowing: shadowingSchema, + ...agentSelectionHintsSchema, + setup: setupSchema.optional(), + projectBinding: projectBindingSchema.optional(), + runtime: runtimeRequirementsSchema.optional(), + requestTimeoutMs: z + .number() + .int() + .positive() + .default(60_000) + .describe("Timeout in milliseconds for Google Discovery HTTP requests."), + operationCacheTtlMs: z + .number() + .int() + .nonnegative() + .default(30_000) + .describe( + "Milliseconds Google Discovery operation metadata stays fresh. Set 0 to refresh every time.", + ), + disabled: z + .boolean() + .default(false) + .describe("When true, omit this Google Discovery Caplet from discovery."), + }) + .strict(); + +const normalizedGoogleDiscoveryApiSchema = publicGoogleDiscoveryApiSchema.extend({ + body: z.string().optional(), +}); + const graphQlOperationSchema = z .object({ document: z.string().min(1).optional().describe("Inline GraphQL operation document."), @@ -973,6 +1058,7 @@ const normalizedCapletSetSchema = publicCapletSetSchema.extend({ type ConfigSchemaServerValue = z.infer; type ConfigSchemaOpenApiEndpointValue = z.infer; +type ConfigSchemaGoogleDiscoveryApiValue = z.infer; type ConfigSchemaGraphQlEndpointValue = z.infer; type ConfigSchemaHttpApiValue = z.infer; type ConfigSchemaCliToolsValue = z.infer; @@ -980,6 +1066,7 @@ type ConfigSchemaCapletSetValue = z.infer; type ConfigInput = { mcpServers?: Record; openapiEndpoints?: Record; + googleDiscoveryApis?: Record; graphqlEndpoints?: Record; httpApis?: Record; cliTools?: Record; @@ -990,6 +1077,7 @@ type ConfigInput = { function configSchemaFor( serverValueSchema: z.ZodTypeAny, openApiEndpointValueSchema: z.ZodTypeAny, + googleDiscoveryApiValueSchema: z.ZodTypeAny, graphQlEndpointValueSchema: z.ZodTypeAny, httpApiValueSchema: z.ZodTypeAny, cliToolsValueSchema: z.ZodTypeAny, @@ -1052,6 +1140,10 @@ function configSchemaFor( .record(z.string().regex(SERVER_ID_PATTERN), openApiEndpointValueSchema) .default({}) .describe("OpenAPI endpoints keyed by stable Caplet ID."), + googleDiscoveryApis: z + .record(z.string().regex(SERVER_ID_PATTERN), googleDiscoveryApiValueSchema) + .default({}) + .describe("Google Discovery APIs keyed by stable Caplet ID."), graphqlEndpoints: z .record(z.string().regex(SERVER_ID_PATTERN), graphQlEndpointValueSchema) .default({}) @@ -1189,13 +1281,76 @@ function configSchemaFor( } } + for (const [api, rawValue] of Object.entries(config.googleDiscoveryApis)) { + const raw = rawValue as ConfigSchemaGoogleDiscoveryApiValue; + if (config.mcpServers[api]) { + ctx.addIssue({ + code: "custom", + path: ["googleDiscoveryApis", api], + message: `Caplet ID ${api} is already used by mcpServers`, + }); + } + if (config.openapiEndpoints[api]) { + ctx.addIssue({ + code: "custom", + path: ["googleDiscoveryApis", api], + message: `Caplet ID ${api} is already used by openapiEndpoints`, + }); + } + if (!SERVER_ID_PATTERN.test(api)) { + ctx.addIssue({ + code: "custom", + path: ["googleDiscoveryApis", api], + message: "Google Discovery API ID must match ^[a-zA-Z0-9_-]{1,64}$", + }); + } + if (Boolean(raw.discoveryPath) === Boolean(raw.discoveryUrl)) { + ctx.addIssue({ + code: "custom", + path: ["googleDiscoveryApis", api], + message: + "Google Discovery API must define exactly one discovery source: discoveryPath or discoveryUrl", + }); + } + if (raw.discoveryUrl && !isAllowedRemoteUrl(raw.discoveryUrl)) { + ctx.addIssue({ + code: "custom", + path: ["googleDiscoveryApis", api, "discoveryUrl"], + message: + "Google Discovery API discoveryUrl must use https except loopback development urls", + }); + } + if (raw.baseUrl && !isAllowedHttpBaseUrl(raw.baseUrl)) { + ctx.addIssue({ + code: "custom", + path: ["googleDiscoveryApis", api, "baseUrl"], + message: + "Google Discovery API baseUrl must use https except loopback development urls and must not include credentials, query, or fragment", + }); + } + if (raw.auth?.type === "headers") { + for (const headerName of Object.keys(raw.auth.headers)) { + const normalized = headerName.toLowerCase(); + if (!HEADER_NAME_PATTERN.test(headerName) || FORBIDDEN_HEADERS.has(normalized)) { + ctx.addIssue({ + code: "custom", + path: ["googleDiscoveryApis", api, "auth", "headers", headerName], + message: `header ${headerName} is not allowed`, + }); + } + } + } + } + for (const [endpoint, rawValue] of Object.entries(config.graphqlEndpoints)) { const raw = rawValue as ConfigSchemaGraphQlEndpointValue; const duplicateBackend = config.mcpServers[endpoint] ? "mcpServers" : config.openapiEndpoints[endpoint] ? "openapiEndpoints" - : undefined; + : config.googleDiscoveryApis[endpoint] + ? "googleDiscoveryApis" + : undefined; if (duplicateBackend) { ctx.addIssue({ code: "custom", @@ -1245,9 +1400,11 @@ function configSchemaFor( ? "mcpServers" : config.openapiEndpoints[endpoint] ? "openapiEndpoints" - : config.graphqlEndpoints[endpoint] - ? "graphqlEndpoints" - : undefined; + : config.googleDiscoveryApis[endpoint] + ? "googleDiscoveryApis" + : config.graphqlEndpoints[endpoint] + ? "graphqlEndpoints" + : undefined; if (duplicateBackend) { ctx.addIssue({ code: "custom", @@ -1290,11 +1447,13 @@ function configSchemaFor( ? "mcpServers" : config.openapiEndpoints[server] ? "openapiEndpoints" - : config.graphqlEndpoints[server] - ? "graphqlEndpoints" - : config.httpApis[server] - ? "httpApis" - : undefined; + : config.googleDiscoveryApis[server] + ? "googleDiscoveryApis" + : config.graphqlEndpoints[server] + ? "graphqlEndpoints" + : config.httpApis[server] + ? "httpApis" + : undefined; if (duplicateBackend) { ctx.addIssue({ code: "custom", @@ -1326,13 +1485,15 @@ function configSchemaFor( ? "mcpServers" : config.openapiEndpoints[server] ? "openapiEndpoints" - : config.graphqlEndpoints[server] - ? "graphqlEndpoints" - : config.httpApis[server] - ? "httpApis" - : config.cliTools[server] - ? "cliTools" - : undefined; + : config.googleDiscoveryApis[server] + ? "googleDiscoveryApis" + : config.graphqlEndpoints[server] + ? "graphqlEndpoints" + : config.httpApis[server] + ? "httpApis" + : config.cliTools[server] + ? "cliTools" + : undefined; if (duplicateBackend) { ctx.addIssue({ code: "custom", @@ -1361,6 +1522,7 @@ function configSchemaFor( export const configFileSchema = configSchemaFor( publicServerSchema, publicOpenApiEndpointSchema, + publicGoogleDiscoveryApiSchema, publicGraphQlEndpointSchema, publicHttpApiSchema, publicCliToolsSchema, @@ -1369,6 +1531,7 @@ export const configFileSchema = configSchemaFor( const normalizedConfigFileSchema = configSchemaFor( normalizedServerSchema, normalizedOpenApiEndpointSchema, + normalizedGoogleDiscoveryApiSchema, normalizedGraphQlEndpointSchema, normalizedHttpApiSchema, normalizedCliToolsSchema, @@ -1423,7 +1586,7 @@ export function loadConfigWithSources( : undefined, ], `Caplets config not found at ${path} or ${projectPath}`, - "Caplets config must define at least one MCP server, OpenAPI endpoint, GraphQL endpoint, HTTP API, CLI tools backend, or Caplet set", + "Caplets config must define at least one MCP server, OpenAPI endpoint, Google Discovery API, GraphQL endpoint, HTTP API, CLI tools backend, or Caplet set", ); } @@ -1483,6 +1646,7 @@ function buildConfigWithSources( emptyMessage && Object.keys(config.mcpServers).length === 0 && Object.keys(config.openapiEndpoints).length === 0 && + Object.keys(config.googleDiscoveryApis).length === 0 && Object.keys(config.graphqlEndpoints).length === 0 && Object.keys(config.httpApis).length === 0 && Object.keys(config.cliTools).length === 0 && @@ -1614,6 +1778,7 @@ export function loadIsolatedConfig(options: { if ( Object.keys(config.mcpServers).length === 0 && Object.keys(config.openapiEndpoints).length === 0 && + Object.keys(config.googleDiscoveryApis).length === 0 && Object.keys(config.graphqlEndpoints).length === 0 && Object.keys(config.httpApis).length === 0 && Object.keys(config.cliTools).length === 0 && @@ -1660,6 +1825,11 @@ function normalizeLocalPaths(input: ConfigInput, baseDir: string): ConfigInput { return stripUndefined({ ...input, openapiEndpoints: normalizeEndpointPaths(input.openapiEndpoints, baseDir, normalizeOpenApiPath), + googleDiscoveryApis: normalizeEndpointPaths( + input.googleDiscoveryApis, + baseDir, + normalizeGoogleDiscoveryPath, + ), graphqlEndpoints: normalizeEndpointPaths(input.graphqlEndpoints, baseDir, normalizeGraphQlPath), cliTools: normalizeEndpointPaths(input.cliTools, baseDir, normalizeCliToolsPaths), capletSets: normalizeEndpointPaths(input.capletSets, baseDir, normalizeCapletSetPaths), @@ -1692,6 +1862,16 @@ function normalizeOpenApiPath( }; } +function normalizeGoogleDiscoveryPath( + endpoint: Record, + baseDir: string, +): Record { + return { + ...endpoint, + discoveryPath: normalizeLocalPath(endpoint.discoveryPath, baseDir), + }; +} + function normalizeGraphQlPath( endpoint: Record, baseDir: string, @@ -1765,6 +1945,12 @@ function rejectProjectConfigExecutableBackendMaps(input: ConfigInput, path: stri `Project config at ${path} cannot define executable backend map openapiEndpoints; use project Markdown Caplet files or user config instead`, ); } + if (input.googleDiscoveryApis && Object.keys(input.googleDiscoveryApis).length > 0) { + throw new CapletsError( + "CONFIG_INVALID", + `Project config at ${path} cannot define executable backend map googleDiscoveryApis; use project Markdown Caplet files or user config instead`, + ); + } if (input.graphqlEndpoints && Object.keys(input.graphqlEndpoints).length > 0) { throw new CapletsError( "CONFIG_INVALID", @@ -1809,6 +1995,10 @@ function mergeConfigInputs(...inputs: Array): ConfigInp ...merged?.openapiEndpoints, ...input.openapiEndpoints, }, + googleDiscoveryApis: { + ...merged?.googleDiscoveryApis, + ...input.googleDiscoveryApis, + }, graphqlEndpoints: { ...merged?.graphqlEndpoints, ...input.graphqlEndpoints, @@ -1860,6 +2050,7 @@ function mergeConfigInputsWithSources(...inputs: Array = {}; + for (const [server, raw] of Object.entries(parsed.data.googleDiscoveryApis)) { + const interpolated = raw as ConfigSchemaGoogleDiscoveryApiValue; + googleDiscoveryApis[server] = stripUndefined({ + ...interpolated, + server, + backend: "googleDiscovery", + }) as GoogleDiscoveryApiConfig; + } + const graphqlEndpoints: Record = {}; for (const [server, raw] of Object.entries(parsed.data.graphqlEndpoints)) { const interpolated = raw as ConfigSchemaGraphQlEndpointValue; @@ -1973,6 +2176,7 @@ export function parseConfig(input: unknown): CapletsConfig { }, mcpServers: servers, openapiEndpoints, + googleDiscoveryApis, graphqlEndpoints, httpApis, cliTools, @@ -2032,6 +2236,7 @@ function isPublicMetadataPath(path: string[]): boolean { path.length < 3 || (path[0] !== "mcpServers" && path[0] !== "openapiEndpoints" && + path[0] !== "googleDiscoveryApis" && path[0] !== "graphqlEndpoints" && path[0] !== "httpApis" && path[0] !== "cliTools" && diff --git a/packages/core/src/config/paths.ts b/packages/core/src/config/paths.ts index c09b9c29..65768be1 100644 --- a/packages/core/src/config/paths.ts +++ b/packages/core/src/config/paths.ts @@ -74,6 +74,15 @@ export function defaultAuthDir( return pathJoin(defaultStateBaseDir(env, home, platform), "caplets", "auth"); } +export function defaultArtifactDir( + env: PathEnv = process.env, + home = homedir(), + platform: Platform = process.platform, +): string { + const pathJoin = platform === "win32" ? win32.join : posix.join; + return pathJoin(defaultStateBaseDir(env, home, platform), "caplets", "artifacts"); +} + export function defaultCompletionCacheDir( env: PathEnv = process.env, home = homedir(), @@ -98,6 +107,7 @@ export function defaultObservedOutputShapeCacheDir( export const DEFAULT_CONFIG_PATH = defaultConfigPath(); export const DEFAULT_AUTH_DIR = defaultAuthDir(); +export const DEFAULT_ARTIFACT_DIR = defaultArtifactDir(); export const DEFAULT_COMPLETION_CACHE_DIR = defaultCompletionCacheDir(); export const DEFAULT_OBSERVED_OUTPUT_SHAPE_CACHE_DIR = defaultObservedOutputShapeCacheDir(); export const PROJECT_CONFIG_FILE = join(".caplets", "config.json"); diff --git a/packages/core/src/engine.ts b/packages/core/src/engine.ts index a3a52253..ea9b06fc 100644 --- a/packages/core/src/engine.ts +++ b/packages/core/src/engine.ts @@ -15,6 +15,7 @@ import { DEFAULT_OBSERVED_OUTPUT_SHAPE_CACHE_DIR } from "./config/paths"; import { DownstreamManager } from "./downstream"; import { CapletsError, errorResult, toSafeError } from "./errors"; import { GraphQLManager } from "./graphql"; +import { GoogleDiscoveryManager } from "./google-discovery"; import { HttpActionManager } from "./http-actions"; import { OpenApiManager } from "./openapi"; import { @@ -32,6 +33,7 @@ export type CapletsEngineOptions = { configPath?: string; projectConfigPath?: string; authDir?: string; + artifactDir?: string; watchDebounceMs?: number; watch?: boolean; writeErr?: (value: string) => void; @@ -62,6 +64,7 @@ export class CapletsEngine { private registry: ServerRegistry; private readonly downstream: DownstreamManager; private readonly openapi: OpenApiManager; + private readonly googleDiscovery: GoogleDiscoveryManager; private readonly graphql: GraphQLManager; private readonly http: HttpActionManager; private readonly cli: CliToolsManager; @@ -92,9 +95,13 @@ export class CapletsEngine { const config = this.configLoader(this.paths.configPath, this.paths.projectConfigPath); this.registry = new ServerRegistry(config); this.downstream = new DownstreamManager(this.registry, selectAuthOptions(options.authDir)); - this.openapi = new OpenApiManager(this.registry, selectAuthOptions(options.authDir)); + this.openapi = new OpenApiManager(this.registry, selectHttpLikeOptions(options)); + this.googleDiscovery = new GoogleDiscoveryManager( + this.registry, + selectHttpLikeOptions(options), + ); this.graphql = new GraphQLManager(this.registry, selectAuthOptions(options.authDir)); - this.http = new HttpActionManager(this.registry, selectAuthOptions(options.authDir)); + this.http = new HttpActionManager(this.registry, selectHttpLikeOptions(options)); this.cli = new CliToolsManager(this.registry); this.capletSets = new CapletSetManager(this.registry, selectAuthOptions(options.authDir)); this.watchDebounceMs = options.watchDebounceMs ?? 250; @@ -200,6 +207,7 @@ export class CapletsEngine { observedOutputShapeScope: this.observedOutputShapeScope, projectFingerprint: this.projectFingerprint, }, + this.googleDiscovery, ); } catch (error) { return errorResult(error); @@ -307,13 +315,15 @@ export class CapletsEngine { ? await this.downstream.listTools(server) : server.backend === "openapi" ? await this.openapi.listTools(server) - : server.backend === "graphql" - ? await this.graphql.listTools(server) - : server.backend === "http" - ? await this.http.listTools(server) - : server.backend === "cli" - ? await this.cli.listTools(server) - : await this.capletSets.listTools(server); + : server.backend === "googleDiscovery" + ? await this.googleDiscovery.listTools(server) + : server.backend === "graphql" + ? await this.graphql.listTools(server) + : server.backend === "http" + ? await this.http.listTools(server) + : server.backend === "cli" + ? await this.cli.listTools(server) + : await this.capletSets.listTools(server); return tools.map((tool) => ({ name: tool.name, ...(tool.description ? { description: tool.description } : {}), @@ -325,13 +335,15 @@ export class CapletsEngine { ? await this.downstream.listTools(server) : server.backend === "openapi" ? await this.openapi.listTools(server) - : server.backend === "graphql" - ? await this.graphql.listTools(server) - : server.backend === "http" - ? await this.http.listTools(server) - : server.backend === "cli" - ? await this.cli.listTools(server) - : await this.capletSets.listTools(server); + : server.backend === "googleDiscovery" + ? await this.googleDiscovery.listTools(server) + : server.backend === "graphql" + ? await this.graphql.listTools(server) + : server.backend === "http" + ? await this.http.listTools(server) + : server.backend === "cli" + ? await this.cli.listTools(server) + : await this.capletSets.listTools(server); } private async callTool(server: CapletConfig, toolName: string, args: Record) { @@ -339,13 +351,15 @@ export class CapletsEngine { ? await this.downstream.callTool(server, toolName, args) : server.backend === "openapi" ? await this.openapi.callTool(server, toolName, args) - : server.backend === "graphql" - ? await this.graphql.callTool(server, toolName, args) - : server.backend === "http" - ? await this.http.callTool(server, toolName, args) - : server.backend === "cli" - ? await this.cli.callTool(server, toolName, args) - : await this.capletSets.callTool(server, toolName, args); + : server.backend === "googleDiscovery" + ? await this.googleDiscovery.callTool(server, toolName, args) + : server.backend === "graphql" + ? await this.graphql.callTool(server, toolName, args) + : server.backend === "http" + ? await this.http.callTool(server, toolName, args) + : server.backend === "cli" + ? await this.cli.callTool(server, toolName, args) + : await this.capletSets.callTool(server, toolName, args); } private async optionalMcpList( @@ -381,6 +395,7 @@ export class CapletsEngine { this.registry = nextRegistry; this.downstream.updateRegistry(nextRegistry); this.openapi.updateRegistry(nextRegistry); + this.googleDiscovery.updateRegistry(nextRegistry); this.graphql.updateRegistry(nextRegistry); this.http.updateRegistry(nextRegistry); this.cli.updateRegistry(nextRegistry); @@ -451,6 +466,9 @@ export class CapletsEngine { if (before?.backend === "openapi" || after?.backend === "openapi" || !after) { this.openapi.invalidate(serverId); } + if (before?.backend === "googleDiscovery" || after?.backend === "googleDiscovery" || !after) { + this.googleDiscovery.invalidate(serverId); + } if (before?.backend === "graphql" || after?.backend === "graphql" || !after) { this.graphql.invalidate(serverId); } @@ -549,6 +567,16 @@ function selectAuthOptions(authDir: string | undefined): { authDir?: string } { return authDir ? { authDir } : {}; } +function selectHttpLikeOptions(options: CapletsEngineOptions): { + authDir?: string; + artifactDir?: string; +} { + return { + ...selectAuthOptions(options.authDir), + ...(options.artifactDir ? { artifactDir: options.artifactDir } : {}), + }; +} + function safeProjectFingerprint(): string | undefined { try { return fingerprintProjectRoot(findProjectRoot()); @@ -584,6 +612,7 @@ function allCaplets(config: CapletsConfig): CapletConfig[] { return [ ...Object.values(config.mcpServers), ...Object.values(config.openapiEndpoints), + ...Object.values(config.googleDiscoveryApis ?? {}), ...Object.values(config.graphqlEndpoints), ...Object.values(config.httpApis), ...Object.values(config.cliTools), diff --git a/packages/core/src/google-discovery/index.ts b/packages/core/src/google-discovery/index.ts new file mode 100644 index 00000000..6449d8d2 --- /dev/null +++ b/packages/core/src/google-discovery/index.ts @@ -0,0 +1,5 @@ +export * from "./types"; +export * from "./schema"; +export * from "./operations"; +export * from "./request"; +export * from "./manager"; diff --git a/packages/core/src/google-discovery/manager.ts b/packages/core/src/google-discovery/manager.ts new file mode 100644 index 00000000..d632f7b6 --- /dev/null +++ b/packages/core/src/google-discovery/manager.ts @@ -0,0 +1,607 @@ +import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { CompatibilityCallToolResult, Tool } from "@modelcontextprotocol/sdk/types"; +import { genericOAuthHeaders } from "../auth"; +import type { GoogleDiscoveryApiConfig } from "../config"; +import { + compactToolSafetyHints, + compactToolSchemaHints, + compactToolSelectionHints, + type CompactTool, +} from "../downstream"; +import { CapletsError, toSafeError } from "../errors"; +import { readHttpLikeResponse } from "../http/response"; +import { isAbortError, readLimitedText } from "../http/utils"; +import { readMediaInput, type ResolvedMediaInput } from "../media"; +import type { ServerRegistry } from "../registry"; +import { markdownStructuredContent } from "../result-content"; +import { searchToolList } from "../tool-search"; +import { + discoveryOperations, + googleDiscoveryScopesForOperations, + type GoogleDiscoveryOperation, +} from "./operations"; +import { buildGoogleDiscoveryUrl, buildJsonRequestInit } from "./request"; +import type { GoogleDiscoveryDocument } from "./types"; + +const DEFAULT_RESUMABLE_THRESHOLD_BYTES = 8 * 1024 * 1024; +const DEFAULT_MEDIA_RESPONSE_MAX_BYTES = 100 * 1024 * 1024; + +type ManagedGoogleDiscovery = { + operations?: GoogleDiscoveryOperation[]; + baseUrl?: string; + fetchedAt?: number; + cacheKey: string; +}; + +export class GoogleDiscoveryManager { + private readonly cache = new Map(); + + constructor( + private registry: ServerRegistry, + private readonly options: { authDir?: string; artifactDir?: string } = {}, + ) {} + + updateRegistry(registry: ServerRegistry): void { + this.registry = registry; + } + + invalidate(serverId: string): void { + this.cache.delete(serverId); + } + + async checkApi(api: GoogleDiscoveryApiConfig): Promise<{ + id: string; + status: string; + toolCount?: number; + elapsedMs: number; + error?: unknown; + }> { + const startedAt = Date.now(); + try { + const operations = await this.refreshOperations(api, true); + this.registry.setStatus(api.server, "available"); + return { + id: api.server, + status: "available", + toolCount: operations.length, + elapsedMs: Date.now() - startedAt, + }; + } catch (error) { + const safe = toSafeError(error, "SERVER_UNAVAILABLE"); + this.registry.setStatus(api.server, "unavailable", safe); + return { + id: api.server, + status: "unavailable", + elapsedMs: Date.now() - startedAt, + error: safe, + }; + } + } + + async listTools(api: GoogleDiscoveryApiConfig): Promise { + const operations = await this.refreshOperations(api, false); + return operations.map((operation) => this.toTool(operation)); + } + + async getTool(api: GoogleDiscoveryApiConfig, toolName: string): Promise { + return this.toTool(await this.getOperation(api, toolName)); + } + + async callTool( + api: GoogleDiscoveryApiConfig, + toolName: string, + args: Record, + ): Promise { + const operation = await this.getOperation(api, toolName); + const requestApi = await this.resolveRequestApi(api); + if (operation.supportsMediaUpload && "media" in args) { + return this.callMediaUpload(requestApi, operation, args); + } + const url = buildGoogleDiscoveryUrl(requestApi, operation, args); + const headers = new Headers( + await authHeaders(requestApi, this.options.authDir, operation.scopes), + ); + const init = buildJsonRequestInit(operation, args, headers); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), requestApi.requestTimeoutMs); + try { + const response = await fetch(url, { ...init, signal: controller.signal }); + if (response.status >= 300 && response.status < 400) { + throw new CapletsError( + "DOWNSTREAM_PROTOCOL_ERROR", + "Google Discovery request returned a redirect", + { + server: requestApi.server, + status: response.status, + location: response.headers.get("location") ? "[REDACTED]" : undefined, + }, + ); + } + const parsed = await readHttpLikeResponse(response, { + capletId: requestApi.server, + ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), + ...(typeof args.filename === "string" ? { filename: args.filename } : {}), + ...(typeof args.outputPath === "string" ? { outputPath: args.outputPath } : {}), + ...(operation.supportsMediaDownload ? { maxBytes: DEFAULT_MEDIA_RESPONSE_MAX_BYTES } : {}), + }); + return { + content: markdownStructuredContent(parsed, { + title: `${requestApi.name} call_tool ${toolName}`, + backend: "googleDiscovery", + operation: "call_tool", + tool: toolName, + }), + structuredContent: parsed as Record, + isError: !response.ok, + }; + } catch (error) { + if (isAbortError(error)) { + throw new CapletsError( + "TOOL_CALL_TIMEOUT", + `Google Discovery request timed out for ${requestApi.server}/${toolName}`, + ); + } + if (error instanceof CapletsError) throw error; + throw new CapletsError( + "DOWNSTREAM_TOOL_ERROR", + `Google Discovery request failed for ${requestApi.server}/${toolName}`, + toSafeError(error), + ); + } finally { + clearTimeout(timeout); + } + } + + private async callMediaUpload( + api: GoogleDiscoveryApiConfig, + operation: GoogleDiscoveryOperation, + args: Record, + ): Promise { + const media = await readMediaInput( + args.media, + this.options.artifactDir ? { artifactRoot: this.options.artifactDir } : {}, + ); + const headers = new Headers(await authHeaders(api, this.options.authDir, operation.scopes)); + const protocol = selectUploadProtocol(operation, media, args); + const response = + protocol === "resumable" + ? await this.callResumableUpload(api, operation, args, media, headers) + : await this.callSingleUpload(api, operation, args, media, headers, protocol); + const parsed = await readHttpLikeResponse(response, { + capletId: api.server, + ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), + }); + return { + content: markdownStructuredContent(parsed, { + title: `${api.name} call_tool ${operation.name}`, + backend: "googleDiscovery", + operation: "call_tool", + tool: operation.name, + }), + structuredContent: parsed as Record, + isError: !response.ok, + }; + } + + private async callSingleUpload( + api: GoogleDiscoveryApiConfig, + operation: GoogleDiscoveryOperation, + args: Record, + media: ResolvedMediaInput, + headers: Headers, + protocol: "simple" | "multipart", + ): Promise { + const upload = operation.mediaUploadProtocols[protocol]; + if (!upload?.path) { + throw new CapletsError( + "CONFIG_INVALID", + `Google Discovery ${protocol} upload path is missing`, + ); + } + const url = uploadUrl(api, upload.path, protocol === "simple" ? "media" : "multipart"); + const init = + protocol === "simple" + ? simpleUploadInit(operation, media, headers) + : multipartUploadInit(operation, args.body, media, headers); + return fetchGoogleRequest(api, operation, url, init); + } + + private async callResumableUpload( + api: GoogleDiscoveryApiConfig, + operation: GoogleDiscoveryOperation, + args: Record, + media: ResolvedMediaInput, + headers: Headers, + ): Promise { + const upload = operation.mediaUploadProtocols.resumable; + if (!upload?.path) { + throw new CapletsError("CONFIG_INVALID", "Google Discovery resumable upload path is missing"); + } + const startUrl = uploadUrl(api, upload.path, "resumable"); + headers.set("content-type", "application/json; charset=UTF-8"); + headers.set("x-upload-content-type", media.mimeType ?? "application/octet-stream"); + headers.set("x-upload-content-length", String(media.bytes.byteLength)); + const started = await fetchGoogleRequest(api, operation, startUrl, { + method: operation.method.toUpperCase(), + headers, + body: JSON.stringify(args.body ?? {}), + redirect: "manual", + }); + const location = started.headers.get("location"); + if (!location) { + throw new CapletsError( + "DOWNSTREAM_PROTOCOL_ERROR", + "Google resumable upload missing Location", + ); + } + const uploadHeaders = new Headers(); + uploadHeaders.set("content-type", media.mimeType ?? "application/octet-stream"); + uploadHeaders.set("content-length", String(media.bytes.byteLength)); + uploadHeaders.set( + "content-range", + `bytes 0-${media.bytes.byteLength - 1}/${media.bytes.byteLength}`, + ); + return fetchGoogleRequest(api, operation, new URL(location), { + method: "PUT", + headers: uploadHeaders, + body: media.bytes, + redirect: "manual", + }); + } + + compact(_api: GoogleDiscoveryApiConfig, tool: Tool): CompactTool { + return { + name: tool.name, + ...(tool.description ? { description: tool.description } : {}), + hasInputSchema: Boolean(tool.inputSchema), + hasOutputSchema: Boolean(tool.outputSchema), + supportsFields: Boolean(tool.outputSchema), + ...compactToolSelectionHints(tool), + ...compactToolSchemaHints(tool), + ...compactToolSafetyHints(tool), + }; + } + + search( + api: GoogleDiscoveryApiConfig, + tools: Tool[], + query: string, + limit: number, + ): CompactTool[] { + return searchToolList(tools, query, limit, (tool) => this.compact(api, tool)); + } + + async resolveAuthScopes(api: GoogleDiscoveryApiConfig): Promise { + return googleDiscoveryScopesForOperations(await this.refreshOperations(api, false)); + } + + private async getOperation( + api: GoogleDiscoveryApiConfig, + toolName: string, + ): Promise { + const operations = await this.refreshOperations(api, false); + const operation = operations.find((candidate) => candidate.name === toolName); + if (!operation) { + throw new CapletsError("TOOL_NOT_FOUND", `Tool ${toolName} was not found on ${api.server}`, { + server: api.server, + tool: toolName, + suggestions: operations + .map((candidate) => candidate.name) + .filter((name) => + name.toLocaleLowerCase().includes(toolName.toLocaleLowerCase()[0] ?? ""), + ) + .slice(0, 5), + }); + } + return operation; + } + + private async refreshOperations( + api: GoogleDiscoveryApiConfig, + force: boolean, + ): Promise { + const cached = this.cache.get(api.server); + const cacheKey = googleDiscoveryCacheKey(api); + const now = Date.now(); + const isFresh = + cached?.operations && + cached.cacheKey === cacheKey && + cached.fetchedAt !== undefined && + api.operationCacheTtlMs > 0 && + now - cached.fetchedAt <= api.operationCacheTtlMs; + if (!force && isFresh) return cached.operations ?? []; + + try { + const document = await loadGoogleDiscoveryDocument(api, this.options.authDir); + const baseUrl = googleDiscoveryBaseUrl(api, document); + const operations = discoveryOperations({ + server: api.server, + document, + ...(api.includeOperations ? { includeOperations: api.includeOperations } : {}), + ...(api.excludeOperations ? { excludeOperations: api.excludeOperations } : {}), + }); + this.cache.set(api.server, { + operations, + ...(baseUrl ? { baseUrl } : {}), + fetchedAt: Date.now(), + cacheKey, + }); + this.registry.setStatus(api.server, "available"); + return operations; + } catch (error) { + const safe = toSafeError(error, "DOWNSTREAM_PROTOCOL_ERROR"); + this.registry.setStatus(api.server, "unavailable", safe); + throw new CapletsError( + safe.code, + `Could not load Google Discovery operations for ${api.server}`, + safe, + ); + } + } + + private toTool(operation: GoogleDiscoveryOperation): Tool { + return { + name: operation.name, + ...(operation.description ? { description: operation.description } : {}), + inputSchema: operation.inputSchema as Tool["inputSchema"], + ...(operation.outputSchema + ? { outputSchema: operation.outputSchema as Tool["outputSchema"] } + : {}), + annotations: { + readOnlyHint: operation.readOnlyHint, + destructiveHint: operation.destructiveHint, + }, + }; + } + + private async resolveRequestApi( + api: GoogleDiscoveryApiConfig, + ): Promise { + if (api.baseUrl) return api; + await this.refreshOperations(api, false); + const baseUrl = this.cache.get(api.server)?.baseUrl; + if (!baseUrl) { + throw new CapletsError("CONFIG_INVALID", `${api.server} is missing Google Discovery baseUrl`); + } + return { ...api, baseUrl }; + } +} + +function selectUploadProtocol( + operation: GoogleDiscoveryOperation, + media: ResolvedMediaInput, + args: Record, +): "simple" | "multipart" | "resumable" { + if ( + media.bytes.byteLength > DEFAULT_RESUMABLE_THRESHOLD_BYTES && + operation.mediaUploadProtocols.resumable + ) { + return "resumable"; + } + if ("body" in args && operation.mediaUploadProtocols.multipart) return "multipart"; + if (operation.mediaUploadProtocols.simple) return "simple"; + if (operation.mediaUploadProtocols.resumable) return "resumable"; + throw new CapletsError( + "CONFIG_INVALID", + "Google Discovery media upload has no supported protocol", + ); +} + +function simpleUploadInit( + operation: GoogleDiscoveryOperation, + media: ResolvedMediaInput, + headers: Headers, +): RequestInit { + headers.set("content-type", media.mimeType ?? "application/octet-stream"); + headers.set("content-length", String(media.bytes.byteLength)); + return { + method: operation.method.toUpperCase(), + headers, + body: media.bytes, + redirect: "manual", + }; +} + +function multipartUploadInit( + operation: GoogleDiscoveryOperation, + body: unknown, + media: ResolvedMediaInput, + headers: Headers, +): RequestInit { + const boundary = `caplets_${randomUUID().replace(/-/gu, "")}`; + const contentType = media.mimeType ?? "application/octet-stream"; + const payload = Buffer.concat([ + Buffer.from( + `--${boundary}\r\ncontent-type: application/json; charset=UTF-8\r\n\r\n${JSON.stringify( + body ?? {}, + )}\r\n`, + ), + Buffer.from(`--${boundary}\r\ncontent-type: ${contentType}\r\n\r\n`), + media.bytes, + Buffer.from(`\r\n--${boundary}--\r\n`), + ]); + headers.set("content-type", `multipart/related; boundary=${boundary}`); + headers.set("content-length", String(payload.byteLength)); + return { + method: operation.method.toUpperCase(), + headers, + body: payload, + redirect: "manual", + }; +} + +function uploadUrl(api: GoogleDiscoveryApiConfig, uploadPath: string, uploadType: string): URL { + if (!api.baseUrl) { + throw new CapletsError("CONFIG_INVALID", `${api.server} is missing Google Discovery baseUrl`); + } + const base = new URL(api.baseUrl); + const url = uploadPath.startsWith("/") + ? new URL(uploadPath, base.origin) + : new URL(uploadPath, api.baseUrl); + url.searchParams.set("uploadType", uploadType); + return url; +} + +function googleDiscoveryBaseUrl( + api: GoogleDiscoveryApiConfig, + document: GoogleDiscoveryDocument, +): string | undefined { + if (api.baseUrl) return api.baseUrl; + if (document.baseUrl) return document.baseUrl; + if (document.rootUrl && document.servicePath) { + return new URL(document.servicePath, document.rootUrl).toString(); + } + return undefined; +} + +async function fetchGoogleRequest( + api: GoogleDiscoveryApiConfig, + operation: GoogleDiscoveryOperation, + url: URL, + init: RequestInit, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), api.requestTimeoutMs); + try { + const response = await fetch(url, { ...init, signal: controller.signal }); + if (response.status >= 300 && response.status < 400) { + throw new CapletsError( + "DOWNSTREAM_PROTOCOL_ERROR", + "Google Discovery request returned a redirect", + { + server: api.server, + status: response.status, + location: response.headers.get("location") ? "[REDACTED]" : undefined, + }, + ); + } + return response; + } catch (error) { + if (isAbortError(error)) { + throw new CapletsError( + "TOOL_CALL_TIMEOUT", + `Google Discovery request timed out for ${api.server}/${operation.name}`, + ); + } + throw error; + } finally { + clearTimeout(timeout); + } +} + +async function loadGoogleDiscoveryDocument( + api: GoogleDiscoveryApiConfig, + authDir?: string, +): Promise { + const source = await loadGoogleDiscoverySource(api, authDir); + let parsed: unknown; + try { + parsed = JSON.parse(source); + } catch (error) { + throw new CapletsError("DOWNSTREAM_PROTOCOL_ERROR", "Google Discovery document is not JSON", { + server: api.server, + error: error instanceof Error ? error.message : String(error), + }); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new CapletsError( + "DOWNSTREAM_PROTOCOL_ERROR", + "Google Discovery document is not an object", + ); + } + return parsed as GoogleDiscoveryDocument; +} + +async function loadGoogleDiscoverySource( + api: GoogleDiscoveryApiConfig, + authDir?: string, +): Promise { + if (api.discoveryPath) { + return readFileSync(api.discoveryPath, "utf8"); + } + if (!api.discoveryUrl) { + throw new CapletsError( + "CONFIG_INVALID", + `${api.server} is missing Google Discovery document source`, + ); + } + return fetchDiscoverySource( + api, + shouldSendDiscoveryAuth(api) ? await authHeaders(api, authDir) : {}, + ); +} + +async function fetchDiscoverySource( + api: GoogleDiscoveryApiConfig, + headers: Record, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), api.requestTimeoutMs); + try { + const response = await fetch(api.discoveryUrl!, { + headers, + redirect: "manual", + signal: controller.signal, + }); + if (response.status >= 300 && response.status < 400) { + throw new CapletsError( + "DOWNSTREAM_PROTOCOL_ERROR", + "Google Discovery document request returned a redirect", + ); + } + if (!response.ok) { + throw new CapletsError( + "DOWNSTREAM_PROTOCOL_ERROR", + "Google Discovery document request failed", + { status: response.status }, + ); + } + return readLimitedText(response, { + errorMessage: "Google Discovery document exceeded byte limit", + }); + } catch (error) { + if (isAbortError(error)) { + throw new CapletsError("TOOL_CALL_TIMEOUT", "Google Discovery document request timed out"); + } + throw error; + } finally { + clearTimeout(timeout); + } +} + +async function authHeaders( + api: GoogleDiscoveryApiConfig, + authDir?: string, + resolvedScopes?: string[], +): Promise> { + switch (api.auth.type) { + case "none": + return {}; + case "bearer": + return { authorization: `Bearer ${api.auth.token}` }; + case "headers": + return api.auth.headers; + case "oauth2": + case "oidc": + return genericOAuthHeaders({ ...api, resolvedScopes }, authDir); + } +} + +function shouldSendDiscoveryAuth(api: GoogleDiscoveryApiConfig): boolean { + return Boolean( + api.discoveryUrl && + api.baseUrl && + new URL(api.discoveryUrl).origin === new URL(api.baseUrl).origin, + ); +} + +function googleDiscoveryCacheKey(api: GoogleDiscoveryApiConfig): string { + return JSON.stringify({ + discoveryPath: api.discoveryPath, + discoveryUrl: api.discoveryUrl, + baseUrl: api.baseUrl, + includeOperations: api.includeOperations, + excludeOperations: api.excludeOperations, + }); +} diff --git a/packages/core/src/google-discovery/operations.ts b/packages/core/src/google-discovery/operations.ts new file mode 100644 index 00000000..ce457481 --- /dev/null +++ b/packages/core/src/google-discovery/operations.ts @@ -0,0 +1,279 @@ +import { googleDiscoverySchemaToJsonSchema } from "./schema"; +import type { + GoogleDiscoveryDocument, + GoogleDiscoveryMethod, + GoogleDiscoveryParameter, + GoogleDiscoveryResource, + GoogleDiscoverySchema, +} from "./types"; + +type GoogleDiscoveryHttpMethod = "get" | "put" | "post" | "delete" | "patch" | "head"; +type ParameterLocation = "path" | "query" | "header" | "body" | "media"; + +export type GoogleDiscoveryOperation = { + name: string; + method: GoogleDiscoveryHttpMethod; + path: string; + description?: string; + inputSchema: Record; + outputSchema?: Record; + readOnlyHint: boolean; + destructiveHint: boolean; + scopes: string[]; + supportsMediaUpload: boolean; + supportsMediaDownload: boolean; + mediaUpload?: { + accept?: string[]; + maxSize?: string; + }; + mediaUploadProtocols: Record; + parameterOrder: string[]; +}; + +export type DiscoveryOperationsOptions = { + server: string; + document: unknown; + includeOperations?: string[]; + excludeOperations?: string[]; +}; + +type MethodEntry = { + resourcePath: string[]; + methodKey: string; + method: GoogleDiscoveryMethod; +}; + +export function discoveryOperations( + options: DiscoveryOperationsOptions, +): GoogleDiscoveryOperation[] { + const document = validateGoogleDiscoveryDocument(options.document); + const schemas = document.schemas ?? {}; + const operations = collectDocumentMethods(document) + .map((entry) => operationFromMethod(options.server, document, schemas, entry)) + .filter((operation) => isIncluded(operation.name, options.includeOperations)) + .filter((operation) => !isExcluded(operation.name, options.excludeOperations)); + + return operations.sort((left, right) => left.name.localeCompare(right.name)); +} + +export function googleDiscoveryScopesForOperations( + operations: GoogleDiscoveryOperation[], +): string[] { + return [...new Set(operations.flatMap((operation) => operation.scopes))].sort(); +} + +function validateGoogleDiscoveryDocument(value: unknown): GoogleDiscoveryDocument { + if (!isRecord(value)) { + throw new Error("Invalid Google Discovery document: expected an object"); + } + if (value.kind !== undefined && value.kind !== "discovery#restDescription") { + throw new Error("Invalid Google Discovery document: expected kind discovery#restDescription"); + } + if (value.resources !== undefined && !isRecord(value.resources)) { + throw new Error("Invalid Google Discovery document: expected resources object"); + } + if (value.methods !== undefined && !isRecord(value.methods)) { + throw new Error("Invalid Google Discovery document: expected methods object"); + } + if (!isRecord(value.resources) && !isRecord(value.methods)) { + throw new Error("Invalid Google Discovery document: expected resources or methods object"); + } + if (value.schemas !== undefined && !isRecord(value.schemas)) { + throw new Error("Invalid Google Discovery document: expected schemas object"); + } + if (value.parameters !== undefined && !isRecord(value.parameters)) { + throw new Error("Invalid Google Discovery document: expected parameters object"); + } + return value as GoogleDiscoveryDocument; +} + +function collectDocumentMethods(document: GoogleDiscoveryDocument): MethodEntry[] { + const topLevel = Object.entries(document.methods ?? {}) + .filter((entry): entry is [string, GoogleDiscoveryMethod] => isRecord(entry[1])) + .map(([methodKey, method]) => ({ resourcePath: [], methodKey, method })); + return [...topLevel, ...collectMethods(document.resources ?? {})]; +} + +function collectMethods( + resources: Record, + resourcePath: string[] = [], +): MethodEntry[] { + const entries: MethodEntry[] = []; + for (const [resourceName, resource] of Object.entries(resources)) { + if (!isRecord(resource)) continue; + const nextPath = [...resourcePath, resourceName]; + for (const [methodKey, method] of Object.entries(resource.methods ?? {})) { + if (isRecord(method)) { + entries.push({ resourcePath: nextPath, methodKey, method }); + } + } + entries.push(...collectMethods(resource.resources ?? {}, nextPath)); + } + return entries; +} + +function operationFromMethod( + server: string, + document: GoogleDiscoveryDocument, + schemas: Record, + entry: MethodEntry, +): GoogleDiscoveryOperation { + const method = normalizedHttpMethod(entry.method.httpMethod); + const name = entry.method.id ?? [server, ...entry.resourcePath, entry.methodKey].join("."); + const scopes = [...new Set(entry.method.scopes ?? [])].sort(); + const inputSchema = buildInputSchema(document.parameters ?? {}, entry.method, schemas); + const outputSchema = entry.method.response?.$ref + ? googleDiscoverySchemaToJsonSchema(entry.method.response, schemas) + : undefined; + const mediaUpload = + entry.method.mediaUpload?.accept || entry.method.mediaUpload?.maxSize + ? { + ...(entry.method.mediaUpload.accept ? { accept: entry.method.mediaUpload.accept } : {}), + ...(entry.method.mediaUpload.maxSize + ? { maxSize: entry.method.mediaUpload.maxSize } + : {}), + } + : undefined; + + return { + name, + method, + path: entry.method.path ?? entry.method.flatPath ?? "", + ...(entry.method.description + ? { description: collapseWhitespace(entry.method.description) } + : {}), + inputSchema, + ...(outputSchema ? { outputSchema } : {}), + readOnlyHint: method === "get" || method === "head", + destructiveHint: method === "delete" || /\.(delete|emptyTrash)$/u.test(name), + scopes, + supportsMediaUpload: entry.method.supportsMediaUpload === true, + supportsMediaDownload: entry.method.supportsMediaDownload === true, + ...(mediaUpload ? { mediaUpload } : {}), + mediaUploadProtocols: entry.method.mediaUpload?.protocols ?? {}, + parameterOrder: entry.method.parameterOrder ?? [], + }; +} + +function buildInputSchema( + globalParameters: Record, + method: GoogleDiscoveryMethod, + schemas: Record, +): Record { + const groups = new Map>(); + const requiredByGroup = new Map(); + const parameters = { ...globalParameters, ...method.parameters }; + + for (const [name, parameter] of Object.entries(parameters)) { + const location = parameter.location ?? "query"; + const group = groups.get(location) ?? {}; + group[name] = googleDiscoverySchemaToJsonSchema(parameter, schemas); + groups.set(location, group); + if (parameter.required === true) { + const required = requiredByGroup.get(location) ?? []; + required.push(name); + requiredByGroup.set(location, required); + } + } + + if (method.request?.$ref) { + groups.set("body", googleDiscoverySchemaToJsonSchema(method.request, schemas)); + } + if (method.supportsMediaUpload === true) { + groups.set("media", { + type: "object", + additionalProperties: false, + properties: { + path: { type: "string" }, + artifact: { type: "string" }, + dataUrl: { type: "string" }, + mimeType: { type: "string" }, + filename: { type: "string" }, + }, + }); + } + + const properties: Record = {}; + const required: string[] = []; + for (const location of ["path", "query", "header", "body", "media"] as const) { + const group = groups.get(location); + if (!group) continue; + if ((location === "body" || location === "media") && isJsonSchemaObject(group)) { + properties[location] = group; + } else { + const groupRequired = requiredByGroup.get(location) ?? []; + properties[location] = { + type: "object", + ...(groupRequired.length > 0 ? { required: groupRequired } : {}), + properties: group, + additionalProperties: false, + }; + } + if (location === "path" || requiredByGroup.has(location)) { + required.push(location); + } + } + if (method.supportsMediaDownload === true) { + properties.filename = { type: "string" }; + properties.outputPath = { type: "string" }; + } + + return { + type: "object", + ...(required.length > 0 ? { required } : {}), + properties, + additionalProperties: false, + }; +} + +function normalizedHttpMethod(method: string | undefined): GoogleDiscoveryHttpMethod { + const normalized = method?.toLowerCase(); + if ( + normalized === "get" || + normalized === "put" || + normalized === "post" || + normalized === "delete" || + normalized === "patch" || + normalized === "head" + ) { + return normalized; + } + return "get"; +} + +function isIncluded(name: string, includeOperations: string[] | undefined): boolean { + return ( + !includeOperations?.length || includeOperations.some((pattern) => globMatches(pattern, name)) + ); +} + +function isExcluded(name: string, excludeOperations: string[] | undefined): boolean { + return excludeOperations?.some((pattern) => globMatches(pattern, name)) === true; +} + +function globMatches(pattern: string, name: string): boolean { + const patternSegments = pattern.split("."); + const nameSegments = name.split("."); + if (patternSegments[0] === "*" && patternSegments.length < nameSegments.length) { + const suffix = patternSegments.slice(1); + return suffix.every( + (segment, index) => segment === nameSegments[nameSegments.length - suffix.length + index], + ); + } + if (patternSegments.length !== nameSegments.length) return false; + return patternSegments.every( + (segment, index) => segment === "*" || segment === nameSegments[index], + ); +} + +function isJsonSchemaObject(value: Record): boolean { + return value.type === "object" || "properties" in value || "additionalProperties" in value; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function collapseWhitespace(value: string): string { + return value.replace(/\s+/gu, " ").trim(); +} diff --git a/packages/core/src/google-discovery/request.ts b/packages/core/src/google-discovery/request.ts new file mode 100644 index 00000000..0860009d --- /dev/null +++ b/packages/core/src/google-discovery/request.ts @@ -0,0 +1,169 @@ +import type { GoogleDiscoveryApiConfig } from "../config"; +import { FORBIDDEN_HEADERS, isAllowedRemoteUrl } from "../config/validation"; +import { CapletsError } from "../errors"; +import type { GoogleDiscoveryOperation } from "./operations"; + +export function buildGoogleDiscoveryUrl( + api: GoogleDiscoveryApiConfig, + operation: GoogleDiscoveryOperation, + args: Record, +): URL { + const base = api.baseUrl; + validateBaseUrl(api, base); + const url = buildOperationUrl( + base, + substitutePath(operation.path, asRecord(args.path), operation), + ); + for (const [key, value] of Object.entries(asRecord(args.query))) { + if (value !== undefined && value !== null) { + url.searchParams.append(key, serializeGoogleDiscoveryValue("query", key, value)); + } + } + return url; +} + +export function buildJsonRequestInit( + operation: GoogleDiscoveryOperation, + args: Record, + headers: Headers, +): RequestInit { + for (const [key, value] of Object.entries(asRecord(args.header))) { + if (value !== undefined && value !== null) { + const normalized = key.toLowerCase(); + if (FORBIDDEN_HEADERS.has(normalized)) { + throw new CapletsError("REQUEST_INVALID", `Header ${key} cannot be supplied by arguments`); + } + headers.set(key, serializeGoogleDiscoveryValue("header", key, value)); + } + } + if ("body" in args) { + headers.set("content-type", "application/json"); + return { + method: operation.method.toUpperCase(), + headers, + body: JSON.stringify(args.body), + redirect: "manual", + }; + } + return { method: operation.method.toUpperCase(), headers, redirect: "manual" }; +} + +function validateBaseUrl( + api: GoogleDiscoveryApiConfig, + base: string | undefined, +): asserts base is string { + if (!base) { + throw new CapletsError("CONFIG_INVALID", `${api.server} is missing Google Discovery baseUrl`); + } + if (!isAllowedRemoteUrl(base)) { + throw new CapletsError( + "CONFIG_INVALID", + `${api.server} Google Discovery baseUrl is not allowed`, + ); + } + const url = new URL(base); + if (url.username || url.password || url.search || url.hash) { + throw new CapletsError( + "CONFIG_INVALID", + `${api.server} Google Discovery baseUrl must not include credentials, query, or fragment`, + ); + } +} + +function buildOperationUrl(base: string, operationPath: string): URL { + if (/^[a-z][a-z0-9+.-]*:/iu.test(operationPath) || operationPath.startsWith("//")) { + throw new CapletsError( + "CONFIG_INVALID", + "Google Discovery operation path cannot change origin", + ); + } + const baseUrl = new URL(base); + const basePath = baseUrl.pathname.replace(/\/+$/u, ""); + const relativePath = operationPath.replace(/^\/+/u, ""); + assertSafeRelativePath(relativePath); + baseUrl.pathname = [basePath, relativePath].filter(Boolean).join("/"); + assertInsideBasePath(baseUrl, basePath); + return baseUrl; +} + +function substitutePath( + path: string, + values: Record, + operation: GoogleDiscoveryOperation, +): string { + return path.replace(/\{([^}]+)\}/gu, (_match, expression: string) => { + const reserved = expression.startsWith("+"); + const name = reserved ? expression.slice(1) : expression; + const value = values[name]; + if (value === undefined || value === null || value === "") { + throw new CapletsError("REQUEST_INVALID", `Missing required path parameter ${name}`, { + tool: operation.name, + }); + } + const serialized = serializeGoogleDiscoveryValue("path", name, value); + return reserved ? encodeReservedPathValue(serialized) : encodeURIComponent(serialized); + }); +} + +function encodeReservedPathValue(value: string): string { + return value + .split("/") + .map((segment) => encodeURIComponent(segment)) + .join("/"); +} + +function assertSafeRelativePath(path: string): void { + for (const segment of path.split("/")) { + const decoded = safeDecodePathSegment(segment); + if (decoded === "." || decoded === "..") { + throw new CapletsError( + "CONFIG_INVALID", + "Google Discovery operation path cannot escape baseUrl", + ); + } + } +} + +function assertInsideBasePath(url: URL, basePath: string): void { + const normalizedBase = basePath === "" ? "/" : `${basePath}/`; + if (normalizedBase === "/") return; + const pathname = url.pathname.endsWith("/") ? url.pathname : `${url.pathname}/`; + if (pathname !== normalizedBase && !pathname.startsWith(normalizedBase)) { + throw new CapletsError( + "CONFIG_INVALID", + "Google Discovery operation path cannot escape baseUrl", + ); + } +} + +function safeDecodePathSegment(segment: string): string { + try { + return decodeURIComponent(segment); + } catch { + return segment; + } +} + +function serializeGoogleDiscoveryValue( + location: "path" | "query" | "header", + name: string, + value: unknown, +): string { + switch (typeof value) { + case "string": + case "number": + case "boolean": + return String(value); + default: + throw new CapletsError( + "REQUEST_INVALID", + `Google Discovery ${location} parameter ${name} must be a string, number, or boolean`, + ); + } +} + +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} diff --git a/packages/core/src/google-discovery/schema.ts b/packages/core/src/google-discovery/schema.ts new file mode 100644 index 00000000..0437049e --- /dev/null +++ b/packages/core/src/google-discovery/schema.ts @@ -0,0 +1,77 @@ +import type { GoogleDiscoverySchema } from "./types"; + +export function googleDiscoverySchemaToJsonSchema( + value: GoogleDiscoverySchema | undefined, + schemas: Record = {}, + seen = new Set(), +): Record { + if (!value) return {}; + if (value.$ref) { + const target = schemas[value.$ref]; + if (!target || seen.has(value.$ref)) return { type: "object", additionalProperties: true }; + return googleDiscoverySchemaToJsonSchema(target, schemas, new Set([...seen, value.$ref])); + } + + const type = discoveryTypeToJsonSchemaType(value.type); + const converted: Record = {}; + if (value.description) converted.description = collapseWhitespace(value.description); + if (type) converted.type = type; + if (value.format) converted.format = value.format; + if (value.enum) converted.enum = value.enum; + const defaultValue = convertedDefault(value.default, type); + if (defaultValue !== undefined) converted.default = defaultValue; + + if (value.repeated) { + return { + ...(converted.description ? { description: converted.description } : {}), + type: "array", + items: omit(converted, ["description", "default"]), + }; + } + + if (value.items) converted.items = googleDiscoverySchemaToJsonSchema(value.items, schemas, seen); + if (value.properties) { + converted.type = converted.type ?? "object"; + converted.properties = Object.fromEntries( + Object.entries(value.properties).map(([key, schema]) => [ + key, + googleDiscoverySchemaToJsonSchema(schema, schemas, seen), + ]), + ); + converted.additionalProperties = false; + } + if (typeof value.additionalProperties === "boolean") { + converted.additionalProperties = value.additionalProperties; + } else if (value.additionalProperties) { + converted.additionalProperties = googleDiscoverySchemaToJsonSchema( + value.additionalProperties, + schemas, + seen, + ); + } + + return converted; +} + +function discoveryTypeToJsonSchemaType(type: string | undefined): string | undefined { + if (type === "any") return "object"; + return type; +} + +function convertedDefault(value: unknown, type: string | undefined): unknown { + if (value === undefined) return undefined; + if (type === "boolean" && typeof value === "string") return value === "true"; + if ((type === "integer" || type === "number") && typeof value === "string") { + const number = Number(value); + return Number.isFinite(number) ? number : value; + } + return value; +} + +function collapseWhitespace(value: string): string { + return value.replace(/\s+/gu, " ").trim(); +} + +function omit(value: Record, keys: string[]): Record { + return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key))); +} diff --git a/packages/core/src/google-discovery/types.ts b/packages/core/src/google-discovery/types.ts new file mode 100644 index 00000000..d4be7293 --- /dev/null +++ b/packages/core/src/google-discovery/types.ts @@ -0,0 +1,61 @@ +export type GoogleDiscoveryDocument = { + kind?: string; + id?: string; + name?: string; + version?: string; + title?: string; + rootUrl?: string; + servicePath?: string; + baseUrl?: string; + auth?: { oauth2?: { scopes?: Record } }; + parameters?: Record; + schemas?: Record; + methods?: Record; + resources?: Record; +}; + +export type GoogleDiscoveryResource = { + methods?: Record; + resources?: Record; +}; + +export type GoogleDiscoveryMethod = { + id?: string; + path?: string; + flatPath?: string; + httpMethod?: string; + description?: string; + parameters?: Record; + parameterOrder?: string[]; + request?: { $ref?: string }; + response?: { $ref?: string }; + scopes?: string[]; + supportsMediaUpload?: boolean; + supportsMediaDownload?: boolean; + mediaUpload?: { + accept?: string[]; + maxSize?: string; + protocols?: Record; + }; +}; + +export type GoogleDiscoveryParameter = GoogleDiscoverySchema & { + location?: "path" | "query" | "header" | "body" | "media"; + required?: boolean; + repeated?: boolean; + deprecated?: boolean; +}; + +export type GoogleDiscoverySchema = { + id?: string; + $ref?: string; + type?: string; + format?: string; + description?: string; + default?: unknown; + enum?: string[]; + repeated?: boolean; + properties?: Record; + items?: GoogleDiscoverySchema; + additionalProperties?: GoogleDiscoverySchema | boolean; +}; diff --git a/packages/core/src/http-actions.ts b/packages/core/src/http-actions.ts index 854c8d86..af6641d6 100644 --- a/packages/core/src/http-actions.ts +++ b/packages/core/src/http-actions.ts @@ -9,7 +9,8 @@ import { type CompactTool, } from "./downstream"; import { CapletsError, toSafeError } from "./errors"; -import { isAbortError, parseHttpBody, readLimitedText } from "./http/utils"; +import { readHttpLikeResponse } from "./http/response"; +import { isAbortError } from "./http/utils"; import type { ServerRegistry } from "./registry"; import { markdownStructuredContent } from "./result-content"; import { searchToolList } from "./tool-search"; @@ -20,7 +21,11 @@ type HttpActionOperation = HttpActionConfig & { name: string }; export class HttpActionManager { constructor( private registry: ServerRegistry, - private readonly options: { authDir?: string } = {}, + private readonly options: { + authDir?: string; + artifactDir?: string; + maxInlineBytes?: number; + } = {}, ) {} updateRegistry(registry: ServerRegistry): void { @@ -100,7 +105,15 @@ export class HttpActionManager { }, ); } - const parsed = await readResponse(response, api, Date.now() - startedAt); + const parsed = { + ...(await readHttpLikeResponse(response, { + capletId: api.server, + ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), + maxInlineBytes: this.options.maxInlineBytes ?? api.maxResponseBytes, + maxBytes: api.maxResponseBytes, + })), + elapsedMs: Date.now() - startedAt, + }; return { content: markdownStructuredContent(parsed, { title: `${api.name} call_tool ${toolName}`, @@ -371,32 +384,6 @@ async function authHeaders(api: HttpApiConfig, authDir?: string): Promise> { - const contentType = response.headers.get("content-type") ?? ""; - const text = await readLimitedText(response, { - maxBytes: maxResponseBytes(api), - errorMessage: "HTTP action response exceeded byte limit", - }); - const body = parseHttpBody(contentType, text); - return { - status: response.status, - statusText: response.statusText, - headers: { - "content-type": contentType, - }, - ...(body === undefined ? {} : { body }), - elapsedMs, - }; -} - -function maxResponseBytes(api: HttpApiConfig): number { - return api.maxResponseBytes; -} - function validateBaseUrl(api: HttpApiConfig): void { if (!isAllowedRemoteUrl(api.baseUrl)) { throw new CapletsError("CONFIG_INVALID", `${api.server} HTTP API baseUrl is not allowed`); diff --git a/packages/core/src/http/response.ts b/packages/core/src/http/response.ts new file mode 100644 index 00000000..b50c3216 --- /dev/null +++ b/packages/core/src/http/response.ts @@ -0,0 +1,188 @@ +import type { MediaArtifact } from "../media"; +import { CapletsError } from "../errors"; +import { writeMediaArtifact } from "../media"; +import { DEFAULT_MAX_RESPONSE_BYTES, parseHttpBody } from "./utils"; + +export type ReadHttpLikeResponseOptions = { + capletId: string; + artifactDir?: string; + outputPath?: string; + filename?: string; + maxInlineBytes?: number; + maxBytes?: number; +}; + +export async function readHttpLikeResponse( + response: Response, + options: ReadHttpLikeResponseOptions, +): Promise> { + const contentType = response.headers.get("content-type") ?? ""; + const mimeType = mimeFromContentType(contentType); + const maxInlineBytes = options.maxInlineBytes ?? DEFAULT_MAX_RESPONSE_BYTES; + const maxBytes = options.maxBytes ?? DEFAULT_MAX_RESPONSE_BYTES; + rejectOversizedContentLength(response, maxBytes); + + if (shouldInline(response, mimeType)) { + const inline = await readInlineCandidate(response, { maxInlineBytes, maxBytes }); + if (!inline.exceeded) { + const body = parseHttpBody(contentType, new TextDecoder().decode(inline.bytes)); + return responseEnvelope(response, contentType, body); + } + const artifact = await writeResponseArtifact(response, options, mimeType, inline.bytes); + return responseEnvelope(response, contentType, { artifact }); + } + + const bytes = await readBoundedBytes(response, maxBytes); + const artifact = await writeResponseArtifact(response, options, mimeType, bytes); + return responseEnvelope(response, contentType, { artifact }); +} + +async function readInlineCandidate( + response: Response, + options: { maxInlineBytes: number; maxBytes: number }, +): Promise<{ bytes: Buffer; exceeded: boolean }> { + if (!response.body) { + return { bytes: Buffer.alloc(0), exceeded: false }; + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let bytes = 0; + let exceeded = false; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (value) { + bytes += value.byteLength; + if (bytes > options.maxBytes) { + await reader.cancel(); + throw responseExceededLimit(options.maxBytes); + } + if (bytes > options.maxInlineBytes) exceeded = true; + chunks.push(value); + } + } + return { bytes: Buffer.concat(chunks), exceeded }; +} + +async function readBoundedBytes(response: Response, maxBytes: number): Promise { + if (!response.body) { + return Buffer.alloc(0); + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let bytes = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (value) { + bytes += value.byteLength; + if (bytes > maxBytes) { + await reader.cancel(); + throw responseExceededLimit(maxBytes); + } + chunks.push(value); + } + } + return Buffer.concat(chunks); +} + +function rejectOversizedContentLength(response: Response, maxBytes: number): void { + const contentLength = response.headers.get("content-length"); + if (!contentLength) return; + const byteLength = Number.parseInt(contentLength, 10); + if (Number.isFinite(byteLength) && byteLength > maxBytes) { + throw responseExceededLimit(maxBytes); + } +} + +function responseExceededLimit(maxBytes: number): CapletsError { + return new CapletsError( + "DOWNSTREAM_PROTOCOL_ERROR", + `HTTP response exceeded byte limit ${maxBytes}`, + ); +} + +async function writeResponseArtifact( + response: Response, + options: ReadHttpLikeResponseOptions, + mimeType: string, + bytes: Buffer, +): Promise { + return await writeMediaArtifact({ + capletId: options.capletId, + ...(options.artifactDir ? { rootDir: options.artifactDir } : {}), + ...(options.outputPath ? { outputPath: options.outputPath } : {}), + suggestedFilename: + options.filename ?? filenameFromContentDisposition(response) ?? "response.bin", + ...(mimeType ? { mimeType } : {}), + bytes, + }); +} + +function responseEnvelope( + response: Response, + contentType: string, + body?: unknown, +): Record { + return { + status: response.status, + statusText: response.statusText, + headers: { + "content-type": contentType, + }, + ...(body === undefined ? {} : { body }), + }; +} + +function shouldInline(response: Response, mimeType: string): boolean { + if (isAttachment(response)) { + return false; + } + return ( + mimeType === "" || + mimeType === "application/json" || + mimeType.endsWith("+json") || + mimeType.endsWith("/json") || + mimeType.startsWith("text/") + ); +} + +function isAttachment(response: Response): boolean { + return /\battachment\b/iu.test(response.headers.get("content-disposition") ?? ""); +} + +function mimeFromContentType(contentType: string): string { + return contentType.split(";")[0]?.toLowerCase().trim() ?? ""; +} + +function filenameFromContentDisposition(response: Response): string | undefined { + const contentDisposition = response.headers.get("content-disposition"); + if (!contentDisposition) { + return undefined; + } + return parseRfc5987Filename(contentDisposition) ?? parseQuotedFilename(contentDisposition); +} + +function parseRfc5987Filename(contentDisposition: string): string | undefined { + const match = /(?:^|;)\s*filename\*=([^;]+)/iu.exec(contentDisposition); + const value = match?.[1]?.trim(); + if (!value) { + return undefined; + } + const encoded = value.replace(/^UTF-8''/iu, ""); + try { + return decodeURIComponent(encoded.replace(/^"|"$/gu, "")); + } catch { + return encoded; + } +} + +function parseQuotedFilename(contentDisposition: string): string | undefined { + const quoted = /(?:^|;)\s*filename="([^"]+)"/iu.exec(contentDisposition)?.[1]; + if (quoted) { + return quoted; + } + return /(?:^|;)\s*filename=([^;]+)/iu.exec(contentDisposition)?.[1]?.trim(); +} + +export type { MediaArtifact }; diff --git a/packages/core/src/media/artifacts.ts b/packages/core/src/media/artifacts.ts new file mode 100644 index 00000000..ba7d7a40 --- /dev/null +++ b/packages/core/src/media/artifacts.ts @@ -0,0 +1,319 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + chmodSync, + existsSync, + lstatSync, + mkdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; +import { DEFAULT_ARTIFACT_DIR } from "../config/paths"; +import { CapletsError } from "../errors"; + +export type MediaArtifact = { + uri: string; + path: string; + filename: string; + mimeType?: string; + byteLength: number; + sha256: string; +}; + +export type WriteMediaArtifactInput = { + rootDir?: string; + capletId: string; + callId?: string; + suggestedFilename?: string; + outputPath?: string; + mimeType?: string; + bytes: Uint8Array | Buffer; +}; + +type StoredMediaArtifactMetadata = { + mimeType?: string; +}; + +type ParsedArtifactUri = { + capletId: string; + callId: string; + filename: string; +}; + +export function artifactUri(capletId: string, callId: string, filename: string): string { + return `caplets://artifacts/${encodeURIComponent(capletId)}/${encodeURIComponent( + callId, + )}/${encodeURIComponent(filename)}`; +} + +export async function writeMediaArtifact(input: WriteMediaArtifactInput): Promise { + const rootDir = resolve(input.rootDir ?? DEFAULT_ARTIFACT_DIR); + const capletId = requiredSafePathSegment(input.capletId, "capletId"); + const callId = safePathSegment(input.callId ?? defaultCallId(), "call"); + const filename = safeFilename( + input.suggestedFilename ?? (input.outputPath ? basename(input.outputPath) : "response.bin"), + ); + const target = input.outputPath + ? assertInsideRoot(rootDir, input.outputPath) + : assertInsideRoot(rootDir, resolve(rootDir, capletId, callId, filename)); + rejectSymlinkPathComponents(rootDir, target, true); + const uriParts = input.outputPath + ? uriPartsForOutputPath(rootDir, target) + : { capletId, callId, filename: safeFilename(basename(target)) }; + const bytes = Buffer.from(input.bytes); + + mkdirSync(dirname(target), { recursive: true, mode: 0o700 }); + writeFileSync(target, bytes, { mode: 0o600 }); + chmodSync(target, 0o600); + + const artifactFilename = uriParts.filename; + writeArtifactMetadata(target, input.mimeType ? { mimeType: input.mimeType } : {}); + return { + uri: artifactUri(uriParts.capletId, uriParts.callId, artifactFilename), + path: target, + filename: artifactFilename, + ...(input.mimeType ? { mimeType: input.mimeType } : {}), + byteLength: bytes.byteLength, + sha256: sha256(bytes), + }; +} + +export function resolveMediaArtifact( + uri: string, + options: { artifactRoot?: string; maxBytes?: number } = {}, +): MediaArtifact { + const parsed = parseArtifactUri(uri); + const rootDir = resolve(options.artifactRoot ?? DEFAULT_ARTIFACT_DIR); + const path = assertInsideRoot( + rootDir, + resolve(rootDir, parsed.capletId, parsed.callId, parsed.filename), + ); + rejectSymlinkPathComponents(rootDir, path, true); + + if (!existsSync(path)) { + throw new CapletsError("REQUEST_INVALID", "Media artifact was not found"); + } + + const stat = statSync(path); + if (!stat.isFile()) { + throw new CapletsError("REQUEST_INVALID", "Media artifact must resolve to a file"); + } + if (options.maxBytes !== undefined && stat.size > options.maxBytes) { + throw new CapletsError("REQUEST_INVALID", `media exceeds byte limit ${options.maxBytes}`); + } + + const bytes = readFileSync(path); + const metadata = readArtifactMetadata(path); + return { + uri, + path, + filename: parsed.filename, + ...(metadata?.mimeType ? { mimeType: metadata.mimeType } : {}), + byteLength: bytes.byteLength, + sha256: sha256(bytes), + }; +} + +function parseArtifactUri(uri: string): ParsedArtifactUri { + let url: URL; + try { + url = new URL(uri); + } catch { + throw new CapletsError("REQUEST_INVALID", "Media artifact URI is invalid"); + } + + if (url.protocol !== "caplets:" || url.hostname !== "artifacts") { + throw new CapletsError( + "REQUEST_INVALID", + "Media artifact URI must start with caplets://artifacts/", + ); + } + + const parts = url.pathname.split("/").filter(Boolean); + if (parts.length !== 3) { + throw new CapletsError("REQUEST_INVALID", "Media artifact URI is missing required parts"); + } + + return { + capletId: decodeSafePathSegment(parts[0]!, "capletId"), + callId: decodeSafePathSegment(parts[1]!, "callId"), + filename: decodeSafeFilename(parts[2]!), + }; +} + +function decodeSafePathSegment(value: string, label: string): string { + const decoded = decodeArtifactUriPart(value); + const safe = safePathSegment(decoded, ""); + if (!safe || safe !== decoded) { + throw new CapletsError("REQUEST_INVALID", `Media artifact URI ${label} is invalid`); + } + return safe; +} + +function decodeSafeFilename(value: string): string { + const decoded = decodeArtifactUriPart(value); + const safe = safeFilename(decoded); + if (safe !== decoded) { + throw new CapletsError("REQUEST_INVALID", "Media artifact URI filename is invalid"); + } + return safe; +} + +function decodeArtifactUriPart(value: string): string { + try { + return decodeURIComponent(value); + } catch { + throw new CapletsError("REQUEST_INVALID", "Media artifact URI contains invalid encoding"); + } +} + +function assertInsideRoot(rootDir: string, candidate: string): string { + if (!isAbsolute(candidate)) { + throw new CapletsError("REQUEST_INVALID", "Media artifact outputPath must be absolute"); + } + + const resolvedRoot = resolve(rootDir); + const resolved = resolve(candidate); + const rel = relative(resolvedRoot, resolved); + if (rel.startsWith("..") || isAbsolute(rel)) { + throw new CapletsError( + "REQUEST_INVALID", + "Media artifact outputPath must stay inside the artifact root", + ); + } + return resolved; +} + +function rejectSymlinkPathComponents( + rootDir: string, + target: string, + includeTarget: boolean, +): void { + const resolvedRoot = resolve(rootDir); + rejectSymlinkRoot(resolvedRoot); + const rel = relative(resolvedRoot, resolve(target)); + const parts = rel.split(/[\\/]+/u).filter(Boolean); + let current = resolvedRoot; + const limit = includeTarget ? parts.length : Math.max(0, parts.length - 1); + for (let index = 0; index < limit; index += 1) { + current = resolve(current, parts[index]!); + try { + if (lstatSync(current).isSymbolicLink()) { + throw new CapletsError("REQUEST_INVALID", "Media artifact path must not contain symlinks"); + } + } catch (error) { + if (error instanceof CapletsError) throw error; + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + } +} + +function rejectSymlinkRoot(rootDir: string): void { + try { + if (lstatSync(rootDir).isSymbolicLink()) { + throw new CapletsError("REQUEST_INVALID", "Media artifact root must not be a symlink"); + } + } catch (error) { + if (error instanceof CapletsError) throw error; + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } +} + +function uriPartsForOutputPath( + rootDir: string, + target: string, +): { capletId: string; callId: string; filename: string } { + const rel = relative(resolve(rootDir), target); + const parts = rel.split(/[\\/]+/u).filter(Boolean); + if (parts.length !== 3) { + throw new CapletsError( + "REQUEST_INVALID", + "Media artifact outputPath must be under ///", + ); + } + return { + capletId: requireAlreadySafePathSegment(parts[0]!, "capletId"), + callId: requireAlreadySafePathSegment(parts[1]!, "callId"), + filename: requireAlreadySafeFilename(parts[2]!), + }; +} + +function requiredSafePathSegment(value: string, label: string): string { + const safe = safePathSegment(value, ""); + if (!safe) { + throw new CapletsError("REQUEST_INVALID", `Media artifact ${label} is required`); + } + return safe; +} + +function requireAlreadySafePathSegment(value: string, label: string): string { + const safe = requiredSafePathSegment(value, label); + if (safe !== value) { + throw new CapletsError("REQUEST_INVALID", `Media artifact outputPath ${label} is invalid`); + } + return safe; +} + +function requireAlreadySafeFilename(value: string): string { + const safe = safeFilename(value); + if (safe !== value) { + throw new CapletsError("REQUEST_INVALID", "Media artifact outputPath filename is invalid"); + } + return safe; +} + +function safePathSegment(value: string, fallback: string): string { + return safeFilename(value, fallback); +} + +function safeFilename(value: string, fallback = "response.bin"): string { + const name = basename(value) + .trim() + .replace(/[^\w.-]+/gu, "_"); + return name && name !== "." && name !== ".." ? name : fallback; +} + +function defaultCallId(): string { + return `${new Date().toISOString().replace(/[:.]/gu, "-")}-${randomUUID()}`; +} + +function sha256(bytes: Buffer): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function artifactMetadataPath(path: string): string { + return `${path}.caplets.json`; +} + +function writeArtifactMetadata(path: string, metadata: StoredMediaArtifactMetadata): void { + const metadataPath = artifactMetadataPath(path); + if (!metadata.mimeType) { + rmSync(metadataPath, { force: true }); + return; + } + writeFileSync(metadataPath, `${JSON.stringify(metadata)}\n`, { mode: 0o600 }); + chmodSync(metadataPath, 0o600); +} + +function readArtifactMetadata(path: string): StoredMediaArtifactMetadata | undefined { + const metadataPath = artifactMetadataPath(path); + if (!existsSync(metadataPath)) return undefined; + + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(metadataPath, "utf8")); + } catch { + throw new CapletsError("REQUEST_INVALID", "Media artifact metadata is invalid"); + } + + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new CapletsError("REQUEST_INVALID", "Media artifact metadata is invalid"); + } + + const value = parsed as Record; + return typeof value.mimeType === "string" && value.mimeType ? { mimeType: value.mimeType } : {}; +} diff --git a/packages/core/src/media/index.ts b/packages/core/src/media/index.ts new file mode 100644 index 00000000..61b743f2 --- /dev/null +++ b/packages/core/src/media/index.ts @@ -0,0 +1,2 @@ +export * from "./artifacts"; +export * from "./input"; diff --git a/packages/core/src/media/input.ts b/packages/core/src/media/input.ts new file mode 100644 index 00000000..c6e0215e --- /dev/null +++ b/packages/core/src/media/input.ts @@ -0,0 +1,131 @@ +import { readFileSync, statSync } from "node:fs"; +import type { Stats } from "node:fs"; +import { basename } from "node:path"; +import { CapletsError } from "../errors"; +import { resolveMediaArtifact } from "./artifacts"; + +export type MediaInput = + | { path: string; artifact?: never; dataUrl?: never; filename?: string; mimeType?: string } + | { artifact: string; path?: never; dataUrl?: never; filename?: string; mimeType?: string } + | { dataUrl: string; path?: never; artifact?: never; filename?: string; mimeType?: string }; + +export type ResolvedMediaInput = { + bytes: Buffer; + filename: string; + mimeType?: string; +}; + +const DEFAULT_MAX_MEDIA_BYTES = 100 * 1024 * 1024; + +export async function readMediaInput( + input: unknown, + options: { artifactRoot?: string; maxBytes?: number } = {}, +): Promise { + if (!input || typeof input !== "object" || Array.isArray(input)) { + throw new CapletsError("REQUEST_INVALID", "media must be an object"); + } + + const media = input as Record; + const sources = ["path", "artifact", "dataUrl"].filter((key) => typeof media[key] === "string"); + if (sources.length !== 1) { + throw new CapletsError( + "REQUEST_INVALID", + "media must define exactly one of path, artifact, or dataUrl", + ); + } + + const filename = typeof media.filename === "string" ? media.filename : undefined; + const mimeType = typeof media.mimeType === "string" ? media.mimeType : undefined; + + if (typeof media.path === "string") { + const stat = statMediaFile(media.path); + enforceSize(stat.size, options.maxBytes); + return { + bytes: readMediaFile(media.path), + filename: filename ?? basename(media.path), + ...(mimeType ? { mimeType } : {}), + }; + } + + if (typeof media.artifact === "string") { + const artifactOptions: { artifactRoot?: string; maxBytes?: number } = {}; + if (options.artifactRoot !== undefined) artifactOptions.artifactRoot = options.artifactRoot; + artifactOptions.maxBytes = options.maxBytes ?? DEFAULT_MAX_MEDIA_BYTES; + const artifact = resolveMediaArtifact(media.artifact, artifactOptions); + const resolvedMimeType = mimeType ?? artifact.mimeType; + return { + bytes: readMediaFile(artifact.path), + filename: filename ?? artifact.filename, + ...(resolvedMimeType ? { mimeType: resolvedMimeType } : {}), + }; + } + + const dataUrlOptions: { filename?: string; mimeType?: string; maxBytes?: number } = {}; + if (filename !== undefined) dataUrlOptions.filename = filename; + if (mimeType !== undefined) dataUrlOptions.mimeType = mimeType; + if (options.maxBytes !== undefined) dataUrlOptions.maxBytes = options.maxBytes; + return readDataUrl(media.dataUrl as string, dataUrlOptions); +} + +function readDataUrl( + dataUrl: string, + options: { filename?: string; mimeType?: string; maxBytes?: number }, +): ResolvedMediaInput { + const match = /^data:([^;,]+);base64,([A-Za-z0-9+/=]+)$/u.exec(dataUrl); + if (!match) { + throw new CapletsError("REQUEST_INVALID", "media.dataUrl must be a base64 data URL"); + } + + const dataMimeType = match[1]; + const base64 = match[2]; + if (!dataMimeType || !base64 || !isStrictBase64(base64)) { + throw new CapletsError("REQUEST_INVALID", "media.dataUrl must be a base64 data URL"); + } + + enforceSize(decodedBase64Length(base64), options.maxBytes); + const bytes = Buffer.from(base64, "base64"); + return { + bytes, + filename: options.filename ?? "media.bin", + mimeType: options.mimeType ?? dataMimeType, + }; +} + +function isStrictBase64(value: string): boolean { + return ( + value.length % 4 === 0 && + /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value) + ); +} + +function decodedBase64Length(value: string): number { + const padding = value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0; + return (value.length / 4) * 3 - padding; +} + +function statMediaFile(path: string): Stats { + try { + const stat = statSync(path); + if (!stat.isFile()) { + throw new CapletsError("REQUEST_INVALID", "media.path must reference a file"); + } + return stat; + } catch (error) { + if (error instanceof CapletsError) throw error; + throw new CapletsError("REQUEST_INVALID", "media.path could not be read"); + } +} + +function readMediaFile(path: string): Buffer { + try { + return readFileSync(path); + } catch { + throw new CapletsError("REQUEST_INVALID", "media file could not be read"); + } +} + +function enforceSize(size: number, maxBytes = DEFAULT_MAX_MEDIA_BYTES): void { + if (size > maxBytes) { + throw new CapletsError("REQUEST_INVALID", `media exceeds byte limit ${maxBytes}`); + } +} diff --git a/packages/core/src/native/service.ts b/packages/core/src/native/service.ts index 87dbf9d6..5be708e1 100644 --- a/packages/core/src/native/service.ts +++ b/packages/core/src/native/service.ts @@ -138,7 +138,7 @@ class DefaultNativeCapletsService implements NativeCapletsService { writeErr: this.writeErr, }); this.postReloadRefresh = this.refreshExposureSnapshot({ - emitToolsChanged: this.hasDirectMcpExposure(), + emitToolsChanged: this.hasSnapshotBackedDirectExposure(), }); this.unsubscribeEngineReload = this.engine.onReload(() => { this.postReloadRefresh = this.refreshExposureSnapshot({ emitToolsChanged: true }); @@ -251,7 +251,7 @@ class DefaultNativeCapletsService implements NativeCapletsService { const directTools = snapshot?.directTools .filter((entry) => entry.caplet.server === caplet.server) - .map((entry) => this.directMcpTool(caplet, entry)) ?? []; + .map((entry) => this.directDiscoveredTool(caplet, entry)) ?? []; return [ ...directTools, ...mcpPrimitiveNativeTools(caplet, snapshot).map((operationName) => @@ -262,10 +262,14 @@ class DefaultNativeCapletsService implements NativeCapletsService { ), ]; } - return []; + return ( + snapshot?.directTools + .filter((entry) => entry.caplet.server === caplet.server) + .map((entry) => this.directDiscoveredTool(caplet, entry)) ?? [] + ); } - private directMcpTool( + private directDiscoveredTool( caplet: ReturnType[number], entry: DirectToolRegistration, ): NativeCapletTool { @@ -325,9 +329,10 @@ class DefaultNativeCapletsService implements NativeCapletsService { } } - private hasDirectMcpExposure(): boolean { + private hasSnapshotBackedDirectExposure(): boolean { return this.engine.enabledServers().some((caplet) => { - if (caplet.backend !== "mcp" || caplet.setup || caplet.projectBinding?.required) return false; + if (caplet.setup || caplet.projectBinding?.required) return false; + if (caplet.backend === "http" || caplet.backend === "cli") return false; return resolveExposure(caplet.exposure, this.engine.currentConfig().options.exposure).direct; }); } diff --git a/packages/core/src/openapi.ts b/packages/core/src/openapi.ts index 86d828db..930d1024 100644 --- a/packages/core/src/openapi.ts +++ b/packages/core/src/openapi.ts @@ -11,7 +11,8 @@ import { type CompactTool, } from "./downstream"; import { CapletsError, toSafeError } from "./errors"; -import { isAbortError, parseHttpBody, readLimitedText } from "./http/utils"; +import { readHttpLikeResponse } from "./http/response"; +import { isAbortError, readLimitedText } from "./http/utils"; import type { ServerRegistry } from "./registry"; import { markdownStructuredContent } from "./result-content"; import { searchToolList } from "./tool-search"; @@ -60,7 +61,7 @@ export class OpenApiManager { constructor( private registry: ServerRegistry, - private readonly options: { authDir?: string } = {}, + private readonly options: { authDir?: string; artifactDir?: string } = {}, ) {} updateRegistry(registry: ServerRegistry): void { @@ -151,7 +152,10 @@ export class OpenApiManager { }, ); } - const parsed = await readResponse(response); + const parsed = await readHttpLikeResponse(response, { + capletId: endpoint.server, + ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), + }); return { content: markdownStructuredContent(parsed, { title: `${endpoint.name} call_tool ${toolName}`, @@ -717,22 +721,6 @@ function shouldSendSpecAuth(endpoint: OpenApiEndpointConfig): boolean { ); } -async function readResponse(response: Response): Promise> { - const contentType = response.headers.get("content-type") ?? ""; - const text = await readLimitedText(response, { - errorMessage: "OpenAPI response exceeded byte limit", - }); - const body = parseHttpBody(contentType, text); - return { - status: response.status, - statusText: response.statusText, - headers: { - "content-type": contentType, - }, - ...(body === undefined ? {} : { body }), - }; -} - async function fetchWithLimit( url: string, timeoutMs: number, diff --git a/packages/core/src/registry.ts b/packages/core/src/registry.ts index deb1e480..47655bea 100644 --- a/packages/core/src/registry.ts +++ b/packages/core/src/registry.ts @@ -2,6 +2,7 @@ import type { CapletConfig, CapletsConfig, CapletServerConfig, + GoogleDiscoveryApiConfig, GraphQlEndpointConfig, } from "./config"; import type { SafeErrorSummary } from "./errors"; @@ -43,6 +44,13 @@ export type CapletServerDetail = { operationCacheTtlMs: number; source: "specPath" | "specUrl"; } + | { + type: "googleDiscovery"; + disabled: boolean; + requestTimeoutMs: number; + operationCacheTtlMs: number; + source: "discoveryPath" | "discoveryUrl"; + } | { type: "graphql"; disabled: boolean; @@ -94,6 +102,7 @@ export class ServerRegistry { const server = this.config.mcpServers[serverId] ?? this.config.openapiEndpoints[serverId] ?? + this.config.googleDiscoveryApis?.[serverId] ?? this.config.graphqlEndpoints[serverId] ?? this.config.httpApis[serverId] ?? this.config.cliTools[serverId] ?? @@ -148,6 +157,7 @@ export class ServerRegistry { return [ ...Object.values(this.config.mcpServers), ...Object.values(this.config.openapiEndpoints), + ...Object.values(this.config.googleDiscoveryApis ?? {}), ...Object.values(this.config.graphqlEndpoints), ...Object.values(this.config.httpApis), ...Object.values(this.config.cliTools), @@ -167,6 +177,16 @@ function backendDetail(server: CapletConfig): CapletServerDetail["backend"] { }; } + if (server.backend === "googleDiscovery") { + return { + type: "googleDiscovery", + disabled: server.disabled, + requestTimeoutMs: server.requestTimeoutMs, + operationCacheTtlMs: server.operationCacheTtlMs, + source: googleDiscoverySource(server), + }; + } + if (server.backend === "graphql") { return { type: "graphql", @@ -215,6 +235,10 @@ function backendDetail(server: CapletConfig): CapletServerDetail["backend"] { }; } +function googleDiscoverySource(server: GoogleDiscoveryApiConfig): "discoveryPath" | "discoveryUrl" { + return server.discoveryPath ? "discoveryPath" : "discoveryUrl"; +} + function capletSetSource( server: Extract, ): "configPath" | "capletsRoot" | "both" { diff --git a/packages/core/src/remote-control/dispatch.ts b/packages/core/src/remote-control/dispatch.ts index 52f28eb9..15871583 100644 --- a/packages/core/src/remote-control/dispatch.ts +++ b/packages/core/src/remote-control/dispatch.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import { addCliCaplet, + addGoogleDiscoveryCaplet, addGraphqlCaplet, addHttpCaplet, addMcpCaplet, @@ -30,7 +31,14 @@ export type RemoteControlDispatchContext = CapletsEngineOptions & { controlCallbackBaseUrl?: string; }; -type AddKind = "cli" | "mcp" | "openapi" | "graphql" | "http"; +type AddKind = + | "cli" + | "mcp" + | "openapi" + | "google-discovery" + | "googleDiscovery" + | "graphql" + | "http"; const ENGINE_COMMANDS = new Set([ "inspect", @@ -252,6 +260,17 @@ function dispatchAdd(args: Record, context: RemoteControlDispat print: false, }), }; + case "google-discovery": + case "googleDiscovery": + return { + remote: true, + label: "Google Discovery", + ...addGoogleDiscoveryCaplet(id, { + ...options, + destinationRoot: context.projectCapletsRoot, + print: false, + }), + }; case "graphql": return { remote: true, @@ -275,7 +294,7 @@ function dispatchAdd(args: Record, context: RemoteControlDispat default: throw new CapletsError( "REQUEST_INVALID", - "add.kind must be cli, mcp, openapi, graphql, or http", + "add.kind must be cli, mcp, openapi, google-discovery, googleDiscovery, graphql, or http", ); } } @@ -369,6 +388,15 @@ function remoteAddOptions( tokenEnv: "string", force: "boolean", }); + case "google-discovery": + case "googleDiscovery": + return pickOptions(options, { + discovery: "string", + discoveryUrl: "string", + baseUrl: "string", + tokenEnv: "string", + force: "boolean", + }); case "graphql": return pickOptions(options, { endpointUrl: "string", diff --git a/packages/core/src/runtime-plan/planner.ts b/packages/core/src/runtime-plan/planner.ts index 5751548f..5e53ecd0 100644 --- a/packages/core/src/runtime-plan/planner.ts +++ b/packages/core/src/runtime-plan/planner.ts @@ -69,7 +69,12 @@ export function classifyCapletRuntimeRoute(caplet: Record): Run if (caplet.backend === "mcp") { return caplet.transport === "stdio" || Boolean(caplet.command) ? "process" : "worker_safe"; } - if (caplet.backend === "openapi" || caplet.backend === "graphql" || caplet.backend === "http") { + if ( + caplet.backend === "openapi" || + caplet.backend === "googleDiscovery" || + caplet.backend === "graphql" || + caplet.backend === "http" + ) { return "worker_safe"; } if (caplet.backend === "caplets") { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 2f3b9394..e7df176b 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -7,6 +7,7 @@ type CapletsRuntimeOptions = { configPath?: string; projectConfigPath?: string; authDir?: string; + artifactDir?: string; watchDebounceMs?: number; server?: ToolServer; writeErr?: (value: string) => void; @@ -71,6 +72,9 @@ function engineOptions(options: CapletsRuntimeOptions): CapletsEngineOptions { if (options.authDir !== undefined) { engineOptions.authDir = options.authDir; } + if (options.artifactDir !== undefined) { + engineOptions.artifactDir = options.artifactDir; + } if (options.watchDebounceMs !== undefined) { engineOptions.watchDebounceMs = options.watchDebounceMs; } diff --git a/packages/core/src/tools.ts b/packages/core/src/tools.ts index 9346b85d..2c5fd762 100644 --- a/packages/core/src/tools.ts +++ b/packages/core/src/tools.ts @@ -5,6 +5,7 @@ import type { CapletConfig } from "./config"; import type { CliToolsManager } from "./cli-tools"; import type { DownstreamManager } from "./downstream"; import { CapletsError } from "./errors"; +import type { GoogleDiscoveryManager } from "./google-discovery"; import type { GraphQLManager } from "./graphql"; import type { HttpActionManager } from "./http-actions"; import type { OpenApiManager } from "./openapi"; @@ -61,6 +62,7 @@ export async function handleServerTool( cli?: CliToolsManager, caplets?: CapletSetManager, options: HandleServerToolOptions = {}, + googleDiscovery?: GoogleDiscoveryManager, ): Promise { const startedAt = Date.now(); const parsed = validateOperationRequest( @@ -84,11 +86,21 @@ export async function handleServerTool( http, cli, caplets, + googleDiscovery, ).check(server as never); return jsonResult(result, metadataFor(server, "check", undefined, startedAt)); } case "tools": { - const backend = backendFor(server, downstream, openapi, graphql, http, cli, caplets); + const backend = backendFor( + server, + downstream, + openapi, + graphql, + http, + cli, + caplets, + googleDiscovery, + ); const tools = await backend.listTools(server as never); const page = pageItems( tools.map((tool) => backend.compact(server as never, tool)), @@ -105,7 +117,16 @@ export async function handleServerTool( ); } case "search_tools": { - const backend = backendFor(server, downstream, openapi, graphql, http, cli, caplets); + const backend = backendFor( + server, + downstream, + openapi, + graphql, + http, + cli, + caplets, + googleDiscovery, + ); const tools = await backend.listTools(server as never); const limit = parsed.limit ?? registry.config.options.defaultSearchLimit; const matches = backend.search(server as never, tools, parsed.query, limit); @@ -121,7 +142,16 @@ export async function handleServerTool( ); } case "describe_tool": { - const backend = backendFor(server, downstream, openapi, graphql, http, cli, caplets); + const backend = backendFor( + server, + downstream, + openapi, + graphql, + http, + cli, + caplets, + googleDiscovery, + ); const tool = await backend.getTool(server as never, parsed.name); const observedOutputShape = await readObservedOutputShape( options, @@ -140,7 +170,16 @@ export async function handleServerTool( ); } case "call_tool": { - const backend = backendFor(server, downstream, openapi, graphql, http, cli, caplets); + const backend = backendFor( + server, + downstream, + openapi, + graphql, + http, + cli, + caplets, + googleDiscovery, + ); const tool = await maybeGetToolForValidation(backend, server, parsed.name); validateToolArgsForAgent(tool, parsed.name, parsed.args); if (parsed.fields === undefined) { @@ -1265,6 +1304,7 @@ function backendFor( http?: HttpActionManager, cli?: CliToolsManager, caplets?: CapletSetManager, + googleDiscovery?: GoogleDiscoveryManager, ) { if (server.backend === "mcp") { return { @@ -1279,6 +1319,25 @@ function backendFor( search: (...args: Parameters) => downstream.search(...args), }; } + if (server.backend === "googleDiscovery") { + if (!googleDiscovery) { + throw new CapletsError("INTERNAL_ERROR", "Google Discovery manager is not configured"); + } + return { + check: (...args: Parameters) => + googleDiscovery.checkApi(...args), + listTools: (...args: Parameters) => + googleDiscovery.listTools(...args), + getTool: (...args: Parameters) => + googleDiscovery.getTool(...args), + callTool: (...args: Parameters) => + googleDiscovery.callTool(...args), + compact: (...args: Parameters) => + googleDiscovery.compact(...args), + search: (...args: Parameters) => + googleDiscovery.search(...args), + }; + } if (server.backend === "graphql") { if (!graphql) { throw new CapletsError("INTERNAL_ERROR", "GraphQL manager is not configured"); diff --git a/packages/core/test/auth.test.ts b/packages/core/test/auth.test.ts index 4fad4e48..e377856a 100644 --- a/packages/core/test/auth.test.ts +++ b/packages/core/test/auth.test.ts @@ -1102,6 +1102,115 @@ describe("auth helpers", () => { } }); + it("includes resolved Google Discovery scopes in generic OIDC authorization", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-auth-google-scopes-")); + let baseUrl = ""; + let authorizationUrl = ""; + const server = createServer((request: IncomingMessage, response: ServerResponse) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { + body += chunk; + }); + request.on("end", () => { + response.setHeader("content-type", "application/json"); + if (request.url === "/.well-known/oauth-protected-resource") { + response.end(JSON.stringify({ authorization_servers: [baseUrl] })); + return; + } + if (request.url === "/.well-known/oauth-authorization-server") { + response.statusCode = 404; + response.end(JSON.stringify({ error: "missing" })); + return; + } + if (request.url === "/.well-known/openid-configuration") { + response.end( + JSON.stringify({ + issuer: baseUrl, + authorization_endpoint: `${baseUrl}/authorize`, + token_endpoint: `${baseUrl}/token`, + registration_endpoint: `${baseUrl}/register`, + }), + ); + return; + } + if (request.url === "/register") { + response.statusCode = 201; + response.end(JSON.stringify({ client_id: "dynamic-client" })); + return; + } + if (request.url === "/token") { + const idToken = [ + "header", + Buffer.from( + JSON.stringify({ iss: baseUrl, sub: "subject-123", aud: "dynamic-client" }), + ).toString("base64url"), + "signature", + ].join("."); + response.end( + JSON.stringify({ + access_token: "new-access-token", + refresh_token: "new-refresh-token", + id_token: idToken, + token_type: "Bearer", + expires_in: 3600, + }), + ); + return; + } + response.end("{}"); + }); + }); + try { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("test server did not bind"); + } + baseUrl = `http://127.0.0.1:${address.port}`; + + await runGenericOAuthFlow( + { + server: "drive", + backend: "googleDiscovery", + baseUrl, + auth: { type: "oidc" }, + resolvedScopes: [ + "https://www.googleapis.com/auth/drive.readonly", + "https://www.googleapis.com/auth/drive", + ], + }, + { + authDir: dir, + noOpen: true, + print: (line) => { + authorizationUrl = line.match(/https?:\/\/\S+/)?.[0] ?? ""; + }, + readManualInput: async () => { + const url = new URL(authorizationUrl); + return `http://127.0.0.1/callback?code=auth-code&state=${url.searchParams.get("state")}`; + }, + }, + ); + + expect(new URL(authorizationUrl).searchParams.get("scope")).toBe( + "openid profile email https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/drive.readonly", + ); + expect(readTokenBundle("drive", dir)?.metadata).toMatchObject({ + requestedScopes: [ + "openid", + "profile", + "email", + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/drive.readonly", + ], + }); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + rmSync(dir, { recursive: true, force: true }); + } + }); + it("rejects non-loopback plaintext OAuth endpoints before token exchange", async () => { await expect( runGenericOAuthFlow( diff --git a/packages/core/test/caplet-sets.test.ts b/packages/core/test/caplet-sets.test.ts index d47e210f..276bca2b 100644 --- a/packages/core/test/caplet-sets.test.ts +++ b/packages/core/test/caplet-sets.test.ts @@ -116,6 +116,48 @@ describe("CapletSetManager", () => { ]); }); + it("routes child Google Discovery Caplets through nested tool calls", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-set-google-discovery-")); + dirs.push(dir); + const configPath = join(dir, "child.json"); + writeFileSync( + configPath, + JSON.stringify({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryPath: join(__dirname, "fixtures/google-discovery/drive.discovery.json"), + baseUrl: "https://www.googleapis.com/drive/v3/", + auth: { type: "none" }, + includeOperations: ["drive.files.list"], + }, + }, + }), + ); + const config = parseConfig({ + capletSets: { + nested: { + name: "Nested Caplets", + description: "Expose child Caplets through a nested collection.", + configPath, + }, + }, + }); + const caplet = config.capletSets.nested!; + const manager = new CapletSetManager(new ServerRegistry(config)); + + const result = await manager.callTool(caplet, "drive", { operation: "tools" }); + + expect(result.isError).toBeUndefined(); + expect(result.structuredContent).toMatchObject({ + result: { + id: "drive", + items: [{ name: "drive.files.list" }], + }, + }); + }); + it("serializes concurrent refreshes for one parent Caplet set", async () => { const { dir, childConfigPath } = childCliConfig(); dirs.push(dir); diff --git a/packages/core/test/caplet-source.test.ts b/packages/core/test/caplet-source.test.ts index 49b86f82..25268913 100644 --- a/packages/core/test/caplet-source.test.ts +++ b/packages/core/test/caplet-source.test.ts @@ -30,6 +30,32 @@ info: title: Weather version: 1.0.0 paths: {} +`, + }, + { + path: "drive/CAPLET.md", + content: `--- +name: Drive +description: Query Google Drive metadata. +googleDiscoveryApi: + discoveryPath: ./drive.discovery.json + auth: + type: oauth2 + issuer: https://accounts.google.com + scopes: + - https://www.googleapis.com/auth/drive.metadata.readonly +--- + +# Drive +`, + }, + { + path: "drive/drive.discovery.json", + content: `{ + "kind": "discovery#restDescription", + "name": "drive", + "version": "v3" +} `, }, { @@ -78,6 +104,8 @@ describe("CapletSource adapters", () => { const source = new BundleCapletSource(fixtureFiles); await expect(source.listFiles()).resolves.toEqual([ + expect.objectContaining({ path: "drive/CAPLET.md" }), + expect.objectContaining({ path: "drive/drive.discovery.json" }), expect.objectContaining({ path: "tools/CAPLET.md" }), expect.objectContaining({ path: "tools/scripts/list-files.js" }), expect.objectContaining({ path: "weather/CAPLET.md" }), @@ -95,6 +123,8 @@ describe("CapletSource adapters", () => { const source = new FilesystemCapletSource(root); await expect(source.listFiles()).resolves.toEqual([ + expect.objectContaining({ path: "drive/CAPLET.md" }), + expect.objectContaining({ path: "drive/drive.discovery.json" }), expect.objectContaining({ path: "tools/CAPLET.md" }), expect.objectContaining({ path: "tools/scripts/list-files.js" }), expect.objectContaining({ path: "weather/CAPLET.md" }), @@ -102,7 +132,7 @@ describe("CapletSource adapters", () => { ]); await expect(source.readFile("./tools/scripts/list-files.js")).resolves.toEqual({ path: "tools/scripts/list-files.js", - content: fixtureFiles[3]!.content, + content: fixtureFiles[5]!.content, }); await expect(source.readFile("/absolute.js")).resolves.toBeUndefined(); }); @@ -128,6 +158,21 @@ describe("CapletSource adapters", () => { }, localReferences: [{ path: "weather/openapi.yaml", exists: true }], }, + { + id: "drive", + backend: "googleDiscovery", + shadowing: "forbid", + setupRequired: false, + authRequired: true, + projectBindingRequired: false, + runtime: { + route: "worker_safe", + setupTarget: undefined, + features: [], + resources: { class: "small", cpu: 1, memoryMb: 1024, diskMb: 4096 }, + }, + localReferences: [{ path: "drive/drive.discovery.json", exists: true }], + }, { id: "tools", backend: "cli", diff --git a/packages/core/test/cli-completion.test.ts b/packages/core/test/cli-completion.test.ts index 16f398bd..2176a6a6 100644 --- a/packages/core/test/cli-completion.test.ts +++ b/packages/core/test/cli-completion.test.ts @@ -66,6 +66,7 @@ describe("CLI completion resolver", () => { "cli", "mcp", "openapi", + "google-discovery", "graphql", "http", ]); diff --git a/packages/core/test/cli-remote.test.ts b/packages/core/test/cli-remote.test.ts index 20b07bd6..665dfb85 100644 --- a/packages/core/test/cli-remote.test.ts +++ b/packages/core/test/cli-remote.test.ts @@ -90,6 +90,40 @@ describe("remote CLI routing", () => { expect(out.join("")).toBe("shared.echo\n"); }); + it("does not append remote completions for locally shadowed Google Discovery caplets", async () => { + const context = testContext("caplets-cli-remote-complete-google-shadowed-"); + const requests: unknown[] = []; + const out: string[] = []; + writeFileSync( + context.configPath, + JSON.stringify({ + googleDiscoveryApis: { + drive: { + name: "Drive API", + description: "Manage Drive files through Google Discovery.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "none" }, + }, + }, + }), + ); + const fetch = vi.fn( + async (_url: Parameters[0], init?: RequestInit) => { + requests.push(JSON.parse(String(init?.body ?? "{}"))); + return Response.json({ ok: true, result: ["drive.remote_only"] }); + }, + ); + + await runCli(["__complete", "--shell", "bash", "--", "call-tool", "drive."], { + env: remoteEnv(context), + fetch, + writeOut: (value) => out.push(value), + }); + + expect(requests).toEqual([]); + expect(out.join("")).not.toContain("remote_only"); + }); + it("uses remote completions when a matching local overlay caplet is disabled", async () => { const context = testContext("caplets-cli-remote-complete-disabled-shadow-"); const requests: unknown[] = []; @@ -832,6 +866,24 @@ describe("remote CLI routing", () => { ], { spec: "./openapi.yaml", baseUrl: "https://api.example.com", tokenEnv: "API_TOKEN" }, ], + [ + "googleDiscovery", + [ + "google-discovery", + "drive", + "--discovery-url", + "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + "--base-url", + "https://www.googleapis.com/drive/v3", + "--token-env", + "GOOGLE_TOKEN", + ], + { + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + baseUrl: "https://www.googleapis.com/drive/v3", + tokenEnv: "GOOGLE_TOKEN", + }, + ], [ "graphql", [ diff --git a/packages/core/test/cli.test.ts b/packages/core/test/cli.test.ts index c2288a40..0a178185 100644 --- a/packages/core/test/cli.test.ts +++ b/packages/core/test/cli.test.ts @@ -256,7 +256,7 @@ describe("cli init", () => { await runCli(["list"], { writeOut: (value) => out.push(value) }); const text = out.join(""); - expect(text).toContain("Configured Caplets (3)"); + expect(text).toContain("Configured Caplets (4)"); expect(text).toContain("Source:"); expect(text).toContain("filesystem"); expect(text).toContain("mcp"); @@ -267,6 +267,8 @@ describe("cli init", () => { expect(text).toContain("openapi"); expect(text).toContain("catalog"); expect(text).toContain("graphql"); + expect(text).toContain("drive"); + expect(text).toContain("googleDiscovery"); expect(text).not.toContain("disabled_remote"); expect(text).not.toContain("secret-access-token"); expect(text).not.toContain("openapi-client"); @@ -324,6 +326,15 @@ describe("cli init", () => { path: configPath, shadows: [], }), + expect.objectContaining({ + server: "drive", + backend: "googleDiscovery", + disabled: false, + status: "not_started", + source: "global-config", + path: configPath, + shadows: [], + }), expect.objectContaining({ server: "filesystem", backend: "mcp", @@ -1061,7 +1072,7 @@ describe("cli init", () => { } }); - it("prints added MCP and OpenAPI backend Caplets", async () => { + it("prints added MCP, OpenAPI, and Google Discovery backend Caplets", async () => { const out: string[] = []; await runCli( @@ -1092,12 +1103,32 @@ describe("cli init", () => { ], { writeOut: (value) => out.push(value) }, ); + await runCli( + [ + "add", + "google-discovery", + "drive", + "--discovery-url", + "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + "--base-url", + "https://www.googleapis.com/drive/v3", + "--token-env", + "GOOGLE_TOKEN", + "--print", + ], + { writeOut: (value) => out.push(value) }, + ); expect(out.join("\n")).toContain("mcpServer:"); expect(out.join("\n")).toContain('transport: "sse"'); expect(out.join("\n")).toContain('token: "$env:MCP_TOKEN"'); expect(out.join("\n")).toContain("openapiEndpoint:"); expect(out.join("\n")).toContain('baseUrl: "https://api.example.com/v1"'); + expect(out.join("\n")).toContain("googleDiscoveryApi:"); + expect(out.join("\n")).toContain( + 'discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest"', + ); + expect(out.join("\n")).toContain('token: "$env:GOOGLE_TOKEN"'); }); it("adds GraphQL and HTTP backend Caplets to the project root", async () => { @@ -1168,6 +1199,35 @@ describe("cli init", () => { } }); + it("writes Google Discovery local discovery paths that load from the original project file", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-add-google-discovery-path-")); + const projectRoot = join(dir, "project"); + const cwd = process.cwd(); + try { + mkdirSync(projectRoot, { recursive: true }); + writeFileSync( + join(projectRoot, "drive.discovery.json"), + JSON.stringify({ kind: "discovery#restDescription", name: "drive", version: "v3" }), + ); + process.chdir(projectRoot); + + await runCli(["add", "google-discovery", "drive", "--discovery", "./drive.discovery.json"], { + writeOut: () => {}, + }); + + const config = loadConfig( + join(dir, "user", "config.json"), + join(projectRoot, ".caplets", "config.json"), + ); + expect(config.googleDiscoveryApis.drive?.discoveryPath).toBe( + join(projectRoot, "drive.discovery.json"), + ); + } finally { + process.chdir(cwd); + rmSync(dir, { recursive: true, force: true }); + } + }); + it("writes GraphQL local schema paths that load from the original project file", async () => { const dir = mkdtempSync(join(tmpdir(), "caplets-add-graphql-path-")); const projectRoot = join(dir, "project"); @@ -2203,6 +2263,72 @@ describe("cli init", () => { } }); + it("refreshes Google Discovery OAuth credentials with explicit scopes without loading discovery", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-auth-google-refresh-cli-")); + const authDir = join(dir, "auth"); + const configPath = join(dir, "config.json"); + const out: string[] = []; + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue( + Response.json({ + access_token: "new-access-token", + refresh_token: "new-refresh-token", + token_type: "Bearer", + expires_in: 3600, + }), + ); + try { + writeFileSync( + configPath, + JSON.stringify({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: "https://discovery.example.invalid/drive/v3/rest", + auth: { + type: "oauth2", + clientId: "client", + tokenUrl: "https://auth.example.com/token", + scopes: ["https://www.googleapis.com/auth/drive.readonly"], + }, + }, + }, + }), + ); + process.env.CAPLETS_CONFIG = configPath; + writeTokenBundle( + { + server: "drive", + authType: "oauth2", + accessToken: "old-access-token", + refreshToken: "old-refresh-token", + expiresAt: "2999-01-01T00:00:00.000Z", + metadata: { + requestedScopes: ["https://www.googleapis.com/auth/drive.readonly"], + }, + }, + authDir, + ); + + await runCli(["auth", "refresh", "drive"], { + writeOut: (value) => out.push(value), + authDir, + }); + + expect(out.join("")).toBe("Refreshed OAuth credentials for `drive`.\n"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith( + "https://auth.example.com/token", + expect.objectContaining({ + method: "POST", + body: expect.stringContaining("refresh_token=old-refresh-token"), + }), + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("logs out configured OpenAPI OAuth endpoints", async () => { const dir = mkdtempSync(join(tmpdir(), "caplets-auth-")); const authDir = join(dir, "auth"); @@ -2255,6 +2381,36 @@ describe("cli setup", () => { expect(text).not.toContain("caplets@caplets"); }); + it("resolves Google Discovery Caplets for Caplet setup", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-setup-google-discovery-")); + const configPath = join(dir, "config.json"); + const out: string[] = []; + try { + writeFileSync( + configPath, + JSON.stringify({ + googleDiscoveryApis: { + drive: { + name: "Drive API", + description: "Manage Drive files through Google Discovery.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "none" }, + }, + }, + }), + ); + + await runCli(["setup", "drive"], { + env: { CAPLETS_CONFIG: configPath }, + writeOut: (value) => out.push(value), + }); + + expect(out.join("")).toBe("No setup metadata is defined for Drive API (drive).\n"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("prompts for integrations when stdin is available", async () => { const out: string[] = []; const commands: Array<{ command: string; args: string[] }> = []; @@ -2644,6 +2800,7 @@ describe("cli completion commands", () => { "cli", "mcp", "openapi", + "google-discovery", "graphql", "http", ]); @@ -2666,7 +2823,12 @@ describe("cli completion commands", () => { }, ); - expect(out.join("").split("\n").filter(Boolean)).toEqual(["catalog", "filesystem", "users"]); + expect(out.join("").split("\n").filter(Boolean)).toEqual([ + "catalog", + "drive", + "filesystem", + "users", + ]); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -2683,7 +2845,12 @@ describe("cli completion commands", () => { writeOut: (value) => out.push(value), }); - expect(out.join("").split("\n").filter(Boolean)).toEqual(["catalog", "filesystem", "users"]); + expect(out.join("").split("\n").filter(Boolean)).toEqual([ + "catalog", + "drive", + "filesystem", + "users", + ]); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -2736,6 +2903,14 @@ function writeInspectionConfig(path: string): void { auth: { type: "oauth2", clientId: "openapi-client" }, }, }, + googleDiscoveryApis: { + drive: { + name: "Drive API", + description: "Manage Drive files through Google Discovery.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "none" }, + }, + }, graphqlEndpoints: { catalog: { name: "Catalog GraphQL", diff --git a/packages/core/test/config.test.ts b/packages/core/test/config.test.ts index a00496e5..0ab66cd0 100644 --- a/packages/core/test/config.test.ts +++ b/packages/core/test/config.test.ts @@ -25,6 +25,7 @@ import { } from "../src/config"; import { listCaplets } from "../src/cli/inspection"; import { CapletsError } from "../src/errors"; +import { ServerRegistry } from "../src/registry"; describe("config", () => { const originalEnv = process.env.EXAMPLE_TOKEN; @@ -375,6 +376,35 @@ describe("config", () => { delete process.env.PROJECT_OPENAPI_SECRET; }); + it("rejects Google Discovery executable backend maps from project config", () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-project-google-discovery-")); + const projectConfigPath = join(dir, ".caplets", "config.json"); + mkdirSync(join(dir, ".caplets"), { recursive: true }); + writeFileSync( + projectConfigPath, + JSON.stringify({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Attempt to load Google Drive from project config.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "none" }, + }, + }, + }), + ); + + expect(() => loadConfig(join(dir, "missing-user-config.json"), projectConfigPath)).toThrow( + expect.objectContaining({ + code: "CONFIG_INVALID", + message: expect.stringContaining( + "cannot define executable backend map googleDiscoveryApis; use project Markdown Caplet files or user config instead", + ) as string, + }) as CapletsError, + ); + rmSync(dir, { recursive: true, force: true }); + }); + it("rejects Caplet set executable backend maps from project config", () => { const dir = mkdtempSync(join(tmpdir(), "caplets-project-capletsets-")); const projectConfigPath = join(dir, ".caplets", "config.json"); @@ -1217,6 +1247,46 @@ describe("config", () => { rmSync(dir, { recursive: true, force: true }); }); + it("loads Google Discovery API-backed Caplet files", () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-google-discovery-files-")); + const root = join(dir, ".caplets"); + mkdirSync(root, { recursive: true }); + writeFileSync( + join(root, "drive.md"), + [ + "---", + "name: Drive API", + "description: Manage Drive files through Google Discovery.", + "googleDiscoveryApi:", + " discoveryPath: ./drive.discovery.json", + " auth:", + " type: oauth2", + " issuer: https://accounts.google.com", + " scopes:", + " - https://www.googleapis.com/auth/drive.metadata.readonly", + " includeOperations:", + " - files.*", + "---", + "# Drive API", + ].join("\n"), + ); + + const config = loadConfig(join(root, "config.json"), join(dir, "missing", "config.json")); + expect(config.googleDiscoveryApis.drive).toMatchObject({ + server: "drive", + backend: "googleDiscovery", + discoveryPath: join(root, "drive.discovery.json"), + auth: { + type: "oauth2", + issuer: "https://accounts.google.com", + scopes: ["https://www.googleapis.com/auth/drive.metadata.readonly"], + }, + includeOperations: ["files.*"], + body: "# Drive API", + }); + rmSync(dir, { recursive: true, force: true }); + }); + it("loads GraphQL config and GraphQL-backed Caplet files", () => { const dir = mkdtempSync(join(tmpdir(), "caplets-graphql-files-")); const root = join(dir, ".caplets"); @@ -1941,6 +2011,49 @@ describe("config", () => { delete process.env.OPENAPI_PUBLIC_SECRET; }); + it("loads Google Discovery APIs with defaults and safe registry details", () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files and permissions.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "oidc", issuer: "https://accounts.google.com", clientId: "client" }, + includeOperations: ["drive.files.*"], + excludeOperations: ["drive.files.delete"], + }, + }, + }); + + expect(config.googleDiscoveryApis.drive).toMatchObject({ + server: "drive", + backend: "googleDiscovery", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + requestTimeoutMs: 60000, + operationCacheTtlMs: 30000, + disabled: false, + }); + expect(new ServerRegistry(config).detail(config.googleDiscoveryApis.drive!)).toEqual({ + id: "drive", + name: "Google Drive", + description: "Access Google Drive files and permissions.", + backend: { + type: "googleDiscovery", + disabled: false, + requestTimeoutMs: 60000, + operationCacheTtlMs: 30000, + source: "discoveryUrl", + }, + }); + expect( + JSON.stringify(new ServerRegistry(config).detail(config.googleDiscoveryApis.drive!)), + ).not.toContain("client"); + expect( + JSON.stringify(new ServerRegistry(config).detail(config.googleDiscoveryApis.drive!)), + ).not.toContain("googleapis.com"); + expect(JSON.stringify(configJsonSchema())).toContain("googleDiscoveryApis"); + }); + it("rejects nested Caplets options", () => { expect(() => parseConfig({ @@ -2073,6 +2186,61 @@ describe("config", () => { } }); + it("rejects invalid Google Discovery sources and duplicate Caplet IDs", () => { + expect(() => + parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Drive", + description: "Access Google Drive files.", + discoveryUrl: "ftp://example.com/discovery.json", + auth: { type: "none" }, + }, + }, + }), + ).toThrow(CapletsError); + expect(() => + parseConfig({ + openapiEndpoints: { + drive: { + name: "Drive OpenAPI", + description: "OpenAPI Drive wrapper.", + specUrl: "https://example.com/openapi.json", + baseUrl: "https://example.com", + auth: { type: "none" }, + }, + }, + googleDiscoveryApis: { + drive: { + name: "Drive", + description: "Access Google Drive files.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "none" }, + }, + }, + }), + ).toThrow( + expect.objectContaining({ + details: expect.arrayContaining([ + expect.objectContaining({ message: expect.stringContaining("already used") }), + ]) as unknown, + }) as CapletsError, + ); + expect(() => + parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Drive", + description: "Access Google Drive files.", + discoveryPath: "/tmp/drive.discovery.json", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "none" }, + }, + }, + }), + ).toThrow(CapletsError); + }); + it("validates server IDs, required names, descriptions, and disabled default", () => { const valid = parseConfig({ mcpServers: { diff --git a/packages/core/test/engine.test.ts b/packages/core/test/engine.test.ts index 8f592c15..71c2937d 100644 --- a/packages/core/test/engine.test.ts +++ b/packages/core/test/engine.test.ts @@ -91,6 +91,24 @@ describe("CapletsEngine", () => { expect(engine.enabledServers().map((caplet) => caplet.server)).toEqual(["gamma"]); }); + it("includes enabled Google Discovery API Caplets", () => { + const { dir, configPath, projectConfigPath } = tempConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files and permissions.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "none" }, + }, + }, + }); + dirs.push(dir); + const engine = new CapletsEngine({ configPath, projectConfigPath, watch: false }); + engines.push(engine); + + expect(engine.enabledServers().map((caplet) => caplet.server)).toEqual(["drive"]); + }); + it("keeps last known-good config when reload validation fails", async () => { const { dir, configPath, projectConfigPath } = tempConfig({ mcpServers: { diff --git a/packages/core/test/exposure-discovery.test.ts b/packages/core/test/exposure-discovery.test.ts index b5f868bb..b02048eb 100644 --- a/packages/core/test/exposure-discovery.test.ts +++ b/packages/core/test/exposure-discovery.test.ts @@ -120,6 +120,7 @@ function configFor( caplets.filter((caplet) => caplet.backend === "mcp").map((caplet) => [caplet.server, caplet]), ) as CapletsConfig["mcpServers"], openapiEndpoints: {}, + googleDiscoveryApis: {}, graphqlEndpoints: {}, httpApis: Object.fromEntries( caplets diff --git a/packages/core/test/fixtures/google-discovery/drive.discovery.json b/packages/core/test/fixtures/google-discovery/drive.discovery.json new file mode 100644 index 00000000..ee669ece --- /dev/null +++ b/packages/core/test/fixtures/google-discovery/drive.discovery.json @@ -0,0 +1,132 @@ +{ + "kind": "discovery#restDescription", + "id": "drive:v3", + "name": "drive", + "version": "v3", + "title": "Drive API", + "rootUrl": "https://www.googleapis.com/", + "servicePath": "drive/v3/", + "baseUrl": "https://www.googleapis.com/drive/v3/", + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/drive": { "description": "Full Drive access." }, + "https://www.googleapis.com/auth/drive.readonly": { "description": "Read Drive files." } + } + } + }, + "parameters": { + "fields": { + "type": "string", + "location": "query", + "description": "Partial response selector." + }, + "prettyPrint": { "type": "boolean", "location": "query", "default": "true" }, + "quotaUser": { "type": "string", "location": "header" } + }, + "schemas": { + "File": { + "id": "File", + "type": "object", + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "parents": { "type": "array", "items": { "type": "string" } } + } + }, + "FileList": { + "id": "FileList", + "type": "object", + "properties": { + "files": { "type": "array", "items": { "$ref": "File" } }, + "nextPageToken": { "type": "string" } + } + } + }, + "resources": { + "files": { + "methods": { + "list": { + "id": "drive.files.list", + "path": "files", + "httpMethod": "GET", + "description": "Lists files.", + "scopes": ["https://www.googleapis.com/auth/drive.readonly"], + "parameters": { + "pageSize": { + "type": "integer", + "format": "int32", + "location": "query", + "default": "100" + } + }, + "response": { "$ref": "FileList" } + }, + "delete": { + "id": "drive.files.delete", + "path": "files/{fileId}", + "httpMethod": "DELETE", + "description": "Permanently deletes a file.", + "parameters": { + "fileId": { "type": "string", "location": "path", "required": true } + }, + "scopes": ["https://www.googleapis.com/auth/drive"] + }, + "create": { + "id": "drive.files.create", + "path": "files", + "httpMethod": "POST", + "description": "Creates a file.", + "request": { "$ref": "File" }, + "response": { "$ref": "File" }, + "scopes": ["https://www.googleapis.com/auth/drive"], + "supportsMediaUpload": true, + "mediaUpload": { + "accept": ["image/png"], + "maxSize": "10MB", + "protocols": { + "simple": { "path": "/upload/drive/v3/files", "multipart": false }, + "multipart": { "path": "/upload/drive/v3/files", "multipart": true }, + "resumable": { "path": "/upload/drive/v3/files", "multipart": true } + } + } + }, + "download": { + "id": "drive.files.download", + "path": "files/{fileId}/download", + "httpMethod": "GET", + "description": "Downloads file media.", + "supportsMediaDownload": true, + "parameters": { + "fileId": { "type": "string", "location": "path", "required": true } + }, + "scopes": ["https://www.googleapis.com/auth/drive.readonly"] + } + }, + "resources": { + "permissions": { + "methods": { + "list": { + "id": "drive.permissions.list", + "path": "files/{fileId}/permissions", + "httpMethod": "GET", + "parameters": { + "fileId": { "type": "string", "location": "path", "required": true } + }, + "scopes": ["https://www.googleapis.com/auth/drive.readonly"] + } + } + } + } + }, + "changes": { + "methods": { + "getStartPageToken": { + "path": "changes/startPageToken", + "httpMethod": "GET", + "scopes": ["https://www.googleapis.com/auth/drive.readonly"] + } + } + } + } +} diff --git a/packages/core/test/google-discovery.test.ts b/packages/core/test/google-discovery.test.ts new file mode 100644 index 00000000..6d3fb9ff --- /dev/null +++ b/packages/core/test/google-discovery.test.ts @@ -0,0 +1,624 @@ +import { readFileSync } from "node:fs"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + buildGoogleDiscoveryUrl, + discoveryOperations, + GoogleDiscoveryManager, + googleDiscoveryScopesForOperations, +} from "../src/google-discovery"; +import { parseConfig } from "../src/config"; +import { DownstreamManager } from "../src/downstream"; +import { ServerRegistry } from "../src/registry"; +import { handleServerTool } from "../src/tools"; + +const fixture = JSON.parse( + readFileSync(join(__dirname, "fixtures/google-discovery/drive.discovery.json"), "utf8"), +); + +let server: ReturnType | undefined; +let baseUrl = ""; +const requests: Array<{ + method: string; + url: string; + body: string; + headers: Record; +}> = []; + +beforeEach(async () => { + requests.length = 0; + server = createServer((request: IncomingMessage, response: ServerResponse) => { + const bodyChunks: Buffer[] = []; + request.on("data", (chunk) => bodyChunks.push(Buffer.from(chunk))); + request.on("end", () => { + const url = request.url ?? "/"; + requests.push({ + method: request.method ?? "GET", + url, + body: Buffer.concat(bodyChunks).toString("utf8"), + headers: request.headers, + }); + response.setHeader("content-type", "application/json"); + if (url === "/drive.discovery.json") { + response.end(JSON.stringify(fixture)); + return; + } + if (url === "/drive-inferred.discovery.json") { + response.end( + JSON.stringify({ + ...fixture, + baseUrl: undefined, + rootUrl: `${baseUrl}/`, + servicePath: "drive/v3/", + }), + ); + return; + } + if (url === "/reserved.discovery.json") { + response.end( + JSON.stringify({ + kind: "discovery#restDescription", + rootUrl: `${baseUrl}/`, + servicePath: "drive/v3/", + schemas: { + File: { + id: "File", + type: "object", + properties: { id: { type: "string" } }, + }, + }, + resources: { + files: { + methods: { + getReserved: { + id: "drive.files.getReserved", + path: "files/{+name}", + httpMethod: "GET", + parameters: { + name: { type: "string", location: "path", required: true }, + }, + response: { $ref: "File" }, + }, + }, + }, + }, + }), + ); + return; + } + if (url === "/redirect.discovery.json") { + response.statusCode = 302; + response.setHeader("location", "/drive.discovery.json"); + response.end("{}"); + return; + } + if (url.startsWith("/drive/v3/files?")) { + response.end(JSON.stringify({ files: [{ id: "1", name: "Report" }] })); + return; + } + if (url === "/drive/v3/files" && request.method === "POST") { + response.statusCode = 201; + response.end(JSON.stringify({ id: "2", name: "Created" })); + return; + } + if (url === "/drive/v3/files/1/download") { + response.setHeader("content-type", "application/pdf"); + response.end("%PDF bytes"); + return; + } + if (url === "/drive/v3/files/large/download") { + const bytes = Buffer.alloc(1024 * 1024 + 1, "x"); + response.setHeader("content-type", "application/pdf"); + response.setHeader("content-length", String(bytes.byteLength)); + response.end(bytes); + return; + } + if (url === "/drive/v3/files/folders/1") { + response.end(JSON.stringify({ id: "folders/1" })); + return; + } + if (url === "/upload/drive/v3/files?uploadType=media" && request.method === "POST") { + response.end(JSON.stringify({ id: "uploaded-media" })); + return; + } + if (url === "/upload/drive/v3/files?uploadType=multipart" && request.method === "POST") { + response.end(JSON.stringify({ id: "uploaded" })); + return; + } + if (url === "/upload/drive/v3/files?uploadType=resumable" && request.method === "POST") { + response.statusCode = 200; + response.setHeader("location", `${baseUrl}/upload/session/abc`); + response.end("{}"); + return; + } + if (url === "/upload/session/abc" && request.method === "PUT") { + response.end(JSON.stringify({ id: "uploaded-resumable" })); + return; + } + response.statusCode = 404; + response.end(JSON.stringify({ error: "not found" })); + }); + }); + await new Promise((resolve) => server!.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Google Discovery test server did not bind"); + } + baseUrl = `http://127.0.0.1:${address.port}`; +}); + +afterEach(async () => { + if (server) { + await new Promise((resolve) => server!.close(() => resolve())); + server = undefined; + } +}); + +describe("Google Discovery parser", () => { + it("maps resources and methods to Caplets operations", () => { + const operations = discoveryOperations({ + server: "drive", + document: fixture, + includeOperations: ["drive.files.*"], + excludeOperations: ["drive.files.delete"], + }); + + expect(operations.map((operation) => operation.name)).toEqual([ + "drive.files.create", + "drive.files.download", + "drive.files.list", + ]); + expect(operations.find((operation) => operation.name === "drive.files.list")).toMatchObject({ + method: "get", + path: "files", + readOnlyHint: true, + destructiveHint: false, + inputSchema: { + type: "object", + properties: { + header: { + properties: { + quotaUser: { type: "string" }, + }, + }, + query: { + properties: { + fields: { type: "string" }, + pageSize: { type: "integer", default: 100 }, + prettyPrint: { type: "boolean", default: true }, + }, + }, + }, + }, + outputSchema: { + properties: { + files: { + items: { + properties: { + id: { type: "string" }, + name: { type: "string" }, + }, + }, + }, + }, + }, + }); + }); + + it("marks destructive operations and resolves filtered scopes", () => { + const operations = discoveryOperations({ + server: "drive", + document: fixture, + excludeOperations: ["*.delete"], + }); + + expect(operations.find((operation) => operation.name === "drive.files.delete")).toBeUndefined(); + expect(googleDiscoveryScopesForOperations(operations)).toEqual([ + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/drive.readonly", + ]); + + const deleteOperation = discoveryOperations({ server: "drive", document: fixture }).find( + (operation) => operation.name === "drive.files.delete", + ); + expect(deleteOperation).toMatchObject({ + destructiveHint: true, + inputSchema: { + required: ["path"], + properties: { + path: { + required: ["fileId"], + properties: { + fileId: { type: "string" }, + }, + }, + }, + }, + }); + }); + + it("walks nested resources and uses stable fallback operation names", () => { + const operations = discoveryOperations({ + server: "drive", + document: fixture, + includeOperations: ["drive.permissions.*", "drive.changes.*"], + }); + + expect(operations.map((operation) => operation.name)).toEqual([ + "drive.changes.getStartPageToken", + "drive.permissions.list", + ]); + }); + + it("preserves media upload and download metadata", () => { + const operations = discoveryOperations({ server: "drive", document: fixture }); + + expect(operations.find((operation) => operation.name === "drive.files.create")).toMatchObject({ + supportsMediaUpload: true, + supportsMediaDownload: false, + mediaUpload: { + accept: ["image/png"], + maxSize: "10MB", + }, + mediaUploadProtocols: { + simple: { path: "/upload/drive/v3/files", multipart: false }, + multipart: { path: "/upload/drive/v3/files", multipart: true }, + resumable: { path: "/upload/drive/v3/files", multipart: true }, + }, + inputSchema: { + properties: { + body: { + properties: { + name: { type: "string" }, + parents: { + type: "array", + items: { type: "string" }, + }, + }, + }, + media: { + type: "object", + additionalProperties: false, + properties: { + path: { type: "string" }, + artifact: { type: "string" }, + dataUrl: { type: "string" }, + mimeType: { type: "string" }, + filename: { type: "string" }, + }, + }, + }, + }, + }); + expect(operations.find((operation) => operation.name === "drive.files.download")).toMatchObject( + { + supportsMediaDownload: true, + inputSchema: { + properties: { + filename: { type: "string" }, + outputPath: { type: "string" }, + }, + }, + }, + ); + }); + + it("rejects invalid discovery documents clearly", () => { + expect(() => + discoveryOperations({ + server: "drive", + document: { kind: "not-discovery", resources: {} }, + }), + ).toThrow(/Invalid Google Discovery document/); + }); + + it("maps top-level Discovery methods", () => { + const operations = discoveryOperations({ + server: "oauth2", + document: { + kind: "discovery#restDescription", + methods: { + tokeninfo: { + path: "tokeninfo", + httpMethod: "GET", + scopes: ["openid"], + }, + }, + }, + }); + + expect(operations).toEqual([ + expect.objectContaining({ + name: "oauth2.tokeninfo", + method: "get", + path: "tokeninfo", + scopes: ["openid"], + }), + ]); + }); +}); + +describe("GoogleDiscoveryManager", () => { + it("lists, describes, searches, resolves scopes, and calls Google Discovery operations", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + includeOperations: ["drive.files.*"], + excludeOperations: ["drive.files.delete"], + }, + }, + }); + const registry = new ServerRegistry(config); + const manager = new GoogleDiscoveryManager(registry); + const caplet = config.googleDiscoveryApis.drive!; + + await expect(manager.listTools(caplet)).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "drive.files.list" }), + expect.objectContaining({ name: "drive.files.create" }), + ]), + ); + await expect(manager.getTool(caplet, "drive.files.list")).resolves.toMatchObject({ + inputSchema: { properties: { query: { properties: { pageSize: { type: "integer" } } } } }, + annotations: { readOnlyHint: true, destructiveHint: false }, + }); + expect( + manager.search(caplet, await manager.listTools(caplet), "list", 5).map((tool) => tool.name), + ).toContain("drive.files.list"); + await expect(manager.resolveAuthScopes(caplet)).resolves.toEqual([ + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/drive.readonly", + ]); + + await expect( + manager.callTool(caplet, "drive.files.list", { query: { pageSize: 2 } }), + ).resolves.toMatchObject({ + structuredContent: { status: 200, body: { files: [{ id: "1", name: "Report" }] } }, + isError: false, + }); + expect(requests.find((request) => request.url.startsWith("/drive/v3/files?"))?.url).toContain( + "pageSize=2", + ); + }); + + it("infers the request base URL from Discovery rootUrl and servicePath", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive-inferred.discovery.json`, + auth: { type: "none" }, + includeOperations: ["drive.files.list"], + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + + await expect( + manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.list", { + query: { pageSize: 2 }, + }), + ).resolves.toMatchObject({ + structuredContent: { status: 200, body: { files: [{ id: "1", name: "Report" }] } }, + }); + expect(requests.find((request) => request.url.startsWith("/drive/v3/files?"))?.url).toContain( + "pageSize=2", + ); + }); + + it("expands reserved Discovery path templates without flattening resource names", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/reserved.discovery.json`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + + await expect( + manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.getReserved", { + path: { name: "folders/1" }, + }), + ).resolves.toMatchObject({ + structuredContent: { status: 200, body: { id: "folders/1" } }, + }); + expect(requests.find((request) => request.url === "/drive/v3/files/folders/1")).toBeDefined(); + }); + + it("rejects Discovery operation paths that escape the configured base path", () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const caplet = config.googleDiscoveryApis.drive!; + const operation = { + name: "drive.escape", + method: "get" as const, + path: "../admin", + description: "Escape", + inputSchema: {}, + readOnlyHint: true, + destructiveHint: false, + scopes: [], + supportsMediaUpload: false, + supportsMediaDownload: false, + mediaUploadProtocols: {}, + parameterOrder: [], + }; + + expect(() => buildGoogleDiscoveryUrl(caplet, operation, {})).toThrow(/cannot escape baseUrl/u); + expect(() => + buildGoogleDiscoveryUrl(caplet, { ...operation, path: "%2e%2e/admin" }, {}), + ).toThrow(/cannot escape baseUrl/u); + expect(() => + buildGoogleDiscoveryUrl( + caplet, + { ...operation, path: "files/{+name}" }, + { + path: { name: "../admin" }, + }, + ), + ).toThrow(/cannot escape baseUrl/u); + }); + + it("rejects redirected discovery documents", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/redirect.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + + await expect(manager.listTools(config.googleDiscoveryApis.drive!)).rejects.toMatchObject({ + code: "DOWNSTREAM_PROTOCOL_ERROR", + }); + }); + + it("executes Google Discovery operations through handleServerTool", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const registry = new ServerRegistry(config); + const manager = new GoogleDiscoveryManager(registry); + const downstream = new DownstreamManager(registry); + const caplet = config.googleDiscoveryApis.drive!; + + const list = await handleServerTool( + caplet, + { operation: "tools" }, + registry, + downstream, + undefined, + undefined, + undefined, + undefined, + undefined, + {}, + manager, + ); + + expect( + list.structuredContent.result.items.map((tool: { name: string }) => tool.name), + ).toContain("drive.files.list"); + + await downstream.close(); + }); + + it("writes Google media downloads as artifacts", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + const result = await manager.callTool( + config.googleDiscoveryApis.drive!, + "drive.files.download", + { + path: { fileId: "1" }, + filename: "report.pdf", + }, + ); + + expect(result.structuredContent).toMatchObject({ + status: 200, + body: { artifact: { filename: "report.pdf", mimeType: "application/pdf" } }, + }); + }); + + it("writes large Google media downloads as artifacts", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + const result = await manager.callTool( + config.googleDiscoveryApis.drive!, + "drive.files.download", + { + path: { fileId: "large" }, + filename: "large.pdf", + }, + ); + + expect(result.structuredContent).toMatchObject({ + status: 200, + body: { + artifact: { + filename: "large.pdf", + mimeType: "application/pdf", + byteLength: 1024 * 1024 + 1, + }, + }, + }); + }); + + it("uploads media from dataUrl using multipart when metadata body is present", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + const result = await manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.create", { + body: { name: "report.pdf" }, + media: { dataUrl: "data:application/pdf;base64,cGRm", filename: "report.pdf" }, + }); + + expect(result.structuredContent).toMatchObject({ status: 200, body: { id: "uploaded" } }); + const upload = requests.find((request) => request.url.includes("uploadType=multipart")); + expect(upload?.headers["content-type"]).toContain("multipart/related"); + expect(upload?.body).not.toContain("cGRm"); + }); +}); diff --git a/packages/core/test/http-actions.test.ts b/packages/core/test/http-actions.test.ts index 7d8d118f..af0eb906 100644 --- a/packages/core/test/http-actions.test.ts +++ b/packages/core/test/http-actions.test.ts @@ -1,5 +1,8 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { parseConfig, type HttpApiConfig } from "../src/config"; import { DownstreamManager } from "../src/downstream"; import { HttpActionManager } from "../src/http-actions"; @@ -45,6 +48,23 @@ describe("HttpActionManager", () => { response.end("x".repeat(2 * 1024 * 1024)); return; } + if (request.url === "/large-stream") { + response.write("x".repeat(128)); + response.write("x".repeat(128)); + response.end(); + return; + } + if (request.url === "/pdf") { + response.setHeader("content-type", "application/pdf"); + response.end(Buffer.from("%PDF-1.7 test")); + return; + } + if (request.url === "/attachment") { + response.removeHeader("content-type"); + response.setHeader("content-disposition", 'attachment; filename="report.bin"'); + response.end(Buffer.from([0, 1, 2, 3])); + return; + } if (request.url === "/missing") { response.statusCode = 404; response.statusMessage = "Not Found"; @@ -280,6 +300,150 @@ describe("HttpActionManager", () => { expect(result.structuredContent).toHaveProperty("elapsedMs"); }); + it("writes binary HTTP responses as media artifacts", async () => { + const artifactDir = mkdtempSync(join(tmpdir(), "caplets-http-artifacts-")); + const manager = new HttpActionManager(registry(), { artifactDir }); + const api = httpApi({ actions: { pdf: { method: "GET", path: "/pdf" } } }); + + try { + const result = await manager.callTool(api, "pdf", {}); + const structured = result.structuredContent as { + status: number; + headers: { "content-type": string }; + body: { artifact: { path: string; mimeType: string; byteLength: number } }; + }; + + expect(structured).toMatchObject({ + status: 200, + headers: { "content-type": "application/pdf" }, + body: { + artifact: { + mimeType: "application/pdf", + byteLength: 13, + }, + }, + }); + expect(readFileSync(structured.body.artifact.path, "utf8")).toBe("%PDF-1.7 test"); + } finally { + rmSync(artifactDir, { recursive: true, force: true }); + } + }); + + it("writes attachment responses without content type as media artifacts", async () => { + const artifactDir = mkdtempSync(join(tmpdir(), "caplets-http-attachment-artifacts-")); + const manager = new HttpActionManager(registry(), { artifactDir }); + const api = httpApi({ actions: { attachment: { method: "GET", path: "/attachment" } } }); + + try { + const result = await manager.callTool(api, "attachment", {}); + const structured = result.structuredContent as { + body: { + artifact: { + uri: string; + path: string; + filename: string; + byteLength: number; + sha256: string; + }; + }; + }; + + expect(structured.body.artifact).toMatchObject({ + filename: "report.bin", + byteLength: 4, + }); + expect(structured.body.artifact.uri).toContain("caplets://artifacts/"); + expect(structured.body.artifact.sha256).toHaveLength(64); + expect(readFileSync(structured.body.artifact.path)).toEqual(Buffer.from([0, 1, 2, 3])); + } finally { + rmSync(artifactDir, { recursive: true, force: true }); + } + }); + + it("writes HTTP artifacts through handleServerTool", async () => { + const artifactDir = mkdtempSync(join(tmpdir(), "caplets-http-tool-artifacts-")); + const config = parseConfig({ + httpApis: { + http: { + name: "HTTP API", + description: "Call configured HTTP service actions.", + baseUrl, + auth: { type: "none" }, + actions: { pdf: { method: "GET", path: "/pdf" } }, + }, + }, + }); + const registry = new ServerRegistry(config); + const http = new HttpActionManager(registry, { artifactDir }); + const downstream = new DownstreamManager(registry); + + try { + const result = (await handleServerTool( + config.httpApis.http!, + { operation: "call_tool", name: "pdf", args: {} }, + registry, + downstream, + undefined, + undefined, + http, + )) as any; + + expect(result.structuredContent.body.artifact.path).toContain(artifactDir); + expect(readFileSync(result.structuredContent.body.artifact.path, "utf8")).toBe( + "%PDF-1.7 test", + ); + } finally { + rmSync(artifactDir, { recursive: true, force: true }); + } + }); + + it("writes oversized streamed text responses as media artifacts", async () => { + const artifactDir = mkdtempSync(join(tmpdir(), "caplets-http-stream-artifacts-")); + const manager = new HttpActionManager(registry(), { artifactDir, maxInlineBytes: 100 }); + const api = httpApi({ + actions: { large_stream: { method: "GET", path: "/large-stream" } }, + }); + + try { + const result = await manager.callTool(api, "large_stream", {}); + const structured = result.structuredContent as { + status: number; + headers: { "content-type": string }; + body: { artifact: { path: string; mimeType: string; byteLength: number } }; + }; + + expect(structured).toMatchObject({ + status: 200, + body: { + artifact: { + mimeType: "application/json", + byteLength: 256, + }, + }, + }); + expect(readFileSync(structured.body.artifact.path, "utf8")).toBe("x".repeat(256)); + } finally { + rmSync(artifactDir, { recursive: true, force: true }); + } + }); + + it("enforces maxResponseBytes as a hard response cap", async () => { + const artifactDir = mkdtempSync(join(tmpdir(), "caplets-http-cap-artifacts-")); + const manager = new HttpActionManager(registry(), { artifactDir, maxInlineBytes: 100 }); + const api = httpApi({ + maxResponseBytes: 200, + actions: { large_stream: { method: "GET", path: "/large-stream" } }, + }); + + try { + await expect(manager.callTool(api, "large_stream", {})).rejects.toMatchObject({ + code: "DOWNSTREAM_PROTOCOL_ERROR", + }); + } finally { + rmSync(artifactDir, { recursive: true, force: true }); + } + }); + it("rejects query and header mappings that resolve to non-objects", async () => { const manager = new HttpActionManager(registry()); const badQueryApi = httpApi({ actions: { bad_query: { method: "GET", path: "/ok" } } }); diff --git a/packages/core/test/media-artifacts.test.ts b/packages/core/test/media-artifacts.test.ts new file mode 100644 index 00000000..155206de --- /dev/null +++ b/packages/core/test/media-artifacts.test.ts @@ -0,0 +1,249 @@ +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + truncateSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + artifactUri, + readMediaInput, + resolveMediaArtifact, + writeMediaArtifact, +} from "../src/media"; + +describe("media artifacts", () => { + const dirs: string[] = []; + + afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + + function tempDir(prefix: string) { + const dir = mkdtempSync(join(tmpdir(), prefix)); + dirs.push(dir); + return dir; + } + + it("writes artifact files with stable metadata", async () => { + const root = tempDir("caplets-artifacts-"); + const artifact = await writeMediaArtifact({ + rootDir: root, + capletId: "google-drive", + suggestedFilename: "report.pdf", + mimeType: "application/pdf", + bytes: Buffer.from("pdf-bytes"), + }); + + expect(artifact).toMatchObject({ + mimeType: "application/pdf", + byteLength: 9, + filename: "report.pdf", + }); + expect(artifact.path).toContain(join(root, "google-drive")); + expect(artifact.sha256).toHaveLength(64); + expect(readFileSync(artifact.path, "utf8")).toBe("pdf-bytes"); + }); + + it("rejects output paths outside an allowed root", async () => { + const root = tempDir("caplets-artifacts-"); + await expect( + writeMediaArtifact({ + rootDir: root, + capletId: "drive", + outputPath: join(root, "..", "escape.bin"), + bytes: Buffer.from("x"), + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); + + it("resolves explicit output paths and rejects unsafe artifact path segments", async () => { + const root = tempDir("caplets-artifacts-"); + const outputPath = join(root, "drive", "call-1", "report.pdf"); + const artifact = await writeMediaArtifact({ + rootDir: root, + capletId: "drive", + outputPath, + mimeType: "application/pdf", + bytes: Buffer.from("pdf"), + }); + + expect(resolveMediaArtifact(artifact.uri, { artifactRoot: root })).toMatchObject({ + path: outputPath, + filename: "report.pdf", + mimeType: "application/pdf", + byteLength: 3, + }); + + await expect( + writeMediaArtifact({ + rootDir: root, + capletId: "drive", + outputPath: join(root, "drive", "bad call", "report.pdf"), + bytes: Buffer.from("x"), + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); + + it("clears stale artifact metadata when overwriting without a MIME type", async () => { + const root = tempDir("caplets-artifacts-"); + const outputPath = join(root, "drive", "call-1", "report.pdf"); + const first = await writeMediaArtifact({ + rootDir: root, + capletId: "drive", + outputPath, + mimeType: "application/pdf", + bytes: Buffer.from("pdf"), + }); + await writeMediaArtifact({ + rootDir: root, + capletId: "drive", + outputPath, + bytes: Buffer.from("plain"), + }); + + expect(resolveMediaArtifact(first.uri, { artifactRoot: root })).toMatchObject({ + filename: "report.pdf", + byteLength: 5, + }); + expect(resolveMediaArtifact(first.uri, { artifactRoot: root }).mimeType).toBeUndefined(); + }); + + it("rejects oversized artifact and data URL inputs before reading decoded bytes", async () => { + const root = tempDir("caplets-artifacts-"); + const artifact = await writeMediaArtifact({ + rootDir: root, + capletId: "drive", + suggestedFilename: "large.txt", + bytes: Buffer.from("large"), + }); + + await expect( + readMediaInput({ artifact: artifact.uri }, { artifactRoot: root, maxBytes: 4 }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + await expect( + readMediaInput( + { dataUrl: "data:text/plain;base64,bGFyZ2U=", filename: "large.txt" }, + { artifactRoot: root, maxBytes: 4 }, + ), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); + + it("applies the default size limit to artifact inputs", async () => { + const root = tempDir("caplets-artifacts-"); + const artifactPath = join(root, "drive", "call-1", "large.txt"); + mkdirSync(join(root, "drive", "call-1"), { recursive: true }); + writeFileSync(artifactPath, ""); + truncateSync(artifactPath, 100 * 1024 * 1024 + 1); + const artifact = artifactUri("drive", "call-1", "large.txt"); + + await expect(readMediaInput({ artifact }, { artifactRoot: root })).rejects.toMatchObject({ + code: "REQUEST_INVALID", + }); + }); + + it("rejects invalid base64 data URL padding", async () => { + const root = tempDir("caplets-artifacts-"); + await expect( + readMediaInput({ dataUrl: "data:text/plain;base64,====" }, { artifactRoot: root }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + await expect( + readMediaInput({ dataUrl: "data:text/plain;base64,a===" }, { artifactRoot: root }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); + + it("rejects artifact paths that escape the root through symlinks", async () => { + const root = tempDir("caplets-artifacts-"); + const outside = tempDir("caplets-artifacts-outside-"); + mkdirSync(join(root, "drive"), { recursive: true }); + symlinkSync(outside, join(root, "drive", "linked")); + + await expect( + writeMediaArtifact({ + rootDir: root, + capletId: "drive", + outputPath: join(root, "drive", "linked", "escape.bin"), + bytes: Buffer.from("x"), + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); + + it("rejects symlinked artifact roots", async () => { + const realRoot = tempDir("caplets-artifacts-real-"); + const parent = tempDir("caplets-artifacts-parent-"); + const linkedRoot = join(parent, "linked-root"); + symlinkSync(realRoot, linkedRoot); + + await expect( + writeMediaArtifact({ + rootDir: linkedRoot, + capletId: "drive", + suggestedFilename: "file.bin", + bytes: Buffer.from("x"), + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); + + it("reads media input from path, artifact reference, and data URL", async () => { + const root = tempDir("caplets-artifacts-"); + const file = join(root, "image.png"); + writeFileSync(file, Buffer.from("png")); + const artifact = await writeMediaArtifact({ + rootDir: root, + capletId: "drive", + suggestedFilename: "existing.png", + mimeType: "image/png", + bytes: Buffer.from("artifact"), + }); + expect(resolveMediaArtifact(artifact.uri, { artifactRoot: root })).toMatchObject({ + filename: "existing.png", + byteLength: 8, + }); + await expect(readMediaInput({ path: file }, { artifactRoot: root })).resolves.toMatchObject({ + bytes: Buffer.from("png"), + filename: "image.png", + }); + await expect( + readMediaInput({ artifact: artifact.uri }, { artifactRoot: root }), + ).resolves.toMatchObject({ + bytes: Buffer.from("artifact"), + filename: "existing.png", + mimeType: "image/png", + }); + await expect( + readMediaInput( + { dataUrl: "data:text/plain;base64,aGVsbG8=", filename: "hello.txt" }, + { artifactRoot: root }, + ), + ).resolves.toMatchObject({ + bytes: Buffer.from("hello"), + filename: "hello.txt", + mimeType: "text/plain", + }); + }); + + it("rejects multiple media input sources and non-base64 data URLs", async () => { + const root = tempDir("caplets-artifacts-"); + await expect( + readMediaInput( + { path: "/tmp/a", dataUrl: "data:text/plain;base64,eA==" }, + { + artifactRoot: root, + }, + ), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + await expect( + readMediaInput( + { dataUrl: "data:text/plain,hello" }, + { + artifactRoot: root, + }, + ), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); +}); diff --git a/packages/core/test/native.test.ts b/packages/core/test/native.test.ts index 99359dcc..62003aca 100644 --- a/packages/core/test/native.test.ts +++ b/packages/core/test/native.test.ts @@ -242,6 +242,76 @@ describe("native Caplets service", () => { } }); + it("discovers direct Google Discovery tools for native integrations", async () => { + const { dir, configPath, projectConfigPath } = tempConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + exposure: "direct", + discoveryPath: join(fixturesDir, "google-discovery/drive.discovery.json"), + baseUrl: "http://127.0.0.1:1/drive/v3/", + auth: { type: "none" }, + includeOperations: ["drive.files.list"], + }, + }, + }); + dirs.push(dir); + const service = createNativeCapletsService({ configPath, projectConfigPath, watch: false }); + + try { + await expect(service.reload()).resolves.toBe(true); + expect(service.listTools()).toEqual([ + expect.objectContaining({ + caplet: "drive__drive.files.list", + toolName: "caplets__drive__drive.files.list", + title: "drive.files.list", + inputSchema: expect.objectContaining({ + properties: expect.objectContaining({ query: expect.any(Object) }), + }), + annotations: { readOnlyHint: true, destructiveHint: false }, + }), + ]); + await expect( + service.execute("drive__drive.files.list", { query: { pageSize: 1 } }), + ).resolves.toMatchObject({ + isError: true, + }); + } finally { + await service.close(); + } + }); + + it("notifies native tool listeners after cold-start direct Google Discovery refresh", async () => { + const { dir, configPath, projectConfigPath } = tempConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + exposure: "direct", + discoveryPath: join(fixturesDir, "google-discovery/drive.discovery.json"), + baseUrl: "http://127.0.0.1:1/drive/v3/", + auth: { type: "none" }, + includeOperations: ["drive.files.list"], + }, + }, + }); + dirs.push(dir); + const service = createNativeCapletsService({ configPath, projectConfigPath, watch: false }); + const events: string[][] = []; + service.onToolsChanged((tools) => { + events.push(configuredCapletIds(tools)); + }); + + try { + await expect + .poll(() => events.at(-1), { timeout: 5_000 }) + .toEqual(["drive__drive.files.list"]); + } finally { + await service.close(); + } + }); + it("lists Code Mode only when exposure includes Code Mode", async () => { const { dir, configPath, projectConfigPath } = tempConfig({ httpApis: { diff --git a/packages/core/test/openapi.test.ts b/packages/core/test/openapi.test.ts index e3ae56fe..89b7dadd 100644 --- a/packages/core/test/openapi.test.ts +++ b/packages/core/test/openapi.test.ts @@ -1,6 +1,6 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { parseConfig } from "../src/config"; @@ -94,6 +94,11 @@ describe("native OpenAPI Caplets", () => { response.end(JSON.stringify({ public: "ok", secret: "hidden" })); return; } + if (request.url === "/reports/42") { + response.setHeader("content-type", "application/pdf"); + response.end(Buffer.from("%PDF-1.7 test")); + return; + } if (request.url === "/protected") { response.statusCode = 401; response.statusMessage = "Unauthorized"; @@ -186,7 +191,7 @@ describe("native OpenAPI Caplets", () => { )) as any; expect( list.structuredContent.result.items.map((tool: { name: string }) => tool.name), - ).toEqual(["createUser", "GET /users/{id}"]); + ).toEqual(["createUser", "GET /users/{id}", "getReport"]); expect( list.structuredContent.result.items.find( (candidate: { name: string }) => candidate.name === "GET /users/{id}", @@ -690,7 +695,7 @@ describe("native OpenAPI Caplets", () => { const openapi = new OpenApiManager(registry); const remote = registry.config.openapiEndpoints.remote!; - await expect(openapi.listTools(remote)).resolves.toHaveLength(2); + await expect(openapi.listTools(remote)).resolves.toHaveLength(3); await expect( openapi.listTools({ ...remote, specUrl: `${baseUrl}/slow-openapi.json` }), ).rejects.toMatchObject({ code: "TOOL_CALL_TIMEOUT" }); @@ -821,6 +826,7 @@ describe("native OpenAPI Caplets", () => { }, mcpServers: {}, openapiEndpoints: { remote: endpoint }, + googleDiscoveryApis: {}, graphqlEndpoints: {}, httpApis: {}, cliTools: {}, @@ -940,6 +946,51 @@ describe("native OpenAPI Caplets", () => { rmSync(dir, { recursive: true, force: true }); } }); + + it("writes binary OpenAPI responses as media artifacts", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-openapi-artifacts-")); + const specPath = join(dir, "openapi.json"); + const artifactDir = join(dir, "artifacts"); + writeFileSync(specPath, JSON.stringify(openApiSpec(baseUrl))); + const config = parseConfig({ + openapiEndpoints: { + reports: { + name: "Reports API", + description: "Download reports from the internal HTTP API.", + specPath, + baseUrl, + auth: { type: "none" }, + }, + }, + }); + const registry = new ServerRegistry(config); + const openapi = new OpenApiManager(registry, { artifactDir }); + + try { + const result = await openapi.callTool(config.openapiEndpoints.reports!, "getReport", { + path: { id: "42" }, + }); + const structured = result.structuredContent as { + status: number; + headers: { "content-type": string }; + body: { artifact: { path: string; mimeType: string; byteLength: number } }; + }; + + expect(structured).toMatchObject({ + status: 200, + headers: { "content-type": "application/pdf" }, + body: { + artifact: { + mimeType: "application/pdf", + byteLength: 13, + }, + }, + }); + expect(readFileSync(structured.body.artifact.path, "utf8")).toBe("%PDF-1.7 test"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); function openApiSpec(baseUrl: string) { @@ -1003,6 +1054,28 @@ function openApiSpec(baseUrl: string) { responses: { "201": { description: "Created" } }, }, }, + "/reports/{id}": { + get: { + operationId: "getReport", + summary: "Download a report", + parameters: [ + { + name: "id", + in: "path", + required: true, + schema: { type: "string" }, + }, + ], + responses: { + "200": { + description: "OK", + content: { + "application/pdf": {}, + }, + }, + }, + }, + }, }, }; } diff --git a/packages/core/test/tools.test.ts b/packages/core/test/tools.test.ts index e67667fe..775fb007 100644 --- a/packages/core/test/tools.test.ts +++ b/packages/core/test/tools.test.ts @@ -282,6 +282,33 @@ describe("generated tool handlers", () => { }); }); + it("fails explicitly for Google Discovery tools until the manager is configured", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files and permissions.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "none" }, + }, + }, + }); + const googleRegistry = new ServerRegistry(config); + const downstream = {} as unknown as DownstreamManager; + + await expect( + handleServerTool( + config.googleDiscoveryApis.drive!, + { operation: "tools" }, + googleRegistry, + downstream, + ), + ).rejects.toMatchObject({ + code: "INTERNAL_ERROR", + message: "Google Discovery manager is not configured", + }); + }); + it("returns HTTP inspect without requiring an HTTP manager", async () => { const httpConfig = parseConfig({ httpApis: { diff --git a/schemas/caplet.schema.json b/schemas/caplet.schema.json index 7200d110..8baeb063 100644 --- a/schemas/caplet.schema.json +++ b/schemas/caplet.schema.json @@ -717,6 +717,265 @@ "additionalProperties": false, "description": "OpenAPI endpoint backend configuration for this Caplet." }, + "googleDiscoveryApi": { + "type": "object", + "properties": { + "discoveryPath": { + "description": "Local Google Discovery document path.", + "type": "string", + "minLength": 1 + }, + "discoveryUrl": { + "description": "Remote Google Discovery document URL.", + "type": "string", + "minLength": 1 + }, + "baseUrl": { + "description": "Override base URL for Google API requests.", + "type": "string", + "minLength": 1 + }, + "auth": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "none" + } + }, + "required": ["type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "bearer" + }, + "token": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "token"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "headers" + }, + "headers": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + } + }, + "required": ["type", "headers"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "oauth2" + }, + "authorizationUrl": { + "type": "string", + "minLength": 1 + }, + "tokenUrl": { + "type": "string", + "minLength": 1 + }, + "issuer": { + "type": "string", + "minLength": 1 + }, + "resourceMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "authorizationServerMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "openidConfigurationUrl": { + "type": "string", + "minLength": 1 + }, + "clientMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "clientId": { + "type": "string", + "minLength": 1 + }, + "clientSecret": { + "type": "string", + "minLength": 1 + }, + "scopes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "redirectUri": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "oidc" + }, + "authorizationUrl": { + "type": "string", + "minLength": 1 + }, + "tokenUrl": { + "type": "string", + "minLength": 1 + }, + "issuer": { + "type": "string", + "minLength": 1 + }, + "resourceMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "authorizationServerMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "openidConfigurationUrl": { + "type": "string", + "minLength": 1 + }, + "clientMetadataUrl": { + "type": "string", + "minLength": 1 + }, + "clientId": { + "type": "string", + "minLength": 1 + }, + "clientSecret": { + "type": "string", + "minLength": 1 + }, + "scopes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "redirectUri": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type"], + "additionalProperties": false + } + ], + "description": "Explicit Google API request auth config. Use {\"type\":\"none\"} for public APIs." + }, + "requestTimeoutMs": { + "description": "Timeout in milliseconds for Google API HTTP requests.", + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "operationCacheTtlMs": { + "description": "Milliseconds Google Discovery operation metadata stays fresh. Set 0 to refresh every time.", + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "includeOperations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 160 + } + }, + "excludeOperations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 160 + } + }, + "disabled": { + "description": "When true, omit this Caplet from discovery.", + "type": "boolean" + }, + "projectBinding": { + "type": "object", + "properties": { + "required": { + "type": "boolean", + "const": true + } + }, + "required": ["required"], + "additionalProperties": false, + "description": "Project Binding requirements for Caplets that need an attached project." + }, + "runtime": { + "type": "object", + "properties": { + "features": { + "type": "array", + "items": { + "type": "string", + "enum": ["docker", "browser"] + } + }, + "resources": { + "type": "object", + "properties": { + "class": { + "type": "string", + "enum": ["standard", "large", "heavy"] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "description": "Runtime feature and resource requirements for hosted execution." + } + }, + "required": ["auth"], + "additionalProperties": false, + "description": "Google Discovery API backend configuration for this Caplet." + }, "graphqlEndpoint": { "type": "object", "properties": { diff --git a/schemas/caplets-config.schema.json b/schemas/caplets-config.schema.json index 3ec7d1ce..06d0eaff 100644 --- a/schemas/caplets-config.schema.json +++ b/schemas/caplets-config.schema.json @@ -962,6 +962,437 @@ "additionalProperties": false } }, + "googleDiscoveryApis": { + "default": {}, + "description": "Google Discovery APIs keyed by stable Caplet ID.", + "type": "object", + "propertyNames": { + "type": "string", + "pattern": "^[a-zA-Z0-9_-]{1,64}$" + }, + "additionalProperties": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 80, + "description": "Human-readable Google Discovery API display name." + }, + "description": { + "type": "string", + "description": "Capability description shown to agents before Google Discovery operations are disclosed." + }, + "discoveryPath": { + "description": "Local Google Discovery document path.", + "type": "string", + "minLength": 1 + }, + "discoveryUrl": { + "description": "Remote Google Discovery document URL.", + "type": "string", + "format": "uri" + }, + "baseUrl": { + "description": "Override base URL for Google API requests.", + "type": "string", + "format": "uri" + }, + "includeOperations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 160 + } + }, + "excludeOperations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 160 + } + }, + "auth": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "none" + } + }, + "required": ["type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "bearer" + }, + "token": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "token"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "headers" + }, + "headers": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + } + }, + "required": ["type", "headers"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "oauth2" + }, + "authorizationUrl": { + "type": "string", + "format": "uri" + }, + "tokenUrl": { + "type": "string", + "format": "uri" + }, + "issuer": { + "type": "string", + "format": "uri" + }, + "resourceMetadataUrl": { + "type": "string", + "format": "uri" + }, + "authorizationServerMetadataUrl": { + "type": "string", + "format": "uri" + }, + "openidConfigurationUrl": { + "type": "string", + "format": "uri" + }, + "clientMetadataUrl": { + "type": "string", + "format": "uri" + }, + "clientId": { + "type": "string", + "minLength": 1 + }, + "clientSecret": { + "type": "string", + "minLength": 1 + }, + "scopes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "redirectUri": { + "type": "string", + "format": "uri" + } + }, + "required": ["type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "oidc" + }, + "authorizationUrl": { + "type": "string", + "format": "uri" + }, + "tokenUrl": { + "type": "string", + "format": "uri" + }, + "issuer": { + "type": "string", + "format": "uri" + }, + "resourceMetadataUrl": { + "type": "string", + "format": "uri" + }, + "authorizationServerMetadataUrl": { + "type": "string", + "format": "uri" + }, + "openidConfigurationUrl": { + "type": "string", + "format": "uri" + }, + "clientMetadataUrl": { + "type": "string", + "format": "uri" + }, + "clientId": { + "type": "string", + "minLength": 1 + }, + "clientSecret": { + "type": "string", + "minLength": 1 + }, + "scopes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "redirectUri": { + "type": "string", + "format": "uri" + } + }, + "required": ["type"], + "additionalProperties": false + } + ], + "description": "Explicit Google API request auth config. Use {\"type\":\"none\"} for public APIs." + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 80 + } + }, + "exposure": { + "type": "string", + "enum": [ + "direct", + "progressive", + "code_mode", + "direct_and_code_mode", + "progressive_and_code_mode" + ], + "description": "How this Caplet is exposed to agents." + }, + "shadowing": { + "default": "forbid", + "description": "Whether attached local Caplets may shadow this remote Caplet ID.", + "type": "string", + "enum": ["forbid", "allow"] + }, + "useWhen": { + "description": "When agents should prefer this Caplet or configured action.", + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "avoidWhen": { + "description": "When agents should avoid this Caplet or configured action.", + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "setup": { + "type": "object", + "properties": { + "commands": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string", + "minLength": 1, + "description": "Human-readable setup or verification step label." + }, + "command": { + "type": "string", + "minLength": 1, + "description": "Executable command to spawn without a shell." + }, + "args": { + "description": "Arguments passed to the command.", + "type": "array", + "items": { + "type": "string" + } + }, + "env": { + "description": "Additional environment variables.", + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + } + }, + "cwd": { + "description": "Working directory for this command.", + "type": "string", + "minLength": 1 + }, + "timeoutMs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "maxOutputBytes": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": ["label", "command"], + "additionalProperties": false + } + }, + "verify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string", + "minLength": 1, + "description": "Human-readable setup or verification step label." + }, + "command": { + "type": "string", + "minLength": 1, + "description": "Executable command to spawn without a shell." + }, + "args": { + "description": "Arguments passed to the command.", + "type": "array", + "items": { + "type": "string" + } + }, + "env": { + "description": "Additional environment variables.", + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + } + }, + "cwd": { + "description": "Working directory for this command.", + "type": "string", + "minLength": 1 + }, + "timeoutMs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "maxOutputBytes": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": ["label", "command"], + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "projectBinding": { + "type": "object", + "properties": { + "required": { + "type": "boolean", + "const": true, + "description": "Requires Project Binding before this Caplet can run." + } + }, + "required": ["required"], + "additionalProperties": false, + "description": "Project Binding requirements for Caplets that need an attached project." + }, + "runtime": { + "type": "object", + "properties": { + "features": { + "type": "array", + "items": { + "type": "string", + "enum": ["docker", "browser"] + }, + "description": "Runtime features required by this Caplet." + }, + "resources": { + "description": "Hosted sandbox resource requirements.", + "type": "object", + "properties": { + "class": { + "description": "Requested hosted sandbox resource class.", + "type": "string", + "enum": ["standard", "large", "heavy"] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "description": "Runtime feature and resource requirements for hosted execution." + }, + "requestTimeoutMs": { + "default": 60000, + "description": "Timeout in milliseconds for Google Discovery HTTP requests.", + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "operationCacheTtlMs": { + "default": 30000, + "description": "Milliseconds Google Discovery operation metadata stays fresh. Set 0 to refresh every time.", + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "disabled": { + "default": false, + "description": "When true, omit this Google Discovery Caplet from discovery.", + "type": "boolean" + } + }, + "required": ["name", "description", "auth"], + "additionalProperties": false + } + }, "graphqlEndpoints": { "default": {}, "description": "GraphQL endpoints keyed by stable Caplet ID.", From ff689417662c87c82b37e330e15b9e2e08cb64ba Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Tue, 16 Jun 2026 15:44:44 -0400 Subject: [PATCH 2/9] fix: address google discovery review feedback --- packages/core/src/caplet-sets.ts | 27 ++- packages/core/src/cli/auth.ts | 6 +- packages/core/src/cloud/runtime-adapter.ts | 1 + packages/core/src/engine.ts | 5 +- packages/core/src/google-discovery/manager.ts | 52 +++-- packages/core/src/google-discovery/request.ts | 49 ++++- packages/core/src/http-actions.ts | 2 + packages/core/src/http/response.ts | 5 +- packages/core/src/media/artifacts.ts | 5 +- packages/core/src/media/input.ts | 3 + packages/core/src/openapi.ts | 7 +- packages/core/src/runtime.ts | 4 + packages/core/src/serve/http.ts | 10 +- packages/core/test/caplet-sets.test.ts | 12 +- packages/core/test/cli.test.ts | 82 ++++++++ packages/core/test/google-discovery.test.ts | 194 +++++++++++++++++- packages/core/test/media-artifacts.test.ts | 22 +- 17 files changed, 440 insertions(+), 46 deletions(-) diff --git a/packages/core/src/caplet-sets.ts b/packages/core/src/caplet-sets.ts index 34242473..34e97342 100644 --- a/packages/core/src/caplet-sets.ts +++ b/packages/core/src/caplet-sets.ts @@ -39,7 +39,12 @@ export class CapletSetManager { constructor( private registry: ServerRegistry, - private readonly options: { authDir?: string; ancestry?: Set } = {}, + private readonly options: { + authDir?: string; + artifactDir?: string; + exposeLocalArtifactPaths?: boolean; + ancestry?: Set; + } = {}, ) {} updateRegistry(registry: ServerRegistry): void { @@ -215,18 +220,24 @@ export class CapletSetManager { maxSearchLimit: config.maxSearchLimit, }); const registry = new ServerRegistry(childConfig); - const authOptions = this.options.authDir ? { authDir: this.options.authDir } : {}; + const sharedOptions = { + ...(this.options.authDir ? { authDir: this.options.authDir } : {}), + ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), + ...(this.options.exposeLocalArtifactPaths === false + ? { exposeLocalArtifactPaths: false } + : {}), + }; const childAncestry = new Set([...ancestry, cacheKey]); child = { registry, - downstream: new DownstreamManager(registry, authOptions), - openapi: new OpenApiManager(registry, authOptions), - graphql: new GraphQLManager(registry, authOptions), - http: new HttpActionManager(registry, authOptions), + downstream: new DownstreamManager(registry, sharedOptions), + openapi: new OpenApiManager(registry, sharedOptions), + graphql: new GraphQLManager(registry, sharedOptions), + http: new HttpActionManager(registry, sharedOptions), cli: new CliToolsManager(registry), - googleDiscovery: new GoogleDiscoveryManager(registry, authOptions), + googleDiscovery: new GoogleDiscoveryManager(registry, sharedOptions), capletSets: new CapletSetManager(registry, { - ...authOptions, + ...sharedOptions, ancestry: childAncestry, }), cacheKey, diff --git a/packages/core/src/cli/auth.ts b/packages/core/src/cli/auth.ts index ceef9213..f974c9ff 100644 --- a/packages/core/src/cli/auth.ts +++ b/packages/core/src/cli/auth.ts @@ -280,7 +280,11 @@ async function resolveAuthTarget( new ServerRegistry(config), authDir ? { authDir } : {}, ); - const baseUrl = api.baseUrl ?? api.discoveryUrl; + const baseUrl = + api.baseUrl ?? + (api.discoveryPath || !api.auth.scopes?.length + ? await manager.resolveBaseUrl(api).catch(() => api.discoveryUrl) + : api.discoveryUrl); return { ...target, ...(baseUrl ? { baseUrl } : {}), diff --git a/packages/core/src/cloud/runtime-adapter.ts b/packages/core/src/cloud/runtime-adapter.ts index d8bd82e8..aeacdb17 100644 --- a/packages/core/src/cloud/runtime-adapter.ts +++ b/packages/core/src/cloud/runtime-adapter.ts @@ -45,6 +45,7 @@ class DefaultCloudRuntimeAdapter implements CloudRuntimeAdapter { ? {} : { projectConfigPath: options.projectConfigPath }), ...(options.authDir === undefined ? {} : { authDir: options.authDir }), + exposeLocalArtifactPaths: false, watch: false, }); this.setupStore = options.setupStore ?? new LocalSetupStore(); diff --git a/packages/core/src/engine.ts b/packages/core/src/engine.ts index ea9b06fc..dd5a4ed5 100644 --- a/packages/core/src/engine.ts +++ b/packages/core/src/engine.ts @@ -34,6 +34,7 @@ export type CapletsEngineOptions = { projectConfigPath?: string; authDir?: string; artifactDir?: string; + exposeLocalArtifactPaths?: boolean; watchDebounceMs?: number; watch?: boolean; writeErr?: (value: string) => void; @@ -103,7 +104,7 @@ export class CapletsEngine { this.graphql = new GraphQLManager(this.registry, selectAuthOptions(options.authDir)); this.http = new HttpActionManager(this.registry, selectHttpLikeOptions(options)); this.cli = new CliToolsManager(this.registry); - this.capletSets = new CapletSetManager(this.registry, selectAuthOptions(options.authDir)); + this.capletSets = new CapletSetManager(this.registry, selectHttpLikeOptions(options)); this.watchDebounceMs = options.watchDebounceMs ?? 250; this.watchEnabled = options.watch ?? true; this.writeErr = options.writeErr ?? ((value: string) => process.stderr.write(value)); @@ -570,10 +571,12 @@ function selectAuthOptions(authDir: string | undefined): { authDir?: string } { function selectHttpLikeOptions(options: CapletsEngineOptions): { authDir?: string; artifactDir?: string; + exposeLocalArtifactPaths?: boolean; } { return { ...selectAuthOptions(options.authDir), ...(options.artifactDir ? { artifactDir: options.artifactDir } : {}), + ...(options.exposeLocalArtifactPaths === false ? { exposeLocalArtifactPaths: false } : {}), }; } diff --git a/packages/core/src/google-discovery/manager.ts b/packages/core/src/google-discovery/manager.ts index d632f7b6..a91b456d 100644 --- a/packages/core/src/google-discovery/manager.ts +++ b/packages/core/src/google-discovery/manager.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { readFileSync } from "node:fs"; +import { readFile } from "node:fs/promises"; import type { CompatibilityCallToolResult, Tool } from "@modelcontextprotocol/sdk/types"; import { genericOAuthHeaders } from "../auth"; import type { GoogleDiscoveryApiConfig } from "../config"; @@ -21,7 +21,11 @@ import { googleDiscoveryScopesForOperations, type GoogleDiscoveryOperation, } from "./operations"; -import { buildGoogleDiscoveryUrl, buildJsonRequestInit } from "./request"; +import { + buildGoogleDiscoveryUploadUrl, + buildGoogleDiscoveryUrl, + buildJsonRequestInit, +} from "./request"; import type { GoogleDiscoveryDocument } from "./types"; const DEFAULT_RESUMABLE_THRESHOLD_BYTES = 8 * 1024 * 1024; @@ -39,7 +43,11 @@ export class GoogleDiscoveryManager { constructor( private registry: ServerRegistry, - private readonly options: { authDir?: string; artifactDir?: string } = {}, + private readonly options: { + authDir?: string; + artifactDir?: string; + exposeLocalArtifactPaths?: boolean; + } = {}, ) {} updateRegistry(registry: ServerRegistry): void { @@ -121,9 +129,14 @@ export class GoogleDiscoveryManager { const parsed = await readHttpLikeResponse(response, { capletId: requestApi.server, ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), + ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), ...(typeof args.filename === "string" ? { filename: args.filename } : {}), ...(typeof args.outputPath === "string" ? { outputPath: args.outputPath } : {}), ...(operation.supportsMediaDownload ? { maxBytes: DEFAULT_MEDIA_RESPONSE_MAX_BYTES } : {}), + ...(operation.supportsMediaDownload && + (typeof args.filename === "string" || typeof args.outputPath === "string") + ? { forceArtifact: true } + : {}), }); return { content: markdownStructuredContent(parsed, { @@ -171,6 +184,7 @@ export class GoogleDiscoveryManager { const parsed = await readHttpLikeResponse(response, { capletId: api.server, ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), + ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), }); return { content: markdownStructuredContent(parsed, { @@ -199,7 +213,13 @@ export class GoogleDiscoveryManager { `Google Discovery ${protocol} upload path is missing`, ); } - const url = uploadUrl(api, upload.path, protocol === "simple" ? "media" : "multipart"); + const url = buildGoogleDiscoveryUploadUrl( + api, + operation, + upload.path, + protocol === "simple" ? "media" : "multipart", + args, + ); const init = protocol === "simple" ? simpleUploadInit(operation, media, headers) @@ -218,7 +238,7 @@ export class GoogleDiscoveryManager { if (!upload?.path) { throw new CapletsError("CONFIG_INVALID", "Google Discovery resumable upload path is missing"); } - const startUrl = uploadUrl(api, upload.path, "resumable"); + const startUrl = buildGoogleDiscoveryUploadUrl(api, operation, upload.path, "resumable", args); headers.set("content-type", "application/json; charset=UTF-8"); headers.set("x-upload-content-type", media.mimeType ?? "application/octet-stream"); headers.set("x-upload-content-length", String(media.bytes.byteLength)); @@ -355,12 +375,16 @@ export class GoogleDiscoveryManager { }; } + async resolveBaseUrl(api: GoogleDiscoveryApiConfig): Promise { + await this.refreshOperations(api, false); + return this.cache.get(api.server)?.baseUrl; + } + private async resolveRequestApi( api: GoogleDiscoveryApiConfig, ): Promise { if (api.baseUrl) return api; - await this.refreshOperations(api, false); - const baseUrl = this.cache.get(api.server)?.baseUrl; + const baseUrl = await this.resolveBaseUrl(api); if (!baseUrl) { throw new CapletsError("CONFIG_INVALID", `${api.server} is missing Google Discovery baseUrl`); } @@ -431,18 +455,6 @@ function multipartUploadInit( }; } -function uploadUrl(api: GoogleDiscoveryApiConfig, uploadPath: string, uploadType: string): URL { - if (!api.baseUrl) { - throw new CapletsError("CONFIG_INVALID", `${api.server} is missing Google Discovery baseUrl`); - } - const base = new URL(api.baseUrl); - const url = uploadPath.startsWith("/") - ? new URL(uploadPath, base.origin) - : new URL(uploadPath, api.baseUrl); - url.searchParams.set("uploadType", uploadType); - return url; -} - function googleDiscoveryBaseUrl( api: GoogleDiscoveryApiConfig, document: GoogleDiscoveryDocument, @@ -518,7 +530,7 @@ async function loadGoogleDiscoverySource( authDir?: string, ): Promise { if (api.discoveryPath) { - return readFileSync(api.discoveryPath, "utf8"); + return readFile(api.discoveryPath, "utf8"); } if (!api.discoveryUrl) { throw new CapletsError( diff --git a/packages/core/src/google-discovery/request.ts b/packages/core/src/google-discovery/request.ts index 0860009d..bdf6c5db 100644 --- a/packages/core/src/google-discovery/request.ts +++ b/packages/core/src/google-discovery/request.ts @@ -14,11 +14,25 @@ export function buildGoogleDiscoveryUrl( base, substitutePath(operation.path, asRecord(args.path), operation), ); - for (const [key, value] of Object.entries(asRecord(args.query))) { - if (value !== undefined && value !== null) { - url.searchParams.append(key, serializeGoogleDiscoveryValue("query", key, value)); - } - } + appendQueryArgs(url, args); + return url; +} + +export function buildGoogleDiscoveryUploadUrl( + api: GoogleDiscoveryApiConfig, + operation: GoogleDiscoveryOperation, + uploadPath: string, + uploadType: "media" | "multipart" | "resumable", + args: Record, +): URL { + const base = api.baseUrl; + validateBaseUrl(api, base); + const url = buildUploadOperationUrl( + base, + substitutePath(uploadPath, asRecord(args.path), operation), + ); + appendQueryArgs(url, args); + url.searchParams.set("uploadType", uploadType); return url; } @@ -86,6 +100,31 @@ function buildOperationUrl(base: string, operationPath: string): URL { return baseUrl; } +function buildUploadOperationUrl(base: string, uploadPath: string): URL { + if (/^[a-z][a-z0-9+.-]*:/iu.test(uploadPath) || uploadPath.startsWith("//")) { + throw new CapletsError("CONFIG_INVALID", "Google Discovery upload path cannot change origin"); + } + const baseUrl = new URL(base); + const relativePath = uploadPath.replace(/^\/+/u, ""); + assertSafeRelativePath(relativePath); + return uploadPath.startsWith("/") + ? new URL(`/${relativePath}`, baseUrl.origin) + : buildOperationUrl(base, uploadPath); +} + +function appendQueryArgs(url: URL, args: Record): void { + for (const [key, value] of Object.entries(asRecord(args.query))) { + if (value === undefined || value === null) continue; + if (Array.isArray(value)) { + for (const entry of value) { + url.searchParams.append(key, serializeGoogleDiscoveryValue("query", key, entry)); + } + continue; + } + url.searchParams.append(key, serializeGoogleDiscoveryValue("query", key, value)); + } +} + function substitutePath( path: string, values: Record, diff --git a/packages/core/src/http-actions.ts b/packages/core/src/http-actions.ts index af6641d6..e636c8a7 100644 --- a/packages/core/src/http-actions.ts +++ b/packages/core/src/http-actions.ts @@ -24,6 +24,7 @@ export class HttpActionManager { private readonly options: { authDir?: string; artifactDir?: string; + exposeLocalArtifactPaths?: boolean; maxInlineBytes?: number; } = {}, ) {} @@ -109,6 +110,7 @@ export class HttpActionManager { ...(await readHttpLikeResponse(response, { capletId: api.server, ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), + ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), maxInlineBytes: this.options.maxInlineBytes ?? api.maxResponseBytes, maxBytes: api.maxResponseBytes, })), diff --git a/packages/core/src/http/response.ts b/packages/core/src/http/response.ts index b50c3216..cbfb1e94 100644 --- a/packages/core/src/http/response.ts +++ b/packages/core/src/http/response.ts @@ -10,6 +10,8 @@ export type ReadHttpLikeResponseOptions = { filename?: string; maxInlineBytes?: number; maxBytes?: number; + forceArtifact?: boolean; + exposeLocalPath?: boolean; }; export async function readHttpLikeResponse( @@ -22,7 +24,7 @@ export async function readHttpLikeResponse( const maxBytes = options.maxBytes ?? DEFAULT_MAX_RESPONSE_BYTES; rejectOversizedContentLength(response, maxBytes); - if (shouldInline(response, mimeType)) { + if (!options.forceArtifact && shouldInline(response, mimeType)) { const inline = await readInlineCandidate(response, { maxInlineBytes, maxBytes }); if (!inline.exceeded) { const body = parseHttpBody(contentType, new TextDecoder().decode(inline.bytes)); @@ -112,6 +114,7 @@ async function writeResponseArtifact( capletId: options.capletId, ...(options.artifactDir ? { rootDir: options.artifactDir } : {}), ...(options.outputPath ? { outputPath: options.outputPath } : {}), + ...(options.exposeLocalPath === false ? { exposeLocalPath: false } : {}), suggestedFilename: options.filename ?? filenameFromContentDisposition(response) ?? "response.bin", ...(mimeType ? { mimeType } : {}), diff --git a/packages/core/src/media/artifacts.ts b/packages/core/src/media/artifacts.ts index ba7d7a40..d59a24dd 100644 --- a/packages/core/src/media/artifacts.ts +++ b/packages/core/src/media/artifacts.ts @@ -15,7 +15,7 @@ import { CapletsError } from "../errors"; export type MediaArtifact = { uri: string; - path: string; + path?: string; filename: string; mimeType?: string; byteLength: number; @@ -30,6 +30,7 @@ export type WriteMediaArtifactInput = { outputPath?: string; mimeType?: string; bytes: Uint8Array | Buffer; + exposeLocalPath?: boolean; }; type StoredMediaArtifactMetadata = { @@ -72,7 +73,7 @@ export async function writeMediaArtifact(input: WriteMediaArtifactInput): Promis writeArtifactMetadata(target, input.mimeType ? { mimeType: input.mimeType } : {}); return { uri: artifactUri(uriParts.capletId, uriParts.callId, artifactFilename), - path: target, + ...(input.exposeLocalPath === false ? {} : { path: target }), filename: artifactFilename, ...(input.mimeType ? { mimeType: input.mimeType } : {}), byteLength: bytes.byteLength, diff --git a/packages/core/src/media/input.ts b/packages/core/src/media/input.ts index c6e0215e..08c6ed8c 100644 --- a/packages/core/src/media/input.ts +++ b/packages/core/src/media/input.ts @@ -52,6 +52,9 @@ export async function readMediaInput( if (options.artifactRoot !== undefined) artifactOptions.artifactRoot = options.artifactRoot; artifactOptions.maxBytes = options.maxBytes ?? DEFAULT_MAX_MEDIA_BYTES; const artifact = resolveMediaArtifact(media.artifact, artifactOptions); + if (!artifact.path) { + throw new CapletsError("REQUEST_INVALID", "Media artifact cannot be read from this runtime"); + } const resolvedMimeType = mimeType ?? artifact.mimeType; return { bytes: readMediaFile(artifact.path), diff --git a/packages/core/src/openapi.ts b/packages/core/src/openapi.ts index 930d1024..49377fb2 100644 --- a/packages/core/src/openapi.ts +++ b/packages/core/src/openapi.ts @@ -61,7 +61,11 @@ export class OpenApiManager { constructor( private registry: ServerRegistry, - private readonly options: { authDir?: string; artifactDir?: string } = {}, + private readonly options: { + authDir?: string; + artifactDir?: string; + exposeLocalArtifactPaths?: boolean; + } = {}, ) {} updateRegistry(registry: ServerRegistry): void { @@ -155,6 +159,7 @@ export class OpenApiManager { const parsed = await readHttpLikeResponse(response, { capletId: endpoint.server, ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), + ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), }); return { content: markdownStructuredContent(parsed, { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index e7df176b..62f2da84 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -8,6 +8,7 @@ type CapletsRuntimeOptions = { projectConfigPath?: string; authDir?: string; artifactDir?: string; + exposeLocalArtifactPaths?: boolean; watchDebounceMs?: number; server?: ToolServer; writeErr?: (value: string) => void; @@ -75,6 +76,9 @@ function engineOptions(options: CapletsRuntimeOptions): CapletsEngineOptions { if (options.artifactDir !== undefined) { engineOptions.artifactDir = options.artifactDir; } + if (options.exposeLocalArtifactPaths !== undefined) { + engineOptions.exposeLocalArtifactPaths = options.exposeLocalArtifactPaths; + } if (options.watchDebounceMs !== undefined) { engineOptions.watchDebounceMs = options.watchDebounceMs; } diff --git a/packages/core/src/serve/http.ts b/packages/core/src/serve/http.ts index 63676436..9e242b29 100644 --- a/packages/core/src/serve/http.ts +++ b/packages/core/src/serve/http.ts @@ -440,12 +440,16 @@ export async function serveHttp( engineOptions: CapletsEngineOptions = {}, writeErr: (value: string) => void = (value) => process.stderr.write(value), ): Promise { - const engine = new CapletsEngine(engineOptions); + const resolvedEngineOptions = { + exposeLocalArtifactPaths: false, + ...engineOptions, + }; + const engine = new CapletsEngine(resolvedEngineOptions); const app = createHttpServeApp(options, engine, { writeErr, control: { - ...engineOptions, - projectCapletsRoot: projectCapletsRootForEngineOptions(engineOptions), + ...resolvedEngineOptions, + projectCapletsRoot: projectCapletsRootForEngineOptions(resolvedEngineOptions), }, }); const paths = servicePaths(options.path); diff --git a/packages/core/test/caplet-sets.test.ts b/packages/core/test/caplet-sets.test.ts index 276bca2b..74904bd3 100644 --- a/packages/core/test/caplet-sets.test.ts +++ b/packages/core/test/caplet-sets.test.ts @@ -108,7 +108,8 @@ describe("CapletSetManager", () => { }, }); const caplet = config.capletSets.nested!; - const manager = new CapletSetManager(new ServerRegistry(config)); + const artifactDir = join(dir, "artifacts"); + const manager = new CapletSetManager(new ServerRegistry(config), { artifactDir }); await expect(manager.listTools(caplet)).resolves.toMatchObject([ { name: "configured" }, @@ -145,7 +146,8 @@ describe("CapletSetManager", () => { }, }); const caplet = config.capletSets.nested!; - const manager = new CapletSetManager(new ServerRegistry(config)); + const artifactDir = join(dir, "artifacts"); + const manager = new CapletSetManager(new ServerRegistry(config), { artifactDir }); const result = await manager.callTool(caplet, "drive", { operation: "tools" }); @@ -156,6 +158,12 @@ describe("CapletSetManager", () => { items: [{ name: "drive.files.list" }], }, }); + const child = ( + manager as unknown as { + children: Map; + } + ).children.get("nested"); + expect(child?.googleDiscovery.options.artifactDir).toBe(artifactDir); }); it("serializes concurrent refreshes for one parent Caplet set", async () => { diff --git a/packages/core/test/cli.test.ts b/packages/core/test/cli.test.ts index 0a178185..e463bc7f 100644 --- a/packages/core/test/cli.test.ts +++ b/packages/core/test/cli.test.ts @@ -2329,6 +2329,88 @@ describe("cli init", () => { } }); + it("uses Discovery-derived base URL for Google Discovery OAuth refresh", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-auth-google-base-url-cli-")); + const authDir = join(dir, "auth"); + const configPath = join(dir, "config.json"); + const discoveryPath = join(dir, "drive.discovery.json"); + const out: string[] = []; + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue( + Response.json({ + access_token: "new-access-token", + refresh_token: "new-refresh-token", + token_type: "Bearer", + expires_in: 3600, + }), + ); + try { + writeFileSync( + discoveryPath, + JSON.stringify({ + kind: "discovery#restDescription", + baseUrl: "https://api.example.com/drive/v3/", + resources: { + files: { + methods: { + list: { + id: "drive.files.list", + path: "files", + httpMethod: "GET", + scopes: ["https://www.googleapis.com/auth/drive.readonly"], + }, + }, + }, + }, + }), + ); + writeFileSync( + configPath, + JSON.stringify({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryPath, + auth: { + type: "oauth2", + clientId: "client", + tokenUrl: "https://auth.example.com/token", + }, + }, + }, + }), + ); + process.env.CAPLETS_CONFIG = configPath; + writeTokenBundle( + { + server: "drive", + authType: "oauth2", + accessToken: "old-access-token", + refreshToken: "old-refresh-token", + expiresAt: "2999-01-01T00:00:00.000Z", + protectedResourceOrigin: "https://api.example.com", + metadata: { + requestedScopes: ["https://www.googleapis.com/auth/drive.readonly"], + }, + }, + authDir, + ); + + await runCli(["auth", "refresh", "drive"], { + writeOut: (value) => out.push(value), + authDir, + }); + + expect(out.join("")).toBe("Refreshed OAuth credentials for `drive`.\n"); + expect(fetchMock).toHaveBeenCalledWith( + "https://auth.example.com/token", + expect.objectContaining({ method: "POST" }), + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("logs out configured OpenAPI OAuth endpoints", async () => { const dir = mkdtempSync(join(tmpdir(), "caplets-auth-")); const authDir = join(dir, "auth"); diff --git a/packages/core/test/google-discovery.test.ts b/packages/core/test/google-discovery.test.ts index 6d3fb9ff..e19d884a 100644 --- a/packages/core/test/google-discovery.test.ts +++ b/packages/core/test/google-discovery.test.ts @@ -1,8 +1,10 @@ -import { readFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { + buildGoogleDiscoveryUploadUrl, buildGoogleDiscoveryUrl, discoveryOperations, GoogleDiscoveryManager, @@ -87,6 +89,48 @@ beforeEach(async () => { ); return; } + if (url === "/upload-path.discovery.json") { + response.end( + JSON.stringify({ + kind: "discovery#restDescription", + rootUrl: `${baseUrl}/`, + servicePath: "drive/v3/", + schemas: { + File: { + id: "File", + type: "object", + properties: { id: { type: "string" } }, + }, + }, + resources: { + files: { + methods: { + update: { + id: "drive.files.update", + path: "files/{fileId}", + httpMethod: "PATCH", + supportsMediaUpload: true, + parameters: { + fileId: { type: "string", location: "path", required: true }, + fields: { type: "string", location: "query" }, + }, + mediaUpload: { + protocols: { + simple: { + path: "/upload/drive/v3/files/{fileId}", + multipart: false, + }, + }, + }, + response: { $ref: "File" }, + }, + }, + }, + }, + }), + ); + return; + } if (url === "/redirect.discovery.json") { response.statusCode = 302; response.setHeader("location", "/drive.discovery.json"); @@ -107,6 +151,11 @@ beforeEach(async () => { response.end("%PDF bytes"); return; } + if (url === "/drive/v3/files/text/download") { + response.setHeader("content-type", "text/plain"); + response.end("plain text export"); + return; + } if (url === "/drive/v3/files/large/download") { const bytes = Buffer.alloc(1024 * 1024 + 1, "x"); response.setHeader("content-type", "application/pdf"); @@ -122,6 +171,15 @@ beforeEach(async () => { response.end(JSON.stringify({ id: "uploaded-media" })); return; } + if ( + url.startsWith("/upload/drive/v3/files/1?") && + url.includes("uploadType=media") && + url.includes("fields=id") && + request.method === "PATCH" + ) { + response.end(JSON.stringify({ id: "1" })); + return; + } if (url === "/upload/drive/v3/files?uploadType=multipart" && request.method === "POST") { response.end(JSON.stringify({ id: "uploaded" })); return; @@ -479,6 +537,81 @@ describe("GoogleDiscoveryManager", () => { ).toThrow(/cannot escape baseUrl/u); }); + it("serializes repeated query parameters from arrays", () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const caplet = config.googleDiscoveryApis.drive!; + const operation = { + name: "drive.files.list", + method: "get" as const, + path: "files", + inputSchema: {}, + readOnlyHint: true, + destructiveHint: false, + scopes: [], + supportsMediaUpload: false, + supportsMediaDownload: false, + mediaUploadProtocols: {}, + parameterOrder: [], + }; + + const url = buildGoogleDiscoveryUrl(caplet, operation, { + query: { label: ["starred", "ownedByMe"] }, + }); + + expect(url.searchParams.getAll("label")).toEqual(["starred", "ownedByMe"]); + }); + + it("builds safe media upload URLs with path and query arguments", () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const caplet = config.googleDiscoveryApis.drive!; + const operation = { + name: "drive.files.update", + method: "patch" as const, + path: "files/{fileId}", + inputSchema: {}, + readOnlyHint: false, + destructiveHint: false, + scopes: [], + supportsMediaUpload: true, + supportsMediaDownload: false, + mediaUploadProtocols: {}, + parameterOrder: [], + }; + + const url = buildGoogleDiscoveryUploadUrl( + caplet, + operation, + "/upload/drive/v3/files/{fileId}", + "media", + { path: { fileId: "1" }, query: { fields: "id" } }, + ); + + expect(url.toString()).toBe(`${baseUrl}/upload/drive/v3/files/1?fields=id&uploadType=media`); + expect(() => + buildGoogleDiscoveryUploadUrl(caplet, operation, "https://evil.example/upload", "media", {}), + ).toThrow(/cannot change origin/u); + }); + it("rejects redirected discovery documents", async () => { const config = parseConfig({ googleDiscoveryApis: { @@ -564,6 +697,41 @@ describe("GoogleDiscoveryManager", () => { }); }); + it("honors outputPath for inlineable Google media downloads", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-google-download-")); + try { + const outputPath = join(dir, "drive", "call", "export.txt"); + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config), { artifactDir: dir }); + const result = await manager.callTool( + config.googleDiscoveryApis.drive!, + "drive.files.download", + { + path: { fileId: "text" }, + outputPath, + }, + ); + + expect(existsSync(outputPath)).toBe(true); + expect(result.structuredContent).toMatchObject({ + status: 200, + body: { artifact: { path: outputPath, filename: "export.txt", mimeType: "text/plain" } }, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("writes large Google media downloads as artifacts", async () => { const config = parseConfig({ googleDiscoveryApis: { @@ -621,4 +789,28 @@ describe("GoogleDiscoveryManager", () => { expect(upload?.headers["content-type"]).toContain("multipart/related"); expect(upload?.body).not.toContain("cGRm"); }); + + it("substitutes path and query args into media upload URLs", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/upload-path.discovery.json`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + const result = await manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.update", { + path: { fileId: "1" }, + query: { fields: "id" }, + media: { dataUrl: "data:text/plain;base64,aGVsbG8=", filename: "hello.txt" }, + }); + + expect(result.structuredContent).toMatchObject({ status: 200, body: { id: "1" } }); + expect( + requests.find((request) => request.url.startsWith("/upload/drive/v3/files/1?"))?.url, + ).toContain("fields=id"); + }); }); diff --git a/packages/core/test/media-artifacts.test.ts b/packages/core/test/media-artifacts.test.ts index 155206de..5188ad6b 100644 --- a/packages/core/test/media-artifacts.test.ts +++ b/packages/core/test/media-artifacts.test.ts @@ -47,7 +47,27 @@ describe("media artifacts", () => { }); expect(artifact.path).toContain(join(root, "google-drive")); expect(artifact.sha256).toHaveLength(64); - expect(readFileSync(artifact.path, "utf8")).toBe("pdf-bytes"); + expect(readFileSync(artifact.path!, "utf8")).toBe("pdf-bytes"); + }); + + it("can omit local artifact paths from returned metadata", async () => { + const root = tempDir("caplets-artifacts-"); + const artifact = await writeMediaArtifact({ + rootDir: root, + capletId: "google-drive", + suggestedFilename: "report.pdf", + mimeType: "application/pdf", + bytes: Buffer.from("pdf-bytes"), + exposeLocalPath: false, + }); + + expect(artifact).toMatchObject({ + uri: expect.stringMatching(/^caplets:\/\/artifacts\//u), + filename: "report.pdf", + byteLength: 9, + sha256: expect.any(String), + }); + expect(artifact).not.toHaveProperty("path"); }); it("rejects output paths outside an allowed root", async () => { From b137ec3386fdc734a3495f76c3ef8e7f0e1ba523 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Tue, 16 Jun 2026 15:48:06 -0400 Subject: [PATCH 3/9] fix: resolve discovery scopes for remote auth --- packages/core/src/cli/auth.ts | 2 +- packages/core/src/remote-control/dispatch.ts | 4 +- .../core/test/remote-control-dispatch.test.ts | 108 ++++++++++++++++++ 3 files changed, 111 insertions(+), 3 deletions(-) diff --git a/packages/core/src/cli/auth.ts b/packages/core/src/cli/auth.ts index f974c9ff..cc9404e3 100644 --- a/packages/core/src/cli/auth.ts +++ b/packages/core/src/cli/auth.ts @@ -267,7 +267,7 @@ export function findAuthTarget(serverId: string, config = loadConfig()): AuthTar return authTargets(config).find((server) => server.server === serverId); } -async function resolveAuthTarget( +export async function resolveAuthTarget( serverId: string, config: CapletsConfig, authDir?: string, diff --git a/packages/core/src/remote-control/dispatch.ts b/packages/core/src/remote-control/dispatch.ts index 15871583..d2b7de62 100644 --- a/packages/core/src/remote-control/dispatch.ts +++ b/packages/core/src/remote-control/dispatch.ts @@ -9,10 +9,10 @@ import { } from "./../cli/add"; import { assertLoginTarget, - findAuthTarget, listAuthRows, logoutAuthResult, refreshAuthResult, + resolveAuthTarget, } from "./../cli/auth"; import { completionShells, type CompletionShell } from "./../cli/completion"; import { initConfig } from "./../cli/init"; @@ -180,7 +180,7 @@ async function startRemoteAuthLogin(serverId: string, context: RemoteControlDisp throw new CapletsError("REQUEST_INVALID", "Remote auth login is not available on this server"); } const config = loadConfigWithSources(context.configPath, context.projectConfigPath).config; - const target = findAuthTarget(serverId, config); + const target = await resolveAuthTarget(serverId, config, context.authDir); assertLoginTarget(target, serverId); const flowId = randomUUID(); const baseUrl = context.controlCallbackBaseUrl.endsWith("/") diff --git a/packages/core/test/remote-control-dispatch.test.ts b/packages/core/test/remote-control-dispatch.test.ts index 57152373..d7bd533a 100644 --- a/packages/core/test/remote-control-dispatch.test.ts +++ b/packages/core/test/remote-control-dispatch.test.ts @@ -428,6 +428,114 @@ describe("dispatchRemoteCliRequest", () => { }); }); + it("resolves Google Discovery scopes before starting remote OAuth login", async () => { + const context = testContext(); + const authFlowStore = new RemoteAuthFlowStore(); + const authDir = join(context.tempRoot, "auth"); + mkdirSync(authDir, { recursive: true }); + const discoveryPath = join(context.tempRoot, "drive.discovery.json"); + writeFileSync( + discoveryPath, + JSON.stringify({ + kind: "discovery#restDescription", + name: "drive", + version: "v3", + title: "Drive API", + rootUrl: "https://www.googleapis.com/", + servicePath: "drive/v3/", + baseUrl: "https://www.googleapis.com/drive/v3/", + resources: { + files: { + methods: { + list: { + id: "drive.files.list", + path: "files", + httpMethod: "GET", + scopes: ["https://www.googleapis.com/auth/drive.metadata.readonly"], + }, + }, + }, + }, + }), + ); + writeFileSync( + context.configPath, + JSON.stringify({ + googleDiscoveryApis: { + drive: { + name: "Drive", + description: "Drive API.", + discoveryPath, + auth: { + type: "oauth2", + clientId: "client", + tokenUrl: "https://oauth2.googleapis.com/token", + authorizationUrl: "https://accounts.google.com/o/oauth2/v2/auth", + }, + }, + }, + }), + ); + + const response = await dispatchRemoteCliRequest( + { command: "auth_login_start", arguments: { server: "drive" } }, + { + ...context, + authDir, + controlCallbackBaseUrl: "http://127.0.0.1:5387/control", + authFlowStore, + }, + ); + + expect(response).toMatchObject({ ok: true }); + const result = response.ok + ? (response.result as { authorizationUrl: string; flowId: string }) + : undefined; + expect(result?.flowId).toBeTruthy(); + const authorizationUrl = new URL(result?.authorizationUrl ?? ""); + expect(authorizationUrl.searchParams.get("scope")).toBe( + "https://www.googleapis.com/auth/drive.metadata.readonly", + ); + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue( + Response.json({ + access_token: "drive-access-token", + token_type: "Bearer", + expires_in: 3600, + }), + ); + + await dispatchRemoteCliRequest( + { + command: "auth_login_complete", + arguments: { + flowId: result?.flowId, + callbackUrl: `http://127.0.0.1/callback?code=code&state=${authorizationUrl.searchParams.get("state")}`, + }, + }, + { + ...context, + authDir, + authFlowStore, + controlCallbackBaseUrl: "http://127.0.0.1:5387/control", + }, + ); + + expect(fetchMock).toHaveBeenCalledWith( + "https://oauth2.googleapis.com/token", + expect.objectContaining({ + method: "POST", + body: expect.stringContaining("code=code"), + }), + ); + expect(readTokenBundle("drive", authDir)).toMatchObject({ + protectedResourceOrigin: "https://www.googleapis.com", + metadata: { + protectedResource: "https://www.googleapis.com/drive/v3/", + requestedScopes: ["https://www.googleapis.com/auth/drive.metadata.readonly"], + }, + }); + }); + it("does not create a pending auth flow when MCP auth is already authorized", async () => { const fixture = remoteFixtureWithOAuth(); const authFlowStore = new RemoteAuthFlowStore(); From 8f39ac67f9760f715c9b7b92ae8111a41d3a0b3c Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Tue, 16 Jun 2026 16:46:16 -0400 Subject: [PATCH 4/9] fix: harden google discovery review paths --- packages/core/src/auth.ts | 2 +- packages/core/src/cli/auth.ts | 5 +- packages/core/src/google-discovery/manager.ts | 45 +++++- packages/core/src/http-actions.ts | 1 + packages/core/src/http/response.ts | 10 +- packages/core/src/media/input.ts | 5 +- packages/core/src/openapi.ts | 3 + packages/core/test/auth.test.ts | 44 ++++++ packages/core/test/cli.test.ts | 110 +++++++++++-- packages/core/test/google-discovery.test.ts | 147 ++++++++++++++++++ packages/core/test/media-artifacts.test.ts | 23 +++ packages/core/test/openapi.test.ts | 115 +++++++++++++- 12 files changed, 485 insertions(+), 25 deletions(-) diff --git a/packages/core/src/auth.ts b/packages/core/src/auth.ts index 486ed679..c3799bff 100644 --- a/packages/core/src/auth.ts +++ b/packages/core/src/auth.ts @@ -1255,7 +1255,7 @@ function tokenBundleMissingScopes( const required = requiredStoredScopes(authConfig, resolvedScopes); if (required.length === 0) return false; const metadataScopes = requestedScopesFromMetadata(bundle.metadata); - const actual = new Set(metadataScopes ?? bundle.scope?.split(/\s+/u).filter(Boolean) ?? []); + const actual = new Set(bundle.scope?.split(/\s+/u).filter(Boolean) ?? metadataScopes ?? []); return required.some((scope) => !actual.has(scope)); } diff --git a/packages/core/src/cli/auth.ts b/packages/core/src/cli/auth.ts index cc9404e3..3b6ca006 100644 --- a/packages/core/src/cli/auth.ts +++ b/packages/core/src/cli/auth.ts @@ -281,10 +281,7 @@ export async function resolveAuthTarget( authDir ? { authDir } : {}, ); const baseUrl = - api.baseUrl ?? - (api.discoveryPath || !api.auth.scopes?.length - ? await manager.resolveBaseUrl(api).catch(() => api.discoveryUrl) - : api.discoveryUrl); + api.baseUrl ?? (await manager.resolveBaseUrl(api).catch(() => undefined)) ?? api.discoveryUrl; return { ...target, ...(baseUrl ? { baseUrl } : {}), diff --git a/packages/core/src/google-discovery/manager.ts b/packages/core/src/google-discovery/manager.ts index a91b456d..d60de36d 100644 --- a/packages/core/src/google-discovery/manager.ts +++ b/packages/core/src/google-discovery/manager.ts @@ -126,8 +126,12 @@ export class GoogleDiscoveryManager { }, ); } + if (response.status === 401 || response.status === 403) { + throw googleAuthError(requestApi, response); + } const parsed = await readHttpLikeResponse(response, { capletId: requestApi.server, + method: operation.method, ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), ...(typeof args.filename === "string" ? { filename: args.filename } : {}), @@ -171,10 +175,13 @@ export class GoogleDiscoveryManager { operation: GoogleDiscoveryOperation, args: Record, ): Promise { - const media = await readMediaInput( - args.media, - this.options.artifactDir ? { artifactRoot: this.options.artifactDir } : {}, - ); + const mediaOptions: { + artifactRoot?: string; + allowLocalPaths?: boolean; + } = {}; + if (this.options.artifactDir) mediaOptions.artifactRoot = this.options.artifactDir; + if (this.options.exposeLocalArtifactPaths === false) mediaOptions.allowLocalPaths = false; + const media = await readMediaInput(args.media, mediaOptions); const headers = new Headers(await authHeaders(api, this.options.authDir, operation.scopes)); const protocol = selectUploadProtocol(operation, media, args); const response = @@ -183,6 +190,7 @@ export class GoogleDiscoveryManager { : await this.callSingleUpload(api, operation, args, media, headers, protocol); const parsed = await readHttpLikeResponse(response, { capletId: api.server, + method: operation.method, ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), }); @@ -403,7 +411,14 @@ function selectUploadProtocol( ) { return "resumable"; } - if ("body" in args && operation.mediaUploadProtocols.multipart) return "multipart"; + if ("body" in args) { + if (operation.mediaUploadProtocols.multipart) return "multipart"; + if (operation.mediaUploadProtocols.resumable) return "resumable"; + throw new CapletsError( + "CONFIG_INVALID", + "Google Discovery media upload metadata requires multipart or resumable upload", + ); + } if (operation.mediaUploadProtocols.simple) return "simple"; if (operation.mediaUploadProtocols.resumable) return "resumable"; throw new CapletsError( @@ -488,6 +503,9 @@ async function fetchGoogleRequest( }, ); } + if (response.status === 401 || response.status === 403) { + throw googleAuthError(api, response); + } return response; } catch (error) { if (isAbortError(error)) { @@ -600,6 +618,23 @@ async function authHeaders( } } +function googleAuthError(api: GoogleDiscoveryApiConfig, response: Response): CapletsError { + return new CapletsError( + response.status === 401 ? "AUTH_REQUIRED" : "AUTH_FAILED", + "Google Discovery authentication failed", + { + server: api.server, + status: response.status, + message: response.statusText, + authType: api.auth.type, + challenge: response.headers.get("www-authenticate") ? "[REDACTED]" : undefined, + ...(api.auth.type === "oauth2" || api.auth.type === "oidc" + ? { nextAction: "run_caplets_auth_login" } + : {}), + }, + ); +} + function shouldSendDiscoveryAuth(api: GoogleDiscoveryApiConfig): boolean { return Boolean( api.discoveryUrl && diff --git a/packages/core/src/http-actions.ts b/packages/core/src/http-actions.ts index e636c8a7..cdb37596 100644 --- a/packages/core/src/http-actions.ts +++ b/packages/core/src/http-actions.ts @@ -109,6 +109,7 @@ export class HttpActionManager { const parsed = { ...(await readHttpLikeResponse(response, { capletId: api.server, + method: operation.method, ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), maxInlineBytes: this.options.maxInlineBytes ?? api.maxResponseBytes, diff --git a/packages/core/src/http/response.ts b/packages/core/src/http/response.ts index cbfb1e94..e9b6328b 100644 --- a/packages/core/src/http/response.ts +++ b/packages/core/src/http/response.ts @@ -5,6 +5,7 @@ import { DEFAULT_MAX_RESPONSE_BYTES, parseHttpBody } from "./utils"; export type ReadHttpLikeResponseOptions = { capletId: string; + method?: string; artifactDir?: string; outputPath?: string; filename?: string; @@ -22,7 +23,11 @@ export async function readHttpLikeResponse( const mimeType = mimeFromContentType(contentType); const maxInlineBytes = options.maxInlineBytes ?? DEFAULT_MAX_RESPONSE_BYTES; const maxBytes = options.maxBytes ?? DEFAULT_MAX_RESPONSE_BYTES; - rejectOversizedContentLength(response, maxBytes); + const method = options.method?.toUpperCase(); + rejectOversizedContentLength(response, maxBytes, method); + if (method === "HEAD") { + return responseEnvelope(response, contentType); + } if (!options.forceArtifact && shouldInline(response, mimeType)) { const inline = await readInlineCandidate(response, { maxInlineBytes, maxBytes }); @@ -88,7 +93,8 @@ async function readBoundedBytes(response: Response, maxBytes: number): Promise { if (!input || typeof input !== "object" || Array.isArray(input)) { throw new CapletsError("REQUEST_INVALID", "media must be an object"); @@ -38,6 +38,9 @@ export async function readMediaInput( const mimeType = typeof media.mimeType === "string" ? media.mimeType : undefined; if (typeof media.path === "string") { + if (options.allowLocalPaths === false) { + throw new CapletsError("REQUEST_INVALID", "media.path is not available in this runtime"); + } const stat = statMediaFile(media.path); enforceSize(stat.size, options.maxBytes); return { diff --git a/packages/core/src/openapi.ts b/packages/core/src/openapi.ts index 49377fb2..cbc22db4 100644 --- a/packages/core/src/openapi.ts +++ b/packages/core/src/openapi.ts @@ -18,6 +18,7 @@ import { markdownStructuredContent } from "./result-content"; import { searchToolList } from "./tool-search"; const HTTP_METHODS = ["get", "put", "post", "delete", "options", "head", "patch", "trace"] as const; +const DEFAULT_OPENAPI_RESPONSE_MAX_BYTES = 100 * 1024 * 1024; const JSON_CONTENT_TYPES = ["application/json"]; const FORBIDDEN_ARGUMENT_HEADERS = new Set([ "accept", @@ -158,8 +159,10 @@ export class OpenApiManager { } const parsed = await readHttpLikeResponse(response, { capletId: endpoint.server, + method: operation.method, ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), + maxBytes: DEFAULT_OPENAPI_RESPONSE_MAX_BYTES, }); return { content: markdownStructuredContent(parsed, { diff --git a/packages/core/test/auth.test.ts b/packages/core/test/auth.test.ts index e377856a..914bb758 100644 --- a/packages/core/test/auth.test.ts +++ b/packages/core/test/auth.test.ts @@ -354,6 +354,50 @@ describe("auth helpers", () => { } }); + it("rejects token bundles whose granted scopes omit a required scope", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-auth-scope-mismatch-")); + try { + writeTokenBundle( + { + server: "drive", + authType: "oauth2", + accessToken: "access-token", + tokenType: "Bearer", + expiresAt: "2999-01-01T00:00:00.000Z", + scope: "https://www.googleapis.com/auth/drive.metadata.readonly", + protectedResourceOrigin: "https://www.googleapis.com", + metadata: { + requestedScopes: [ + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/drive.metadata.readonly", + ], + }, + }, + dir, + ); + + await expect( + genericOAuthHeaders( + { + server: "drive", + backend: "googleDiscovery", + baseUrl: "https://www.googleapis.com/drive/v3/", + auth: { + type: "oauth2", + scopes: ["https://www.googleapis.com/auth/drive"], + }, + }, + dir, + ), + ).rejects.toMatchObject({ + code: "AUTH_REQUIRED", + details: { nextAction: "run_caplets_auth_login" }, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("rejects generic OAuth headers when refresh returns an expired token", async () => { const dir = mkdtempSync(join(tmpdir(), "caplets-auth-refresh-expired-")); const server = createServer((_request: IncomingMessage, response: ServerResponse) => { diff --git a/packages/core/test/cli.test.ts b/packages/core/test/cli.test.ts index e463bc7f..8065bf4a 100644 --- a/packages/core/test/cli.test.ts +++ b/packages/core/test/cli.test.ts @@ -2263,19 +2263,22 @@ describe("cli init", () => { } }); - it("refreshes Google Discovery OAuth credentials with explicit scopes without loading discovery", async () => { + it("refreshes Google Discovery OAuth credentials with explicit scopes when discovery is unavailable", async () => { const dir = mkdtempSync(join(tmpdir(), "caplets-auth-google-refresh-cli-")); const authDir = join(dir, "auth"); const configPath = join(dir, "config.json"); const out: string[] = []; - const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue( - Response.json({ - access_token: "new-access-token", - refresh_token: "new-refresh-token", - token_type: "Bearer", - expires_in: 3600, - }), - ); + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockRejectedValueOnce(new Error("discovery unavailable")) + .mockResolvedValueOnce( + Response.json({ + access_token: "new-access-token", + refresh_token: "new-refresh-token", + token_type: "Bearer", + expires_in: 3600, + }), + ); try { writeFileSync( configPath, @@ -2316,7 +2319,7 @@ describe("cli init", () => { }); expect(out.join("")).toBe("Refreshed OAuth credentials for `drive`.\n"); - expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledTimes(2); expect(fetchMock).toHaveBeenCalledWith( "https://auth.example.com/token", expect.objectContaining({ @@ -2411,6 +2414,93 @@ describe("cli init", () => { } }); + it("uses Discovery-derived base URL for explicit Google Discovery OAuth scopes", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-auth-google-proxy-cli-")); + const authDir = join(dir, "auth"); + const configPath = join(dir, "config.json"); + const out: string[] = []; + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockResolvedValueOnce( + Response.json({ + kind: "discovery#restDescription", + baseUrl: "https://api.example.com/drive/v3/", + resources: { + files: { + methods: { + list: { + id: "drive.files.list", + path: "files", + httpMethod: "GET", + }, + }, + }, + }, + }), + ) + .mockResolvedValueOnce( + Response.json({ + access_token: "new-access-token", + refresh_token: "new-refresh-token", + token_type: "Bearer", + expires_in: 3600, + scope: "https://www.googleapis.com/auth/drive.readonly", + }), + ); + try { + writeFileSync( + configPath, + JSON.stringify({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: "https://discovery-proxy.example.com/drive/v3/rest", + auth: { + type: "oauth2", + clientId: "client", + tokenUrl: "https://auth.example.com/token", + scopes: ["https://www.googleapis.com/auth/drive.readonly"], + }, + }, + }, + }), + ); + process.env.CAPLETS_CONFIG = configPath; + writeTokenBundle( + { + server: "drive", + authType: "oauth2", + accessToken: "old-access-token", + refreshToken: "old-refresh-token", + expiresAt: "2999-01-01T00:00:00.000Z", + protectedResourceOrigin: "https://api.example.com", + scope: "https://www.googleapis.com/auth/drive.readonly", + }, + authDir, + ); + + await runCli(["auth", "refresh", "drive"], { + writeOut: (value) => out.push(value), + authDir, + }); + + expect(out.join("")).toBe("Refreshed OAuth credentials for `drive`.\n"); + expect(fetchMock).toHaveBeenNthCalledWith( + 1, + "https://discovery-proxy.example.com/drive/v3/rest", + expect.anything(), + ); + expect(fetchMock).toHaveBeenNthCalledWith( + 2, + "https://auth.example.com/token", + expect.objectContaining({ method: "POST" }), + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("logs out configured OpenAPI OAuth endpoints", async () => { const dir = mkdtempSync(join(tmpdir(), "caplets-auth-")); const authDir = join(dir, "auth"); diff --git a/packages/core/test/google-discovery.test.ts b/packages/core/test/google-discovery.test.ts index e19d884a..c8742b89 100644 --- a/packages/core/test/google-discovery.test.ts +++ b/packages/core/test/google-discovery.test.ts @@ -10,6 +10,7 @@ import { GoogleDiscoveryManager, googleDiscoveryScopesForOperations, } from "../src/google-discovery"; +import { writeTokenBundle } from "../src/auth"; import { parseConfig } from "../src/config"; import { DownstreamManager } from "../src/downstream"; import { ServerRegistry } from "../src/registry"; @@ -131,6 +132,65 @@ beforeEach(async () => { ); return; } + if (url === "/upload-resumable.discovery.json") { + response.end( + JSON.stringify({ + kind: "discovery#restDescription", + rootUrl: `${baseUrl}/`, + servicePath: "drive/v3/", + schemas: { + File: { + id: "File", + type: "object", + properties: { id: { type: "string" } }, + }, + }, + resources: { + files: { + methods: { + create: { + id: "drive.files.create", + path: "files", + httpMethod: "POST", + request: { $ref: "File" }, + supportsMediaUpload: true, + mediaUpload: { + protocols: { + simple: { path: "/upload/drive/v3/files", multipart: false }, + resumable: { path: "/upload/drive/v3/files", multipart: true }, + }, + }, + response: { $ref: "File" }, + }, + }, + }, + }, + }), + ); + return; + } + if (url === "/auth-error.discovery.json") { + response.end( + JSON.stringify({ + kind: "discovery#restDescription", + rootUrl: `${baseUrl}/`, + servicePath: "drive/v3/", + resources: { + files: { + methods: { + protected: { + id: "drive.files.protected", + path: "protected", + httpMethod: "GET", + scopes: ["https://www.googleapis.com/auth/drive.readonly"], + }, + }, + }, + }, + }), + ); + return; + } if (url === "/redirect.discovery.json") { response.statusCode = 302; response.setHeader("location", "/drive.discovery.json"); @@ -167,6 +227,16 @@ beforeEach(async () => { response.end(JSON.stringify({ id: "folders/1" })); return; } + if (url === "/drive/v3/protected") { + response.statusCode = 401; + response.statusMessage = "Unauthorized"; + response.setHeader( + "www-authenticate", + 'Bearer error="invalid_token", access_token="secret-google-token"', + ); + response.end(JSON.stringify({ error: "secret-google-token" })); + return; + } if (url === "/upload/drive/v3/files?uploadType=media" && request.method === "POST") { response.end(JSON.stringify({ id: "uploaded-media" })); return; @@ -790,6 +860,33 @@ describe("GoogleDiscoveryManager", () => { expect(upload?.body).not.toContain("cGRm"); }); + it("uses resumable upload to preserve metadata when multipart is unavailable", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/upload-resumable.discovery.json`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + const result = await manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.create", { + body: { name: "report.pdf" }, + media: { dataUrl: "data:application/pdf;base64,cGRm", filename: "report.pdf" }, + }); + + expect(result.structuredContent).toMatchObject({ + status: 200, + body: { id: "uploaded-resumable" }, + }); + const start = requests.find((request) => request.url.includes("uploadType=resumable")); + expect(start?.body).toBe(JSON.stringify({ name: "report.pdf" })); + expect(start?.headers["x-upload-content-type"]).toBe("application/pdf"); + expect(requests.find((request) => request.url.includes("uploadType=media"))).toBeUndefined(); + }); + it("substitutes path and query args into media upload URLs", async () => { const config = parseConfig({ googleDiscoveryApis: { @@ -813,4 +910,54 @@ describe("GoogleDiscoveryManager", () => { requests.find((request) => request.url.startsWith("/upload/drive/v3/files/1?"))?.url, ).toContain("fields=id"); }); + + it("surfaces Google OAuth failures as auth-required errors", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-google-auth-error-")); + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/auth-error.discovery.json`, + auth: { + type: "oauth2", + tokenUrl: `${baseUrl}/token`, + clientId: "client", + }, + }, + }, + }); + writeTokenBundle( + { + server: "drive", + authType: "oauth2", + accessToken: "expired-access-token", + tokenType: "Bearer", + expiresAt: "2999-01-01T00:00:00.000Z", + clientId: "client", + protectedResourceOrigin: baseUrl, + metadata: { + requestedScopes: ["https://www.googleapis.com/auth/drive.readonly"], + }, + }, + dir, + ); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config), { authDir: dir }); + + try { + await expect( + manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.protected", {}), + ).rejects.toMatchObject({ + code: "AUTH_REQUIRED", + details: { + server: "drive", + status: 401, + nextAction: "run_caplets_auth_login", + challenge: "[REDACTED]", + }, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/packages/core/test/media-artifacts.test.ts b/packages/core/test/media-artifacts.test.ts index 5188ad6b..82f38e9d 100644 --- a/packages/core/test/media-artifacts.test.ts +++ b/packages/core/test/media-artifacts.test.ts @@ -247,6 +247,29 @@ describe("media artifacts", () => { }); }); + it("can forbid local media file paths for remote runtimes", async () => { + const root = tempDir("caplets-artifacts-"); + const file = join(root, "local.txt"); + writeFileSync(file, "local"); + + await expect( + readMediaInput({ path: file }, { artifactRoot: root, allowLocalPaths: false }), + ).rejects.toMatchObject({ + code: "REQUEST_INVALID", + message: "media.path is not available in this runtime", + }); + + await expect( + readMediaInput( + { dataUrl: "data:text/plain;base64,bG9jYWw=", filename: "local.txt" }, + { artifactRoot: root, allowLocalPaths: false }, + ), + ).resolves.toMatchObject({ + filename: "local.txt", + bytes: Buffer.from("local"), + }); + }); + it("rejects multiple media input sources and non-base64 data URLs", async () => { const root = tempDir("caplets-artifacts-"); await expect( diff --git a/packages/core/test/openapi.test.ts b/packages/core/test/openapi.test.ts index 89b7dadd..f472b9bc 100644 --- a/packages/core/test/openapi.test.ts +++ b/packages/core/test/openapi.test.ts @@ -94,6 +94,19 @@ describe("native OpenAPI Caplets", () => { response.end(JSON.stringify({ public: "ok", secret: "hidden" })); return; } + if (request.url === "/reports/large") { + const bytes = Buffer.alloc(1024 * 1024 + 1, "x"); + response.setHeader("content-type", "application/pdf"); + response.setHeader("content-length", String(bytes.byteLength)); + response.end(bytes); + return; + } + if (request.url === "/reports/42" && request.method === "HEAD") { + response.setHeader("content-type", "application/pdf"); + response.setHeader("content-length", String(1024 * 1024 + 1)); + response.end(); + return; + } if (request.url === "/reports/42") { response.setHeader("content-type", "application/pdf"); response.end(Buffer.from("%PDF-1.7 test")); @@ -191,7 +204,7 @@ describe("native OpenAPI Caplets", () => { )) as any; expect( list.structuredContent.result.items.map((tool: { name: string }) => tool.name), - ).toEqual(["createUser", "GET /users/{id}", "getReport"]); + ).toEqual(["createUser", "GET /users/{id}", "getReport", "headReport"]); expect( list.structuredContent.result.items.find( (candidate: { name: string }) => candidate.name === "GET /users/{id}", @@ -695,7 +708,7 @@ describe("native OpenAPI Caplets", () => { const openapi = new OpenApiManager(registry); const remote = registry.config.openapiEndpoints.remote!; - await expect(openapi.listTools(remote)).resolves.toHaveLength(3); + await expect(openapi.listTools(remote)).resolves.toHaveLength(4); await expect( openapi.listTools({ ...remote, specUrl: `${baseUrl}/slow-openapi.json` }), ).rejects.toMatchObject({ code: "TOOL_CALL_TIMEOUT" }); @@ -991,6 +1004,84 @@ describe("native OpenAPI Caplets", () => { rmSync(dir, { recursive: true, force: true }); } }); + + it("writes large OpenAPI responses as media artifacts", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-openapi-large-artifacts-")); + const specPath = join(dir, "openapi.json"); + const artifactDir = join(dir, "artifacts"); + writeFileSync(specPath, JSON.stringify(openApiSpec(baseUrl))); + const config = parseConfig({ + openapiEndpoints: { + reports: { + name: "Reports API", + description: "Download reports from the internal HTTP API.", + specPath, + baseUrl, + auth: { type: "none" }, + }, + }, + }); + const registry = new ServerRegistry(config); + const openapi = new OpenApiManager(registry, { artifactDir }); + + try { + const result = await openapi.callTool(config.openapiEndpoints.reports!, "getReport", { + path: { id: "large" }, + }); + const structured = result.structuredContent as { + status: number; + body: { artifact: { path: string; mimeType: string; byteLength: number } }; + }; + + expect(structured).toMatchObject({ + status: 200, + body: { + artifact: { + mimeType: "application/pdf", + byteLength: 1024 * 1024 + 1, + }, + }, + }); + expect(readFileSync(structured.body.artifact.path).byteLength).toBe(1024 * 1024 + 1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("does not reject OpenAPI HEAD responses by content length", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-openapi-head-")); + const specPath = join(dir, "openapi.json"); + writeFileSync(specPath, JSON.stringify(openApiSpec(baseUrl))); + const config = parseConfig({ + openapiEndpoints: { + reports: { + name: "Reports API", + description: "Download reports from the internal HTTP API.", + specPath, + baseUrl, + auth: { type: "none" }, + }, + }, + }); + const registry = new ServerRegistry(config); + const openapi = new OpenApiManager(registry); + + try { + const result = await openapi.callTool(config.openapiEndpoints.reports!, "headReport", { + path: { id: "42" }, + }); + + expect(result.structuredContent).toMatchObject({ + status: 200, + headers: { + "content-type": "application/pdf", + }, + }); + expect(result.structuredContent).not.toHaveProperty("body"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); function openApiSpec(baseUrl: string) { @@ -1075,6 +1166,26 @@ function openApiSpec(baseUrl: string) { }, }, }, + head: { + operationId: "headReport", + summary: "Inspect a report", + parameters: [ + { + name: "id", + in: "path", + required: true, + schema: { type: "string" }, + }, + ], + responses: { + "200": { + description: "OK", + content: { + "application/pdf": {}, + }, + }, + }, + }, }, }, }; From 4d6c21ba9a1f61ff17b77bd16fa96441cf375bb9 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Tue, 16 Jun 2026 19:07:50 -0400 Subject: [PATCH 5/9] fix: finish google discovery review follow-ups --- packages/core/src/google-discovery/manager.ts | 33 ++- .../core/src/google-discovery/operations.ts | 24 +- packages/core/src/media/input.ts | 37 +++- packages/core/test/google-discovery.test.ts | 206 +++++++++++++++++- packages/core/test/media-artifacts.test.ts | 1 + 5 files changed, 287 insertions(+), 14 deletions(-) diff --git a/packages/core/src/google-discovery/manager.ts b/packages/core/src/google-discovery/manager.ts index d60de36d..3c13174d 100644 --- a/packages/core/src/google-discovery/manager.ts +++ b/packages/core/src/google-discovery/manager.ts @@ -30,6 +30,7 @@ import type { GoogleDiscoveryDocument } from "./types"; const DEFAULT_RESUMABLE_THRESHOLD_BYTES = 8 * 1024 * 1024; const DEFAULT_MEDIA_RESPONSE_MAX_BYTES = 100 * 1024 * 1024; +const DEFAULT_DISCOVERY_DOCUMENT_MAX_BYTES = 20 * 1024 * 1024; type ManagedGoogleDiscovery = { operations?: GoogleDiscoveryOperation[]; @@ -106,7 +107,10 @@ export class GoogleDiscoveryManager { if (operation.supportsMediaUpload && "media" in args) { return this.callMediaUpload(requestApi, operation, args); } - const url = buildGoogleDiscoveryUrl(requestApi, operation, args); + const requestArgs = shouldRequestMediaDownload(operation, args) + ? withMediaDownloadQuery(args) + : args; + const url = buildGoogleDiscoveryUrl(requestApi, operation, requestArgs); const headers = new Headers( await authHeaders(requestApi, this.options.authDir, operation.scopes), ); @@ -136,7 +140,7 @@ export class GoogleDiscoveryManager { ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), ...(typeof args.filename === "string" ? { filename: args.filename } : {}), ...(typeof args.outputPath === "string" ? { outputPath: args.outputPath } : {}), - ...(operation.supportsMediaDownload ? { maxBytes: DEFAULT_MEDIA_RESPONSE_MAX_BYTES } : {}), + maxBytes: DEFAULT_MEDIA_RESPONSE_MAX_BYTES, ...(operation.supportsMediaDownload && (typeof args.filename === "string" || typeof args.outputPath === "string") ? { forceArtifact: true } @@ -588,6 +592,7 @@ async function fetchDiscoverySource( ); } return readLimitedText(response, { + maxBytes: DEFAULT_DISCOVERY_DOCUMENT_MAX_BYTES, errorMessage: "Google Discovery document exceeded byte limit", }); } catch (error) { @@ -618,6 +623,30 @@ async function authHeaders( } } +function shouldRequestMediaDownload( + operation: GoogleDiscoveryOperation, + args: Record, +): boolean { + return ( + operation.supportsMediaDownload && + (typeof args.filename === "string" || typeof args.outputPath === "string") + ); +} + +function withMediaDownloadQuery(args: Record): Record { + const query = + args.query && typeof args.query === "object" && !Array.isArray(args.query) + ? (args.query as Record) + : {}; + return { + ...args, + query: { + ...query, + alt: "media", + }, + }; +} + function googleAuthError(api: GoogleDiscoveryApiConfig, response: Response): CapletsError { return new CapletsError( response.status === 401 ? "AUTH_REQUIRED" : "AUTH_FAILED", diff --git a/packages/core/src/google-discovery/operations.ts b/packages/core/src/google-discovery/operations.ts index ce457481..de2d71ed 100644 --- a/packages/core/src/google-discovery/operations.ts +++ b/packages/core/src/google-discovery/operations.ts @@ -122,9 +122,10 @@ function operationFromMethod( const name = entry.method.id ?? [server, ...entry.resourcePath, entry.methodKey].join("."); const scopes = [...new Set(entry.method.scopes ?? [])].sort(); const inputSchema = buildInputSchema(document.parameters ?? {}, entry.method, schemas); - const outputSchema = entry.method.response?.$ref + const bodyOutputSchema = entry.method.response?.$ref ? googleDiscoverySchemaToJsonSchema(entry.method.response, schemas) : undefined; + const outputSchema = bodyOutputSchema ? structuredOutputSchema(bodyOutputSchema) : undefined; const mediaUpload = entry.method.mediaUpload?.accept || entry.method.mediaUpload?.maxSize ? { @@ -155,6 +156,27 @@ function operationFromMethod( }; } +function structuredOutputSchema(bodySchema: Record): Record { + return { + type: "object", + additionalProperties: false, + required: ["status", "statusText", "headers"], + properties: { + status: { type: "number" }, + statusText: { type: "string" }, + headers: { + type: "object", + additionalProperties: false, + required: ["content-type"], + properties: { + "content-type": { type: "string" }, + }, + }, + body: bodySchema, + }, + }; +} + function buildInputSchema( globalParameters: Record, method: GoogleDiscoveryMethod, diff --git a/packages/core/src/media/input.ts b/packages/core/src/media/input.ts index 4f2a4e29..aba77b8a 100644 --- a/packages/core/src/media/input.ts +++ b/packages/core/src/media/input.ts @@ -1,6 +1,6 @@ import { readFileSync, statSync } from "node:fs"; import type { Stats } from "node:fs"; -import { basename } from "node:path"; +import { basename, extname } from "node:path"; import { CapletsError } from "../errors"; import { resolveMediaArtifact } from "./artifacts"; @@ -43,10 +43,12 @@ export async function readMediaInput( } const stat = statMediaFile(media.path); enforceSize(stat.size, options.maxBytes); + const resolvedFilename = filename ?? basename(media.path); + const resolvedMimeType = mimeType ?? mimeTypeFromFilename(resolvedFilename); return { bytes: readMediaFile(media.path), - filename: filename ?? basename(media.path), - ...(mimeType ? { mimeType } : {}), + filename: resolvedFilename, + ...(resolvedMimeType ? { mimeType: resolvedMimeType } : {}), }; } @@ -73,6 +75,35 @@ export async function readMediaInput( return readDataUrl(media.dataUrl as string, dataUrlOptions); } +function mimeTypeFromFilename(filename: string): string | undefined { + switch (extname(filename).toLowerCase()) { + case ".csv": + return "text/csv"; + case ".gif": + return "image/gif"; + case ".htm": + case ".html": + return "text/html"; + case ".jpeg": + case ".jpg": + return "image/jpeg"; + case ".json": + return "application/json"; + case ".pdf": + return "application/pdf"; + case ".png": + return "image/png"; + case ".txt": + return "text/plain"; + case ".webp": + return "image/webp"; + case ".xml": + return "application/xml"; + default: + return undefined; + } +} + function readDataUrl( dataUrl: string, options: { filename?: string; mimeType?: string; maxBytes?: number }, diff --git a/packages/core/test/google-discovery.test.ts b/packages/core/test/google-discovery.test.ts index c8742b89..d3d957f5 100644 --- a/packages/core/test/google-discovery.test.ts +++ b/packages/core/test/google-discovery.test.ts @@ -191,6 +191,74 @@ beforeEach(async () => { ); return; } + if (url === "/download-alt.discovery.json") { + response.end( + JSON.stringify({ + kind: "discovery#restDescription", + rootUrl: `${baseUrl}/`, + servicePath: "drive/v3/", + resources: { + files: { + methods: { + export: { + id: "drive.files.export", + path: "files/{fileId}", + httpMethod: "GET", + supportsMediaDownload: true, + parameters: { + fileId: { type: "string", location: "path", required: true }, + }, + }, + }, + }, + }, + }), + ); + return; + } + if (url === "/large-response.discovery.json") { + response.end( + JSON.stringify({ + kind: "discovery#restDescription", + rootUrl: `${baseUrl}/`, + servicePath: "drive/v3/", + resources: { + files: { + methods: { + largeList: { + id: "drive.files.largeList", + path: "files/large-list", + httpMethod: "GET", + }, + }, + }, + }, + }), + ); + return; + } + if (url === "/large-document.discovery.json") { + response.end( + JSON.stringify({ + kind: "discovery#restDescription", + description: "x".repeat(1024 * 1024 + 1), + rootUrl: `${baseUrl}/`, + servicePath: "drive/v3/", + resources: { + files: { + methods: { + list: { + id: "drive.files.list", + path: "files", + httpMethod: "GET", + }, + }, + }, + }, + }), + ); + return; + } if (url === "/redirect.discovery.json") { response.statusCode = 302; response.setHeader("location", "/drive.discovery.json"); @@ -206,17 +274,17 @@ beforeEach(async () => { response.end(JSON.stringify({ id: "2", name: "Created" })); return; } - if (url === "/drive/v3/files/1/download") { + if (url === "/drive/v3/files/1/download?alt=media") { response.setHeader("content-type", "application/pdf"); response.end("%PDF bytes"); return; } - if (url === "/drive/v3/files/text/download") { + if (url === "/drive/v3/files/text/download?alt=media") { response.setHeader("content-type", "text/plain"); response.end("plain text export"); return; } - if (url === "/drive/v3/files/large/download") { + if (url === "/drive/v3/files/large/download?alt=media") { const bytes = Buffer.alloc(1024 * 1024 + 1, "x"); response.setHeader("content-type", "application/pdf"); response.setHeader("content-length", String(bytes.byteLength)); @@ -227,6 +295,25 @@ beforeEach(async () => { response.end(JSON.stringify({ id: "folders/1" })); return; } + if (url === "/drive/v3/files/1?alt=media") { + response.setHeader("content-type", "application/pdf"); + response.end("%PDF alt media"); + return; + } + if (url === "/drive/v3/files/1") { + response.end(JSON.stringify({ id: "1", name: "Metadata only" })); + return; + } + if (url === "/drive/v3/files/large-list") { + const body = JSON.stringify({ + files: [{ id: "1", name: "Report" }], + padding: "x".repeat(1024 * 1024 + 1), + }); + response.setHeader("content-type", "application/json"); + response.setHeader("content-length", String(Buffer.byteLength(body))); + response.end(body); + return; + } if (url === "/drive/v3/protected") { response.statusCode = 401; response.statusMessage = "Unauthorized"; @@ -321,14 +408,21 @@ describe("Google Discovery parser", () => { }, outputSchema: { properties: { - files: { - items: { - properties: { - id: { type: "string" }, - name: { type: "string" }, + body: { + properties: { + files: { + items: { + properties: { + id: { type: "string" }, + name: { type: "string" }, + }, + }, }, }, }, + headers: { type: "object" }, + status: { type: "number" }, + statusText: { type: "string" }, }, }, }); @@ -515,6 +609,24 @@ describe("GoogleDiscoveryManager", () => { ); }); + it("loads Discovery documents larger than the default HTTP text cap", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/large-document.discovery.json`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + + await expect(manager.listTools(config.googleDiscoveryApis.drive!)).resolves.toEqual([ + expect.objectContaining({ name: "drive.files.list" }), + ]); + }); + it("infers the request base URL from Discovery rootUrl and servicePath", async () => { const config = parseConfig({ googleDiscoveryApis: { @@ -736,6 +848,27 @@ describe("GoogleDiscoveryManager", () => { list.structuredContent.result.items.map((tool: { name: string }) => tool.name), ).toContain("drive.files.list"); + const call = await handleServerTool( + caplet, + { + operation: "call_tool", + name: "drive.files.list", + args: { query: { pageSize: 2 } }, + fields: ["body.files"], + }, + registry, + downstream, + undefined, + undefined, + undefined, + undefined, + undefined, + {}, + manager, + ); + + expect(call.structuredContent).toEqual({ body: { files: [{ id: "1", name: "Report" }] } }); + await downstream.close(); }); @@ -802,6 +935,30 @@ describe("GoogleDiscoveryManager", () => { } }); + it("adds alt=media when writing Google media downloads as artifacts", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/download-alt.discovery.json`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + const result = await manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.export", { + path: { fileId: "1" }, + filename: "export.pdf", + }); + + expect(result.structuredContent).toMatchObject({ + status: 200, + body: { artifact: { filename: "export.pdf", mimeType: "application/pdf" } }, + }); + expect(requests.find((request) => request.url === "/drive/v3/files/1?alt=media")).toBeDefined(); + }); + it("writes large Google media downloads as artifacts", async () => { const config = parseConfig({ googleDiscoveryApis: { @@ -836,6 +993,39 @@ describe("GoogleDiscoveryManager", () => { }); }); + it("writes large non-media Google responses as artifacts", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/large-response.discovery.json`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + const result = await manager.callTool( + config.googleDiscoveryApis.drive!, + "drive.files.largeList", + {}, + ); + + expect(result.structuredContent).toMatchObject({ + status: 200, + body: { + artifact: { + mimeType: "application/json", + byteLength: expect.any(Number), + }, + }, + }); + expect( + (result.structuredContent as { body: { artifact: { byteLength: number } } }).body.artifact + .byteLength, + ).toBeGreaterThan(1024 * 1024); + }); + it("uploads media from dataUrl using multipart when metadata body is present", async () => { const config = parseConfig({ googleDiscoveryApis: { diff --git a/packages/core/test/media-artifacts.test.ts b/packages/core/test/media-artifacts.test.ts index 82f38e9d..f5a7677c 100644 --- a/packages/core/test/media-artifacts.test.ts +++ b/packages/core/test/media-artifacts.test.ts @@ -227,6 +227,7 @@ describe("media artifacts", () => { await expect(readMediaInput({ path: file }, { artifactRoot: root })).resolves.toMatchObject({ bytes: Buffer.from("png"), filename: "image.png", + mimeType: "image/png", }); await expect( readMediaInput({ artifact: artifact.uri }, { artifactRoot: root }), From a9e2812fd6223ce3bd6b13d4d0ba23a9be544e5e Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Tue, 16 Jun 2026 20:08:57 -0400 Subject: [PATCH 6/9] fix: harden google discovery auth follow-ups --- packages/core/src/google-discovery/manager.ts | 56 ++++++++- .../core/src/google-discovery/operations.ts | 24 +++- packages/core/src/http/response.ts | 2 +- packages/core/test/google-discovery.test.ts | 110 ++++++++++++++++++ packages/core/test/media-artifacts.test.ts | 28 +++++ 5 files changed, 212 insertions(+), 8 deletions(-) diff --git a/packages/core/src/google-discovery/manager.ts b/packages/core/src/google-discovery/manager.ts index 3c13174d..8ef4bf03 100644 --- a/packages/core/src/google-discovery/manager.ts +++ b/packages/core/src/google-discovery/manager.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { readFile } from "node:fs/promises"; import type { CompatibilityCallToolResult, Tool } from "@modelcontextprotocol/sdk/types"; -import { genericOAuthHeaders } from "../auth"; +import { genericOAuthHeaders, readTokenBundle } from "../auth"; import type { GoogleDiscoveryApiConfig } from "../config"; import { compactToolSafetyHints, @@ -267,6 +267,7 @@ export class GoogleDiscoveryManager { "Google resumable upload missing Location", ); } + const uploadUrl = new URL(location); const uploadHeaders = new Headers(); uploadHeaders.set("content-type", media.mimeType ?? "application/octet-stream"); uploadHeaders.set("content-length", String(media.bytes.byteLength)); @@ -274,7 +275,8 @@ export class GoogleDiscoveryManager { "content-range", `bytes 0-${media.bytes.byteLength - 1}/${media.bytes.byteLength}`, ); - return fetchGoogleRequest(api, operation, new URL(location), { + copySessionAuthorization(api, startUrl, uploadUrl, headers, uploadHeaders); + return fetchGoogleRequest(api, operation, uploadUrl, { method: "PUT", headers: uploadHeaders, body: media.bytes, @@ -560,10 +562,7 @@ async function loadGoogleDiscoverySource( `${api.server} is missing Google Discovery document source`, ); } - return fetchDiscoverySource( - api, - shouldSendDiscoveryAuth(api) ? await authHeaders(api, authDir) : {}, - ); + return fetchDiscoverySource(api, await discoveryAuthHeaders(api, authDir)); } async function fetchDiscoverySource( @@ -605,6 +604,20 @@ async function fetchDiscoverySource( } } +async function discoveryAuthHeaders( + api: GoogleDiscoveryApiConfig, + authDir?: string, +): Promise> { + if (!shouldSendDiscoveryAuth(api)) return {}; + if ( + (api.auth.type === "oauth2" || api.auth.type === "oidc") && + !hasStoredOAuthBundle(api, authDir) + ) { + return {}; + } + return authHeaders(api, authDir); +} + async function authHeaders( api: GoogleDiscoveryApiConfig, authDir?: string, @@ -623,6 +636,37 @@ async function authHeaders( } } +function hasStoredOAuthBundle(api: GoogleDiscoveryApiConfig, authDir?: string): boolean { + const bundle = readTokenBundle(api.server, authDir); + return Boolean(bundle?.accessToken || bundle?.refreshToken); +} + +function copySessionAuthorization( + api: GoogleDiscoveryApiConfig, + startUrl: URL, + uploadUrl: URL, + source: Headers, + target: Headers, +): void { + const authorization = source.get("authorization"); + if (!authorization || !isAllowedUploadSessionOrigin(api, startUrl, uploadUrl)) return; + target.set("authorization", authorization); +} + +function isAllowedUploadSessionOrigin( + api: GoogleDiscoveryApiConfig, + startUrl: URL, + uploadUrl: URL, +): boolean { + if (uploadUrl.origin === startUrl.origin) return true; + if (api.baseUrl && uploadUrl.origin === new URL(api.baseUrl).origin) return true; + return isGoogleApiOrigin(uploadUrl); +} + +function isGoogleApiOrigin(url: URL): boolean { + return url.protocol === "https:" && url.hostname.endsWith(".googleapis.com"); +} + function shouldRequestMediaDownload( operation: GoogleDiscoveryOperation, args: Record, diff --git a/packages/core/src/google-discovery/operations.ts b/packages/core/src/google-discovery/operations.ts index de2d71ed..4cbd64ab 100644 --- a/packages/core/src/google-discovery/operations.ts +++ b/packages/core/src/google-discovery/operations.ts @@ -120,7 +120,7 @@ function operationFromMethod( ): GoogleDiscoveryOperation { const method = normalizedHttpMethod(entry.method.httpMethod); const name = entry.method.id ?? [server, ...entry.resourcePath, entry.methodKey].join("."); - const scopes = [...new Set(entry.method.scopes ?? [])].sort(); + const scopes = selectGoogleDiscoveryScopes(entry.method.scopes); const inputSchema = buildInputSchema(document.parameters ?? {}, entry.method, schemas); const bodyOutputSchema = entry.method.response?.$ref ? googleDiscoverySchemaToJsonSchema(entry.method.response, schemas) @@ -156,6 +156,28 @@ function operationFromMethod( }; } +function selectGoogleDiscoveryScopes(scopes: string[] | undefined): string[] { + const unique = [...new Set(scopes ?? [])].sort(); + const preferred = unique.toSorted(compareScopePreference)[0]; + return preferred ? [preferred] : []; +} + +function compareScopePreference(left: string, right: string): number { + const leftRank = scopePreferenceRank(left); + const rightRank = scopePreferenceRank(right); + return leftRank - rightRank || right.length - left.length || left.localeCompare(right); +} + +function scopePreferenceRank(scope: string): number { + const suffix = scope.toLowerCase().split("/").pop() ?? scope.toLowerCase(); + const tokens = suffix.split(/[._:-]+/u); + if (tokens.includes("readonly")) return 0; + if (tokens.includes("file")) return 1; + if (tokens.includes("metadata") || tokens.includes("appdata")) return 2; + if (tokens.includes("read")) return 3; + return 4; +} + function structuredOutputSchema(bodySchema: Record): Record { return { type: "object", diff --git a/packages/core/src/http/response.ts b/packages/core/src/http/response.ts index e9b6328b..e63330e1 100644 --- a/packages/core/src/http/response.ts +++ b/packages/core/src/http/response.ts @@ -119,7 +119,7 @@ async function writeResponseArtifact( return await writeMediaArtifact({ capletId: options.capletId, ...(options.artifactDir ? { rootDir: options.artifactDir } : {}), - ...(options.outputPath ? { outputPath: options.outputPath } : {}), + ...(response.ok && options.outputPath ? { outputPath: options.outputPath } : {}), ...(options.exposeLocalPath === false ? { exposeLocalPath: false } : {}), suggestedFilename: options.filename ?? filenameFromContentDisposition(response) ?? "response.bin", diff --git a/packages/core/test/google-discovery.test.ts b/packages/core/test/google-discovery.test.ts index d3d957f5..8b5fdfa7 100644 --- a/packages/core/test/google-discovery.test.ts +++ b/packages/core/test/google-discovery.test.ts @@ -460,6 +460,40 @@ describe("Google Discovery parser", () => { }); }); + it("selects a minimal Google scope alternative instead of requiring every method scope", () => { + const operations = discoveryOperations({ + server: "drive", + document: { + kind: "discovery#restDescription", + resources: { + files: { + methods: { + list: { + id: "drive.files.list", + path: "files", + httpMethod: "GET", + scopes: [ + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/drive.metadata.readonly", + ], + }, + }, + }, + }, + }, + }); + + expect(operations).toEqual([ + expect.objectContaining({ + name: "drive.files.list", + scopes: ["https://www.googleapis.com/auth/drive.metadata.readonly"], + }), + ]); + expect(googleDiscoveryScopesForOperations(operations)).toEqual([ + "https://www.googleapis.com/auth/drive.metadata.readonly", + ]); + }); + it("walks nested resources and uses stable fallback operation names", () => { const operations = discoveryOperations({ server: "drive", @@ -627,6 +661,37 @@ describe("GoogleDiscoveryManager", () => { ]); }); + it("fetches public remote Discovery documents before requiring OAuth credentials", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-google-public-discovery-")); + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { + type: "oauth2", + tokenUrl: `${baseUrl}/token`, + clientId: "client", + }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config), { authDir: dir }); + + try { + await expect(manager.listTools(config.googleDiscoveryApis.drive!)).resolves.toEqual( + expect.arrayContaining([expect.objectContaining({ name: "drive.files.list" })]), + ); + expect( + requests.find((request) => request.url === "/drive.discovery.json")?.headers, + ).not.toHaveProperty("authorization"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("infers the request base URL from Discovery rootUrl and servicePath", async () => { const config = parseConfig({ googleDiscoveryApis: { @@ -1077,6 +1142,51 @@ describe("GoogleDiscoveryManager", () => { expect(requests.find((request) => request.url.includes("uploadType=media"))).toBeUndefined(); }); + it("preserves OAuth authorization on resumable upload session requests", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-google-resumable-auth-")); + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/upload-resumable.discovery.json`, + auth: { + type: "oauth2", + tokenUrl: `${baseUrl}/token`, + clientId: "client", + }, + }, + }, + }); + writeTokenBundle( + { + server: "drive", + authType: "oauth2", + accessToken: "upload-token", + tokenType: "Bearer", + expiresAt: "2999-01-01T00:00:00.000Z", + clientId: "client", + protectedResourceOrigin: baseUrl, + }, + dir, + ); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config), { authDir: dir }); + + try { + await manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.create", { + body: { name: "report.pdf" }, + media: { dataUrl: "data:application/pdf;base64,cGRm", filename: "report.pdf" }, + }); + const start = requests.find((request) => request.url.includes("uploadType=resumable")); + const upload = requests.find((request) => request.url === "/upload/session/abc"); + + expect(start?.headers.authorization).toBe("Bearer upload-token"); + expect(upload?.headers.authorization).toBe("Bearer upload-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("substitutes path and query args into media upload URLs", async () => { const config = parseConfig({ googleDiscoveryApis: { diff --git a/packages/core/test/media-artifacts.test.ts b/packages/core/test/media-artifacts.test.ts index f5a7677c..3ee5a67b 100644 --- a/packages/core/test/media-artifacts.test.ts +++ b/packages/core/test/media-artifacts.test.ts @@ -16,6 +16,7 @@ import { resolveMediaArtifact, writeMediaArtifact, } from "../src/media"; +import { readHttpLikeResponse } from "../src/http/response"; describe("media artifacts", () => { const dirs: string[] = []; @@ -134,6 +135,33 @@ describe("media artifacts", () => { expect(resolveMediaArtifact(first.uri, { artifactRoot: root }).mimeType).toBeUndefined(); }); + it("does not overwrite explicit output paths with failed forced responses", async () => { + const root = tempDir("caplets-artifacts-"); + const outputDir = join(root, "drive", "call-1"); + const outputPath = join(outputDir, "report.pdf"); + mkdirSync(outputDir, { recursive: true }); + writeFileSync(outputPath, "previous-pdf"); + + const result = await readHttpLikeResponse( + new Response(JSON.stringify({ error: "missing" }), { + status: 404, + statusText: "Not Found", + headers: { "content-type": "application/json" }, + }), + { + capletId: "drive", + artifactDir: root, + outputPath, + forceArtifact: true, + }, + ); + + const artifact = (result.body as { artifact: { path?: string } }).artifact; + expect(result.status).toBe(404); + expect(readFileSync(outputPath, "utf8")).toBe("previous-pdf"); + expect(artifact.path).not.toBe(outputPath); + }); + it("rejects oversized artifact and data URL inputs before reading decoded bytes", async () => { const root = tempDir("caplets-artifacts-"); const artifact = await writeMediaArtifact({ From 0f8e13f6b98e97a426431628eade4c08db99f7a3 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Tue, 16 Jun 2026 21:12:50 -0400 Subject: [PATCH 7/9] fix: cover google discovery upload edge cases --- packages/core/src/auth.ts | 13 +- packages/core/src/google-discovery/manager.ts | 125 ++++++++++++------ packages/core/src/tools.ts | 18 +++ packages/core/test/auth.test.ts | 36 +++++ packages/core/test/google-discovery.test.ts | 102 +++++++++++++- packages/core/test/openapi.test.ts | 47 +++++++ 6 files changed, 298 insertions(+), 43 deletions(-) diff --git a/packages/core/src/auth.ts b/packages/core/src/auth.ts index c3799bff..c03812bc 100644 --- a/packages/core/src/auth.ts +++ b/packages/core/src/auth.ts @@ -1256,7 +1256,18 @@ function tokenBundleMissingScopes( if (required.length === 0) return false; const metadataScopes = requestedScopesFromMetadata(bundle.metadata); const actual = new Set(bundle.scope?.split(/\s+/u).filter(Boolean) ?? metadataScopes ?? []); - return required.some((scope) => !actual.has(scope)); + return required.some( + (scope) => ![...actual].some((grantedScope) => oauthScopeSatisfies(grantedScope, scope)), + ); +} + +function oauthScopeSatisfies(grantedScope: string, requiredScope: string): boolean { + if (grantedScope === requiredScope) return true; + const googleScopePrefix = "https://www.googleapis.com/auth/"; + if (!grantedScope.startsWith(googleScopePrefix) || !requiredScope.startsWith(googleScopePrefix)) { + return false; + } + return requiredScope.startsWith(`${grantedScope}.`); } function requiredStoredScopes( diff --git a/packages/core/src/google-discovery/manager.ts b/packages/core/src/google-discovery/manager.ts index 8ef4bf03..f42f57e0 100644 --- a/packages/core/src/google-discovery/manager.ts +++ b/packages/core/src/google-discovery/manager.ts @@ -188,26 +188,48 @@ export class GoogleDiscoveryManager { const media = await readMediaInput(args.media, mediaOptions); const headers = new Headers(await authHeaders(api, this.options.authDir, operation.scopes)); const protocol = selectUploadProtocol(operation, media, args); - const response = - protocol === "resumable" - ? await this.callResumableUpload(api, operation, args, media, headers) - : await this.callSingleUpload(api, operation, args, media, headers, protocol); - const parsed = await readHttpLikeResponse(response, { - capletId: api.server, - method: operation.method, - ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), - ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), - }); - return { - content: markdownStructuredContent(parsed, { - title: `${api.name} call_tool ${operation.name}`, - backend: "googleDiscovery", - operation: "call_tool", - tool: operation.name, - }), - structuredContent: parsed as Record, - isError: !response.ok, - }; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), api.requestTimeoutMs); + try { + const response = + protocol === "resumable" + ? await this.callResumableUpload(api, operation, args, media, headers, controller.signal) + : await this.callSingleUpload( + api, + operation, + args, + media, + headers, + protocol, + controller.signal, + ); + const parsed = await readHttpLikeResponse(response, { + capletId: api.server, + method: operation.method, + ...(this.options.artifactDir ? { artifactDir: this.options.artifactDir } : {}), + ...(this.options.exposeLocalArtifactPaths === false ? { exposeLocalPath: false } : {}), + }); + return { + content: markdownStructuredContent(parsed, { + title: `${api.name} call_tool ${operation.name}`, + backend: "googleDiscovery", + operation: "call_tool", + tool: operation.name, + }), + structuredContent: parsed as Record, + isError: !response.ok, + }; + } catch (error) { + if (isAbortError(error)) { + throw new CapletsError( + "TOOL_CALL_TIMEOUT", + `Google Discovery request timed out for ${api.server}/${operation.name}`, + ); + } + throw error; + } finally { + clearTimeout(timeout); + } } private async callSingleUpload( @@ -217,6 +239,7 @@ export class GoogleDiscoveryManager { media: ResolvedMediaInput, headers: Headers, protocol: "simple" | "multipart", + signal?: AbortSignal, ): Promise { const upload = operation.mediaUploadProtocols[protocol]; if (!upload?.path) { @@ -236,7 +259,7 @@ export class GoogleDiscoveryManager { protocol === "simple" ? simpleUploadInit(operation, media, headers) : multipartUploadInit(operation, args.body, media, headers); - return fetchGoogleRequest(api, operation, url, init); + return fetchGoogleRequest(api, operation, url, init, { signal }); } private async callResumableUpload( @@ -245,6 +268,7 @@ export class GoogleDiscoveryManager { args: Record, media: ResolvedMediaInput, headers: Headers, + signal?: AbortSignal, ): Promise { const upload = operation.mediaUploadProtocols.resumable; if (!upload?.path) { @@ -254,12 +278,21 @@ export class GoogleDiscoveryManager { headers.set("content-type", "application/json; charset=UTF-8"); headers.set("x-upload-content-type", media.mimeType ?? "application/octet-stream"); headers.set("x-upload-content-length", String(media.bytes.byteLength)); - const started = await fetchGoogleRequest(api, operation, startUrl, { - method: operation.method.toUpperCase(), - headers, - body: JSON.stringify(args.body ?? {}), - redirect: "manual", - }); + const started = await fetchGoogleRequest( + api, + operation, + startUrl, + { + method: operation.method.toUpperCase(), + headers, + body: JSON.stringify(args.body ?? {}), + redirect: "manual", + }, + { signal }, + ); + if (!started.ok) { + return started; + } const location = started.headers.get("location"); if (!location) { throw new CapletsError( @@ -271,17 +304,20 @@ export class GoogleDiscoveryManager { const uploadHeaders = new Headers(); uploadHeaders.set("content-type", media.mimeType ?? "application/octet-stream"); uploadHeaders.set("content-length", String(media.bytes.byteLength)); - uploadHeaders.set( - "content-range", - `bytes 0-${media.bytes.byteLength - 1}/${media.bytes.byteLength}`, - ); + uploadHeaders.set("content-range", resumableContentRange(media.bytes.byteLength)); copySessionAuthorization(api, startUrl, uploadUrl, headers, uploadHeaders); - return fetchGoogleRequest(api, operation, uploadUrl, { - method: "PUT", - headers: uploadHeaders, - body: media.bytes, - redirect: "manual", - }); + return fetchGoogleRequest( + api, + operation, + uploadUrl, + { + method: "PUT", + headers: uploadHeaders, + body: media.bytes, + redirect: "manual", + }, + { signal }, + ); } compact(_api: GoogleDiscoveryApiConfig, tool: Tool): CompactTool { @@ -476,6 +512,10 @@ function multipartUploadInit( }; } +function resumableContentRange(byteLength: number): string { + return byteLength === 0 ? "bytes */0" : `bytes 0-${byteLength - 1}/${byteLength}`; +} + function googleDiscoveryBaseUrl( api: GoogleDiscoveryApiConfig, document: GoogleDiscoveryDocument, @@ -493,11 +533,14 @@ async function fetchGoogleRequest( operation: GoogleDiscoveryOperation, url: URL, init: RequestInit, + options: { signal?: AbortSignal | undefined } = {}, ): Promise { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), api.requestTimeoutMs); + const controller = options.signal ? undefined : new AbortController(); + const timeout = controller + ? setTimeout(() => controller.abort(), api.requestTimeoutMs) + : undefined; try { - const response = await fetch(url, { ...init, signal: controller.signal }); + const response = await fetch(url, { ...init, signal: options.signal ?? controller!.signal }); if (response.status >= 300 && response.status < 400) { throw new CapletsError( "DOWNSTREAM_PROTOCOL_ERROR", @@ -522,7 +565,7 @@ async function fetchGoogleRequest( } throw error; } finally { - clearTimeout(timeout); + if (timeout) clearTimeout(timeout); } } diff --git a/packages/core/src/tools.ts b/packages/core/src/tools.ts index 2c5fd762..245faebe 100644 --- a/packages/core/src/tools.ts +++ b/packages/core/src/tools.ts @@ -1092,6 +1092,12 @@ export function projectCallToolResult( "Field selection requires the downstream tool to return object structuredContent", ); } + if (hasArtifactPlaceholderForSelectedFields(structuredContent, fields)) { + throw new CapletsError( + "REQUEST_INVALID", + "Field selection cannot project from an artifact response. Retry without fields and read the returned artifact.", + ); + } const projected = projectStructuredContent(structuredContent, outputSchema, fields); return { @@ -1101,6 +1107,18 @@ export function projectCallToolResult( } as T & CallToolResult; } +function hasArtifactPlaceholderForSelectedFields( + structuredContent: Record, + fields: string[], +): boolean { + const body = structuredContent.body; + return ( + isPlainObject(body) && + isPlainObject(body.artifact) && + fields.some((field) => field === "body" || field.startsWith("body.")) + ); +} + export function extractArtifacts(result: unknown): CapletArtifact[] { if (!isPlainObject(result) || !Array.isArray(result.content)) { return []; diff --git a/packages/core/test/auth.test.ts b/packages/core/test/auth.test.ts index 914bb758..fab15c73 100644 --- a/packages/core/test/auth.test.ts +++ b/packages/core/test/auth.test.ts @@ -398,6 +398,42 @@ describe("auth helpers", () => { } }); + it("accepts broader Google Discovery OAuth scope alternatives", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-auth-google-scope-alternative-")); + try { + writeTokenBundle( + { + server: "drive", + authType: "oauth2", + accessToken: "access-token", + tokenType: "Bearer", + expiresAt: "2999-01-01T00:00:00.000Z", + scope: "https://www.googleapis.com/auth/drive", + protectedResourceOrigin: "https://www.googleapis.com", + metadata: { + requestedScopes: ["https://www.googleapis.com/auth/drive"], + }, + }, + dir, + ); + + await expect( + genericOAuthHeaders( + { + server: "drive", + backend: "googleDiscovery", + baseUrl: "https://www.googleapis.com/drive/v3/", + auth: { type: "oauth2" }, + resolvedScopes: ["https://www.googleapis.com/auth/drive.metadata.readonly"], + }, + dir, + ), + ).resolves.toEqual({ authorization: "Bearer access-token" }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("rejects generic OAuth headers when refresh returns an expired token", async () => { const dir = mkdtempSync(join(tmpdir(), "caplets-auth-refresh-expired-")); const server = createServer((_request: IncomingMessage, response: ServerResponse) => { diff --git a/packages/core/test/google-discovery.test.ts b/packages/core/test/google-discovery.test.ts index 8b5fdfa7..06a35bbc 100644 --- a/packages/core/test/google-discovery.test.ts +++ b/packages/core/test/google-discovery.test.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -342,11 +342,31 @@ beforeEach(async () => { return; } if (url === "/upload/drive/v3/files?uploadType=resumable" && request.method === "POST") { + const metadata = bodyChunks.length + ? (JSON.parse(Buffer.concat(bodyChunks).toString("utf8")) as { name?: string }) + : {}; + if (metadata.name === "invalid-metadata.pdf") { + response.statusCode = 400; + response.end(JSON.stringify({ error: "invalid metadata" })); + return; + } + if (metadata.name === "slow-response.pdf") { + response.statusCode = 200; + response.setHeader("location", `${baseUrl}/upload/session/slow`); + response.end("{}"); + return; + } response.statusCode = 200; response.setHeader("location", `${baseUrl}/upload/session/abc`); response.end("{}"); return; } + if (url === "/upload/session/slow" && request.method === "PUT") { + response.writeHead(200, { "content-type": "application/json" }); + response.write("{"); + setTimeout(() => response.end('"id":"uploaded-resumable"}'), 100); + return; + } if (url === "/upload/session/abc" && request.method === "PUT") { response.end(JSON.stringify({ id: "uploaded-resumable" })); return; @@ -1142,6 +1162,86 @@ describe("GoogleDiscoveryManager", () => { expect(requests.find((request) => request.url.includes("uploadType=media"))).toBeUndefined(); }); + it("returns resumable upload initiation errors as downstream responses", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/upload-resumable.discovery.json`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + + const result = await manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.create", { + body: { name: "invalid-metadata.pdf" }, + media: { dataUrl: "data:application/pdf;base64,cGRm", filename: "report.pdf" }, + }); + + expect(result).toMatchObject({ + isError: true, + structuredContent: { + status: 400, + body: { error: "invalid metadata" }, + }, + }); + expect(requests.find((request) => request.url === "/upload/session/abc")).toBeUndefined(); + }); + + it("sends a valid content range for empty resumable uploads", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-google-empty-upload-")); + const path = join(dir, "empty.txt"); + writeFileSync(path, ""); + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/upload-resumable.discovery.json`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + + try { + await manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.create", { + body: { name: "empty.txt" }, + media: { path }, + }); + + const upload = requests.find((request) => request.url === "/upload/session/abc"); + expect(upload?.headers["content-length"]).toBe("0"); + expect(upload?.headers["content-range"]).toBe("bytes */0"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("keeps resumable upload response reads within the request timeout", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/upload-resumable.discovery.json`, + requestTimeoutMs: 20, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + + await expect( + manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.create", { + body: { name: "slow-response.pdf" }, + media: { dataUrl: "data:application/pdf;base64,cGRm", filename: "report.pdf" }, + }), + ).rejects.toMatchObject({ code: "TOOL_CALL_TIMEOUT" }); + }); + it("preserves OAuth authorization on resumable upload session requests", async () => { const dir = mkdtempSync(join(tmpdir(), "caplets-google-resumable-auth-")); const config = parseConfig({ diff --git a/packages/core/test/openapi.test.ts b/packages/core/test/openapi.test.ts index f472b9bc..d3a935c0 100644 --- a/packages/core/test/openapi.test.ts +++ b/packages/core/test/openapi.test.ts @@ -77,6 +77,12 @@ describe("native OpenAPI Caplets", () => { response.end(JSON.stringify({ id: "42", active: true, name: "Ada" })); return; } + if (request.url?.startsWith("/users/large?active=true")) { + response.end( + JSON.stringify({ id: "large", name: "Ada", padding: "x".repeat(1024 * 1024) }), + ); + return; + } if (request.url?.startsWith("/api/v1/users/42?active=true")) { response.end(JSON.stringify({ id: "42", active: true, prefixed: true })); return; @@ -1048,6 +1054,47 @@ describe("native OpenAPI Caplets", () => { } }); + it("rejects field projection when OpenAPI JSON responses become artifacts", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-openapi-large-json-fields-")); + const specPath = join(dir, "openapi.json"); + const artifactDir = join(dir, "artifacts"); + writeFileSync(specPath, JSON.stringify(openApiSpec(baseUrl))); + const config = parseConfig({ + openapiEndpoints: { + users: { + name: "Users API", + description: "Manage users through the internal HTTP API.", + specPath, + baseUrl, + auth: { type: "none" }, + }, + }, + }); + const registry = new ServerRegistry(config); + const caplet = config.openapiEndpoints.users!; + const openapi = new OpenApiManager(registry, { artifactDir }); + const downstream = new DownstreamManager(registry); + + try { + await expect( + handleServerTool( + caplet, + { + operation: "call_tool", + name: "GET /users/{id}", + args: { path: { id: "large" }, query: { active: true } }, + fields: ["body.name"], + }, + registry, + downstream, + openapi, + ), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("does not reject OpenAPI HEAD responses by content length", async () => { const dir = mkdtempSync(join(tmpdir(), "caplets-openapi-head-")); const specPath = join(dir, "openapi.json"); From 995de4878f230fb7b6aae5308ff12d8b8c0f32a1 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Tue, 16 Jun 2026 22:00:42 -0400 Subject: [PATCH 8/9] fix: address remaining google discovery review feedback --- packages/core/src/config.ts | 24 +++-- packages/core/src/google-discovery/manager.ts | 37 +++++-- .../core/src/google-discovery/operations.ts | 1 + packages/core/src/http/response.ts | 9 +- packages/core/test/config.test.ts | 32 ++++++ packages/core/test/google-discovery.test.ts | 99 +++++++++++++++++++ packages/core/test/media-artifacts.test.ts | 23 +++++ 7 files changed, 204 insertions(+), 21 deletions(-) diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index 2c99f70c..f3274a15 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -1283,18 +1283,24 @@ function configSchemaFor( for (const [api, rawValue] of Object.entries(config.googleDiscoveryApis)) { const raw = rawValue as ConfigSchemaGoogleDiscoveryApiValue; - if (config.mcpServers[api]) { - ctx.addIssue({ - code: "custom", - path: ["googleDiscoveryApis", api], - message: `Caplet ID ${api} is already used by mcpServers`, - }); - } - if (config.openapiEndpoints[api]) { + const duplicateBackend = config.mcpServers[api] + ? "mcpServers" + : config.openapiEndpoints[api] + ? "openapiEndpoints" + : config.graphqlEndpoints[api] + ? "graphqlEndpoints" + : config.httpApis[api] + ? "httpApis" + : config.cliTools[api] + ? "cliTools" + : config.capletSets[api] + ? "capletSets" + : undefined; + if (duplicateBackend) { ctx.addIssue({ code: "custom", path: ["googleDiscoveryApis", api], - message: `Caplet ID ${api} is already used by openapiEndpoints`, + message: `Caplet ID ${api} is already used by ${duplicateBackend}`, }); } if (!SERVER_ID_PATTERN.test(api)) { diff --git a/packages/core/src/google-discovery/manager.ts b/packages/core/src/google-discovery/manager.ts index f42f57e0..72cee66e 100644 --- a/packages/core/src/google-discovery/manager.ts +++ b/packages/core/src/google-discovery/manager.ts @@ -274,9 +274,10 @@ export class GoogleDiscoveryManager { if (!upload?.path) { throw new CapletsError("CONFIG_INVALID", "Google Discovery resumable upload path is missing"); } + const contentType = safeMediaContentType(media.mimeType); const startUrl = buildGoogleDiscoveryUploadUrl(api, operation, upload.path, "resumable", args); headers.set("content-type", "application/json; charset=UTF-8"); - headers.set("x-upload-content-type", media.mimeType ?? "application/octet-stream"); + headers.set("x-upload-content-type", contentType); headers.set("x-upload-content-length", String(media.bytes.byteLength)); const started = await fetchGoogleRequest( api, @@ -302,7 +303,7 @@ export class GoogleDiscoveryManager { } const uploadUrl = new URL(location); const uploadHeaders = new Headers(); - uploadHeaders.set("content-type", media.mimeType ?? "application/octet-stream"); + uploadHeaders.set("content-type", contentType); uploadHeaders.set("content-length", String(media.bytes.byteLength)); uploadHeaders.set("content-range", resumableContentRange(media.bytes.byteLength)); copySessionAuthorization(api, startUrl, uploadUrl, headers, uploadHeaders); @@ -474,7 +475,7 @@ function simpleUploadInit( media: ResolvedMediaInput, headers: Headers, ): RequestInit { - headers.set("content-type", media.mimeType ?? "application/octet-stream"); + headers.set("content-type", safeMediaContentType(media.mimeType)); headers.set("content-length", String(media.bytes.byteLength)); return { method: operation.method.toUpperCase(), @@ -491,7 +492,7 @@ function multipartUploadInit( headers: Headers, ): RequestInit { const boundary = `caplets_${randomUUID().replace(/-/gu, "")}`; - const contentType = media.mimeType ?? "application/octet-stream"; + const contentType = safeMediaContentType(media.mimeType); const payload = Buffer.concat([ Buffer.from( `--${boundary}\r\ncontent-type: application/json; charset=UTF-8\r\n\r\n${JSON.stringify( @@ -516,6 +517,14 @@ function resumableContentRange(byteLength: number): string { return byteLength === 0 ? "bytes */0" : `bytes 0-${byteLength - 1}/${byteLength}`; } +function safeMediaContentType(mimeType: string | undefined): string { + const contentType = mimeType ?? "application/octet-stream"; + if (/[\r\n]/u.test(contentType)) { + throw new CapletsError("REQUEST_INVALID", "media.mimeType must not contain line breaks"); + } + return contentType; +} + function googleDiscoveryBaseUrl( api: GoogleDiscoveryApiConfig, document: GoogleDiscoveryDocument, @@ -651,12 +660,10 @@ async function discoveryAuthHeaders( api: GoogleDiscoveryApiConfig, authDir?: string, ): Promise> { - if (!shouldSendDiscoveryAuth(api)) return {}; - if ( - (api.auth.type === "oauth2" || api.auth.type === "oidc") && - !hasStoredOAuthBundle(api, authDir) - ) { - return {}; + if (api.auth.type === "none") return {}; + if (api.auth.type === "oauth2" || api.auth.type === "oidc") { + if (!shouldSendDiscoveryAuth(api) && !hasStoredOAuthBundle(api, authDir)) return {}; + return storedOAuthAccessHeaders(api, authDir); } return authHeaders(api, authDir); } @@ -684,6 +691,16 @@ function hasStoredOAuthBundle(api: GoogleDiscoveryApiConfig, authDir?: string): return Boolean(bundle?.accessToken || bundle?.refreshToken); } +function storedOAuthAccessHeaders( + api: GoogleDiscoveryApiConfig, + authDir?: string, +): Record { + const bundle = readTokenBundle(api.server, authDir); + return bundle?.accessToken + ? { authorization: `${bundle.tokenType ?? "Bearer"} ${bundle.accessToken}` } + : {}; +} + function copySessionAuthorization( api: GoogleDiscoveryApiConfig, startUrl: URL, diff --git a/packages/core/src/google-discovery/operations.ts b/packages/core/src/google-discovery/operations.ts index 4cbd64ab..49842cc3 100644 --- a/packages/core/src/google-discovery/operations.ts +++ b/packages/core/src/google-discovery/operations.ts @@ -175,6 +175,7 @@ function scopePreferenceRank(scope: string): number { if (tokens.includes("file")) return 1; if (tokens.includes("metadata") || tokens.includes("appdata")) return 2; if (tokens.includes("read")) return 3; + if (suffix === "cloud-platform") return 5; return 4; } diff --git a/packages/core/src/http/response.ts b/packages/core/src/http/response.ts index e63330e1..6866ca7b 100644 --- a/packages/core/src/http/response.ts +++ b/packages/core/src/http/response.ts @@ -24,7 +24,7 @@ export async function readHttpLikeResponse( const maxInlineBytes = options.maxInlineBytes ?? DEFAULT_MAX_RESPONSE_BYTES; const maxBytes = options.maxBytes ?? DEFAULT_MAX_RESPONSE_BYTES; const method = options.method?.toUpperCase(); - rejectOversizedContentLength(response, maxBytes, method); + await rejectOversizedContentLength(response, maxBytes, method); if (method === "HEAD") { return responseEnvelope(response, contentType); } @@ -93,12 +93,17 @@ async function readBoundedBytes(response: Response, maxBytes: number): Promise { if (method === "HEAD") return; const contentLength = response.headers.get("content-length"); if (!contentLength) return; const byteLength = Number.parseInt(contentLength, 10); if (Number.isFinite(byteLength) && byteLength > maxBytes) { + await response.body?.cancel().catch(() => {}); throw responseExceededLimit(maxBytes); } } diff --git a/packages/core/test/config.test.ts b/packages/core/test/config.test.ts index 0ab66cd0..f7fc925c 100644 --- a/packages/core/test/config.test.ts +++ b/packages/core/test/config.test.ts @@ -2226,6 +2226,38 @@ describe("config", () => { ]) as unknown, }) as CapletsError, ); + expect(() => + parseConfig({ + httpApis: { + drive: { + name: "Drive HTTP", + description: "HTTP Drive wrapper.", + baseUrl: "https://example.com", + auth: { type: "none" }, + actions: { + list: { method: "GET", path: "/files" }, + }, + }, + }, + googleDiscoveryApis: { + drive: { + name: "Drive", + description: "Access Google Drive files.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + auth: { type: "none" }, + }, + }, + }), + ).toThrow( + expect.objectContaining({ + details: expect.arrayContaining([ + expect.objectContaining({ + path: ["googleDiscoveryApis", "drive"], + message: expect.stringContaining("already used by httpApis"), + }), + ]) as unknown, + }) as CapletsError, + ); expect(() => parseConfig({ googleDiscoveryApis: { diff --git a/packages/core/test/google-discovery.test.ts b/packages/core/test/google-discovery.test.ts index 06a35bbc..e9805312 100644 --- a/packages/core/test/google-discovery.test.ts +++ b/packages/core/test/google-discovery.test.ts @@ -47,6 +47,15 @@ beforeEach(async () => { response.end(JSON.stringify(fixture)); return; } + if (url === "/private.discovery.json") { + if (request.headers.authorization !== "Bearer private-discovery-token") { + response.statusCode = 401; + response.end(JSON.stringify({ error: "auth required" })); + return; + } + response.end(JSON.stringify(fixture)); + return; + } if (url === "/drive-inferred.discovery.json") { response.end( JSON.stringify({ @@ -514,6 +523,28 @@ describe("Google Discovery parser", () => { ]); }); + it("prefers API-specific scopes over catch-all cloud-platform", () => { + expect( + discoveryOperations({ + server: "bigquery", + document: { + kind: "discovery#restDescription", + methods: { + insert: { + id: "bigquery.jobs.insert", + path: "projects/{projectId}/jobs", + httpMethod: "POST", + scopes: [ + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/cloud-platform", + ], + }, + }, + }, + })[0]?.scopes, + ).toEqual(["https://www.googleapis.com/auth/bigquery"]); + }); + it("walks nested resources and uses stable fallback operation names", () => { const operations = discoveryOperations({ server: "drive", @@ -712,6 +743,48 @@ describe("GoogleDiscoveryManager", () => { } }); + it("sends stored OAuth credentials for private remote Discovery documents without baseUrl", async () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-google-private-discovery-")); + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/private.discovery.json`, + auth: { + type: "oauth2", + tokenUrl: `${baseUrl}/token`, + clientId: "client", + }, + }, + }, + }); + writeTokenBundle( + { + server: "drive", + authType: "oauth2", + accessToken: "private-discovery-token", + tokenType: "Bearer", + expiresAt: "2999-01-01T00:00:00.000Z", + clientId: "client", + protectedResourceOrigin: baseUrl, + }, + dir, + ); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config), { authDir: dir }); + + try { + await expect(manager.listTools(config.googleDiscoveryApis.drive!)).resolves.toEqual( + expect.arrayContaining([expect.objectContaining({ name: "drive.files.list" })]), + ); + expect( + requests.find((request) => request.url === "/private.discovery.json")?.headers, + ).toHaveProperty("authorization", "Bearer private-discovery-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("infers the request base URL from Discovery rootUrl and servicePath", async () => { const config = parseConfig({ googleDiscoveryApis: { @@ -1135,6 +1208,32 @@ describe("GoogleDiscoveryManager", () => { expect(upload?.body).not.toContain("cGRm"); }); + it("rejects media MIME types that would inject multipart headers", async () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive files.", + discoveryUrl: `${baseUrl}/drive.discovery.json`, + baseUrl: `${baseUrl}/drive/v3/`, + auth: { type: "none" }, + }, + }, + }); + const manager = new GoogleDiscoveryManager(new ServerRegistry(config)); + + await expect( + manager.callTool(config.googleDiscoveryApis.drive!, "drive.files.create", { + body: { name: "report.pdf" }, + media: { + dataUrl: "data:application/pdf;base64,cGRm", + filename: "report.pdf", + mimeType: "application/pdf\r\n\r\ninjected-body-data", + }, + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); + it("uses resumable upload to preserve metadata when multipart is unavailable", async () => { const config = parseConfig({ googleDiscoveryApis: { diff --git a/packages/core/test/media-artifacts.test.ts b/packages/core/test/media-artifacts.test.ts index 3ee5a67b..6ea0b606 100644 --- a/packages/core/test/media-artifacts.test.ts +++ b/packages/core/test/media-artifacts.test.ts @@ -162,6 +162,29 @@ describe("media artifacts", () => { expect(artifact.path).not.toBe(outputPath); }); + it("cancels oversized responses rejected by content-length before reading", async () => { + let cancelled = false; + const body = new ReadableStream({ + cancel() { + cancelled = true; + }, + }); + + await expect( + readHttpLikeResponse( + new Response(body, { + headers: { + "content-length": "5", + "content-type": "application/octet-stream", + }, + }), + { capletId: "drive", maxBytes: 4 }, + ), + ).rejects.toMatchObject({ code: "DOWNSTREAM_PROTOCOL_ERROR" }); + + expect(cancelled).toBe(true); + }); + it("rejects oversized artifact and data URL inputs before reading decoded bytes", async () => { const root = tempDir("caplets-artifacts-"); const artifact = await writeMediaArtifact({ From f914448fcc1589f6abdfdcd182de6eac0de4f28b Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Tue, 16 Jun 2026 22:04:36 -0400 Subject: [PATCH 9/9] fix: close remaining codex review gaps --- packages/core/src/attach/server.ts | 1 + packages/core/src/native/service.ts | 1 + .../core/src/observed-output-shapes/key.ts | 10 ++++++ packages/core/src/serve/http.ts | 4 ++- packages/core/src/tools.ts | 26 +++++++++++++- .../core/test/observed-output-shapes.test.ts | 28 +++++++++++++++ packages/core/test/tools.test.ts | 36 +++++++++++++++++++ 7 files changed, 104 insertions(+), 2 deletions(-) diff --git a/packages/core/src/attach/server.ts b/packages/core/src/attach/server.ts index 5859ec0f..56b363cb 100644 --- a/packages/core/src/attach/server.ts +++ b/packages/core/src/attach/server.ts @@ -64,6 +64,7 @@ function createAttachNativeService(options: AttachServeOptions, io: AttachServeI url: options.selection.remote.attachUrl, ...(options.selection.remote.fetch ? { fetch: options.selection.remote.fetch } : {}), }), + exposeLocalArtifactPaths: false, ...(io.writeErr ? { writeErr: io.writeErr } : {}), }); } diff --git a/packages/core/src/native/service.ts b/packages/core/src/native/service.ts index 5be708e1..3e86cce4 100644 --- a/packages/core/src/native/service.ts +++ b/packages/core/src/native/service.ts @@ -50,6 +50,7 @@ export type NativeCapletsServiceOptions = NativeCapletsServiceResolutionInput & configPath?: string; projectConfigPath?: string; authDir?: string; + exposeLocalArtifactPaths?: boolean; watchDebounceMs?: number; watch?: boolean; writeErr?: (value: string) => void; diff --git a/packages/core/src/observed-output-shapes/key.ts b/packages/core/src/observed-output-shapes/key.ts index 08283d46..930a8d65 100644 --- a/packages/core/src/observed-output-shapes/key.ts +++ b/packages/core/src/observed-output-shapes/key.ts @@ -67,6 +67,16 @@ function nonSecretBackendIdentity(caplet: CapletConfig): unknown { specUrl: caplet.specUrl, baseUrl: caplet.baseUrl, }; + case "googleDiscovery": + return { + backend: caplet.backend, + server: caplet.server, + discoveryPath: caplet.discoveryPath, + discoveryUrl: caplet.discoveryUrl, + baseUrl: caplet.baseUrl, + includeOperations: caplet.includeOperations, + excludeOperations: caplet.excludeOperations, + }; case "graphql": return { backend: caplet.backend, diff --git a/packages/core/src/serve/http.ts b/packages/core/src/serve/http.ts index 9e242b29..7a9bbc0f 100644 --- a/packages/core/src/serve/http.ts +++ b/packages/core/src/serve/http.ts @@ -474,12 +474,14 @@ export async function serveHttpWithSessionFactory( createSession: HttpMcpSessionFactory, writeErr: (value: string) => void = (value) => process.stderr.write(value), ): Promise { - const engine = new CapletsEngine({}); + const resolvedEngineOptions = { exposeLocalArtifactPaths: false }; + const engine = new CapletsEngine(resolvedEngineOptions); const app = createHttpServeApp(options, engine, { writeErr, exposeAttach: false, sessionFactory: createSession, control: { + ...resolvedEngineOptions, projectCapletsRoot: resolveProjectCapletsRoot(), }, }); diff --git a/packages/core/src/tools.ts b/packages/core/src/tools.ts index 245faebe..0acea0ae 100644 --- a/packages/core/src/tools.ts +++ b/packages/core/src/tools.ts @@ -1120,12 +1120,16 @@ function hasArtifactPlaceholderForSelectedFields( } export function extractArtifacts(result: unknown): CapletArtifact[] { - if (!isPlainObject(result) || !Array.isArray(result.content)) { + if (!isPlainObject(result)) { return []; } const artifacts: CapletArtifact[] = []; const seen = new Set(); + addStructuredArtifact(artifacts, seen, result.structuredContent); + if (!Array.isArray(result.content)) { + return artifacts; + } for (const item of result.content) { if (!isPlainObject(item) || item.type !== "text" || typeof item.text !== "string") { continue; @@ -1167,6 +1171,26 @@ export function extractArtifacts(result: unknown): CapletArtifact[] { return artifacts; } +function addStructuredArtifact( + artifacts: CapletArtifact[], + seen: Set, + structuredContent: unknown, +): void { + if (!isPlainObject(structuredContent)) return; + const body = structuredContent.body; + if (!isPlainObject(body) || !isPlainObject(body.artifact)) return; + const path = typeof body.artifact.path === "string" ? body.artifact.path : undefined; + const uri = typeof body.artifact.uri === "string" ? body.artifact.uri : undefined; + const displayPath = path ?? uri; + if (!displayPath || seen.has(displayPath)) return; + seen.add(displayPath); + artifacts.push({ + kind: "file", + displayPath, + pathResolution: path ? "absolute" : "relative-to-mcp-server", + }); +} + type MarkdownLink = { label: string; destination: string; diff --git a/packages/core/test/observed-output-shapes.test.ts b/packages/core/test/observed-output-shapes.test.ts index c2661e9e..d651b231 100644 --- a/packages/core/test/observed-output-shapes.test.ts +++ b/packages/core/test/observed-output-shapes.test.ts @@ -139,6 +139,34 @@ describe("Observed Output Shapes", () => { expect(JSON.stringify(key)).not.toContain("GH_TOKEN"); }); + it("builds Google Discovery backend fingerprints", () => { + const config = parseConfig({ + googleDiscoveryApis: { + drive: { + name: "Google Drive", + description: "Access Google Drive.", + discoveryUrl: "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest", + baseUrl: "https://www.googleapis.com/drive/v3/", + auth: { type: "none" }, + includeOperations: ["drive.files.list"], + }, + }, + }); + + const key = observedOutputShapeKey({ + scope: "local", + caplet: config.googleDiscoveryApis.drive!, + toolName: "drive.files.list", + }); + + expect(key).toMatchObject({ + capletId: "drive", + backendKind: "googleDiscovery", + toolName: "drive.files.list", + }); + expect(key.backendFingerprint).toHaveLength(64); + }); + it("stores, expires, and prunes filesystem cache entries", async () => { const dir = mkdtempSync(join(tmpdir(), "caplets-observed-shapes-")); const ttlMs = 1_000; diff --git a/packages/core/test/tools.test.ts b/packages/core/test/tools.test.ts index 775fb007..0afe8525 100644 --- a/packages/core/test/tools.test.ts +++ b/packages/core/test/tools.test.ts @@ -1019,6 +1019,42 @@ describe("generated tool handlers", () => { ]); }); + it("extracts structured artifact envelopes from call_tool results", async () => { + const downstream = { + callTool: vi.fn().mockResolvedValue({ + content: [{ type: "text" as const, text: "downloaded" }], + structuredContent: { + status: 200, + body: { + artifact: { + uri: "caplets://artifacts/http/call-1/report.pdf", + path: "/tmp/caplets/report.pdf", + filename: "report.pdf", + mimeType: "application/pdf", + byteLength: 12, + sha256: "a".repeat(64), + }, + }, + }, + }), + } as unknown as DownstreamManager; + + const result = await handleServerTool( + server, + { operation: "call_tool", name: "read", args: {} }, + registry, + downstream, + ); + + expect(result._meta.caplets.artifacts).toEqual([ + { + kind: "file", + displayPath: "/tmp/caplets/report.pdf", + pathResolution: "absolute", + }, + ]); + }); + it("extracts artifact links with spaces, title attributes, and parentheses", async () => { const result = await callToolWithText( 'Saved artifact [Screenshot](./screenshots/final view.png), file [Trace](./trace.zip "trace"), and artifact [Archive](./run(1).zip)',