diff --git a/deploy/helm/llm-api-gateway/README.md b/deploy/helm/llm-api-gateway/README.md index 04eb6256e8..7ec8068fe7 100644 --- a/deploy/helm/llm-api-gateway/README.md +++ b/deploy/helm/llm-api-gateway/README.md @@ -70,6 +70,7 @@ Important settings to review before deployment: - `llmApiGateway.metrics.enabled` to expose a metrics port on the Service and Deployment (default: `false`) - `llmApiGateway.metrics.serviceMonitor.enabled` to create a Prometheus `ServiceMonitor` (requires `metrics.enabled`) - `llmApiGateway.config.lastCluster.*` for session last-cluster hints sent to the LLM Request Router (default: disabled) +- `llmApiGateway.config.decisionImageTokens` for the flat input token estimate of each image on `/v1/decisions` and `/v1/systemone` (default: `1024`) - `llmApiGateway.config.exposeRouterErrorCodes` to return the LLM Request Router rejection reason in `error.code` of OpenAI-format error responses (default: `false`) - `llmApiGateway.olric.*` for embedded rate-limit and last-cluster state and peer discovery. The Olric node starts whenever `olric.enabled` is `true`, even with rate limiting off - `llmApiGateway.vault.*` for JWT authentication path, role, and audience values used by the Vault Agent injector diff --git a/deploy/helm/llm-api-gateway/llm-api-gateway/templates/configmap.yaml b/deploy/helm/llm-api-gateway/llm-api-gateway/templates/configmap.yaml index 7f368b76dc..1a012aa769 100644 --- a/deploy/helm/llm-api-gateway/llm-api-gateway/templates/configmap.yaml +++ b/deploy/helm/llm-api-gateway/llm-api-gateway/templates/configmap.yaml @@ -32,6 +32,7 @@ data: STARGATE_LAST_CLUSTER_LOOKUP_TIMEOUT: {{ $cfg.lastCluster.lookupTimeout | quote }} STARGATE_LAST_CLUSTER_LOCAL_MAX_ENTRIES: {{ $cfg.lastCluster.localMaxEntries | int64 | toString | quote }} NVCF_GATEWAY_INFERENCE_WRITE_TIMEOUT: {{ $cfg.inferenceWriteTimeout | quote }} + NVCF_DECISION_IMAGE_TOKENS: {{ $cfg.decisionImageTokens | int64 | toString | quote }} NVCF_GRPC_ADDR: {{ $cfg.nvcfGrpcAddr | quote }} NVCF_GRPC_INSECURE: {{ $cfg.nvcfGrpcInsecure | toString | quote }} NVCF_GRPC_TIMEOUT: {{ $cfg.nvcfGrpcTimeout | quote }} diff --git a/deploy/helm/llm-api-gateway/llm-api-gateway/values.yaml b/deploy/helm/llm-api-gateway/llm-api-gateway/values.yaml index e2cd2ad382..411f0998b9 100644 --- a/deploy/helm/llm-api-gateway/llm-api-gateway/values.yaml +++ b/deploy/helm/llm-api-gateway/llm-api-gateway/values.yaml @@ -70,6 +70,10 @@ llmApiGateway: # reading. Applies only while a write is in progress, so it does not cap # total response time or upstream pauses. 0s disables the bound. inferenceWriteTimeout: 60s + # Flat input token estimate for each image on /v1/decisions and + # /v1/systemone. The image payload is never counted by size. Must be 0 or + # more; 0 counts images as free. + decisionImageTokens: 1024 nvcfGrpcAddr: "api.nvcf.svc.cluster.local:9090" nvcfGrpcInsecure: false nvcfGrpcTimeout: 2s diff --git a/deploy/helm/llm-api-gateway/scripts/test-default-ownership.sh b/deploy/helm/llm-api-gateway/scripts/test-default-ownership.sh index 1f73269ac9..694e5d0735 100755 --- a/deploy/helm/llm-api-gateway/scripts/test-default-ownership.sh +++ b/deploy/helm/llm-api-gateway/scripts/test-default-ownership.sh @@ -36,11 +36,14 @@ for kv in ENABLED=false TTL=10m LOOKUP_TIMEOUT=20ms LOCAL_MAX_ENTRIES=100000; do test "$(read_config "$default_manifest" "STARGATE_LAST_CLUSTER_${kv%%=*}")" = "${kv#*=}" || fail "STARGATE_LAST_CLUSTER_${kv%%=*} default must be ${kv#*=}" done +test "$(read_config "$default_manifest" NVCF_DECISION_IMAGE_TOKENS)" = 1024 || + fail "NVCF_DECISION_IMAGE_TOKENS default must be 1024" helm template llm-api-gateway "$chart_dir" \ --namespace nvcf \ --set-string llmApiGateway.image.repository=example.invalid/llm-api-gateway \ --set llmApiGateway.config.nvcfGrpcInsecure=true \ + --set llmApiGateway.config.decisionImageTokens=2048 \ --set llmApiGateway.metrics.enabled=true \ --set llmApiGateway.metrics.serviceMonitor.enabled=true \ --set llmApiGateway.config.lastCluster.enabled=true \ @@ -49,6 +52,8 @@ helm template llm-api-gateway "$chart_dir" \ test "$(read_config "$override_manifest" NVCF_GRPC_INSECURE)" = true || fail "explicit plaintext transport override did not reach the ConfigMap" +test "$(read_config "$override_manifest" NVCF_DECISION_IMAGE_TOKENS)" = 2048 || + fail "decisionImageTokens override must render as NVCF_DECISION_IMAGE_TOKENS" # 1000000 checks that a large cap renders as a plain integer. for kv in ENABLED=true LOCAL_MAX_ENTRIES=1000000; do test "$(read_config "$override_manifest" "STARGATE_LAST_CLUSTER_${kv%%=*}")" = "${kv#*=}" || diff --git a/deploy/helm/vanity-gateway/README.md b/deploy/helm/vanity-gateway/README.md index be2153ab9a..4121e57d53 100644 --- a/deploy/helm/vanity-gateway/README.md +++ b/deploy/helm/vanity-gateway/README.md @@ -106,14 +106,18 @@ or the pod is killed mid-drain. ## Route mapping -`vanityGateway.mappingConfig.v2config` has three sections: +`vanityGateway.mappingConfig.v2config` has four sections: - `openai`: per-endpoint model routes, keyed by endpoint (`chatCompletions`, - `completions`, `embeddings`, `responses`, and the image endpoints). Each route - requires `modelName` and `functionID`. + `completions`, `embeddings`, `responses`, `decisions`, and the image + endpoints). Each route requires `modelName` and `functionID`. `decisions` + serves the beta `POST /v1/decisions` route. - `anthropicMessages`: native Anthropic Messages model routes, served on `POST /v1/messages`. Routes use the same fields as `openai.chatCompletions` and share the API host set by `openai.host`. +- `typesafeSystemOne`: beta TypeSafe System One model routes, served on + `POST /v1/systemone`. A top-level sibling of `anthropicMessages`, with the + same fields as `openai.decisions`. - `vanity`: host-based routes, each requiring a `host` and a `paths` map. Each path requires `path` and `functionID`. @@ -123,13 +127,14 @@ across all routes. A model may set `functionType: LLM` to be served by the LLM Gateway instead of the invocation service, supported in `openai.chatCompletions`, -`openai.responses`, `openai.embeddings`, and `anthropicMessages`. Callers still send the public `modelName`; the gateway +`openai.responses`, `openai.embeddings`, `openai.decisions`, `anthropicMessages`, and +`typesafeSystemOne`. Callers still send the public `modelName`; the gateway rewrites the request model to `functionID/modelName` before forwarding. The values schema enforces what the gateway checks at startup, so a values file that would fail the container fails the render instead: -- `functionType` is rejected outside those four endpoints +- `functionType` is rejected outside those endpoints - `usePexec`, `outgoingPathOverride`, and `sessionTimeout` are rejected on such a model, since the LLM Gateway ignores them. An explicit `false`, `""`, or `0` is accepted, because that is what an absent key produces @@ -162,6 +167,43 @@ Messages. Upgrade Pylon workers, the Stargate router, and the LLM Gateway before enabling Messages routes. See `docs/overview/llm-gateway.md` for the request, accounting, and Claude Code details. +Decision models (beta) are mapped per route. A model is enabled for a route only +when listed in that route's section, and a function that serves both formats is +listed in both: + +```yaml +vanityGateway: + mappingConfig: + v2config: + openai: + host: api.example.com + decisions: + example_decision-model: + modelName: example/decision-model + functionID: + functionVersionID: + example_decision-model-llm: + modelName: example/decision-model-llm + functionID: + functionType: LLM + typesafeSystemOne: + example_decision-model: + modelName: example/decision-model + functionID: + functionVersionID: +``` + +Request and response bodies pass through unchanged, and responses are not +streamed. With `usePexec: true` the invocation service normally sends the request +to the default invocation path registered for the function, whichever route the +client called, so a `usePexec: true` function serves only its default path. +A function that serves both routes should set `usePexec: false`. Models with +`functionType: LLM` need LLM Gateway and Stargate builds that serve these routes; +enable them only after those builds run in the environment. Request body +sampling (`promptCacheKey`, `firstMessageHash`) and `promptCacheKeyHeaders` are +rejected in the decision sections. See the service README for error shapes and +`/v1/models` behavior. + Shadow traffic is supported. Each shadow target is resolved from the same model table and routed by its own `functionType`, so a shadow of an LLM model reaches the LLM Gateway, and an LLM model may shadow a model served by the invocation diff --git a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json index 93782c2d7e..def02ac087 100644 --- a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json +++ b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json @@ -145,6 +145,9 @@ "anthropicMessages": { "$ref": "#/definitions/openAIStickyEndpointRoutes" }, + "typesafeSystemOne": { + "$ref": "#/definitions/openAILLMEndpointRoutes" + }, "vanity": { "$ref": "#/definitions/vanityRoutes" } @@ -273,6 +276,29 @@ "required": [ "embeddings" ] + }, + { + "properties": { + "decisions": { + "not": { + "additionalProperties": { + "not": { + "properties": { + "functionType": { + "const": "LLM" + } + }, + "required": [ + "functionType" + ] + } + } + } + } + }, + "required": [ + "decisions" + ] } ] } @@ -303,6 +329,29 @@ "required": [ "anthropicMessages" ] + }, + { + "properties": { + "typesafeSystemOne": { + "not": { + "additionalProperties": { + "not": { + "properties": { + "functionType": { + "const": "LLM" + } + }, + "required": [ + "functionType" + ] + } + } + } + } + }, + "required": [ + "typesafeSystemOne" + ] } ] } @@ -357,6 +406,9 @@ "embeddings": { "$ref": "#/definitions/openAILLMEndpointRoutes" }, + "decisions": { + "$ref": "#/definitions/openAILLMEndpointRoutes" + }, "responses": { "$ref": "#/definitions/openAIStickyEndpointRoutes" }, diff --git a/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh b/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh index 9b853b15b5..ecb89cc12c 100755 --- a/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh +++ b/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh @@ -76,11 +76,21 @@ assert_renders "a model without functionType renders with an empty endpoint" write_values "${ENDPOINT}" chatCompletions " functionType: llmGateway" assert_rejected "value must be 'LLM'" "an unrecognized functionType is rejected" -for section in responses embeddings; do +for section in responses embeddings decisions; do write_values "${ENDPOINT}" "${section}" "${LLM}" assert_renders "an LLM model renders in ${section}" done +write_values "" decisions "${LLM}" +assert_rejected "/vanityGateway/config/llmGatewayEndpoint" "an LLM decisions model requires an endpoint" + +write_values "${ENDPOINT}" decisions "${LLM} + usePexec: true" +assert_rejected "a_model/usePexec" "usePexec is rejected on an LLM decisions model" + +write_values "" decisions "" +assert_renders "a decisions model without functionType renders with an empty endpoint" + for section in completions imageGenerations imageEdits imageVariations; do write_values "${ENDPOINT}" "${section}" "${LLM}" assert_rejected "openai/${section}/a_model" "functionType is rejected in ${section}" @@ -156,6 +166,46 @@ write_messages_values "${ENDPOINT}" "${MESSAGES_LLM} sessionTimeout: 900" assert_rejected "anthropicMessages/a_model/sessionTimeout" "sessionTimeout is rejected on an LLM Messages model" +# typesafeSystemOne is a top-level sibling of anthropicMessages. +write_system_one_values() { + local endpoint="$1" extra="$2" + cat > "${VALUES}" < "${VALUES}" <-` format. Supported units are `S` (seconds), `M` (minutes), `H` (hours), `D` (days), and `W` (weeks). Use `1000-S` for a single limit, or `1000-S,5000-M,100000-H,500000-D,1000000-W` for a combined limit with distinct units. Use JSON input for combined limits because inline CLI model specs use commas as field separators. #### Deploy Function diff --git a/docs/overview/llm-gateway.md b/docs/overview/llm-gateway.md index 3af9b16818..e33ee555c0 100644 --- a/docs/overview/llm-gateway.md +++ b/docs/overview/llm-gateway.md @@ -1,6 +1,6 @@ # LLM Gateway -The LLM Gateway handles OpenAI-compatible and native Anthropic Messages invocation for NVCF LLM functions in +The LLM Gateway handles OpenAI-compatible, native Anthropic Messages, and beta decision model invocation for NVCF LLM functions in self-managed deployments. Use LLM functions when requests should enter through the LLM invocation route and NVCF should route them by function and model. @@ -91,6 +91,8 @@ gateway does not wrap it in a second NVCF envelope. | `/v1/responses` | Supports native Responses API requests. Streaming clients receive server-sent events (SSE). Non-streaming clients receive the terminal Responses JSON object. | | `/v1/embeddings` | Supports embeddings requests with string or string array input. | | `/v1/messages` | Preserves native Anthropic Messages JSON responses and server-sent events, including tool and thinking content blocks. | +| `/v1/decisions` | Beta. Passes OpenAI Decisions requests and responses through unchanged. Not streamed. | +| `/v1/systemone` | Beta. Passes TypeSafe System One requests and responses through unchanged. Not streamed. | `llmConfig.routingMethod` is a routing algorithm name, optionally followed by tuning parameters, such as `pulsar;seed=stable-a;consider_kv_free_tokens=true`. @@ -240,6 +242,7 @@ body. | `/v1/chat/completions` | `model` with the `/` prefix, `stream`, `max_tokens`, `max_completion_tokens`, `prompt_cache_key`, `stream_options`, and a non-empty `messages` array | | `/v1/responses` | `model` with the `/` prefix, `stream`, `max_output_tokens`, `prompt_cache_key`, `conversation.id`, and a non-empty `input` unless `prompt`, `previous_response_id`, or `conversation` is present | | `/v1/embeddings` | `model` with the `/` prefix, and a non-empty `input` with at most 2048 entries | +| `/v1/decisions`, `/v1/systemone` | `model` with the `/` prefix. No other field. | The body must be a JSON object. A checked field that appears twice in different letter cases returns `400`. @@ -249,10 +252,74 @@ The gateway forwards the body unchanged except for these fields: - Chat: sets `model` and `stream`, merges `stream_options`, and removes `debug`. - Responses: sets `model` and `stream`. - Embeddings: sets `model`. +- Decisions and System One: sets `model`. The gateway does not add `service_tier`. The backend receives it only when the client sends it. +### Decision models (beta) + +Decision models answer typed questions about an input and return probabilities +instead of generated text. Two beta routes pass these requests through: + +- `POST /v1/decisions` for the OpenAI Decisions format. +- `POST /v1/systemone` for the TypeSafe System One format. + +The gateway reads only `model`, rewrites it to the backend model name, and +forwards the rest of the body unchanged. It does not translate between the two +formats, so the backend must implement each route that it declares. Responses +are not streamed: `stream` is ignored. Client headers, including +`TypeSafe-Danger-Zone-Images`, reach the backend. + +```bash +curl -sS -X POST "http://${GATEWAY_ADDR}/v1/decisions" \ + -H "Host: llm.invocation.${GATEWAY_ADDR}" \ + -H "Authorization: Bearer ${NVCF_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "/decision-model", + "input": "The sky is clear.", + "questions": [{"type": "predicate", "text": "Is it raining?"}] + }' +``` + +A model is enabled for a route only when its `llmConfig.uris` lists that path. +A model with no `uris`, or without the route in them, returns `404` in the +error shape of the route. These routes have no default-allow, and +`MODEL_URI_ALLOWLIST_ENABLED` does not change this. + +`llmConfig.uris` is fixed when the function is created, so declare the decision +routes at creation. A function that serves both routes should use +`usePexec: false` in the Vanity Gateway mapping, because a `usePexec: true` +function reaches only its default path. + +Token admission counts one request and an estimate of the input tokens. The +estimate counts every string in the body as text. Each image counts a flat +number, `1024` by default, set with `NVCF_DECISION_IMAGE_TOKENS` on the gateway. +Base64 image data is never counted. An image is an object with `type` set to +`input_image`, an element of `images`, a value of `assets`, or any string that +is a base64 data URL. The estimate is sent as `X-Input-Tokens` and +`X-Token-Estimate`. + +Errors that the gateway or the request router generates use the shape of the +route: + +- `/v1/decisions` uses the OpenAI error shape. +- `/v1/systemone` uses `{"error":{"code","message","request_id"},"request_id"}`. + `request_id` equals the `X-Request-ID` response header. Codes are + `invalid_json`, `invalid_request`, `invalid_api_key`, `not_found`, + `request_too_large`, `rate_limit_exceeded`, `internal_error`, + `provider_unavailable`, and `provider_timeout`. + +An error from the backend is relayed unchanged, whatever its shape, size, or +encoding. + +When a successful response reports `usage.input_tokens`, the gateway records it +on `llm_api_gateway_llm_tokens_total` and counts the response in +`llm_api_gateway_decision_usage_observations_total` with `status="observed"`. A +response without it counts as `absent`. The estimate remains the admission +value. + ### Anthropic Messages Use `/v1/messages` with a backend that implements the native Anthropic Messages diff --git a/docs/self-managed/gateway-routing.md b/docs/self-managed/gateway-routing.md index 75798d828e..571f7c6320 100644 --- a/docs/self-managed/gateway-routing.md +++ b/docs/self-managed/gateway-routing.md @@ -609,7 +609,7 @@ These routes attach to the Gateway you prepared in [Gateway quickstart](./gatewa | API Keys | `api-keys.` | 80 | Token generation and API key management | | NVCF API | `api.` | 80 | Function management (create, deploy, delete) | | Invocation | `invocation.`, `*.invocation.` | 80 | Function invocation (wildcard for dynamic routing) | -| LLM Invocation | `llm.invocation.` | 80 | OpenAI-compatible LLM invocation routes such as `/v1/chat/completions`, `/v1/responses`, and `/v1/embeddings` | +| LLM Invocation | `llm.invocation.` | 80 | LLM invocation routes such as `/v1/chat/completions`, `/v1/responses`, `/v1/embeddings`, and the beta `/v1/decisions` and `/v1/systemone` | | Vanity Gateway | `vanity.` | 80 | Optional vanity host/path routing to `vanity-gateway.nvcf:8080`, only in stack packages that include the addon | | NVCF UI | `nvcf-ui.` | 80 | Optional nvcf-ui host/path routing to `nvcf-ui.nvcf-ui:8300`, only in stack packages that include the addon | | gRPC | N/A (TCP routing, no hostname matching) | 10081 | gRPC function invocations | diff --git a/docs/self-managed/llm-function-enablement.md b/docs/self-managed/llm-function-enablement.md index 9b2c602608..5eb38e3265 100644 --- a/docs/self-managed/llm-function-enablement.md +++ b/docs/self-managed/llm-function-enablement.md @@ -23,6 +23,17 @@ Standard HTTP, gRPC, and LLS functions do not require this addon, even when a container exposes paths such as `/v1/chat/completions`, `/v1/responses`, or `/v1/embeddings`. +The addon also serves the beta decision model routes, `POST /v1/decisions` and +`POST /v1/systemone`. The gateway passes their bodies through unchanged and +does not stream them. Declare each route the container serves in +`models[].llmConfig.uris` when you create the function, for example +`--llm-model "name=,uris=/v1/decisions|/v1/systemone"`. The `uris` of a +function cannot change after creation, and a model gets a `404` on a decision +route that it does not declare. Through the Vanity Gateway, a function that +serves both routes should use `usePexec: false`, because a `usePexec: true` +function reaches only its default path. See the +[LLM Gateway guide](../overview/llm-gateway.md#decision-models-beta). + When enabled, the stack creates: - `llm-api-gateway` in the `nvcf` namespace. @@ -1020,7 +1031,7 @@ authentication hop. The LLM API Gateway returns errors in the format OpenAI and Anthropic SDKs read. Errors from the gateway and the LLM request router use the OpenAI format on `/v1/chat/completions`, `/v1/responses`, `/v1/embeddings`, and the -other OpenAI-compatible endpoints: +other OpenAI-compatible endpoints, including `/v1/decisions`: ```json {"error":{"message":"no eligible candidates","type":"invalid_request_error","param":null,"code":null}} @@ -1037,7 +1048,15 @@ The `type` follows the HTTP status. For example, `401` is `overloaded_error`. On OpenAI-compatible endpoints, `529` always has `code: "overloaded_error"`. -Model server errors already in the endpoint's format are returned unchanged, +On `/v1/systemone`, they use the System One format, with `error.code`, +`error.message`, and `error.request_id`. See the +[LLM Gateway guide](../overview/llm-gateway.md#decision-models-beta) for the +codes. + +On `/v1/decisions` and `/v1/systemone`, model server errors are returned +unchanged, whatever their shape, size, or encoding. + +On the other endpoints, model server errors already in the endpoint's format are returned unchanged, including the server's `type` and `code`. Other model server bodies are converted to the endpoint's format, keeping up to 32 KiB of the server's message when the gateway can read it. The gateway reads uncompressed and gzip error bodies up diff --git a/examples/function-samples/openai-compatible-sample/README.md b/examples/function-samples/openai-compatible-sample/README.md index 4753b3a1f2..e623c85c34 100644 --- a/examples/function-samples/openai-compatible-sample/README.md +++ b/examples/function-samples/openai-compatible-sample/README.md @@ -15,6 +15,8 @@ clients. | `POST /v1/responses` | Responses API JSON or SSE output | | `POST /v1/embeddings` | Embeddings for one string or an array of strings | | `POST /v1/messages` | Anthropic Messages JSON or SSE output | +| `POST /v1/decisions` | Canned Decisions answers, one per question, in order | +| `POST /v1/systemone` | Canned System One answers, keyed by question name | | `GET /v1/models` | Lists the sample model | | `GET /v1/models/{id}` | Returns metadata for a model ID | | `GET /health` | Health check returning 200 | @@ -343,6 +345,27 @@ ulimit -n 65536 -e LOAD_TESTER_OUTPUT_CHUNKS=12000 ``` +## Decision routes + +`POST /v1/decisions` and `POST /v1/systemone` return fixed answers for test +use. Decisions returns `answers` as a list in question order. System One +returns `answers` keyed by question name. Both include `model` and `usage`. +Benchmark controls do not apply. Only `X-Load-Tester-Echo` and the +`x_load_tester_echo` body field are read, and they add the request body as +`request_body`. + +Every response has an `echo` object with the request headers +`typesafe-danger-zone-images`, `x-model`, `x-input-tokens`, `x-request-id` +and `x-vendor-test` when present, plus `body_length`. When +`TypeSafe-Danger-Zone-Images` is sent, the response also sets +`x-mock-echo-typesafe-danger-zone-images`. + +These two routes accept bodies up to 16 MiB. The other routes keep 10 MiB. +Bodies must be one JSON object, otherwise the response is 400 with a JSON +error. Add `?mock_error=429` for a 429 with `Retry-After: 7`, or +`?mock_error=400` for a 400. The 400 body is FastAPI style +(`{"detail":[...]}`) on System One and OpenAI style on Decisions. + ## NVCF LLM functions Expose port 8000 and configure the function inference URL as `/`. Declare the @@ -354,6 +377,8 @@ routes required by the workload, such as: /v1/responses /v1/embeddings /v1/messages +/v1/decisions +/v1/systemone ``` See the [LLM Gateway guide](../../../docs/user/llm-gateway.md) for function diff --git a/examples/function-samples/openai-compatible-sample/http-server/BUILD.bazel b/examples/function-samples/openai-compatible-sample/http-server/BUILD.bazel index 317e3ba876..d59ad52a5a 100644 --- a/examples/function-samples/openai-compatible-sample/http-server/BUILD.bazel +++ b/examples/function-samples/openai-compatible-sample/http-server/BUILD.bazel @@ -22,6 +22,7 @@ load("//rules/oci:defs.bzl", "go_oci_image") go_library( name = "http-server_lib", srcs = [ + "decisions.go", "main.go", "messages.go", ], @@ -38,6 +39,7 @@ go_binary( go_test( name = "http-server_test", srcs = [ + "decisions_test.go", "echo_test.go", "main_test.go", "messages_test.go", diff --git a/examples/function-samples/openai-compatible-sample/http-server/decisions.go b/examples/function-samples/openai-compatible-sample/http-server/decisions.go new file mode 100644 index 0000000000..0d782d7b0a --- /dev/null +++ b/examples/function-samples/openai-compatible-sample/http-server/decisions.go @@ -0,0 +1,427 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" +) + +// decisionMaxRequestBytes bounds the decision routes only. A System One image +// request can carry an 8 MB image, which is about 10.7 MB as base64 and does +// not fit the 10 MiB limit the other routes keep. +const decisionMaxRequestBytes = 16 << 20 + +const ( + decisionRoute = "/v1/decisions" + systemOneRoute = "/v1/systemone" + mockErrorQuery = "mock_error" + mockRetryAfter = "7" + mockEchoImagesHdr = "x-mock-echo-typesafe-danger-zone-images" +) + +// decisionEchoHeaders are the request headers reported in the response "echo" +// object so a test can prove they reached the backend. +var decisionEchoHeaders = []string{ + "typesafe-danger-zone-images", + "x-model", + "x-input-tokens", + "x-request-id", + "x-vendor-test", +} + +type decisionRequest struct { + Model string `json:"model"` + Questions json.RawMessage `json:"questions"` + Echo *bool `json:"x_load_tester_echo"` +} + +type decisionsUsage struct { + InputTokens int `json:"input_tokens"` + InputTokensDetails struct { + CachedTokens int `json:"cached_tokens"` + CacheWriteTokens int `json:"cache_write_tokens"` + } `json:"input_tokens_details"` + OutputTokens int `json:"output_tokens"` + OutputTokensDetails struct { + ReasoningTokens int `json:"reasoning_tokens"` + } `json:"output_tokens_details"` + TotalTokens int `json:"total_tokens"` +} + +type systemOneUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` +} + +type decisionEcho map[string]any + +type decisionsResponse struct { + Model string `json:"model"` + Answers []any `json:"answers"` + Usage decisionsUsage `json:"usage"` + Echo decisionEcho `json:"echo"` + RequestBody json.RawMessage `json:"request_body,omitempty"` +} + +type systemOneResponse struct { + Model string `json:"model"` + Answers map[string]any `json:"answers"` + Usage systemOneUsage `json:"usage"` + Echo decisionEcho `json:"echo"` + RequestBody json.RawMessage `json:"request_body,omitempty"` +} + +// decisionQuestion reads only what the canned answers need. Everything else is +// kept raw so an unexpected field never fails a request. +type decisionQuestion struct { + Type string `json:"type"` + Name string `json:"name"` + Choices json.RawMessage `json:"choices"` + Levels json.RawMessage `json:"levels"` + Criteria json.RawMessage `json:"criteria"` +} + +func handleDecisions(w http.ResponseWriter, r *http.Request) { + handleDecisionRoute(w, r, false) +} + +func handleSystemOne(w http.ResponseWriter, r *http.Request) { + handleDecisionRoute(w, r, true) +} + +func handleDecisionRoute(w http.ResponseWriter, r *http.Request, systemOne bool) { + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + writeDecisionError(w, systemOne, http.StatusMethodNotAllowed, "method not allowed") + return + } + raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, decisionMaxRequestBytes)) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeDecisionError(w, systemOne, http.StatusRequestEntityTooLarge, "request body is too large") + return + } + writeDecisionError(w, systemOne, http.StatusBadRequest, "could not read request body") + return + } + request, err := parseDecisionRequest(raw) + if err != nil { + writeDecisionError(w, systemOne, http.StatusBadRequest, err.Error()) + return + } + + switch r.URL.Query().Get(mockErrorQuery) { + case "429": + w.Header().Set("Retry-After", mockRetryAfter) + writeDecisionError(w, systemOne, http.StatusTooManyRequests, "mock rate limit") + return + case "400": + writeDecisionError(w, systemOne, http.StatusBadRequest, "mock validation error") + return + } + + echo := decisionEcho{"body_length": len(raw)} + for _, name := range decisionEchoHeaders { + if value := r.Header.Get(name); value != "" { + echo[name] = value + } + } + if value := r.Header.Get("typesafe-danger-zone-images"); value != "" { + w.Header().Set(mockEchoImagesHdr, value) + } + var echoedBody json.RawMessage + if request.Echo != nil && *request.Echo || strings.EqualFold(r.Header.Get(headerEcho), "true") { + echoedBody = json.RawMessage(raw) + } + model := request.Model + if model == "" { + model = defaultModel + } + inputTokens := estimateDecisionInputTokens(r, raw) + + if systemOne { + writeJSON(w, http.StatusOK, systemOneResponse{ + Model: model, Answers: systemOneAnswers(request.Questions), + Usage: systemOneUsage{InputTokens: inputTokens, OutputTokens: 1}, Echo: echo, RequestBody: echoedBody, + }) + return + } + usage := decisionsUsage{InputTokens: inputTokens, OutputTokens: 1, TotalTokens: inputTokens + 1} + writeJSON(w, http.StatusOK, decisionsResponse{ + Model: model, Answers: decisionAnswers(request.Questions), + Usage: usage, Echo: echo, RequestBody: echoedBody, + }) +} + +func parseDecisionRequest(raw []byte) (decisionRequest, error) { + var request decisionRequest + decoder := json.NewDecoder(bytes.NewReader(raw)) + if err := decoder.Decode(&request); err != nil { + return request, errors.New("request body must be a JSON object") + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return request, errors.New("request body must contain one JSON object") + } + return request, nil +} + +// estimateDecisionInputTokens prefers the gateway estimate when the caller +// sends one, so usage lines up with what the gateway metered. +func estimateDecisionInputTokens(r *http.Request, raw []byte) int { + if value, err := strconv.Atoi(r.Header.Get("X-Input-Tokens")); err == nil && value >= 0 { + return value + } + return len(raw)/4 + 1 +} + +func decisionAnswers(questions json.RawMessage) []any { + var list []json.RawMessage + if err := json.Unmarshal(questions, &list); err != nil || len(list) == 0 { + return []any{predicateAnswer("")} + } + answers := make([]any, 0, len(list)) + for _, item := range list { + var question decisionQuestion + _ = json.Unmarshal(item, &question) + answers = append(answers, cannedAnswer(question, false)) + } + return answers +} + +func systemOneAnswers(questions json.RawMessage) map[string]any { + var byName map[string]json.RawMessage + if err := json.Unmarshal(questions, &byName); err != nil || len(byName) == 0 { + return map[string]any{"question": cannedAnswer(decisionQuestion{Type: "noul"}, true)} + } + answers := make(map[string]any, len(byName)) + for name, item := range byName { + var question decisionQuestion + _ = json.Unmarshal(item, &question) + answers[name] = cannedAnswer(question, true) + } + return answers +} + +func predicateAnswer(name string) map[string]any { + return withName(map[string]any{"type": "predicate", "probability": 0.99}, name) +} + +func withName(answer map[string]any, name string) map[string]any { + if name != "" { + answer["name"] = name + } + return answer +} + +// distribution puts 0.7 on the first of n options and spreads the rest evenly. +func distribution(n int) []float64 { + probabilities := make([]float64, n) + for i := range probabilities { + switch { + case n == 1: + probabilities[i] = 1 + case i == 0: + probabilities[i] = 0.7 + default: + probabilities[i] = 0.3 / float64(n-1) + } + } + return probabilities +} + +// listItems returns the elements of a JSON array, or nil. +func listItems(raw json.RawMessage) []json.RawMessage { + var list []json.RawMessage + if json.Unmarshal(raw, &list) != nil { + return nil + } + return list +} + +// field returns the member of a JSON object item, or the item itself when it +// is not an object, so plain values and objects are both accepted. +func field(item json.RawMessage, key string) json.RawMessage { + var object map[string]json.RawMessage + if json.Unmarshal(item, &object) == nil { + if value, ok := object[key]; ok { + return value + } + } + return item +} + +// objectKeys returns the keys of a JSON object in document order. +func objectKeys(raw json.RawMessage) []string { + decoder := json.NewDecoder(bytes.NewReader(raw)) + if token, err := decoder.Token(); err != nil || token != json.Delim('{') { + return nil + } + var keys []string + for decoder.More() { + token, err := decoder.Token() + name, ok := token.(string) + if err != nil || !ok { + return keys + } + keys = append(keys, name) + var skip json.RawMessage + if decoder.Decode(&skip) != nil { + return keys + } + } + return keys +} + +func labelOf(item json.RawMessage, key string) string { + var label string + if json.Unmarshal(field(item, key), &label) == nil { + return label + } + return strings.Trim(string(field(item, key)), `"`) +} + +func weightedIndex(probabilities []float64) float64 { + var score float64 + for i, probability := range probabilities { + score += float64(i) * probability + } + return score +} + +func cannedAnswer(question decisionQuestion, systemOne bool) map[string]any { + switch question.Type { + case "choice": + if systemOne { + return systemOneChoice(question) + } + return decisionsChoice(question) + case "score": + if systemOne { + return systemOneScore(question) + } + return decisionsScore(question) + case "box": + answer := map[string]any{ + "type": "box", "box": map[string]any{"top": 0.1, "bottom": 0.9, "left": 0.1, "right": 0.9}, "confidence": 0.9, + } + if systemOne { + return answer + } + return withName(answer, question.Name) + } + if systemOne { + return map[string]any{"type": "noul", "noul": 0.99} + } + return predicateAnswer(question.Name) +} + +func decisionsChoice(question decisionQuestion) map[string]any { + values := []json.RawMessage{} + for _, item := range listItems(question.Choices) { + values = append(values, field(item, "value")) + } + if len(values) == 0 { + values = []json.RawMessage{json.RawMessage(`"A"`)} + } + weights := distribution(len(values)) + probabilities := make([]map[string]any, len(values)) + for i, value := range values { + probabilities[i] = map[string]any{"value": value, "probability": weights[i]} + } + return withName(map[string]any{"type": "choice", "choice": values[0], "probabilities": probabilities, "confidence": 0.9}, question.Name) +} + +func decisionsScore(question decisionQuestion) map[string]any { + levels := listItems(question.Levels) + labels := []string{"0", "1"} + if len(levels) > 0 { + labels = labels[:0] + for _, level := range levels { + labels = append(labels, labelOf(level, "label")) + } + } + weights := distribution(len(labels)) + probabilities := make([]map[string]any, len(labels)) + for i, label := range labels { + probabilities[i] = map[string]any{"value": i, "label": label, "probability": weights[i]} + } + return withName(map[string]any{"type": "score", "score": weightedIndex(weights), "probabilities": probabilities, "confidence": 0.9}, question.Name) +} + +func systemOneChoice(question decisionQuestion) map[string]any { + labels := objectKeys(question.Criteria) + if len(labels) == 0 { + for _, item := range listItems(question.Choices) { + labels = append(labels, labelOf(item, "value")) + } + } + if len(labels) == 0 { + labels = []string{"A"} + } + weights := distribution(len(labels)) + probabilities := make(map[string]float64, len(labels)) + for i, label := range labels { + probabilities[label] = weights[i] + } + return map[string]any{"type": "choice", "choice": labels[0], "confidence": 0.9, "probabilities": probabilities} +} + +func systemOneScore(question decisionQuestion) map[string]any { + descriptions := listItems(question.Criteria) + legend := map[string]string{} + for i, description := range descriptions { + legend[strconv.Itoa(i)] = labelOf(description, "") + } + count := len(descriptions) + if count == 0 { + count = 2 + legend = map[string]string{"0": "0", "1": "1"} + } + weights := distribution(count) + probabilities := make(map[string]float64, count) + for i, weight := range weights { + probabilities[strconv.Itoa(i)] = weight + } + return map[string]any{"type": "score", "score": weightedIndex(weights), "confidence": 0.9, "legend": legend, "probabilities": probabilities} +} + +// writeDecisionError writes the error shape of the route: FastAPI style for +// System One, OpenAI style for Decisions. +func writeDecisionError(w http.ResponseWriter, systemOne bool, status int, message string) { + if systemOne { + writeJSON(w, status, map[string]any{"detail": []map[string]any{{ + "loc": []string{"body"}, "msg": message, "type": "value_error", + }}}) + return + } + errorType := "invalid_request_error" + if status == http.StatusTooManyRequests { + errorType = "rate_limit_error" + } else if status >= http.StatusInternalServerError { + errorType = "api_error" + } + writeJSON(w, status, map[string]any{"error": map[string]any{ + "message": message, "type": errorType, "code": fmt.Sprint(status), + }}) +} diff --git a/examples/function-samples/openai-compatible-sample/http-server/decisions_test.go b/examples/function-samples/openai-compatible-sample/http-server/decisions_test.go new file mode 100644 index 0000000000..e8977e5028 --- /dev/null +++ b/examples/function-samples/openai-compatible-sample/http-server/decisions_test.go @@ -0,0 +1,495 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +const decisionsBody = `{"model":"test-model","input":"a photo","questions":[` + + `{"type":"predicate","text":"is it red?"},` + + `{"type":"choice","text":"which?","choices":["cat","dog"]},` + + `{"type":"score","text":"how good?"}]}` + +const systemOneBody = `{"model":"test-model","state":{"text":"hello"},"questions":{` + + `"a":{"type":"noul"},"b":{"type":"choice","choices":["x","y"]},"c":{"type":"score"}}}` + +var decisionHeaders = map[string]string{ + "TypeSafe-Danger-Zone-Images": "true", + "X-Model": "test-model", + "X-Input-Tokens": "42", + "X-Request-Id": "req-abc", + "X-Vendor-Test": "vendor-value", +} + +func TestDecisionsReturnsCannedAnswersInQuestionOrder(t *testing.T) { + recorder := postJSON(t, "/v1/decisions", decisionsBody) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", recorder.Code, recorder.Body.String()) + } + var response struct { + Model string `json:"model"` + Answers []map[string]any `json:"answers"` + Usage map[string]any `json:"usage"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("unmarshal: %v: %s", err, recorder.Body.String()) + } + if response.Model != "test-model" { + t.Fatalf("model = %q", response.Model) + } + if len(response.Answers) != 3 { + t.Fatalf("answers = %v, want 3", response.Answers) + } + for i, key := range []string{"probability", "choice", "score"} { + if _, ok := response.Answers[i][key]; !ok { + t.Fatalf("answer %d = %v, want key %q", i, response.Answers[i], key) + } + } + if response.Answers[1]["choice"] != "cat" { + t.Fatalf("choice = %v, want first listed choice", response.Answers[1]["choice"]) + } + for _, key := range []string{"input_tokens", "output_tokens", "total_tokens"} { + if _, ok := response.Usage[key]; !ok { + t.Fatalf("usage = %v, missing %q", response.Usage, key) + } + } + if response.Usage["total_tokens"].(float64) != response.Usage["input_tokens"].(float64)+response.Usage["output_tokens"].(float64) { + t.Fatalf("usage total mismatch: %v", response.Usage) + } +} + +func TestSystemOneReturnsCannedAnswersByName(t *testing.T) { + recorder := postJSON(t, "/v1/systemone", systemOneBody) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", recorder.Code, recorder.Body.String()) + } + var response struct { + Model string `json:"model"` + Answers map[string]map[string]any `json:"answers"` + Usage map[string]int `json:"usage"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("unmarshal: %v: %s", err, recorder.Body.String()) + } + if response.Model != "test-model" || len(response.Answers) != 3 { + t.Fatalf("response = %s", recorder.Body.String()) + } + if response.Answers["a"]["type"] != "noul" || response.Answers["a"]["noul"] != 0.99 { + t.Fatalf("answer a = %v, want noul 0.99", response.Answers["a"]) + } + if response.Answers["b"]["type"] != "choice" || response.Answers["b"]["choice"] != "x" { + t.Fatalf("answer b = %v", response.Answers["b"]) + } + if response.Answers["c"]["type"] != "score" { + t.Fatalf("answer c = %v", response.Answers["c"]) + } + if _, ok := response.Usage["input_tokens"]; !ok || len(response.Usage) != 2 { + t.Fatalf("usage = %v, want only input_tokens and output_tokens", response.Usage) + } +} + +func TestDecisionRoutesEchoRequestHeadersAndBodyLength(t *testing.T) { + for _, path := range []string{"/v1/decisions", "/v1/systemone"} { + t.Run(path, func(t *testing.T) { + body := decisionsBody + if path == "/v1/systemone" { + body = systemOneBody + } + recorder := postJSONWithHeaders(t, path, body, decisionHeaders) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d: %s", recorder.Code, recorder.Body.String()) + } + var response struct { + Echo map[string]any `json:"echo"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + want := map[string]any{ + "typesafe-danger-zone-images": "true", + "x-model": "test-model", + "x-input-tokens": "42", + "x-request-id": "req-abc", + "x-vendor-test": "vendor-value", + "body_length": float64(len(body)), + } + for key, value := range want { + if response.Echo[key] != value { + t.Fatalf("echo[%q] = %v, want %v (echo %v)", key, response.Echo[key], value, response.Echo) + } + } + if got := recorder.Header().Get("x-mock-echo-typesafe-danger-zone-images"); got != "true" { + t.Fatalf("x-mock-echo-typesafe-danger-zone-images = %q, want true", got) + } + }) + } +} + +func TestDecisionRoutesOmitEchoHeaderWhenRequestHasNone(t *testing.T) { + recorder := postJSON(t, "/v1/decisions", decisionsBody) + if got := recorder.Header().Get("x-mock-echo-typesafe-danger-zone-images"); got != "" { + t.Fatalf("header = %q, want unset", got) + } +} + +func TestDecisionRoutesEchoRequestBodyWhenAsked(t *testing.T) { + for _, path := range []string{"/v1/decisions", "/v1/systemone"} { + t.Run(path, func(t *testing.T) { + body := `{"model":"m","x_load_tester_echo":true,"extra":{"k":[1,2]}}` + recorder := postJSON(t, path, body) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d: %s", recorder.Code, recorder.Body.String()) + } + var response struct { + RequestBody json.RawMessage `json:"request_body"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if string(response.RequestBody) != body { + t.Fatalf("request_body = %s, want %s", response.RequestBody, body) + } + }) + } +} + +func TestDecisionRoutesAcceptEightMegabyteImageAtRealEncodedSize(t *testing.T) { + image := base64.StdEncoding.EncodeToString(make([]byte, 8_000_000)) + if len(image) <= maxRequestBytes { + t.Fatalf("test image encodes to %d, not above the 10 MiB general limit", len(image)) + } + for _, path := range []string{"/v1/decisions", "/v1/systemone"} { + t.Run(path, func(t *testing.T) { + body := `{"model":"m","input":{"image":"` + image + `"}}` + recorder := postJSON(t, path, body) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %.200s", recorder.Code, recorder.Body.String()) + } + }) + } +} + +func TestDecisionRoutesRejectBodiesAbove16MiB(t *testing.T) { + filler := strings.Repeat("a", decisionMaxRequestBytes) + for _, path := range []string{"/v1/decisions", "/v1/systemone"} { + t.Run(path, func(t *testing.T) { + recorder := postJSON(t, path, `{"model":"m","input":"`+filler+`"}`) + if recorder.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status = %d, want 413", recorder.Code) + } + assertJSONError(t, recorder.Body.Bytes()) + }) + } +} + +func TestExistingRoutesKeepTenMiBLimit(t *testing.T) { + if decisionMaxRequestBytes != 16<<20 { + t.Fatalf("decisionMaxRequestBytes = %d, want 16 MiB", decisionMaxRequestBytes) + } + if maxRequestBytes != 10<<20 { + t.Fatalf("maxRequestBytes = %d, want 10 MiB", maxRequestBytes) + } + filler := strings.Repeat("a", 11<<20) + recorder := postJSON(t, "/v1/embeddings", `{"model":"m","input":"`+filler+`"}`) + if recorder.Code == http.StatusOK { + t.Fatalf("embeddings accepted an 11 MiB body") + } + recorder = postJSON(t, "/v1/chat/completions", `{"model":"m","messages":[{"role":"user","content":"`+filler+`"}]}`) + if recorder.Code == http.StatusOK { + t.Fatalf("chat accepted an 11 MiB body") + } +} + +func TestDecisionRoutesRejectInvalidBodies(t *testing.T) { + for _, path := range []string{"/v1/decisions", "/v1/systemone"} { + for name, body := range map[string]string{ + "not json": `not json`, + "array": `[1,2]`, + "string": `"x"`, + "two objects": `{} {}`, + "empty": ``, + } { + t.Run(path+" "+name, func(t *testing.T) { + recorder := postJSON(t, path, body) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", recorder.Code, recorder.Body.String()) + } + assertJSONError(t, recorder.Body.Bytes()) + }) + } + } +} + +func TestDecisionRoutesMockErrorTriggers(t *testing.T) { + for _, path := range []string{"/v1/decisions", "/v1/systemone"} { + t.Run(path+" 429", func(t *testing.T) { + recorder := postJSON(t, path+"?mock_error=429", decisionsBody) + if recorder.Code != http.StatusTooManyRequests { + t.Fatalf("status = %d, want 429", recorder.Code) + } + if got := recorder.Header().Get("Retry-After"); got != "7" { + t.Fatalf("Retry-After = %q, want 7", got) + } + assertJSONError(t, recorder.Body.Bytes()) + }) + } + t.Run("decisions 400 is OpenAI style", func(t *testing.T) { + recorder := postJSON(t, "/v1/decisions?mock_error=400", decisionsBody) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", recorder.Code) + } + var body struct { + Error struct { + Message string `json:"message"` + Type string `json:"type"` + } `json:"error"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil || body.Error.Message == "" || body.Error.Type == "" { + t.Fatalf("body = %s (err %v)", recorder.Body.String(), err) + } + }) + t.Run("systemone 400 is FastAPI style", func(t *testing.T) { + recorder := postJSON(t, "/v1/systemone?mock_error=400", systemOneBody) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", recorder.Code) + } + var body struct { + Detail []struct { + Loc []any `json:"loc"` + Msg string `json:"msg"` + Type string `json:"type"` + } `json:"detail"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil || len(body.Detail) == 0 || body.Detail[0].Msg == "" { + t.Fatalf("body = %s (err %v)", recorder.Body.String(), err) + } + }) +} + +func TestDecisionRoutesRejectNonPost(t *testing.T) { + for _, path := range []string{"/v1/decisions", "/v1/systemone"} { + recorder := getPath(t, path) + if recorder.Code != http.StatusMethodNotAllowed { + t.Fatalf("%s GET status = %d, want 405", path, recorder.Code) + } + } +} + +func assertJSONError(t *testing.T, body []byte) { + t.Helper() + var parsed map[string]any + if err := json.Unmarshal(body, &parsed); err != nil || len(parsed) == 0 { + t.Fatalf("error body is not a JSON object: %q (err %v)", body, err) + } +} + +func getPath(t *testing.T, path string) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequestWithContext(context.Background(), http.MethodGet, path, nil) + recorder := httptest.NewRecorder() + newRouter().ServeHTTP(recorder, request) + return recorder +} + +const realDecisionsBody = `{"model":"m","input":[{"role":"user","content":[` + + `{"type":"input_text","text":"look"},{"type":"input_image","image_url":"data:image/png;base64,AAAA"}]}],"questions":[` + + `{"type":"predicate","name":"p","instructions":"is it red?"},` + + `{"type":"choice","name":"c","instructions":"which?","choices":[{"value":"cat","description":"a cat"},{"value":"dog","description":"a dog"}]},` + + `{"type":"choice","name":"c2","instructions":"which?","choices":["bird","fish"]},` + + `{"type":"score","name":"s","instructions":"rate","levels":[{"label":"bad","description":"d"},{"label":"good","description":"d"}]},` + + `{"type":"score","name":"s2","instructions":"rate","levels":["low","high"]},` + + `{"type":"box","name":"b","instructions":"find it"},` + + `{"type":"future","name":"f","extra":{"x":1}}]}` + +func TestDecisionsAcceptsRealQuestionShapes(t *testing.T) { + recorder := postJSON(t, "/v1/decisions", realDecisionsBody) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d: %s", recorder.Code, recorder.Body.String()) + } + var response struct { + Answers []map[string]any `json:"answers"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if len(response.Answers) != 7 { + t.Fatalf("answers = %s, want 7", recorder.Body.String()) + } + names := []string{"p", "c", "c2", "s", "s2", "b", "f"} + for i, name := range names { + if response.Answers[i]["name"] != name { + t.Fatalf("answer %d name = %v, want %s", i, response.Answers[i]["name"], name) + } + } + a := response.Answers + if a[0]["type"] != "predicate" || a[0]["probability"] == nil { + t.Fatalf("predicate = %v", a[0]) + } + if a[1]["choice"] != "cat" || a[2]["choice"] != "bird" { + t.Fatalf("choices = %v %v", a[1], a[2]) + } + for _, i := range []int{1, 2, 3, 4} { + if _, ok := a[i]["probabilities"].([]any); !ok || a[i]["confidence"] == nil { + t.Fatalf("answer %d lacks probabilities list/confidence: %v", i, a[i]) + } + } + for _, i := range []int{3, 4} { + if _, ok := a[i]["score"].(float64); !ok { + t.Fatalf("score %d = %v, want a number", i, a[i]["score"]) + } + } + box, ok := a[5]["box"].(map[string]any) + if a[5]["type"] != "box" || !ok || box["top"] == nil || box["bottom"] == nil || box["left"] == nil || box["right"] == nil || a[5]["confidence"] == nil { + t.Fatalf("box = %v", a[5]) + } + if a[6]["type"] == "choice" || a[6]["name"] != "f" { + t.Fatalf("unknown type answer = %v", a[6]) + } +} + +const realSystemOneBody = `{"model":"m","state":[{"k":"v"}],"images":["AAAA"],"assets":{"a":{"x":1}},"questions":{` + + `"n":{"type":"noul","criteria":{"true":"yes","false":"no"}},` + + `"c":{"type":"choice","criteria":{"opt_a":"first","opt_b":"second"}},` + + `"s":{"type":"score","criteria":["worst","mid","best"]},` + + `"b":{"type":"box"},` + + `"u":{"type":"mystery","extra":[1,2]}}}` + +func TestSystemOneAcceptsRealQuestionShapes(t *testing.T) { + for _, state := range []string{`"text"`, `{"a":1}`, `[1,2]`} { + body := strings.Replace(realSystemOneBody, `[{"k":"v"}]`, state, 1) + recorder := postJSON(t, "/v1/systemone", body) + if recorder.Code != http.StatusOK { + t.Fatalf("state %s status = %d: %s", state, recorder.Code, recorder.Body.String()) + } + } + recorder := postJSON(t, "/v1/systemone", realSystemOneBody) + var response struct { + Answers map[string]map[string]any `json:"answers"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + a := response.Answers + if len(a) != 5 { + t.Fatalf("answers = %s, want 5", recorder.Body.String()) + } + if a["n"]["type"] != "noul" || a["n"]["noul"] != 0.99 { + t.Fatalf("noul = %v", a["n"]) + } + if a["c"]["choice"] != "opt_a" || a["c"]["probabilities"] == nil || a["c"]["confidence"] == nil { + t.Fatalf("choice = %v", a["c"]) + } + if _, ok := a["s"]["score"].(float64); !ok { + t.Fatalf("score = %v, want a number", a["s"]) + } + if _, ok := a["s"]["legend"].(map[string]any); !ok { + t.Fatalf("legend = %v, want an object", a["s"]["legend"]) + } + if _, ok := a["s"]["probabilities"].(map[string]any); !ok { + t.Fatalf("probabilities = %v, want an object", a["s"]["probabilities"]) + } + if _, ok := a["c"]["probabilities"].(map[string]any); !ok { + t.Fatalf("choice probabilities = %v, want an object", a["c"]["probabilities"]) + } + if a["b"]["box"] == nil { + t.Fatalf("box = %v", a["b"]) + } + if a["u"]["type"] == nil { + t.Fatalf("unknown = %v", a["u"]) + } +} + +func TestDecisionsMatchDocumentedTypes(t *testing.T) { + body := `{"model":"m","input":"x","questions":[` + + `{"type":"choice","name":"flag","choices":[{"value":true,"description":"yes"},{"value":false,"description":"no"}]},` + + `{"type":"choice","name":"num","choices":[3,4]},` + + `{"type":"score","name":"s","levels":[{"label":"lo","description":"d"},{"label":"mid","description":"d"},{"label":"hi","description":"d"}]}]}` + recorder := postJSON(t, "/v1/decisions", body) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d: %s", recorder.Code, recorder.Body.String()) + } + var response struct { + Answers []map[string]any `json:"answers"` + Usage map[string]any `json:"usage"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + a := response.Answers + if a[0]["choice"] != true { + t.Fatalf("boolean choice = %#v, want true", a[0]["choice"]) + } + probs := a[0]["probabilities"].([]any) + if len(probs) != 2 || probs[0].(map[string]any)["value"] != true || probs[1].(map[string]any)["value"] != false || probs[0].(map[string]any)["probability"] == nil { + t.Fatalf("boolean probabilities = %v", probs) + } + if a[1]["choice"] != float64(3) { + t.Fatalf("number choice = %#v, want 3", a[1]["choice"]) + } + score, ok := a[2]["score"].(float64) + if !ok || score < 0 || score > 2 { + t.Fatalf("score = %#v, want number in [0,2]", a[2]["score"]) + } + levels := a[2]["probabilities"].([]any) + if len(levels) != 3 { + t.Fatalf("score probabilities = %v", levels) + } + for i, label := range []string{"lo", "mid", "hi"} { + level := levels[i].(map[string]any) + if level["value"] != float64(i) || level["label"] != label || level["probability"] == nil { + t.Fatalf("level %d = %v", i, level) + } + } + details, _ := response.Usage["input_tokens_details"].(map[string]any) + out, _ := response.Usage["output_tokens_details"].(map[string]any) + if details["cached_tokens"] == nil || details["cache_write_tokens"] == nil || out["reasoning_tokens"] == nil { + t.Fatalf("usage = %v, want token detail objects", response.Usage) + } +} + +func TestSystemOneMatchesDocumentedTypes(t *testing.T) { + body := `{"model":"m","state":"s","questions":{` + + `"c":{"type":"choice","criteria":{"alpha":"first","beta":"second"}},` + + `"s":{"type":"score","criteria":["worst","mid","best"]}}}` + recorder := postJSON(t, "/v1/systemone", body) + var response struct { + Answers map[string]map[string]any `json:"answers"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + c := response.Answers["c"] + probs, ok := c["probabilities"].(map[string]any) + if c["choice"] != "alpha" || !ok || probs["alpha"] == nil || probs["beta"] == nil || c["confidence"] == nil { + t.Fatalf("choice = %v", c) + } + s := response.Answers["s"] + legend, _ := s["legend"].(map[string]any) + sp, _ := s["probabilities"].(map[string]any) + if legend["0"] != "worst" || legend["2"] != "best" || len(legend) != 3 || len(sp) != 3 || sp["1"] == nil { + t.Fatalf("score = %v", s) + } + if score, ok := s["score"].(float64); !ok || score < 0 || score > 2 { + t.Fatalf("score = %#v", s["score"]) + } +} diff --git a/examples/function-samples/openai-compatible-sample/http-server/main.go b/examples/function-samples/openai-compatible-sample/http-server/main.go index 9281352d5a..19a544f9b6 100644 --- a/examples/function-samples/openai-compatible-sample/http-server/main.go +++ b/examples/function-samples/openai-compatible-sample/http-server/main.go @@ -380,6 +380,8 @@ func newRouterWithConfig(config serverConfig) http.Handler { mux.HandleFunc("/v1/completions", func(w http.ResponseWriter, r *http.Request) { handleCompletions(w, r, config) }) mux.HandleFunc("/v1/embeddings", func(w http.ResponseWriter, r *http.Request) { handleEmbeddings(w, r, config) }) mux.HandleFunc("/v1/messages", func(w http.ResponseWriter, r *http.Request) { handleMessages(w, r, config) }) + mux.HandleFunc(decisionRoute, handleDecisions) + mux.HandleFunc(systemOneRoute, handleSystemOne) return mux } diff --git a/src/clis/nvcf-cli/README.md b/src/clis/nvcf-cli/README.md index efa18fc488..01da4f8568 100644 --- a/src/clis/nvcf-cli/README.md +++ b/src/clis/nvcf-cli/README.md @@ -696,9 +696,12 @@ and the CLI prints the API error. `tokenRateLimit` supports positive integer token limits for `S`, `M`, `H`, `D`, and `W`. Use `1000-S` for a single inline CLI limit. Use JSON input for combined limits, such as `1000-S,5000-M,100000-H,500000-D,1000000-W`, because inline model specs use commas as field separators. Supported LLM paths are `/v1/chat/completions`, `/v1/responses`, `/v1/embeddings`, -and native Anthropic `/v1/messages`. Messages requires a compatible gateway, -Stargate, and Pylon build and a backend that serves that protocol. Published -stack releases must qualify and include those builds before use. +native Anthropic `/v1/messages`, and the beta decision model paths +`/v1/decisions` and `/v1/systemone`. Messages and the decision paths require a +compatible gateway, Stargate, and Pylon build and a backend that serves that +protocol. Published stack releases must qualify and include those builds before +use. Declare a decision path in `uris` when you create the function, because +`uris` cannot change afterward. #### Deploy a Function *Uses `NVCF_TOKEN` (with `NVCF_API_KEY` fallback)* diff --git a/src/clis/nvcf-cli/USAGE-GUIDE.md b/src/clis/nvcf-cli/USAGE-GUIDE.md index 4c729a7c04..193cc99bc8 100644 --- a/src/clis/nvcf-cli/USAGE-GUIDE.md +++ b/src/clis/nvcf-cli/USAGE-GUIDE.md @@ -873,9 +873,12 @@ optionally followed by `;name=value` tuning parameters, such as `pulsar;seed=stable-a`. The CLI sends the value unchanged; the API checks it and the CLI prints the API error. Supported LLM paths are `/v1/chat/completions`, `/v1/responses`, `/v1/embeddings`, -and native Anthropic `/v1/messages`. Messages requires a compatible gateway, -Stargate, and Pylon build and a backend that serves that protocol. Published -stack releases must qualify and include those builds before use. +native Anthropic `/v1/messages`, and the beta decision model paths +`/v1/decisions` and `/v1/systemone`. Messages and the decision paths require a +compatible gateway, Stargate, and Pylon build and a backend that serves that +protocol. Published stack releases must qualify and include those builds before +use. Declare a decision path in `uris` when you create the function, because +`uris` cannot change afterward. ### 5. List Functions diff --git a/src/invocation-plane-services/llm-api-gateway/README.md b/src/invocation-plane-services/llm-api-gateway/README.md index ad1e8ee6db..6dac1d808d 100644 --- a/src/invocation-plane-services/llm-api-gateway/README.md +++ b/src/invocation-plane-services/llm-api-gateway/README.md @@ -1,7 +1,8 @@ # llm-api-gateway `llm-api-gateway` is an OpenAI-compatible gateway for routing chat, -responses, and embeddings traffic to NVCF functions through Stargate. +responses, embeddings, and decision model traffic to NVCF functions through +Stargate. Requests reach this gateway as OpenAI-compatible payloads. The gateway does not render Hugging Face or Jinja chat templates, does not tokenize prompts, and @@ -34,6 +35,8 @@ The gateway currently serves: - `POST /v1/chat/completions` - `POST /v1/responses` - `POST /v1/embeddings` +- `POST /v1/decisions` +- `POST /v1/systemone` ## Request Routing @@ -72,6 +75,42 @@ conversation. The gateway preserves the raw body field for the model backend. It forwards only a SHA-256-derived value in the internal `x-cache-affinity-key` header to Stargate. +### Decision routes + +`POST /v1/decisions` and `POST /v1/systemone` forward the request and response +bodies unchanged to a decision model. The gateway reads only `model`, rewrites +it to the function's model name, and does not translate between the two +formats. Responses are not streamed; `stream` is ignored. + +- A model is enabled for a route only when its function declares the route in + the model's `uris`, for example `--llm-model name=,uris=/v1/decisions`. + A model with no `uris`, or without the route in them, gets a 404 in the + route's own error shape. These routes have no default-allow and are not + affected by `MODEL_URI_ALLOWLIST_ENABLED`. +- Admission counts one request and the estimated input tokens, and the + estimate is sent to Stargate as `x-input-tokens` and `x-token-estimate`. The + estimate counts every string in the body as text. Each image counts a flat + `NVCF_DECISION_IMAGE_TOKENS` (default `1024`) and its base64 is never + counted. An image is an object with `type` set to `input_image`, an element + of `images`, a value of `assets`, or any string that is a base64 data URL. +- Client headers, including `TypeSafe-Danger-Zone-Images`, reach the model + server. +- `/v1/decisions` returns errors in the OpenAI shape. `/v1/systemone` returns + `{"error":{"code","message","request_id"},"request_id"}`, where `request_id` + equals the `X-Request-ID` response header. Codes are `invalid_json`, + `invalid_request`, `invalid_api_key`, `not_found`, `request_too_large`, + `rate_limit_exceeded`, `internal_error`, `provider_unavailable`, and + `provider_timeout`. +- An error from the model server is relayed byte for byte, whatever its shape, + size, or encoding. Only errors the gateway or the request router generates + use the shapes above. The router marks its own errors with + `X-Stargate-Error-Code`. +- When a 2xx response reports `usage.input_tokens`, the gateway records it on + `llm_api_gateway_llm_tokens_total` and counts the response in + `llm_api_gateway_decision_usage_observations_total` with `status="observed"`. + A response without it counts as `absent`. The estimate stays the admission + value. + ### Last-cluster hints With `STARGATE_LAST_CLUSTER_ENABLED=true`, the gateway remembers which @@ -213,6 +252,9 @@ Useful overrides: - `NVCF_GATEWAY_ADDR` to bind a specific listen address - `NVCF_GATEWAY_MAX_REQUEST_BODY_BYTES` to reject larger request bodies with 413 (default `0`, no limit) +- `NVCF_DECISION_IMAGE_TOKENS` for the flat input token estimate of each image + on the decision routes (default `1024`, must be `0` or more). The Helm + chart sets it with `llmApiGateway.config.decisionImageTokens`. - `STARGATE_CONNECT_TIMEOUT` to control Stargate dial timeout - `STARGATE_REQUEST_TIMEOUT` to cap end-to-end Stargate request time - `STARGATE_LAST_CLUSTER_ENABLED=true` to send session last-cluster hints to diff --git a/src/invocation-plane-services/llm-api-gateway/api/BUILD.bazel b/src/invocation-plane-services/llm-api-gateway/api/BUILD.bazel index d10a681c79..f6b51a4c2e 100644 --- a/src/invocation-plane-services/llm-api-gateway/api/BUILD.bazel +++ b/src/invocation-plane-services/llm-api-gateway/api/BUILD.bazel @@ -22,6 +22,9 @@ go_library( "body_limit.go", "chat_handler.go", "context.go", + "decision_handler.go", + "decision_token_count.go", + "decision_usage.go", "error_response.go", "handlers.go", "last_cluster_keys.go", @@ -89,6 +92,11 @@ go_test( srcs = [ "auth_middleware_test.go", "body_limit_test.go", + "decision_backend_error_test.go", + "decision_error_test.go", + "decision_handler_test.go", + "decision_token_count_test.go", + "decision_usage_test.go", "embedding_input_test.go", "embeddings_handler_test.go", "fidelity_corpus_test.go", diff --git a/src/invocation-plane-services/llm-api-gateway/api/context.go b/src/invocation-plane-services/llm-api-gateway/api/context.go index 679cc434c1..a5bb3726dc 100644 --- a/src/invocation-plane-services/llm-api-gateway/api/context.go +++ b/src/invocation-plane-services/llm-api-gateway/api/context.go @@ -33,6 +33,9 @@ const ( contextKeyRequestContext = "nvcf.request_context" contextKeyRequestID = "nvcf.request_id" contextKeyRateLimitCtx = "nvcf.rate_limit_context" + // contextKeyBackendErrorRelayed marks a backend error written unchanged, + // so headers the gateway normally owns are left as the backend sent them. + contextKeyBackendErrorRelayed = "nvcf.backend_error_relayed" ) type GatewayContext struct { diff --git a/src/invocation-plane-services/llm-api-gateway/api/decision_backend_error_test.go b/src/invocation-plane-services/llm-api-gateway/api/decision_backend_error_test.go new file mode 100644 index 0000000000..17be0bd2cd --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/decision_backend_error_test.go @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + echo "github.com/labstack/echo/v4" + "github.com/rs/zerolog" + zlog "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/config" + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/provider" +) + +// serveDecisionUpstream runs route against a backend that answers with +// status, headers and body, and returns the gateway response. +func serveDecisionUpstream(t *testing.T, route decisionRoute, cfg *config.Config, clientHeaders http.Header, status int, headers http.Header, body []byte) *httptest.ResponseRecorder { + t.Helper() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + for key, values := range headers { + for _, value := range values { + w.Header().Add(key, value) + } + } + w.WriteHeader(status) + _, _ = w.Write(body) + })) + t.Cleanup(upstream.Close) + + e := decisionTestAPI(t, upstream.URL, cfg, nil, decisionSpecs(route.path)) + req := decisionRequest(route, route.body) + for key, values := range clientHeaders { + req.Header[key] = values + } + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + return rec +} + +// TestDecisionRoutesRelayBackendErrorsUnchanged checks that a non-2xx response +// without X-Stargate-Error-Code is a backend error and reaches the client byte +// for byte, whatever its shape, size or encoding. +func TestDecisionRoutesRelayBackendErrorsUnchanged(t *testing.T) { + t.Parallel() + + gzipped := gzipBytes(t, `{"detail":"compressed validation error"}`) + oversized := []byte(`{"detail":[{"loc":["body","state"],"msg":"too long","input":"` + strings.Repeat("x", provider.MaxUpstreamErrorBodyBytes+1) + `"}]}`) + for _, route := range decisionRoutes { + for _, tc := range []struct { + name string + status int + headers http.Header + body []byte + }{ + {"FastAPI detail list", http.StatusUnprocessableEntity, http.Header{"Content-Type": {"application/json"}}, []byte(`{"detail":[{"type":"missing","loc":["body","questions"],"msg":"Field required","input":{}}]}`)}, + {"OpenAI shape", http.StatusBadRequest, http.Header{"Content-Type": {"application/json"}}, []byte(`{"error":{"message":"bad question","type":"BadRequestError","param":null,"code":400}}`)}, + {"System One shape with backend codes", http.StatusBadRequest, http.Header{"Content-Type": {"application/json"}, "X-System-One-Error-Code": {"invalid_question"}, "X-System-One-Error-Source": {"upstream"}}, []byte(`{"error":{"code":"invalid_question","message":"bad","request_id":"backend-id"},"request_id":"backend-id"}`)}, + {"body over 1 MiB", http.StatusUnprocessableEntity, http.Header{"Content-Type": {"application/json"}}, oversized}, + {"gzip body", http.StatusUnprocessableEntity, http.Header{"Content-Type": {"application/json"}, "Content-Encoding": {"gzip"}}, gzipped}, + {"unsupported encoding", http.StatusBadRequest, http.Header{"Content-Type": {"application/json"}, "Content-Encoding": {"br"}}, []byte("\x1b\x00 compressed")}, + {"plain text", http.StatusInternalServerError, http.Header{"Content-Type": {"text/plain"}}, []byte("upstream connect error")}, + {"backend overload status", 529, http.Header{"Content-Type": {"application/json"}}, []byte(`{"detail":"busy"}`)}, + {"backend request id", http.StatusBadRequest, http.Header{"Content-Type": {"application/json"}, HeaderRequestID: {"backend-request-id"}}, []byte(`{"detail":"x"}`)}, + } { + t.Run(route.name+"/"+tc.name, func(t *testing.T) { + t.Parallel() + + rec := serveDecisionUpstream(t, route, config.Default(), http.Header{echo.HeaderAcceptEncoding: {"gzip, br"}}, tc.status, tc.headers, tc.body) + + require.Equal(t, tc.status, rec.Code) + require.True(t, bytes.Equal(tc.body, rec.Body.Bytes()), "body must be byte for byte; got %d bytes, want %d", rec.Body.Len(), len(tc.body)) + for key, values := range tc.headers { + require.Equal(t, values, rec.Header().Values(key), key) + } + }) + } + } +} + +func TestDecisionRoutesRouterErrorsUseNativeShape(t *testing.T) { + t.Parallel() + + for _, route := range decisionRoutes { + for _, tc := range []struct { + name string + routerCode string + status, wantStatus int + body string + wantMessage string + wantSystemOneCode string + wantOpenAIType string + wantOpenAIOverloadTag bool + }{ + {"no eligible candidates", "no_eligible_candidates", http.StatusNotFound, http.StatusNotFound, `{"error":"no eligible candidates","code":"no_eligible_candidates"}`, "no eligible candidates", "not_found", "invalid_request_error", false}, + {"overloaded", "overloaded_error", http.StatusServiceUnavailable, 529, `{"error":{"code":"overloaded_error","message":"Inference capacity is temporarily unavailable.","param":"","type":"overloaded_error"}}`, "Inference capacity is temporarily unavailable.", "provider_unavailable", "overloaded_error", true}, + {"tunnel failure with empty body", "tunnel_failed", http.StatusBadGateway, http.StatusBadGateway, ``, "Bad Gateway", "provider_unavailable", "server_error", false}, + {"request body over replay limit", "body_too_large", http.StatusRequestEntityTooLarge, http.StatusRequestEntityTooLarge, ``, "Request Entity Too Large", "request_too_large", "invalid_request_error", false}, + {"router rejection with a backend-shaped body", "unknown_method", http.StatusBadRequest, http.StatusBadRequest, `{"detail":[{"msg":"router says no"}]}`, "router says no", "invalid_request", "invalid_request_error", false}, + } { + t.Run(route.name+"/"+tc.name, func(t *testing.T) { + t.Parallel() + + rec := serveDecisionUpstream(t, route, config.Default(), nil, tc.status, + http.Header{"X-Stargate-Error-Code": {tc.routerCode}, "Retry-After": {"2"}, "Content-Type": {"application/json"}}, []byte(tc.body)) + + requireNativeError(t, route, rec, tc.wantStatus, tc.wantOpenAIType, tc.wantSystemOneCode) + require.Empty(t, rec.Header().Values("X-Stargate-Error-Code")) + require.Equal(t, "2", rec.Header().Get("Retry-After")) + require.Contains(t, rec.Body.String(), tc.wantMessage) + if !tc.wantOpenAIOverloadTag { + require.NotContains(t, rec.Body.String(), tc.routerCode, "the router code is only logged unless ExposeRouterErrorCodes is set") + } + }) + } + } +} + +// A router failure that is not a routing rejection is a bare status with no +// body and no X-Stargate-Error-Code. It must not be relayed raw. +func TestDecisionRoutesRenderBareRouterErrorsNatively(t *testing.T) { + t.Parallel() + + for _, route := range decisionRoutes { + for _, tc := range []struct { + status int + openAIType string + systemOneCode string + }{ + {http.StatusRequestEntityTooLarge, "invalid_request_error", "request_too_large"}, + {499, "invalid_request_error", "invalid_request"}, + {http.StatusServiceUnavailable, "server_error", "provider_unavailable"}, + {http.StatusBadGateway, "server_error", "provider_unavailable"}, + {http.StatusGatewayTimeout, "server_error", "provider_timeout"}, + } { + t.Run(route.name+"/"+http.StatusText(tc.status), func(t *testing.T) { + t.Parallel() + rec := serveDecisionUpstream(t, route, config.Default(), nil, tc.status, + http.Header{"Retry-After": {"3"}, "X-System-One-Error-Code": {"backend_code"}}, nil) + requireNativeError(t, route, rec, tc.status, tc.openAIType, tc.systemOneCode) + require.Equal(t, "3", rec.Header().Get("Retry-After")) + require.Empty(t, rec.Header().Values("X-System-One-Error-Code"), "gateway-written errors carry no X-System-One-* headers") + }) + } + } +} + +// An empty body on a status Stargate does not emit as a bare router error is +// the backend's own answer and is relayed unchanged, headers included. +func TestDecisionRoutesRelayEmptyBackendErrorsUnchanged(t *testing.T) { + t.Parallel() + + for _, route := range decisionRoutes { + for _, status := range []int{http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusNotImplemented} { + t.Run(route.name+"/"+http.StatusText(status), func(t *testing.T) { + t.Parallel() + headers := http.Header{"Content-Type": {"application/json"}, "Retry-After": {"7"}, "X-System-One-Error-Code": {"backend_code"}, "X-Vendor": {"kept"}} + rec := serveDecisionUpstream(t, route, config.Default(), nil, status, headers, nil) + + require.Equal(t, status, rec.Code) + require.Empty(t, rec.Body.String()) + for key, values := range headers { + require.Equal(t, values, rec.Header().Values(key), key) + } + }) + } + } +} + +type peekCountingReader struct { + r io.Reader + read int +} + +func (c *peekCountingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.read += n + return n, err +} + +func TestBackendErrorPeekDoesNotBufferBody(t *testing.T) { + t.Parallel() + + payload := strings.Repeat("x", 4<<20) + counter := &peekCountingReader{r: strings.NewReader(payload)} + // 502 is one of the bare router statuses, so the empty-body peek runs. + resp := &provider.ProxyResponse{StatusCode: http.StatusBadGateway, Header: http.Header{}, Body: io.NopCloser(counter)} + + require.True(t, isBackendErrorOnDecisionRoute(resp)) + require.Equal(t, 1, counter.read, "the peek must read exactly one byte") + rest, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, payload, string(rest), "the peeked byte must not be lost") +} + +func TestDecisionRoutesRequestIDHeader(t *testing.T) { + t.Parallel() + + for _, route := range decisionRoutes { + t.Run(route.name+"/success", func(t *testing.T) { + t.Parallel() + + rec := serveDecisionUpstream(t, route, config.Default(), nil, http.StatusOK, + http.Header{"Content-Type": {"application/json"}, HeaderRequestID: {"backend-request-id"}}, []byte(`{}`)) + + require.Equal(t, http.StatusOK, rec.Code) + if route.systemOne { + // System One errors carry the NVCF ID, so every response does. + require.Equal(t, "request-a", rec.Header().Get(HeaderRequestID)) + } else { + require.Equal(t, "backend-request-id", rec.Header().Get(HeaderRequestID)) + } + require.Empty(t, rec.Header().Get(headerMessagesRequestID)) + }) + } +} + +func TestDecisionRoutesRouterRejectionLogsErrorCode(t *testing.T) { + var logs bytes.Buffer + oldLogger := zlog.Logger + zlog.Logger = zerolog.New(&logs) + t.Cleanup(func() { zlog.Logger = oldLogger }) + + for _, route := range decisionRoutes { + logs.Reset() + rec := serveDecisionUpstream(t, route, config.Default(), nil, http.StatusNotFound, + http.Header{"X-Stargate-Error-Code": {"no_eligible_candidates"}}, []byte(`{"error":"no eligible candidates","code":"no_eligible_candidates"}`)) + + require.Equal(t, http.StatusNotFound, rec.Code, route.name) + require.NotContains(t, rec.Body.String(), "no_eligible_candidates", route.name) + require.Contains(t, logs.String(), `"router_error_code":"no_eligible_candidates"`, route.name) + require.Contains(t, logs.String(), `"message":"completed http request"`, route.name) + } +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/decision_error_test.go b/src/invocation-plane-services/llm-api-gateway/api/decision_error_test.go new file mode 100644 index 0000000000..c1b9ce7b9d --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/decision_error_test.go @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + echo "github.com/labstack/echo/v4" + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/config" +) + +// decisionErrorAPI serves path with a handler that sets backendHeaders, as a +// relayed backend response would, and then fails with err. +func decisionErrorAPI(path string, backendHeaders http.Header, err error) *echo.Echo { + e := echo.New() + e.HTTPErrorHandler = newHTTPErrorHandler(false) + e.Use(NewContextMiddleware(config.Default())) + e.POST(path, func(c echo.Context) error { + for key, values := range backendHeaders { + for _, value := range values { + c.Response().Header().Add(key, value) + } + } + return err + }) + return e +} + +func serveDecisionError(t *testing.T, path string, backendHeaders http.Header, err error) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{}`)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + req.Header.Set(HeaderRequestID, "request-a") + rec := httptest.NewRecorder() + decisionErrorAPI(path, backendHeaders, err).ServeHTTP(rec, req) + return rec +} + +func systemOneErrorJSON(t *testing.T, code, message, requestID string) string { + t.Helper() + body, err := json.Marshal(map[string]any{ + "error": map[string]string{"code": code, "message": message, "request_id": requestID}, + "request_id": requestID, + }) + require.NoError(t, err) + return string(body) +} + +func TestErrorShapeFor(t *testing.T) { + t.Parallel() + + for path, want := range map[string]errorShape{ + "/v1/messages": shapeAnthropic, + "/v1/decisions": shapeOpenAI, + "/v1/systemone": shapeSystemOne, + "/v1/chat/completions": shapeOpenAI, + "/v1/embeddings": shapeOpenAI, + "/unknown": shapeOpenAI, + } { + req := httptest.NewRequest(http.MethodPost, path, nil) + require.Equal(t, want, errorShapeFor(req), path) + } +} + +func TestSystemOneGatewayErrorsUseSystemOneShape(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + status int + code string + internal error + }{ + {http.StatusBadRequest, "invalid_request", nil}, + {http.StatusBadRequest, "invalid_json", errInvalidJSON}, + {http.StatusUnauthorized, "invalid_api_key", nil}, + {http.StatusForbidden, "invalid_request", nil}, + {http.StatusNotFound, "not_found", nil}, + {http.StatusRequestEntityTooLarge, "request_too_large", nil}, + {http.StatusUnprocessableEntity, "invalid_request", nil}, + {http.StatusTooManyRequests, "rate_limit_exceeded", nil}, + {http.StatusInternalServerError, "internal_error", nil}, + {http.StatusNotImplemented, "internal_error", nil}, + {http.StatusBadGateway, "provider_unavailable", nil}, + {http.StatusServiceUnavailable, "provider_unavailable", nil}, + {statusOverloaded, "provider_unavailable", nil}, + {http.StatusGatewayTimeout, "provider_timeout", nil}, + } { + t.Run(fmt.Sprintf("%d/%s", tc.status, tc.code), func(t *testing.T) { + t.Parallel() + + err := echo.NewHTTPError(tc.status, "something failed") + if tc.internal != nil { + err = err.SetInternal(tc.internal) + } + rec := serveDecisionError(t, "/v1/systemone", nil, err) + + require.Equal(t, tc.status, rec.Code, rec.Body.String()) + require.Equal(t, echo.MIMEApplicationJSON, rec.Header().Get(echo.HeaderContentType)) + require.JSONEq(t, systemOneErrorJSON(t, tc.code, "something failed", "request-a"), rec.Body.String()) + require.Equal(t, "request-a", rec.Header().Get(HeaderRequestID)) + require.Empty(t, rec.Header().Get(headerMessagesRequestID)) + for name := range rec.Header() { + require.False(t, strings.HasPrefix(strings.ToLower(name), "x-system-one-"), name) + } + }) + } +} + +func TestSystemOneErrorRequestIDMatchesHeaderAfterBackendHeadersCopied(t *testing.T) { + t.Parallel() + + rec := serveDecisionError(t, "/v1/systemone", + http.Header{HeaderRequestID: {"backend-id"}, "X-System-One-Error-Code": {"backend_code"}}, + echo.NewHTTPError(http.StatusBadGateway, "router rejected")) + + require.Equal(t, "request-a", rec.Header().Get(HeaderRequestID)) + require.JSONEq(t, systemOneErrorJSON(t, "provider_unavailable", "router rejected", "request-a"), rec.Body.String()) +} + +func TestDecisionsGatewayErrorsUseOpenAIShape(t *testing.T) { + t.Parallel() + + rec := serveDecisionError(t, "/v1/decisions", nil, echo.NewHTTPError(http.StatusNotFound, "no such model")) + + require.Equal(t, http.StatusNotFound, rec.Code) + require.JSONEq(t, openAIErrorJSON(t, "invalid_request_error", "no such model", nil), rec.Body.String()) + require.Equal(t, "request-a", rec.Header().Get(HeaderRequestID)) + require.Empty(t, rec.Header().Get(headerMessagesRequestID)) +} + +func TestSystemOneErrorForPlainErrorIsInternalError(t *testing.T) { + t.Parallel() + + rec := serveDecisionError(t, "/v1/systemone", nil, errors.New("boom")) + + require.Equal(t, http.StatusInternalServerError, rec.Code) + require.JSONEq(t, systemOneErrorJSON(t, "internal_error", "Internal Server Error", "request-a"), rec.Body.String()) +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/decision_handler.go b/src/invocation-plane-services/llm-api-gateway/api/decision_handler.go new file mode 100644 index 0000000000..076319809f --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/decision_handler.go @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + + echo "github.com/labstack/echo/v4" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/internal/must" + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/requestctx" + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/telemetry" +) + +// decisionRouteRequest binds only what routing needs. The body is otherwise +// opaque: it is forwarded unchanged apart from the model rewrite. +type decisionRouteRequest struct { + Model string `json:"model"` +} + +// Decisions serves POST /v1/decisions as a non-streaming passthrough. +func (h *OpenAIProxyHandlers) Decisions(ec echo.Context) error { + return h.serveDecisionRoute(ec, decisionsEndpointPath) +} + +// SystemOne serves POST /v1/systemone as a non-streaming passthrough. +func (h *OpenAIProxyHandlers) SystemOne(ec echo.Context) error { + return h.serveDecisionRoute(ec, systemOneEndpointPath) +} + +func (h *OpenAIProxyHandlers) serveDecisionRoute(ec echo.Context, endpointPath string) error { + c := must.As[*GatewayContext](ec) + rawBody, err := captureRequestBody(c.Request()) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + // Parse before requiring the function context: a body that is not JSON + // has no routing key, and the client should hear invalid JSON, not a + // missing model prefix. + var request decisionRouteRequest + if err := json.Unmarshal(rawBody, &request); err != nil { + return decisionBodyError(err) + } + if err := rejectAmbiguousMembers(rawBody, decisionRouteRequest{}); err != nil { + return err + } + reqCtx, err := h.handlers.requireFunctionRequestContext(c) + if err != nil { + return err + } + requestModel := request.Model + if err := h.requireDecisionRouteEnabled(c, reqCtx, requestModel, endpointPath); err != nil { + return err + } + + inputTokens, err := estimatedTokenCountForDecisionBody(rawBody, h.decisionImageTokens()) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + if err := h.admitProxyRequest(c, reqCtx, inputTokens, 0); err != nil { + return err + } + + span := trace.SpanFromContext(c.UserContext()) + span.SetAttributes( + attribute.String("gateway.endpoint", endpointPath), + attribute.String("gen_ai.request.model", requestModel), + attribute.String("gateway.routed_model", reqCtx.Model), + attribute.String("nvcf.function.id", requestFunctionID(c)), + attribute.String("nvcf.org.id", reqCtx.OrgID), + attribute.Int("gen_ai.usage.input_tokens", inputTokens), + ) + + return h.forwardJSONRequest(c, reqCtx, rawBody, inputTokens) +} + +// decisionBodyError returns a 400 for a body that could not be bound. Syntax +// errors are marked so System One reports invalid_json rather than +// invalid_request. +func decisionBodyError(err error) error { + httpErr := echo.NewHTTPError(http.StatusBadRequest, err.Error()) + var typeErr *json.UnmarshalTypeError + if !errors.As(err, &typeErr) { + httpErr.SetInternal(fmt.Errorf("%w: %w", errInvalidJSON, err)) + } + return httpErr +} + +// requireDecisionRouteEnabled resolves the routed model and refuses it with a +// 404 unless the function declares endpointPath in the model's uris. A model +// with no spec, or a spec with no uris, is not enabled: unlike the other +// routes, these have no default-allow. +func (h *OpenAIProxyHandlers) requireDecisionRouteEnabled( + c *GatewayContext, + reqCtx *requestctx.RequestContext, + requestModel string, + endpointPath string, +) error { + routedModel, err := normalizeOpenAIRequestModel(reqCtx, requestModel) + if err != nil { + return err + } + reqCtx.Model = routedModel + setRoutingMethodForModel(reqCtx, routedModel) + + if spec, ok := reqCtx.ModelSpecs[routedModel]; ok { + for _, uri := range spec.URIs { + if modelURIMatches(uri, endpointPath) { + return nil + } + } + } + + ctx := c.UserContext() + telemetry.AddWithContext( + ctx, + h.handlers.observability.modelURIAllowlistRejections, + 1, + attribute.String("endpoint", endpointPath), + ) + telemetry.Logger(ctx).Warn(). + Str("model", routedModel). + Str("endpoint", endpointPath). + Str("mode", "enforce"). + Msg("endpoint not in model uris allowlist") + return echo.NewHTTPError( + http.StatusNotFound, + fmt.Sprintf("model %q is not enabled for %s", routedModel, endpointPath), + ) +} + +func (h *OpenAIProxyHandlers) decisionImageTokens() int { + if h == nil || h.handlers == nil || h.handlers.config == nil { + return 0 + } + return h.handlers.config.DecisionImageTokens +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/decision_handler_test.go b/src/invocation-plane-services/llm-api-gateway/api/decision_handler_test.go new file mode 100644 index 0000000000..91bbd8c2d0 --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/decision_handler_test.go @@ -0,0 +1,302 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + + echo "github.com/labstack/echo/v4" + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/config" + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/nvcf" + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/provider" + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/ratelimit" + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/requestctx" +) + +const decisionTestModel = "company-name/model-name" + +// decisionRoute is one of the two decision routes with the body a client sends. +type decisionRoute struct { + name, path, body string + systemOne bool +} + +var decisionRoutes = []decisionRoute{ + {name: "decisions", path: "/v1/decisions", body: `{"model":"fn-alpha/company-name/model-name","input":"hello","predicates":[{"id":"p1","text":"Is it red?"}]}`}, + {name: "systemone", path: "/v1/systemone", body: `{"model":"fn-alpha/company-name/model-name","state":"hello","questions":{"q1":{"type":"choice","options":["a","b"]}}}`, systemOne: true}, +} + +// staticLimitResolver applies the same limits to every request. +type staticLimitResolver []ratelimit.ResourceLimit + +func (r staticLimitResolver) ResolveLimits(_ context.Context, _ *requestctx.RequestContext, _ string) ([]ratelimit.ResourceLimit, error) { + return r, nil +} + +// decisionTestAPI serves both decision routes against upstream. specs is the +// model spec map the auth response declares for function fn-alpha. +func decisionTestAPI(t *testing.T, upstream string, cfg *config.Config, limiter *recordingRateLimiter, specs map[string]nvcf.ModelSpec, opts ...HandlerOption) *echo.Echo { + t.Helper() + p, err := provider.NewStargateProvider(config.StargateConfig{URL: upstream}) + require.NoError(t, err) + auth := &stubInvocationAuthClient{authResponse: &nvcf.InvocationAuthResponse{ + RoutingKey: "fn-alpha", RateLimitKey: "org-test", AuthContext: map[string]string{"ncaId": "org-test"}, + ModelSpecs: specs, + }} + e := echo.New() + e.Use(NewContextMiddleware(cfg)) + e.Use(NewNVCFAuthMiddleware(auth)) + h := NewHandlers(cfg, p, nil, opts...) + if limiter != nil { + h.rateLimiter = limiter + } + RegisterRoutes(e, h) + return e +} + +func decisionSpecs(uris ...string) map[string]nvcf.ModelSpec { + return map[string]nvcf.ModelSpec{decisionTestModel: {URIs: uris, RoutingMethod: "random"}} +} + +func decisionRequest(route decisionRoute, body string) *http.Request { + req := httptest.NewRequest(http.MethodPost, route.path, strings.NewReader(body)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + req.Header.Set("Authorization", "Bearer caller-token") + req.Header.Set(HeaderRequestID, "request-a") + return req +} + +// requireNativeError checks the error body is in the route's native shape. +func requireNativeError(t *testing.T, route decisionRoute, rec *httptest.ResponseRecorder, wantStatus int, wantOpenAIType, wantSystemOneCode string) { + t.Helper() + require.Equal(t, wantStatus, rec.Code, rec.Body.String()) + var body map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body), rec.Body.String()) + errObject, ok := body["error"].(map[string]any) + require.True(t, ok, rec.Body.String()) + if route.systemOne { + require.Equal(t, wantSystemOneCode, errObject["code"], rec.Body.String()) + require.Equal(t, "request-a", errObject["request_id"], rec.Body.String()) + require.Equal(t, "request-a", body["request_id"], rec.Body.String()) + require.Equal(t, "request-a", rec.Header().Get(HeaderRequestID)) + return + } + require.Equal(t, wantOpenAIType, errObject["type"], rec.Body.String()) + require.Contains(t, body["error"], "message") + require.NotContains(t, body, "request_id") +} + +func TestDecisionRoutesForwardBodyAndHeaders(t *testing.T) { + t.Parallel() + + type captured struct { + Path, Model, RoutingKey, InputTokens, TokenEstimate string + Header http.Header + Body []byte + } + const responseBody = `{"results":[{"id":"p1","probability":0.25,"vendor":{"n":9007199254740993}}]}` + for _, route := range decisionRoutes { + t.Run(route.name, func(t *testing.T) { + t.Parallel() + + got := make(chan captured, 1) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + got <- captured{r.URL.Path, r.Header.Get("X-Model"), r.Header.Get("X-Routing-Key"), r.Header.Get("X-Input-Tokens"), r.Header.Get("X-Token-Estimate"), r.Header.Clone(), body} + w.Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + w.Header().Set("X-Vendor-Response", "kept") + _, _ = io.WriteString(w, responseBody) + })) + defer upstream.Close() + + cfg := config.Default() + cfg.DecisionImageTokens = 1000 + e := decisionTestAPI(t, upstream.URL, cfg, nil, decisionSpecs(route.path)) + body := strings.Replace(route.body, `"hello"`, `"hello","images":["QUJD","QUJD"]`, 1) + req := decisionRequest(route, body) + req.Header.Set("TypeSafe-Danger-Zone-Images", "true") + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + require.Equal(t, responseBody, rec.Body.String()) + require.Equal(t, echo.MIMEApplicationJSON, rec.Header().Get(echo.HeaderContentType)) + require.Equal(t, "kept", rec.Header().Get("X-Vendor-Response")) + + upstreamRequest := <-got + require.Equal(t, route.path, upstreamRequest.Path) + require.Equal(t, decisionTestModel, upstreamRequest.Model) + require.Equal(t, "fn-alpha", upstreamRequest.RoutingKey) + require.Equal(t, "true", upstreamRequest.Header.Get("TypeSafe-Danger-Zone-Images")) + require.Equal(t, "Bearer caller-token", upstreamRequest.Header.Get("Authorization")) + outbound := jsonObject(t, upstreamRequest.Body) + require.JSONEq(t, `"`+decisionTestModel+`"`, string(outbound["model"])) + require.JSONEq(t, `["QUJD","QUJD"]`, string(outbound["images"])) + + wantTokens, err := estimatedTokenCountForDecisionBody([]byte(body), 1000) + require.NoError(t, err) + require.GreaterOrEqual(t, wantTokens, 2000, "two images must count") + require.Equal(t, strconv.Itoa(wantTokens), upstreamRequest.InputTokens) + require.Equal(t, upstreamRequest.InputTokens, upstreamRequest.TokenEstimate) + }) + } +} + +func TestDecisionRoutesModelNotEnabled(t *testing.T) { + t.Parallel() + + for _, route := range decisionRoutes { + for _, tc := range []struct { + name string + specs map[string]nvcf.ModelSpec + }{ + {"route not declared", decisionSpecs("/v1/chat/completions")}, + {"only the other decision route declared", decisionSpecs(otherDecisionPath(route.path))}, + {"no uris declared", decisionSpecs()}, + {"no spec for the model", map[string]nvcf.ModelSpec{"company-name/other": {URIs: []string{route.path}}}}, + {"no specs at all", nil}, + } { + t.Run(route.name+"/"+tc.name, func(t *testing.T) { + t.Parallel() + + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { hits.Add(1) })) + defer upstream.Close() + + e := decisionTestAPI(t, upstream.URL, config.Default(), nil, tc.specs) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, decisionRequest(route, route.body)) + + requireNativeError(t, route, rec, http.StatusNotFound, "invalid_request_error", "not_found") + require.Contains(t, rec.Body.String(), route.path) + require.Zero(t, hits.Load(), "a model that is not enabled must not reach the backend") + }) + } + } +} + +func otherDecisionPath(path string) string { + if path == decisionsEndpointPath { + return systemOneEndpointPath + } + return decisionsEndpointPath +} + +func TestDecisionRoutesAcceptLenientlyDeclaredURIs(t *testing.T) { + t.Parallel() + + for _, route := range decisionRoutes { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, `{}`) })) + defer upstream.Close() + + declared := strings.ToUpper(strings.TrimPrefix(route.path, "/")) + "/" + e := decisionTestAPI(t, upstream.URL, config.Default(), nil, decisionSpecs("/v1/chat/completions", declared)) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, decisionRequest(route, route.body)) + require.Equal(t, http.StatusOK, rec.Code, route.name+": "+rec.Body.String()) + } +} + +func TestDecisionRoutesRejectBadRequests(t *testing.T) { + t.Parallel() + + for _, route := range decisionRoutes { + for _, tc := range []struct { + name, body, wantSystemOneCode string + }{ + {"malformed JSON", `{"model":"fn-alpha/company-name/model-name",`, "invalid_json"}, + {"empty body", ``, "invalid_json"}, + {"body not an object", `[1,2]`, "invalid_request"}, + {"model missing", `{"input":"hello"}`, "invalid_request"}, + {"model not a string", `{"model":7}`, "invalid_request"}, + {"model without function prefix", `{"model":"model"}`, "invalid_request"}, + {"nesting deeper than 64 levels", `{"model":"fn-alpha/company-name/model-name","state":` + strings.Repeat(`{"a":`, 70) + `1` + strings.Repeat(`}`, 70) + `}`, "invalid_request"}, + {"model duplicate member case", `{"model":"fn-alpha/company-name/model-name","MODEL":"fn-alpha/company-name/other"}`, "invalid_request"}, + } { + t.Run(route.name+"/"+tc.name, func(t *testing.T) { + t.Parallel() + + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { hits.Add(1) })) + defer upstream.Close() + + e := decisionTestAPI(t, upstream.URL, config.Default(), nil, decisionSpecs(route.path)) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, decisionRequest(route, tc.body)) + + requireNativeError(t, route, rec, http.StatusBadRequest, "invalid_request_error", tc.wantSystemOneCode) + require.Zero(t, hits.Load()) + }) + } + } +} + +func TestDecisionRoutesRequireBearerAuthorization(t *testing.T) { + t.Parallel() + + for _, route := range decisionRoutes { + e := decisionTestAPI(t, "http://127.0.0.1:1", config.Default(), nil, decisionSpecs(route.path)) + req := decisionRequest(route, route.body) + req.Header.Del("Authorization") + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + requireNativeError(t, route, rec, http.StatusUnauthorized, "authentication_error", "invalid_api_key") + } +} + +// TestDecisionRoutesRateLimitRefusal checks that an exhausted request count or +// input token limit refuses the request before the backend is called, with the +// route's native 429 and a Retry-After header. +func TestDecisionRoutesRateLimitRefusal(t *testing.T) { + t.Parallel() + + limits := staticLimitResolver{{SubjectKey: "org-test:fn-alpha", RequestsPerMinute: 10, InputTokensPerMinute: 10_000}} + for _, route := range decisionRoutes { + for _, tc := range []struct { + name string + exhausted ratelimit.LimitDimension + }{ + {"request count", ratelimit.RequestsPerMinute}, + {"input tokens", ratelimit.InputTokensPerMinute}, + } { + t.Run(route.name+"/"+tc.name, func(t *testing.T) { + t.Parallel() + + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { hits.Add(1) })) + defer upstream.Close() + + limiter := &recordingRateLimiter{currentValueFn: func(call rateLimitCall) int64 { + if strings.HasSuffix(call.key, ":"+string(tc.exhausted)) { + return 0 + } + return call.limit.Limit + }} + e := decisionTestAPI(t, upstream.URL, config.Default(), limiter, decisionSpecs(route.path), WithLimitResolver(limits)) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, decisionRequest(route, route.body)) + + requireNativeError(t, route, rec, http.StatusTooManyRequests, "rate_limit_error", "rate_limit_exceeded") + require.NotEmpty(t, rec.Header().Get("Retry-After")) + require.Zero(t, hits.Load(), "a refused request must not reach the backend") + var sawLimit bool + for _, call := range limiter.Calls() { + require.False(t, call.mustConsume, "a refused request must not consume") + sawLimit = sawLimit || strings.HasSuffix(call.key, ":"+string(tc.exhausted)) + } + require.True(t, sawLimit, "the %s dimension must be checked", tc.exhausted) + }) + } + } +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/decision_token_count.go b/src/invocation-plane-services/llm-api-gateway/api/decision_token_count.go new file mode 100644 index 0000000000..8eb1aaf447 --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/decision_token_count.go @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "strings" +) + +// maxDecisionBodyDepth bounds JSON nesting in a decision request body. +const maxDecisionBodyDepth = 64 + +var errDecisionBodyTooDeep = errors.New("request body is nested more than 64 levels deep") + +// estimatedTokenCountForDecisionBody estimates the input tokens of a decision +// route request without knowing its schema. Every string leaf counts as text. +// Each image slot counts a flat imageTokens and its payload is never counted, +// so a large base64 image cannot hide or inflate the estimate. Image slots are +// objects with type "input_image", each element of an "images" array, each +// value of an "assets" object, and any string that is a base64 data URL. +// +// The body is read in one pass over decoder tokens with a stack of at most +// maxDecisionBodyDepth frames, so time is linear and extra memory does not +// grow with nesting. Deeper bodies return an error. +func estimatedTokenCountForDecisionBody(raw []byte, imageTokens int) (int, error) { + trimmed := bytes.TrimSpace(raw) + if !json.Valid(trimmed) { + return estimatedTokenCountForText(string(raw)), nil + } + dec := json.NewDecoder(bytes.NewReader(trimmed)) + var stack []decisionFrame + root := 0 + for { + tok, err := dec.Token() + if err != nil { + if err == io.EOF { + return root, nil + } + return 0, err + } + var ( + value int + done bool + ) + switch t := tok.(type) { + case json.Delim: + switch t { + case '{', '[': + frame := decisionFrame{object: t == '{'} + if len(stack) > 0 { + top := &stack[len(stack)-1] + if top.slots { + // The whole element is one image slot: skip it. + if err := skipContainer(dec, len(stack)+1); err != nil { + return 0, err + } + top.total += imageTokens + top.haveKey = false + continue + } + frame.slots = (top.key == "images" && t == '[') || (top.key == "assets" && t == '{') + } + if len(stack) >= maxDecisionBodyDepth { + return 0, errDecisionBodyTooDeep + } + stack = append(stack, frame) + continue + default: // closing delimiter + frame := stack[len(stack)-1] + stack = stack[:len(stack)-1] + value, done = frame.total, true + if frame.image { + value = imageTokens + } + } + case string: + top := (*decisionFrame)(nil) + if len(stack) > 0 { + top = &stack[len(stack)-1] + } + if top != nil && top.object && !top.haveKey { + top.key, top.haveKey = t, true + continue + } + switch { + case top != nil && top.slots, isBase64DataURL(t): + value = imageTokens + default: + value = estimatedTokenCountForText(t) + } + if top != nil && top.object && top.key == "type" && t == "input_image" { + top.image = true + } + done = true + default: // number, bool, null + if len(stack) > 0 && stack[len(stack)-1].slots { + value = imageTokens + done = true + } else { + done = true + } + } + if !done { + continue + } + if len(stack) == 0 { + root += value + continue + } + top := &stack[len(stack)-1] + top.total += value + top.haveKey = false + } +} + +// decisionFrame is the state of one open object or array. +type decisionFrame struct { + object bool + haveKey bool // an object key was read and its value is next + key string // the current object key + slots bool // each child is an image slot + image bool // this object has type "input_image" + total int +} + +// skipContainer consumes tokens up to the close of the container whose opening +// delimiter was just read, enforcing the depth cap. +func skipContainer(dec *json.Decoder, depth int) error { + for open := 1; open > 0; { + tok, err := dec.Token() + if err != nil { + return err + } + if d, ok := tok.(json.Delim); ok { + switch d { + case '{', '[': + open++ + if depth+open-1 > maxDecisionBodyDepth { + return errDecisionBodyTooDeep + } + default: + open-- + } + } + } + return nil +} + +func isBase64DataURL(text string) bool { + return strings.HasPrefix(text, "data:") && strings.Contains(text, ";base64,") +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/decision_token_count_test.go b/src/invocation-plane-services/llm-api-gateway/api/decision_token_count_test.go new file mode 100644 index 0000000000..e1bad55744 --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/decision_token_count_test.go @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEstimatedTokenCountForDecisionBody(t *testing.T) { + t.Parallel() + + const imageTokens = 1000 + hello := estimatedTokenCountForText("hello") + question := estimatedTokenCountForText("Is it red?") + bigBase64 := strings.Repeat("QUJD", 100_000) + smallBase64 := "QUJD" + + for _, tc := range []struct { + name string + body string + want int + }{ + {"text leaf counts and keys do not", `{"model":"m","input":"hello"}`, estimatedTokenCountForText("m") + hello}, + {"nested strings in arrays and objects count", `{"input":"hello","questions":[{"predicate":"Is it red?"}]}`, hello + question}, + {"numbers booleans and null do not count", `{"n":12345,"b":true,"x":null}`, 0}, + {"empty string counts nothing", `{"input":""}`, 0}, + {"input_image object is one image", `{"input":[{"type":"input_image","image_url":"data:image/png;base64,` + bigBase64 + `","detail":"high"}]}`, imageTokens}, + {"input_image base64 length does not matter", `{"input":[{"type":"input_image","image_url":"data:image/png;base64,` + smallBase64 + `"}]}`, imageTokens}, + {"images array counts each element", `{"images":["` + bigBase64 + `","` + smallBase64 + `",{"mediaType":"image/png","base64":"` + smallBase64 + `"}],"state":"hello"}`, 3*imageTokens + hello}, + {"assets map counts each value", `{"assets":{"a":{"mediaType":"image/png","base64":"` + bigBase64 + `"},"b":"data:image/jpeg;base64,` + smallBase64 + `"}}`, 2 * imageTokens}, + {"assets with several object values", `{"assets":{"a":{"x":"1"},"b":{"y":[1,2]},"c":{"z":{}},"d":"hello"},"state":"hello"}`, 4*imageTokens + hello}, + {"data url anywhere is one image", `{"state":{"screenshot":"data:image/png;base64,` + bigBase64 + `"},"input":"hello"}`, imageTokens + hello}, + {"data url without base64 is text", `{"input":"data:text/plain,hi"}`, estimatedTokenCountForText("data:text/plain,hi")}, + {"images key that is not an array is walked as text", `{"images":"hello"}`, hello}, + {"duplicate keys are all counted", `{"input":"hello","input":"hello"}`, 2 * hello}, + {"duplicate images keys are all counted", `{"images":["a"],"images":["b","c"]}`, 3 * imageTokens}, + {"mixed text and images", `{"input":"hello","images":["x"],"questions":[{"type":"input_image","image_url":"u"},{"predicate":"Is it red?"}]}`, hello + 2*imageTokens + question}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := estimatedTokenCountForDecisionBody([]byte(tc.body), imageTokens) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +func TestEstimatedTokenCountForDecisionBodyImageTokensIsConfigurable(t *testing.T) { + t.Parallel() + + body := []byte(`{"images":["a","b"]}`) + got, err := estimatedTokenCountForDecisionBody(body, 0) + require.NoError(t, err) + require.Equal(t, 0, got) + got, err = estimatedTokenCountForDecisionBody(body, 7) + require.NoError(t, err) + require.Equal(t, 2*7, got) +} + +func TestEstimatedTokenCountForDecisionBodyInvalidJSONFallsBackToText(t *testing.T) { + t.Parallel() + + raw := `{"input":"hello",` + got, err := estimatedTokenCountForDecisionBody([]byte(raw), 1000) + require.NoError(t, err) + require.Equal(t, estimatedTokenCountForText(raw), got) +} + +func nestedDecisionBody(depth int, leaf string) string { + return strings.Repeat(`{"a":`, depth) + `"` + leaf + `"` + strings.Repeat(`}`, depth) +} + +func TestEstimatedTokenCountForDecisionBodyRejectsDeepNesting(t *testing.T) { + t.Parallel() + + _, err := estimatedTokenCountForDecisionBody([]byte(nestedDecisionBody(64, "x")), 1000) + require.NoError(t, err, "64 levels are allowed") + _, err = estimatedTokenCountForDecisionBody([]byte(nestedDecisionBody(65, "x")), 1000) + require.Error(t, err) + _, err = estimatedTokenCountForDecisionBody([]byte(`{"images":`+strings.Repeat("[", 70)+strings.Repeat("]", 70)+`}`), 1000) + require.Error(t, err, "nesting inside an image slot counts too") +} + +// A big leaf nested deep must cost a few copies of the leaf, not one per level. +func TestEstimatedTokenCountForDecisionBodyAllocationIsBounded(t *testing.T) { + body := []byte(nestedDecisionBody(60, strings.Repeat("a", 1<<20))) + + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + got, err := estimatedTokenCountForDecisionBody(body, 1000) + runtime.ReadMemStats(&after) + + require.NoError(t, err) + require.Equal(t, estimatedTokenCountForText(strings.Repeat("a", 1<<20)), got) + allocated := after.TotalAlloc - before.TotalAlloc + require.Less(t, allocated, uint64(16<<20), "allocated %d bytes for a 1 MiB body", allocated) +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/decision_usage.go b/src/invocation-plane-services/llm-api-gateway/api/decision_usage.go new file mode 100644 index 0000000000..da25db4a86 --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/decision_usage.go @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "math" + "net/http" + "strings" + + echo "github.com/labstack/echo/v4" + "go.opentelemetry.io/otel/attribute" + + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/models" + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/provider" + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/telemetry" +) + +// decisionUsageObserver watches a decision route response body for +// usage.input_tokens without buffering it. It reuses the Messages scanner, +// which validates the JSON incrementally and skips large content. +type decisionUsageObserver struct { + scanner messagesJSONUsageScanner +} + +func (o *decisionUsageObserver) Write(p []byte) (int, error) { + o.scanner.Write(p) + return len(p), nil +} + +// inputTokens returns usage.input_tokens when the body was a complete JSON +// object that reported it as a non-negative number. +func (o *decisionUsageObserver) inputTokens() (int, bool) { + s := &o.scanner + if !s.done || s.invalid || s.oversized || s.usage.InputTokens == nil || *s.usage.InputTokens < 0 { + return 0, false + } + return *s.usage.InputTokens, true +} + +// observesDecisionUsage reports whether the response body should be scanned: +// a 2xx response on a decision route that is not content-encoded. +func observesDecisionUsage(r *http.Request, resp *provider.ProxyResponse) bool { + switch r.URL.Path { + case decisionsEndpointPath, systemOneEndpointPath: + default: + return false + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices || resp.Body == nil { + return false + } + encoding := resp.Header.Get(echo.HeaderContentEncoding) + return encoding == "" || strings.EqualFold(encoding, "identity") +} + +// recordDecisionUsage records what the backend reported after the response +// was delivered. The estimate stays the admission value; the reported count is +// only an observation, and a response without usage counts nothing. +func (h *OpenAIProxyHandlers) recordDecisionUsage(c *GatewayContext, observer *decisionUsageObserver) { + ctx := c.UserContext() + endpoint := c.Request().URL.Path + functionID := requestFunctionID(c) + status := "absent" + if tokens, ok := observer.inputTokens(); ok { + status = "observed" + h.handlers.observability.recordLLMUsage(ctx, endpoint, functionID, + &models.ChatCompletionUsage{PromptTokens: uint32(min(int64(tokens), math.MaxUint32))}, false) + } + telemetry.AddWithContext(ctx, h.handlers.observability.decisionUsage, 1, + attribute.String("endpoint", endpoint), + attribute.String("status", status), + telemetry.FunctionIDAttribute(functionID), + ) +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/decision_usage_test.go b/src/invocation-plane-services/llm-api-gateway/api/decision_usage_test.go new file mode 100644 index 0000000000..91179f7944 --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/decision_usage_test.go @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "context" + "net/http" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" + + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/config" + "github.com/NVIDIA/nvcf/src/invocation-plane-services/llm-gateway/telemetry" +) + +// These tests replace the global meter and tracer providers, so they do not +// run in parallel. + +func installDecisionMetricReader(t *testing.T) *sdkmetric.ManualReader { + t.Helper() + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + old := otel.GetMeterProvider() + otel.SetMeterProvider(provider) + t.Cleanup(func() { + otel.SetMeterProvider(old) + _ = provider.Shutdown(context.Background()) + }) + return reader +} + +// counterSeries returns the counter's value for every label set, keyed by the +// value of one label. +func counterSeries(t *testing.T, reader *sdkmetric.ManualReader, name string, labelFilter map[string]string, keyLabel string) map[string]int64 { + t.Helper() + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + out := map[string]int64{} + for _, scope := range rm.ScopeMetrics { + for _, m := range scope.Metrics { + if m.Name != name { + continue + } + for _, dp := range m.Data.(metricdata.Sum[int64]).DataPoints { + match := true + for label, want := range labelFilter { + if got, _ := dp.Attributes.Value(attribute.Key(label)); got.AsString() != want { + match = false + } + } + if match { + key, _ := dp.Attributes.Value(attribute.Key(keyLabel)) + out[key.AsString()] += dp.Value + } + } + } + } + return out +} + +func TestDecisionRoutesObserveResponseUsage(t *testing.T) { + gzipped := gzipBytes(t, `{"usage":{"input_tokens":42}}`) + for _, route := range decisionRoutes { + for _, tc := range []struct { + name string + headers http.Header + body []byte + wantTokens int64 + wantObserved int64 + wantAbsent int64 + }{ + {"usage present", http.Header{"Content-Type": {"application/json"}}, []byte(`{"results":[{"p":0.5}],"usage":{"input_tokens":42,"output_tokens":3}}`), 42, 1, 0}, + {"usage first member", http.Header{"Content-Type": {"application/json"}}, []byte(`{"usage":{"input_tokens":7},"results":[]}`), 7, 1, 0}, + {"usage zero", http.Header{"Content-Type": {"application/json"}}, []byte(`{"usage":{"input_tokens":0}}`), 0, 1, 0}, + {"usage absent", http.Header{"Content-Type": {"application/json"}}, []byte(`{"results":[{"p":0.5}]}`), 0, 0, 1}, + {"usage without input_tokens", http.Header{"Content-Type": {"application/json"}}, []byte(`{"usage":{"output_tokens":3}}`), 0, 0, 1}, + {"usage not an object", http.Header{"Content-Type": {"application/json"}}, []byte(`{"usage":"n/a"}`), 0, 0, 1}, + {"negative input_tokens", http.Header{"Content-Type": {"application/json"}}, []byte(`{"usage":{"input_tokens":-5}}`), 0, 0, 1}, + {"body not JSON", http.Header{"Content-Type": {"text/plain"}}, []byte(`ok`), 0, 0, 1}, + {"encoded body is skipped", http.Header{"Content-Type": {"application/json"}, "Content-Encoding": {"gzip"}}, gzipped, 0, 0, 0}, + } { + t.Run(route.name+"/"+tc.name, func(t *testing.T) { + reader := installDecisionMetricReader(t) + rec := serveDecisionUpstream(t, route, config.Default(), http.Header{"Accept-Encoding": {"gzip"}}, http.StatusOK, tc.headers, tc.body) + + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, string(tc.body), rec.Body.String(), "the response must reach the client unchanged") + + observations := counterSeries(t, reader, "llm_api_gateway_decision_usage_observations_total", map[string]string{"endpoint": route.path}, "status") + require.EqualValues(t, tc.wantObserved, observations["observed"], "observed") + require.EqualValues(t, tc.wantAbsent, observations["absent"], "absent") + + tokens := counterSeries(t, reader, "llm_api_gateway_llm_tokens_total", map[string]string{"endpoint": route.path, "function_id": "fn-alpha"}, "token_type") + require.EqualValues(t, tc.wantTokens, tokens["prompt"], "prompt tokens") + require.Zero(t, tokens["completion"], "only the input side is observed") + require.Zero(t, tokens["total"]) + }) + } + } +} + +func TestDecisionRoutesDoNotObserveErrorResponses(t *testing.T) { + for _, route := range decisionRoutes { + reader := installDecisionMetricReader(t) + rec := serveDecisionUpstream(t, route, config.Default(), nil, http.StatusBadRequest, + http.Header{"Content-Type": {"application/json"}}, []byte(`{"usage":{"input_tokens":42}}`)) + + require.Equal(t, http.StatusBadRequest, rec.Code) + observations := counterSeries(t, reader, "llm_api_gateway_decision_usage_observations_total", map[string]string{"endpoint": route.path}, "status") + require.Zero(t, observations["observed"]+observations["absent"], route.name) + tokens := counterSeries(t, reader, "llm_api_gateway_llm_tokens_total", map[string]string{"endpoint": route.path}, "token_type") + require.Empty(t, tokens, route.name) + } +} + +func TestDecisionRoutesSetSpanAttributes(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + t.Cleanup(func() { _ = tracerProvider.Shutdown(context.Background()) }) + oldTracer := telemetry.Tracer + telemetry.Tracer = sync.OnceValue(func() trace.Tracer { return tracerProvider.Tracer("test") }) + t.Cleanup(func() { telemetry.Tracer = oldTracer }) + + for _, route := range decisionRoutes { + recorder.Reset() + rec := serveDecisionUpstream(t, route, config.Default(), nil, http.StatusOK, http.Header{"Content-Type": {"application/json"}}, []byte(`{}`)) + require.Equal(t, http.StatusOK, rec.Code) + + var found bool + for _, span := range recorder.Ended() { + attrs := attribute.NewSet(span.Attributes()...) + endpoint, ok := attrs.Value("gateway.endpoint") + if !ok { + continue + } + found = true + require.Equal(t, route.path, endpoint.AsString()) + functionID, _ := attrs.Value("nvcf.function.id") + require.Equal(t, "fn-alpha", functionID.AsString()) + orgID, _ := attrs.Value("nvcf.org.id") + require.Equal(t, "org-test", orgID.AsString()) + } + require.True(t, found, "%s: no span carries gateway.endpoint", route.name) + } +} + +func TestDecisionRoutesSeededByTelemetry(t *testing.T) { + require.Equal(t, []string{decisionsEndpointPath, systemOneEndpointPath}, telemetry.DecisionRoutes()) +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/error_response.go b/src/invocation-plane-services/llm-api-gateway/api/error_response.go index 0eeb8a9f42..e9c92ad930 100644 --- a/src/invocation-plane-services/llm-api-gateway/api/error_response.go +++ b/src/invocation-plane-services/llm-api-gateway/api/error_response.go @@ -55,11 +55,51 @@ type anthropicError struct { Message string `json:"message"` } +type systemOneErrorResponse struct { + Error systemOneError `json:"error"` + RequestID string `json:"request_id"` +} + +type systemOneError struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +// errorShape is the error envelope an endpoint speaks. +type errorShape int + +const ( + shapeOpenAI errorShape = iota + shapeAnthropic + shapeSystemOne +) + +// errInvalidJSON marks a 400 caused by a body that is not valid JSON, so the +// System One shape can report invalid_json instead of invalid_request. +var errInvalidJSON = errors.New("request body is not valid JSON") + +// errorShapeFor returns the error envelope for the request's endpoint: +// Anthropic on /v1/messages, System One on /v1/systemone and OpenAI everywhere +// else, including /v1/decisions. +func errorShapeFor(r *http.Request) errorShape { + switch r.URL.Path { + case messagesEndpointPath: + return shapeAnthropic + case systemOneEndpointPath: + return shapeSystemOne + default: + return shapeOpenAI + } +} + // newHTTPErrorHandler renders every error the gateway or the request router -// produces in the endpoint's native envelope: Anthropic on /v1/messages and -// OpenAI everywhere else. Model server errors already in that envelope are -// relayed so their type and code reach the client; Anthropic ones get the -// NVCF request_id. +// produces in the endpoint's native envelope: Anthropic on /v1/messages, +// System One on /v1/systemone and OpenAI everywhere else. Model server errors +// already in the OpenAI or Anthropic envelope are relayed so their type and +// code reach the client; Anthropic ones get the NVCF request_id. System One +// has no relay: its backend errors never reach this handler (they are written +// unchanged) and router errors always get the System One envelope. func newHTTPErrorHandler(exposeRouterErrorCodes bool) echo.HTTPErrorHandler { return func(err error, c echo.Context) { if c.Response().Committed { @@ -84,9 +124,10 @@ func newHTTPErrorHandler(exposeRouterErrorCodes bool) echo.HTTPErrorHandler { // The handler always writes an uncompressed body; drop any encoding // copied from the upstream response. c.Response().Header().Del(echo.HeaderContentEncoding) - anthropic := isMessagesRequest(c.Request()) + shape := errorShapeFor(c.Request()) + anthropic := shape == shapeAnthropic var upstream *provider.UpstreamError - if errors.As(err, &upstream) && upstream.RouterCode == "" { + if errors.As(err, &upstream) && upstream.RouterCode == "" && shape != shapeSystemOne { native := isOpenAIErrorBody if anthropic { native = isAnthropicErrorBody @@ -103,6 +144,14 @@ func newHTTPErrorHandler(exposeRouterErrorCodes bool) echo.HTTPErrorHandler { } c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + if shape == shapeSystemOne { + requestID := errorRequestID(c) + _ = c.JSON(status, systemOneErrorResponse{ + Error: systemOneError{Code: systemOneErrorCode(status, err), Message: message, RequestID: requestID}, + RequestID: requestID, + }) + return + } if anthropic { _ = c.JSON(status, anthropicErrorResponse{ Type: "error", @@ -166,6 +215,31 @@ func anthropicErrorType(status int) string { } } +// systemOneErrorCode maps an HTTP status to the System One error code. Other +// 4xx statuses are invalid_request and other 5xx statuses internal_error. +func systemOneErrorCode(status int, err error) string { + switch { + case status == http.StatusBadRequest && errors.Is(err, errInvalidJSON): + return "invalid_json" + case status == http.StatusUnauthorized: + return "invalid_api_key" + case status == http.StatusNotFound: + return "not_found" + case status == http.StatusRequestEntityTooLarge: + return "request_too_large" + case status == http.StatusTooManyRequests: + return "rate_limit_exceeded" + case status == http.StatusBadGateway, status == http.StatusServiceUnavailable, status == statusOverloaded: + return "provider_unavailable" + case status == http.StatusGatewayTimeout: + return "provider_timeout" + case status >= http.StatusInternalServerError: + return "internal_error" + default: + return "invalid_request" + } +} + // isOpenAIErrorBody reports whether body is {"error":{"message":"..."}}. func isOpenAIErrorBody(body []byte) bool { var envelope struct { diff --git a/src/invocation-plane-services/llm-api-gateway/api/fidelity_corpus_test.go b/src/invocation-plane-services/llm-api-gateway/api/fidelity_corpus_test.go index 2cec620e82..ec66ad80c8 100644 --- a/src/invocation-plane-services/llm-api-gateway/api/fidelity_corpus_test.go +++ b/src/invocation-plane-services/llm-api-gateway/api/fidelity_corpus_test.go @@ -56,6 +56,11 @@ import ( // the X-Input-Tokens and X-Cache-Affinity-Key headers sent upstream must // equal them; an empty affinity key means the header is absent. // +// Decisions and System One rewrite only model and relay the upstream JSON +// unchanged. Their input token estimate counts every string leaf as text and +// each image slot as the flat default of config.Default().DecisionImageTokens +// (1024), never the base64. +// // Chat restores the client model in responses and assembles unary responses // from SSE deltas. Responses relays SSE or extracts the terminal response // object for unary clients. Embeddings relays the upstream JSON unchanged. @@ -101,7 +106,7 @@ func TestGatewayRequestResponseFidelityCorpus(t *testing.T) { require.NoError(t, decoder.Decode(&fixture)) require.Equal(t, io.EOF, decoder.Decode(new(any)), "unexpected trailing fixture data") require.NotEmpty(t, fixture.Description) - require.Contains(t, []string{"/v1/chat/completions", "/v1/responses", "/v1/embeddings", "/v1/messages"}, fixture.Path) + require.Contains(t, []string{"/v1/chat/completions", "/v1/responses", "/v1/embeddings", "/v1/messages", "/v1/decisions", "/v1/systemone"}, fixture.Path) t.Log(fixture.Description) contentType, upstreamBody := fixture.UpstreamResponse.encode(t) @@ -146,7 +151,8 @@ func TestGatewayRequestResponseFidelityCorpus(t *testing.T) { // Checks that the gateway keeps because it needs the field for routing, relay // mode, admission, token estimates or session affinity. Each row is rejected -// before the request reaches the backend. +// before the request reaches the backend. A decision route model that does not +// declare the route is a 404, covered by TestDecisionRoutesModelNotEnabled. func TestGatewayKeptRequestChecks(t *testing.T) { t.Parallel() @@ -155,6 +161,8 @@ func TestGatewayKeptRequestChecks(t *testing.T) { responses = "/v1/responses" embeddings = "/v1/embeddings" messages = "/v1/messages" + decisions = "/v1/decisions" + systemOne = "/v1/systemone" model = `"model":"fn-alpha/company/model"` // newPassthroughAPI allows this model on no route. restricted = `"model":"fn-alpha/company/restricted"` @@ -212,6 +220,18 @@ func TestGatewayKeptRequestChecks(t *testing.T) { {"messages max_tokens missing", messages, `{` + model + `,"messages":[{"role":"user","content":"hi"}]}`}, {"messages stream not a boolean", messages, `{` + model + `,` + msgInput + `,"stream":"yes"}`}, {"messages messages empty", messages, `{` + model + `,"max_tokens":16,"messages":[]}`}, + {"decisions malformed JSON", decisions, `{` + model + `,`}, + {"decisions body not an object", decisions, `[]`}, + {"decisions duplicate model member case", decisions, `{` + model + `,"MODEL":"fn-alpha/company/other"}`}, + {"decisions model missing", decisions, `{"input":"hi"}`}, + {"decisions model not a string", decisions, `{"model":7}`}, + {"decisions model without function prefix", decisions, `{"model":"model","input":"hi"}`}, + {"systemone malformed JSON", systemOne, `{` + model + `,`}, + {"systemone body not an object", systemOne, `"hi"`}, + {"systemone duplicate model member case", systemOne, `{` + model + `,"MODEL":"fn-alpha/company/other"}`}, + {"systemone model missing", systemOne, `{"state":"hi"}`}, + {"systemone model not a string", systemOne, `{"model":7}`}, + {"systemone model without function prefix", systemOne, `{"model":"model","state":"hi"}`}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() diff --git a/src/invocation-plane-services/llm-api-gateway/api/handlers.go b/src/invocation-plane-services/llm-api-gateway/api/handlers.go index fde79225cc..dc9ebc512a 100644 --- a/src/invocation-plane-services/llm-api-gateway/api/handlers.go +++ b/src/invocation-plane-services/llm-api-gateway/api/handlers.go @@ -42,6 +42,7 @@ type Handlers struct { type observabilityMetrics struct { messagesUsage otelmetric.Int64Counter + decisionUsage otelmetric.Int64Counter llmTokens otelmetric.Int64Counter providerTime otelmetric.Float64Histogram streamFirstToken otelmetric.Float64Histogram @@ -52,6 +53,7 @@ type observabilityMetrics struct { func newObservabilityMetrics() observabilityMetrics { return observabilityMetrics{ messagesUsage: telemetry.MessagesUsage(), + decisionUsage: telemetry.DecisionUsage(), llmTokens: telemetry.LLMTokens(), providerTime: telemetry.ProviderTime(), streamFirstToken: telemetry.StreamFirstToken(), diff --git a/src/invocation-plane-services/llm-api-gateway/api/middleware.go b/src/invocation-plane-services/llm-api-gateway/api/middleware.go index 3697678d41..89e953338f 100644 --- a/src/invocation-plane-services/llm-api-gateway/api/middleware.go +++ b/src/invocation-plane-services/llm-api-gateway/api/middleware.go @@ -109,13 +109,23 @@ func NewContextMiddleware(cfg *config.Config) echo.MiddlewareFunc { gc.SetRequest(gc.Request().WithContext(ctx)) logRequestBodyReadFailure(ctx, bodyErr) gc.Response().Header().Set(HeaderRequestID, requestID) - if isMessagesRequest(gc.Request()) { + switch errorShapeFor(gc.Request()) { + case shapeAnthropic: // Runs as headers are written, after backend headers are // copied, so the NVCF ID replaces any backend request ID. gc.Response().Before(func() { gc.Response().Header().Set(HeaderRequestID, requestID) gc.Response().Header().Set(headerMessagesRequestID, requestID) }) + case shapeSystemOne: + // System One errors carry the NVCF ID in the body, so the + // header must match after backend headers are copied. + gc.Response().Before(func() { + if relayed, _ := gc.Get(contextKeyBackendErrorRelayed).(bool); relayed { + return + } + gc.Response().Header().Set(HeaderRequestID, requestID) + }) } metricAttrs := []attribute.KeyValue{ diff --git a/src/invocation-plane-services/llm-api-gateway/api/model_uris.go b/src/invocation-plane-services/llm-api-gateway/api/model_uris.go index 0821d7844a..51d78c1489 100644 --- a/src/invocation-plane-services/llm-api-gateway/api/model_uris.go +++ b/src/invocation-plane-services/llm-api-gateway/api/model_uris.go @@ -33,6 +33,8 @@ const ( responsesEndpointPath = "/v1/responses" embeddingsEndpointPath = "/v1/embeddings" messagesEndpointPath = "/v1/messages" + decisionsEndpointPath = "/v1/decisions" + systemOneEndpointPath = "/v1/systemone" ) // requireModelURIAllowlist checks the model's declared uris allowlist for diff --git a/src/invocation-plane-services/llm-api-gateway/api/passthrough_test.go b/src/invocation-plane-services/llm-api-gateway/api/passthrough_test.go index 049e7b472e..1f9d1b1e9e 100644 --- a/src/invocation-plane-services/llm-api-gateway/api/passthrough_test.go +++ b/src/invocation-plane-services/llm-api-gateway/api/passthrough_test.go @@ -90,7 +90,14 @@ func newPassthroughAPI(t *testing.T, upstreamURL string) *echo.Echo { e := echo.New() e.Use(NewContextMiddleware(cfg)) // Models without a spec skip the allowlist; company/restricted has no allowed route. - e.Use(modelSpecsMiddleware(map[string]nvcf.ModelSpec{"company/restricted": {URIs: []string{"/v1/other"}}})) + // The decision routes have no default-allow, so the models the corpus uses + // declare every route. + allRoutes := []string{chatCompletionsEndpointPath, responsesEndpointPath, embeddingsEndpointPath, messagesEndpointPath, decisionsEndpointPath, systemOneEndpointPath} + e.Use(modelSpecsMiddleware(map[string]nvcf.ModelSpec{ + "company/restricted": {URIs: []string{"/v1/other"}}, + "company/model": {URIs: allRoutes}, + "company-name/model-name": {URIs: allRoutes}, + })) RegisterRoutes(e, NewHandlers(cfg, stargate, nil, nil)) return e } diff --git a/src/invocation-plane-services/llm-api-gateway/api/proxy_handler.go b/src/invocation-plane-services/llm-api-gateway/api/proxy_handler.go index 57f3a50670..0be52d8c06 100644 --- a/src/invocation-plane-services/llm-api-gateway/api/proxy_handler.go +++ b/src/invocation-plane-services/llm-api-gateway/api/proxy_handler.go @@ -44,6 +44,8 @@ var hopByHopHeaders = map[string]struct{}{ func (h *OpenAIProxyHandlers) RegisterRoutes(group *echo.Group) { group.POST("/v1/embeddings", h.Embeddings) + group.POST(decisionsEndpointPath, h.Decisions) + group.POST(systemOneEndpointPath, h.SystemOne) } func (h *OpenAIProxyHandlers) forwardJSONRequest( @@ -78,7 +80,19 @@ func (h *OpenAIProxyHandlers) forwardJSONRequest( return echo.NewHTTPError(http.StatusBadGateway, "proxy provider returned no response") } - return writeProxyResponse(c, resp) + var tee io.Writer + var observer *decisionUsageObserver + if observesDecisionUsage(c.Request(), resp) { + observer = &decisionUsageObserver{} + tee = observer + } + if err := writeProxyResponseObserved(c, resp, tee); err != nil { + return err + } + if observer != nil { + h.recordDecisionUsage(c, observer) + } + return nil } func (h *Handlers) requireFunctionRequestContext( @@ -153,6 +167,12 @@ func (h *Handlers) dispatchEstimatedProxyRequest( } func writeProxyResponse(c *GatewayContext, resp *provider.ProxyResponse) error { + return writeProxyResponseObserved(c, resp, nil) +} + +// writeProxyResponseObserved writes resp to the client. When tee is not nil, +// every body byte written to the client is also written to it. +func writeProxyResponseObserved(c *GatewayContext, resp *provider.ProxyResponse, tee io.Writer) error { if resp == nil { return echo.NewHTTPError(http.StatusBadGateway, "proxy provider returned no response") } @@ -161,17 +181,98 @@ func writeProxyResponse(c *GatewayContext, resp *provider.ProxyResponse) error { } copyProxyHeaders(c.Response().Header(), resp.Header) - if err := proxyUpstreamError(resp); err != nil { + if isDecisionRoute(c.Request()) && resp.StatusCode >= http.StatusBadRequest { + if isBackendErrorOnDecisionRoute(resp) { + // The model server's own error is part of its API: relay it as is. + c.Set(contextKeyBackendErrorRelayed, true) + } else { + // Gateway-written errors carry no backend X-System-One-* headers. + for name := range c.Response().Header() { + if strings.HasPrefix(strings.ToLower(name), "x-system-one-") { + c.Response().Header().Del(name) + } + } + return proxyUpstreamError(resp) + } + } else if err := proxyUpstreamError(resp); err != nil { return err } c.Response().WriteHeader(resp.StatusCode) if resp.Body == nil { return nil } - _, err := io.Copy(c.Response().Writer, resp.Body) + var body io.Reader = resp.Body + if tee != nil { + body = io.TeeReader(body, tee) + } + _, err := io.Copy(c.Response().Writer, body) return err } +func isDecisionRoute(r *http.Request) bool { + return r.URL.Path == decisionsEndpointPath || r.URL.Path == systemOneEndpointPath +} + +// bareRouterErrorStatuses are the statuses Stargate answers with no body and +// no X-Stargate-Error-Code when the router itself fails: a replay buffer +// overflow (413), a client that went away (499) and tunnel or transport +// failures (502, 503, 504). Stargate marks only routing rejections and overload +// with the header. Its bare 400s come from malformed gateway-set headers, which +// the gateway never sends, so 400 is left out and an empty 400 is the +// backend's. Marking every Stargate-generated error would remove this +// heuristic (follow-up). +var bareRouterErrorStatuses = map[int]bool{ + http.StatusRequestEntityTooLarge: true, + 499: true, + http.StatusBadGateway: true, + http.StatusServiceUnavailable: true, + http.StatusGatewayTimeout: true, +} + +// isBackendErrorOnDecisionRoute reports whether the non-2xx resp on a decision +// route is a model server error. Stargate marks the errors it generates with +// X-Stargate-Error-Code, except a bare status with an empty body for the +// statuses in bareRouterErrorStatuses. Anything else, including an empty 401, +// 404 or 429, is the backend's answer. Those routes relay backend errors +// untouched, because the backend defines the error shapes of its own API (for +// example a FastAPI detail list). +func isBackendErrorOnDecisionRoute(resp *provider.ProxyResponse) bool { + if provider.IsRouterRejection(resp.Header) { + return false + } + if bareRouterErrorStatuses[resp.StatusCode] && emptyBody(resp) { + return false + } + return true +} + +// emptyBody reports whether resp has no body, reading at most one byte. A +// byte that is read is put back in front of the stream. Stargate answers some +// router failures, such as a replay buffer overflow or a failed tunnel, with a +// bare status and no body or error code header; those are router errors, not +// backend errors. +func emptyBody(resp *provider.ProxyResponse) bool { + if resp.Body == nil { + return true + } + var first [1]byte + for { + n, err := resp.Body.Read(first[:]) + if n == 1 { + resp.Body = &prefixedBody{Reader: io.MultiReader(bytes.NewReader(first[:]), resp.Body), Closer: resp.Body} + return false + } + if err != nil { + return err == io.EOF + } + } +} + +type prefixedBody struct { + io.Reader + io.Closer +} + func copyProxyHeaders(dst http.Header, src http.Header) { connectionHeaders := make(map[string]struct{}) for key, values := range src { diff --git a/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/decisions-image.json b/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/decisions-image.json new file mode 100644 index 0000000000..b9f569b7e9 --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/decisions-image.json @@ -0,0 +1,82 @@ +{ + "description": "A decisions request with an input_image data URL forwards the image unchanged; the estimate counts the image as the flat default of 1024 tokens whatever the base64 length.", + "path": "/v1/decisions", + "request": { + "model": "fn-alpha/company/model", + "input": [ + { + "type": "input_text", + "text": "What is shown?" + }, + { + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgo=" + } + ], + "questions": [ + { + "id": "c1", + "type": "choice", + "options": [ + "cat", + "dog" + ] + } + ] + }, + "upstream_request": { + "model": "company/model", + "input": [ + { + "type": "input_text", + "text": "What is shown?" + }, + { + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgo=" + } + ], + "questions": [ + { + "id": "c1", + "type": "choice", + "options": [ + "cat", + "dog" + ] + } + ] + }, + "upstream_input_tokens": "1077", + "upstream_affinity_key": "", + "upstream_response": { + "json": { + "results": [ + { + "id": "c1", + "type": "choice", + "probabilities": { + "cat": 0.75, + "dog": 0.25 + } + } + ], + "model": "company/model" + } + }, + "response": { + "json": { + "results": [ + { + "id": "c1", + "type": "choice", + "probabilities": { + "cat": 0.75, + "dog": 0.25 + } + } + ], + "model": "company/model" + } + } +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/decisions-minimal.json b/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/decisions-minimal.json new file mode 100644 index 0000000000..e524d7959e --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/decisions-minimal.json @@ -0,0 +1,74 @@ +{ + "description": "Minimal decisions request only rewrites model; question members, unknown members and response extensions survive, and the input token estimate counts every string leaf.", + "path": "/v1/decisions", + "request": { + "model": "fn-alpha/company/model", + "input": "Is the sky blue?", + "questions": [ + { + "id": "q1", + "type": "predicate", + "text": "Is it daytime?", + "vendor_hint": { + "flag": true, + "revision": 9007199254740993 + } + } + ] + }, + "upstream_request": { + "model": "company/model", + "input": "Is the sky blue?", + "questions": [ + { + "id": "q1", + "type": "predicate", + "text": "Is it daytime?", + "vendor_hint": { + "flag": true, + "revision": 9007199254740993 + } + } + ] + }, + "upstream_input_tokens": "43", + "upstream_affinity_key": "", + "upstream_response": { + "json": { + "results": [ + { + "id": "q1", + "type": "predicate", + "probability": 0.12345678901234567890123456789, + "vendor_result": null + } + ], + "model": "company/model", + "usage": { + "input_tokens": 43, + "vendor_usage": { + "cached_tokens": 0 + } + } + } + }, + "response": { + "json": { + "results": [ + { + "id": "q1", + "type": "predicate", + "probability": 0.12345678901234567890123456789, + "vendor_result": null + } + ], + "model": "company/model", + "usage": { + "input_tokens": 43, + "vendor_usage": { + "cached_tokens": 0 + } + } + } + } +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/systemone-images-assets.json b/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/systemone-images-assets.json new file mode 100644 index 0000000000..979d32a6dc --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/systemone-images-assets.json @@ -0,0 +1,92 @@ +{ + "description": "A System One request with images and assets in both forms and an unknown member forwards them unchanged; the estimate counts each image slot as the flat default of 1024 tokens and never counts the base64.", + "path": "/v1/systemone", + "request": { + "model": "fn-alpha/company/model", + "state": "Compare the images.", + "images": [ + "iVBORw0KGgo=", + "/9j/4AAQ" + ], + "assets": { + "logo": { + "mediaType": "image/png", + "base64": "iVBORw0KGgo=" + }, + "shot": "data:image/jpeg;base64,/9j/4AAQ" + }, + "questions": { + "q1": { + "type": "choice", + "options": [ + "same", + "different" + ] + } + }, + "vendor_extension": { + "flag": true, + "revision": 9007199254740993 + } + }, + "upstream_request": { + "model": "company/model", + "state": "Compare the images.", + "images": [ + "iVBORw0KGgo=", + "/9j/4AAQ" + ], + "assets": { + "logo": { + "mediaType": "image/png", + "base64": "iVBORw0KGgo=" + }, + "shot": "data:image/jpeg;base64,/9j/4AAQ" + }, + "questions": { + "q1": { + "type": "choice", + "options": [ + "same", + "different" + ] + } + }, + "vendor_extension": { + "flag": true, + "revision": 9007199254740993 + } + }, + "upstream_input_tokens": "4138", + "upstream_affinity_key": "", + "upstream_response": { + "json": { + "answers": { + "q1": { + "probabilities": { + "same": 0.0625, + "different": 0.9375 + } + } + }, + "usage": { + "input_tokens": 4100 + } + } + }, + "response": { + "json": { + "answers": { + "q1": { + "probabilities": { + "same": 0.0625, + "different": 0.9375 + } + } + }, + "usage": { + "input_tokens": 4100 + } + } + } +} diff --git a/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/systemone-minimal.json b/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/systemone-minimal.json new file mode 100644 index 0000000000..936350e0ad --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/api/testdata/fidelity/systemone-minimal.json @@ -0,0 +1,54 @@ +{ + "description": "Minimal System One request only rewrites model; state, questions and response extensions survive, and the response may omit usage.", + "path": "/v1/systemone", + "request": { + "model": "fn-alpha/company/model", + "state": "The user opened the settings page.", + "questions": { + "q1": { + "type": "noul", + "instruction": "Describe intent." + } + } + }, + "upstream_request": { + "model": "company/model", + "state": "The user opened the settings page.", + "questions": { + "q1": { + "type": "noul", + "instruction": "Describe intent." + } + } + }, + "upstream_input_tokens": "40", + "upstream_affinity_key": "", + "upstream_response": { + "json": { + "answers": { + "q1": { + "text": "The user wants to change a setting.", + "score": 0.5, + "vendor_answer": { + "revision": 9007199254740993, + "empty": null + } + } + } + } + }, + "response": { + "json": { + "answers": { + "q1": { + "text": "The user wants to change a setting.", + "score": 0.5, + "vendor_answer": { + "revision": 9007199254740993, + "empty": null + } + } + } + } + } +} diff --git a/src/invocation-plane-services/llm-api-gateway/config/config.go b/src/invocation-plane-services/llm-api-gateway/config/config.go index 3c12703dbb..8c7c6dbfa6 100644 --- a/src/invocation-plane-services/llm-api-gateway/config/config.go +++ b/src/invocation-plane-services/llm-api-gateway/config/config.go @@ -48,6 +48,10 @@ type Config struct { // as no_eligible_candidates, in error.code of OpenAI-format error // responses. When false, the reason is only logged. ExposeRouterErrorCodes bool + // DecisionImageTokens is the flat input token estimate for each image in + // a /v1/decisions or /v1/systemone request. The image payload is never + // counted by size. Zero counts images as free. + DecisionImageTokens int } type ServerConfig struct { @@ -206,6 +210,10 @@ func (c ModelCapabilities) SupportsSpeechToSpeech() bool { return c.SpeechToSpeech == nil || *c.SpeechToSpeech } +// defaultDecisionImageTokens is a placeholder estimate per image until the +// decision model owners set a measured value with NVCF_DECISION_IMAGE_TOKENS. +const defaultDecisionImageTokens = 1024 + func Default() *Config { return &Config{ Telemetry: TelemetryConfig{ @@ -248,6 +256,7 @@ func Default() *Config { Enabled: true, FailOpen: true, }, + DecisionImageTokens: defaultDecisionImageTokens, RateLimitSync: RateLimitSynchronizationConfig{ ApplyRemote: true, NATS: RateLimitNATSConfig{ @@ -543,6 +552,14 @@ func applyDefaultModelEnv(cfg *Config, errs *envErrs) { cfg.DefaultRPM = rpm } + if tokens, ok := errs.integer("NVCF_DECISION_IMAGE_TOKENS"); ok { + if tokens < 0 { + errs.add("NVCF_DECISION_IMAGE_TOKENS", strconv.Itoa(tokens), errors.New("must be >= 0")) + } else { + cfg.DecisionImageTokens = tokens + } + } + if raw := os.Getenv("NVCF_MODEL_CAPABILITIES"); raw != "" { var caps map[string]ModelCapabilities if err := json.Unmarshal([]byte(raw), &caps); err != nil { diff --git a/src/invocation-plane-services/llm-api-gateway/config/config_test.go b/src/invocation-plane-services/llm-api-gateway/config/config_test.go index 33b73e6ccf..df585fc7e9 100644 --- a/src/invocation-plane-services/llm-api-gateway/config/config_test.go +++ b/src/invocation-plane-services/llm-api-gateway/config/config_test.go @@ -373,3 +373,47 @@ func TestLoadFromEnvReadsExposeRouterErrorCodes(t *testing.T) { t.Fatal("NVCF_GATEWAY_EXPOSE_ROUTER_ERROR_CODES=true was not applied") } } + +func TestLoadFromEnvDecisionImageTokens(t *testing.T) { + cfg, err := LoadFromEnv() + if err != nil { + t.Fatalf("LoadFromEnv() error = %v", err) + } + if cfg.DecisionImageTokens <= 0 { + t.Fatalf("default decision image tokens = %d, want a positive default", cfg.DecisionImageTokens) + } + + t.Setenv("NVCF_DECISION_IMAGE_TOKENS", "2048") + cfg, err = LoadFromEnv() + if err != nil { + t.Fatalf("LoadFromEnv() error = %v", err) + } + if cfg.DecisionImageTokens != 2048 { + t.Fatalf("decision image tokens = %d, want 2048", cfg.DecisionImageTokens) + } + + t.Setenv("NVCF_DECISION_IMAGE_TOKENS", "0") + cfg, err = LoadFromEnv() + if err != nil { + t.Fatalf("LoadFromEnv() error = %v", err) + } + if cfg.DecisionImageTokens != 0 { + t.Fatalf("decision image tokens = %d, want 0 (images not counted)", cfg.DecisionImageTokens) + } +} + +func TestLoadFromEnvRejectsInvalidDecisionImageTokens(t *testing.T) { + for _, value := range []string{"-1", "many"} { + t.Run(value, func(t *testing.T) { + t.Setenv("NVCF_DECISION_IMAGE_TOKENS", value) + + _, err := LoadFromEnv() + if err == nil { + t.Fatal("LoadFromEnv() error = nil, want error") + } + if !strings.Contains(err.Error(), "NVCF_DECISION_IMAGE_TOKENS") { + t.Fatalf("error %q does not mention NVCF_DECISION_IMAGE_TOKENS", err.Error()) + } + }) + } +} diff --git a/src/invocation-plane-services/llm-api-gateway/provider/upstream_error.go b/src/invocation-plane-services/llm-api-gateway/provider/upstream_error.go index 1b78aab2ae..cbe48b14f0 100644 --- a/src/invocation-plane-services/llm-api-gateway/provider/upstream_error.go +++ b/src/invocation-plane-services/llm-api-gateway/provider/upstream_error.go @@ -79,6 +79,13 @@ func ReadUpstreamErrorBody(body io.Reader, contentEncoding string) []byte { return data } +// IsRouterRejection reports whether a response was generated by the request +// router itself rather than relayed from a model server. The router marks +// every error it generates with X-Stargate-Error-Code. +func IsRouterRejection(header http.Header) bool { + return len(header.Values(headerStargateErrorCode)) > 0 +} + // UpstreamError is a non-2xx response from the request router or the model // server. The API layer renders it in the endpoint's error envelope, or relays // Body unchanged when the model server already used that envelope. diff --git a/src/invocation-plane-services/llm-api-gateway/telemetry/BUILD.bazel b/src/invocation-plane-services/llm-api-gateway/telemetry/BUILD.bazel index 49108c12ee..1cd1d0244c 100644 --- a/src/invocation-plane-services/llm-api-gateway/telemetry/BUILD.bazel +++ b/src/invocation-plane-services/llm-api-gateway/telemetry/BUILD.bazel @@ -63,6 +63,7 @@ go_test( name = "telemetry_test", srcs = [ "auth_metrics_test.go", + "decision_metrics_test.go", "metrics_test.go", "olric_collector_test.go", ], @@ -74,6 +75,7 @@ go_test( "@com_github_prometheus_common//model", "@com_github_stretchr_testify//require", "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//attribute", "@io_opentelemetry_go_otel_metric//:metric", "@io_opentelemetry_go_otel_metric//noop", "@io_opentelemetry_go_otel_sdk//resource", diff --git a/src/invocation-plane-services/llm-api-gateway/telemetry/decision_metrics_test.go b/src/invocation-plane-services/llm-api-gateway/telemetry/decision_metrics_test.go new file mode 100644 index 0000000000..ed148fb8ab --- /dev/null +++ b/src/invocation-plane-services/llm-api-gateway/telemetry/decision_metrics_test.go @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package telemetry + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// firstScrapeSeries returns the label sets of every series metric name holds +// after InitializeMetrics and before any traffic, each with its value. +func firstScrapeSeries(t *testing.T, name string) map[attribute.Distinct]int64 { + t.Helper() + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + oldProvider := otel.GetMeterProvider() + otel.SetMeterProvider(provider) + t.Cleanup(func() { + otel.SetMeterProvider(oldProvider) + _ = provider.Shutdown(context.Background()) + }) + + InitializeMetrics() + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + series := map[attribute.Distinct]int64{} + for _, scope := range rm.ScopeMetrics { + for _, m := range scope.Metrics { + if m.Name != name { + continue + } + switch data := m.Data.(type) { + case metricdata.Sum[int64]: + for _, dp := range data.DataPoints { + series[dp.Attributes.Equivalent()] = dp.Value + } + default: + t.Fatalf("unexpected data type %T for %s", data, name) + } + } + } + return series +} + +func labels(kv ...string) attribute.Distinct { + attrs := make([]attribute.KeyValue, 0, len(kv)/2) + for i := 0; i < len(kv); i += 2 { + attrs = append(attrs, attribute.String(kv[i], kv[i+1])) + } + set := attribute.NewSet(attrs...) + return set.Equivalent() +} + +func TestDecisionUsageObservationsStartAtZero(t *testing.T) { + series := firstScrapeSeries(t, "llm_api_gateway_decision_usage_observations_total") + + for _, endpoint := range []string{"/v1/decisions", "/v1/systemone"} { + for _, status := range []string{"observed", "absent"} { + value, ok := series[labels("endpoint", endpoint, "status", status, "function_id", "none")] + require.True(t, ok, "missing zero series %s %s in %v", endpoint, status, series) + require.Zero(t, value) + } + } + require.Len(t, series, 4, "only the bounded seed series may exist before traffic") +} + +func TestDecisionRoutesHTTPRequestSeriesStartAtZero(t *testing.T) { + requests := firstScrapeSeries(t, "llm_api_gateway_http_requests_total") + active := firstScrapeSeries(t, "llm_api_gateway_http_active_requests") + + for _, route := range []string{"/v1/decisions", "/v1/systemone"} { + for _, status := range []string{"200", "400", "401", "404", "429", "500", "502", "504"} { + value, ok := requests[labels("method", "POST", "route", route, "function_id", "none", "status", status)] + require.True(t, ok, "missing zero request series %s %s", route, status) + require.Zero(t, value) + } + value, ok := active[labels("method", "POST", "route", route, "function_id", "none")] + require.True(t, ok, "missing zero active series %s", route) + require.Zero(t, value) + } +} diff --git a/src/invocation-plane-services/llm-api-gateway/telemetry/metrics.go b/src/invocation-plane-services/llm-api-gateway/telemetry/metrics.go index 64fb9cfc4a..c8b51e1839 100644 --- a/src/invocation-plane-services/llm-api-gateway/telemetry/metrics.go +++ b/src/invocation-plane-services/llm-api-gateway/telemetry/metrics.go @@ -79,6 +79,7 @@ func InitializeMetrics() { _ = ModelURIAllowlistRejections() _ = LLMTokens() _ = MessagesUsage() + _ = DecisionUsage() _ = ProviderTime() _ = StreamFirstToken() _ = StreamDuration() @@ -98,6 +99,7 @@ func InitializeMetrics() { _ = authRequestsTotal() _ = authRequestDuration() preInitAuthMetrics() + preInitDecisionMetrics() _ = lastClusterLookupDuration() preInitLastClusterMetrics() } @@ -202,6 +204,16 @@ func MessagesUsage() otelmetric.Int64Counter { return must.Get(Meter().Int64Counter(metricPrefix+"messages_usage_observations_total", otelmetric.WithDescription("Messages usage accounting outcomes; estimates are not reported as actual tokens."))) } +// DecisionUsage counts response usage observations on the decision routes. +// status is "observed" when the response reported usage.input_tokens and +// "absent" when it did not. +func DecisionUsage() otelmetric.Int64Counter { + return must.Get(Meter().Int64Counter( + metricPrefix+"decision_usage_observations_total", + otelmetric.WithDescription("Decision route responses by whether they reported input token usage."), + )) +} + func ProviderTime() otelmetric.Float64Histogram { return must.Get(Meter().Float64Histogram( metricPrefix+"provider_time_seconds", @@ -559,3 +571,48 @@ func DropReasonRemoteApplyDisabled() attribute.KeyValue { func SynchronizerDropReasonOldMessage() attribute.KeyValue { return attribute.String("reason", synchronizerDropOldMessage) } + +// decisionRoutes are the routes whose series are seeded before traffic. The +// route label comes from the router pattern, so without a seed the series for +// a route that has not been called yet does not exist and absent() and rate() +// have nothing to read. +var decisionRoutes = []string{"/v1/decisions", "/v1/systemone"} + +// DecisionRoutes returns the decision routes seeded by InitializeMetrics. +func DecisionRoutes() []string { return append([]string(nil), decisionRoutes...) } + +// decisionSeedStatuses bounds the status label of the seeded request series +// to the statuses the decision routes commonly return. +var decisionSeedStatuses = []string{"200", "400", "401", "404", "429", "500", "502", "504"} + +// preInitDecisionMetrics emits zero samples for the decision routes on the +// usage counter and the HTTP request instruments, using the missing function +// label so cardinality stays bounded. +func preInitDecisionMetrics() { + ctx := context.Background() + usage := DecisionUsage() + requests := HTTPRequestsTotal() + active := HTTPActiveRequests() + for _, route := range decisionRoutes { + for _, status := range []string{"observed", "absent"} { + usage.Add(ctx, 0, otelmetric.WithAttributes( + attribute.String("endpoint", route), + attribute.String("status", status), + FunctionIDAttribute(""), + )) + } + active.Add(ctx, 0, otelmetric.WithAttributes( + attribute.String("method", "POST"), + attribute.String("route", route), + FunctionIDAttribute(""), + )) + for _, status := range decisionSeedStatuses { + requests.Add(ctx, 0, otelmetric.WithAttributes( + attribute.String("method", "POST"), + attribute.String("route", route), + FunctionIDAttribute(""), + attribute.String("status", status), + )) + } + } +} diff --git a/src/invocation-plane-services/vanity-gateway/README.md b/src/invocation-plane-services/vanity-gateway/README.md index dc99be9951..14f0236db9 100644 --- a/src/invocation-plane-services/vanity-gateway/README.md +++ b/src/invocation-plane-services/vanity-gateway/README.md @@ -22,11 +22,14 @@ Use the [release workflow](../../../docs/dev/github-release-process.md) for sour ## About The Vanity Gateway fronts invocation APIs with route mappings from a YAML config -file. It supports three request shapes: +file. It supports four request shapes: - OpenAI-compatible `/v1/*` endpoints that map public model names to functions. - The native Anthropic Messages endpoint, `POST /v1/messages`, configured under `v2config.anthropicMessages`. +- Beta decision model endpoints, `POST /v1/decisions` and + `POST /v1/systemone`, configured under `v2config.openai.decisions` and + `v2config.typesafeSystemOne`. - Vanity URL routes that forward caller-defined paths to function executions and polling. The gateway is meant to run against an upstream invocation service with pexec @@ -156,6 +159,7 @@ v2config: completions: {} embeddings: {} responses: {} + decisions: {} imageGenerations: {} imageEdits: {} imageVariations: {} @@ -208,7 +212,8 @@ v2config: Shadow traffic is supported for mapping sections with JSON request bodies: `chatCompletions`, `completions`, `embeddings`, `responses`, `imageGenerations`, -and `anthropicMessages`. Shadow targets must be model names in the same mapping +`anthropicMessages`, `decisions`, and `typesafeSystemOne`. The decision sections +support `random` and `perBearerKey` sampling only. Shadow targets must be model names in the same mapping section as the primary model. The gateway rewrites the request `model` field to the shadow target and marks the replay with `NVCF-Shadow: true` so shadow requests do not recursively shadow. @@ -352,8 +357,8 @@ added to logs, traces, or metric labels. Set `functionType: LLM` on a model entry to serve it from the LLM Gateway instead of the invocation service. Supported in `openai.chatCompletions`, -`openai.responses`, `openai.embeddings`, and `anthropicMessages`, the four -routes the LLM Gateway serves. +`openai.responses`, `openai.embeddings`, `anthropicMessages`, `openai.decisions`, +and `typesafeSystemOne`, the routes the LLM Gateway serves. ```yaml v2config: @@ -383,7 +388,7 @@ forwarded: the mapping is what decides which function a caller reaches. Vanity Gateway passes `Authorization`, `NVCF-POLL-SECONDS`, and the caller's other headers to the LLM Gateway. -Config validation rejects `functionType: LLM` outside the four supported +Config validation rejects `functionType: LLM` outside the supported sections, and rejects `usePexec`, `outgoingPathOverride`, and `sessionTimeout` on those entries because the LLM Gateway ignores them. An `X-Priority` entry in `customHeaders` is rejected too: the LLM Gateway answers `400 Bad Request` for @@ -508,7 +513,8 @@ YAML merge defaults and explicit overrides are supported. For a repeated `modelName`, disjoint metadata keys merge. A conflicting key keeps its first value, including null, in this section order: `chatCompletions`, `completions`, `embeddings`, `responses`, `imageGenerations`, -`imageEdits`, `imageVariations`, then `anthropicMessages`. Within each section, +`imageEdits`, `imageVariations`, `anthropicMessages`, `decisions`, then +`typesafeSystemOne`. Within each section, YAML entry order decides precedence. Programmatically constructed configs without source order use alphabetical entry keys. Nested objects are whole values; their contents do not merge. Both OpenAI discovery endpoints use the @@ -583,6 +589,100 @@ The Messages path requires compatible LLM Gateway, Stargate router, and Pylon worker builds. See `docs/overview/llm-gateway.md` for upgrade order, token accounting, session affinity, and Claude Code configuration. +## Invoking Decision Models (Beta) + +Decision models answer typed questions about an input and return +probabilities instead of generated text. Two beta routes pass these requests +through on the shared API host set by `openai.host`: + +- `POST /v1/decisions` serves the OpenAI Decisions format. Map models under + `v2config.openai.decisions`. +- `POST /v1/systemone` serves the TypeSafe System One format. Map models under + `v2config.typesafeSystemOne`, a top-level sibling of `anthropicMessages`. + +Entries use the same fields and validation as `openai.chatCompletions`. A model +is enabled for a route only when it is listed in that route's section. A +function that serves both formats is listed in both sections. A request for a +model that is not listed in the section of the route it called returns `404`. + +```yaml +v2config: + openai: + host: api.example.com + decisions: + example_decision-model: + modelName: example/decision-model + functionID: 00000000-0000-0000-0000-000000000001 + functionVersionID: 11111111-1111-1111-1111-111111111111 + example_decision-model-llm: + modelName: example/decision-model-llm + functionID: 00000000-0000-0000-0000-000000000002 + functionType: LLM + typesafeSystemOne: + example_decision-model: + modelName: example/decision-model + functionID: 00000000-0000-0000-0000-000000000001 + functionVersionID: 11111111-1111-1111-1111-111111111111 +``` + +In this example `example/decision-model` is one function that serves both +formats, so it appears in both sections. `example/decision-model-llm` is served +through the LLM Gateway on `/v1/decisions` only. + +The gateway reads only `model` from the body and never translates between the +two formats, so the backend must implement each route it is configured for. +Request and response bodies pass through unchanged, including backend 4xx and +5xx responses. Responses are not streamed: `stream` is ignored and the gateway +does not ask for an event stream. Vendor headers such as +`TypeSafe-Danger-Zone-Images: true` reach the backend unchanged. Request bodies +may be up to 25 MB. + +```shell +source .env +curl http://localhost:10081/v1/decisions \ +-H "Host: api.example.com" \ +-H "Content-Type: application/json" \ +-H "Authorization: Bearer ${API_TOKEN}" \ +-d '{ +"model": "example/decision-model", +"input": "The sky is clear.", +"questions": [{"type": "predicate", "text": "Is it raining?"}] +}' +``` + +Direct models: with `usePexec: false` and no `outgoingPathOverride`, the +gateway keeps the path, `/v1/decisions` or `/v1/systemone`, and selects the +function with function ID headers. With `usePexec: true` the gateway calls the +pexec path of the function, and the invocation service then normally sends the request to +the default invocation path registered for that function, whichever route the +client called. A `usePexec: true` function therefore serves only its default path. A +function that serves both routes should use `usePexec: false`. + +Errors generated by the Vanity Gateway use the native shape of the route: + +- `/v1/decisions` uses the OpenAI error shape. +- `/v1/systemone` uses + `{"error":{"code":"...","message":"...","request_id":"..."},"request_id":"..."}`, + where `request_id` has the same value as the `X-Request-Id` response header. + The codes are `invalid_json` and `invalid_request` for `400`, `not_found` for + `404` and `410`, `request_too_large` for `413`, `provider_unavailable` for + `502` and `503`, and `internal_error`. The gateway does not set + `X-System-One-*` headers. + +Malformed JSON and a missing `model` return `400`, an unknown model returns +`404`, a body over the size limit returns `413`, an offline model returns `503` +with `Retry-After`, an end-of-life model returns `410`, and a failed upstream +request returns `502`. Upstream status codes and bodies pass through unchanged. + +`/v1/models` and `/v1/models/{model}` list decision models and add +`supported_endpoints`, for example `["/v1/decisions"]`. + +Shadow request body sampling (`promptCacheKey` and `firstMessageHash`) and +`promptCacheKeyHeaders` are not valid in the decision sections. + +A model with `functionType: LLM` needs LLM Gateway and Stargate builds that +serve these routes. Enable it only after those builds run in the environment. + ## Invoking Vanity URL Routes ```shell diff --git a/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel b/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel index 3e1f418c84..a1c2470d47 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel +++ b/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "h2.go", "health.go", "llm_gateway_director.go", + "native_errors.go", "openai_director.go", "shadow.go", "shadow_metrics.go", @@ -62,6 +63,7 @@ go_test( name = "gateway_test", srcs = [ "client_metrics_test.go", + "decision_routes_test.go", "error_origin_test.go", "gateway_test.go", "h2_test.go", diff --git a/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go b/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go new file mode 100644 index 0000000000..dc00146356 --- /dev/null +++ b/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go @@ -0,0 +1,501 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + config "ai-api-gateway-service/gateway_config" + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" +) + +const ( + decisionModel = "acme/decision-model" + imagesHeader = "TypeSafe-Danger-Zone-Images" + decisionFunctionID = "decision-func-1" + fastAPIValidationErr = `{"detail":[{"loc":["body","questions"],"msg":"field required","type":"missing"}]}` +) + +type systemOneErrorBody struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` + RequestID string `json:"request_id"` +} + +// decisionMappings enables decisionModel on the listed routes only. +func decisionMappings(decisions, systemOne bool, mutate func(*config.ModelFunctionDetails)) *config.GatewayConfig { + entry := config.ModelFunctionDetails{ModelName: decisionModel, FunctionID: decisionFunctionID} + if mutate != nil { + mutate(&entry) + } + mappings := &config.GatewayConfig{} + mappings.OpenAI.Host = openAIHost + if decisions { + mappings.OpenAI.Decisions = map[string]config.ModelFunctionDetails{"d": entry} + } + if systemOne { + mappings.TypeSafeSystemOne = map[string]config.ModelFunctionDetails{"s": entry} + } + return mappings +} + +func llmDecision(entry *config.ModelFunctionDetails) { entry.FunctionType = config.FunctionTypeLLM } + +func postDecision(t *testing.T, mux http.Handler, path, body string, headers map[string]string) *httptest.ResponseRecorder { + t.Helper() + req := openAIRequest(t, path, body) + req.Header.Set("Content-Type", "application/json") + for name, value := range headers { + req.Header.Set(name, value) + } + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + return rec +} + +func requireOpenAIErrorBody(t *testing.T, rec *httptest.ResponseRecorder, status int) OpenAIError { + t.Helper() + require.Equal(t, status, rec.Code, rec.Body.String()) + require.Equal(t, "application/json", rec.Header().Get("Content-Type")) + var payload OpenAIError + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &payload), rec.Body.String()) + require.NotEmpty(t, payload.Error.Message) + require.NotEmpty(t, payload.Error.Code) + return payload +} + +func requireSystemOneErrorBody(t *testing.T, rec *httptest.ResponseRecorder, status int, code string) systemOneErrorBody { + t.Helper() + require.Equal(t, status, rec.Code, rec.Body.String()) + require.Equal(t, "application/json", rec.Header().Get("Content-Type")) + var payload systemOneErrorBody + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &payload), rec.Body.String()) + require.Equal(t, code, payload.Error.Code) + require.NotEmpty(t, payload.Error.Message) + requestID := rec.Header().Get("X-Request-Id") + require.NotEmpty(t, requestID) + require.Equal(t, requestID, payload.RequestID, "top-level request_id must equal the X-Request-Id header") + require.Equal(t, requestID, payload.Error.RequestID, "error.request_id must equal the X-Request-Id header") + return payload +} + +func TestDecisionRoutesForwardDirectWithoutTranslation(t *testing.T) { + for _, path := range []string{decisionsPath, systemOnePath} { + t.Run(path, func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"answers":{"q1":{"yes":0.9}}}`)) + }) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + + body := `{"model":"` + decisionModel + `","input":"is it red?","questions":[{"type":"predicate","text":"red?"}],"extra":{"keep":true}}` + rec := postDecision(t, mux, path, body, map[string]string{imagesHeader: "true"}) + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.JSONEq(t, `{"answers":{"q1":{"yes":0.9}}}`, rec.Body.String()) + received := awaitRequest(t, requests) + assert.Equal(t, path, received.path, "the route is kept when usePexec is false") + assert.Equal(t, decisionFunctionID, received.headers.Get("function-id")) + assert.Equal(t, "true", received.headers.Get(imagesHeader), "vendor image header must pass unchanged") + assert.Equal(t, body, received.body, "the body must pass through unchanged") + }) + } +} + +func TestDecisionRoutesForwardThroughLLMGatewayWithImageHeader(t *testing.T) { + for _, path := range []string{decisionsPath, systemOnePath} { + t.Run(path, func(t *testing.T) { + nvcfRequests := make(chan capturedRequest, 1) + nvcfBackend := captureServer(t, nvcfRequests, nil) + llmRequests := make(chan capturedRequest, 1) + llmBackend := captureServer(t, llmRequests, nil) + mux := openAIMux(t, decisionMappings(true, true, llmDecision), nvcfBackend.URL, llmBackend.URL) + + rec := postDecision(t, mux, path, `{"model":"`+decisionModel+`","input":"x"}`, map[string]string{imagesHeader: "true"}) + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + received := awaitRequest(t, llmRequests) + assert.Equal(t, path, received.path) + assert.Contains(t, received.body, `"model":"`+decisionFunctionID+"/"+decisionModel+`"`) + assert.Equal(t, "true", received.headers.Get(imagesHeader)) + assert.Empty(t, received.headers.Get("function-id")) + assert.Empty(t, nvcfRequests) + }) + } +} + +func TestDecisionRoutesIgnoreStreamTrueForAccept(t *testing.T) { + for _, tc := range []struct { + path string + wantAccept string + }{ + {decisionsPath, ""}, + {systemOnePath, ""}, + } { + t.Run(tc.path, func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, tc.path, `{"model":"`+decisionModel+`","stream":true}`, nil) + + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, tc.wantAccept, awaitRequest(t, requests).headers.Get("Accept")) + }) + } + + t.Run("control: chat completions still sets Accept for stream true", func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mappings := decisionMappings(true, true, nil) + mappings.OpenAI.ChatCompletions = mappings.OpenAI.Decisions + mux := openAIMux(t, mappings, backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, "/v1/chat/completions", `{"model":"`+decisionModel+`","stream":true}`, nil) + + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "text/event-stream", awaitRequest(t, requests).headers.Get("Accept")) + }) +} + +func TestDecisionRoutesAcceptLargeImageBodies(t *testing.T) { + image := strings.Repeat("A", 8*1024*1024) + body := `{"model":"` + decisionModel + `","state":{"image":"data:image/png;base64,` + image + `"}}` + for _, path := range []string{decisionsPath, systemOnePath} { + t.Run(path, func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, path, body, map[string]string{imagesHeader: "true"}) + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Equal(t, len(body), len(awaitRequest(t, requests).body)) + }) + } +} + +func TestDecisionRoutesUnknownModelReturnsNativeJSON404(t *testing.T) { + mux := openAIMux(t, decisionMappings(true, true, nil), "http://nvcf.invalid", "http://llm.invalid") + + rec := postDecision(t, mux, decisionsPath, `{"model":"nope"}`, nil) + assert.Equal(t, "model_not_found", requireOpenAIErrorBody(t, rec, http.StatusNotFound).Error.Code) + assert.Equal(t, "gateway_rejected", rec.Header().Get("NVCF-Error-Source")) + + rec = postDecision(t, mux, systemOnePath, `{"model":"nope"}`, nil) + requireSystemOneErrorBody(t, rec, http.StatusNotFound, "not_found") + assert.Equal(t, "gateway_rejected", rec.Header().Get("NVCF-Error-Source")) +} + +func TestDecisionModelEnabledForOneRouteIsNotFoundOnTheOther(t *testing.T) { + t.Run("decisions only", func(t *testing.T) { + requests := make(chan capturedRequest, 2) + backend := captureServer(t, requests, nil) + mux := openAIMux(t, decisionMappings(true, false, nil), backend.URL, "http://llm.invalid") + + requireSystemOneErrorBody(t, postDecision(t, mux, systemOnePath, `{"model":"`+decisionModel+`"}`, nil), http.StatusNotFound, "not_found") + assert.Empty(t, requests, "the other route must not reach the backend") + require.Equal(t, http.StatusOK, postDecision(t, mux, decisionsPath, `{"model":"`+decisionModel+`"}`, nil).Code) + }) + t.Run("systemone only", func(t *testing.T) { + requests := make(chan capturedRequest, 2) + backend := captureServer(t, requests, nil) + mux := openAIMux(t, decisionMappings(false, true, nil), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, decisionsPath, `{"model":"`+decisionModel+`"}`, nil) + requireOpenAIErrorBody(t, rec, http.StatusNotFound) + assert.Empty(t, requests) + require.Equal(t, http.StatusOK, postDecision(t, mux, systemOnePath, `{"model":"`+decisionModel+`"}`, nil).Code) + }) +} + +func TestDecisionRoutesLocalErrorsUseNativeShape(t *testing.T) { + tooLarge := `{"model":"` + strings.Repeat("a", maxRequestSize) + `"}` + for _, tc := range []struct { + name, body string + status int + s1Code string + }{ + {"malformed json", `{`, http.StatusBadRequest, "invalid_json"}, + {"model wrong type", `{"model":1}`, http.StatusBadRequest, "invalid_json"}, + {"missing model", `{"input":"x"}`, http.StatusBadRequest, "invalid_request"}, + {"body too large", tooLarge, http.StatusRequestEntityTooLarge, "request_too_large"}, + } { + t.Run("decisions "+tc.name, func(t *testing.T) { + mux := openAIMux(t, decisionMappings(true, true, nil), "http://nvcf.invalid", "http://llm.invalid") + rec := postDecision(t, mux, decisionsPath, tc.body, nil) + assert.Equal(t, tc.s1Code, requireOpenAIErrorBody(t, rec, tc.status).Error.Code) + }) + t.Run("systemone "+tc.name, func(t *testing.T) { + mux := openAIMux(t, decisionMappings(true, true, nil), "http://nvcf.invalid", "http://llm.invalid") + requireSystemOneErrorBody(t, postDecision(t, mux, systemOnePath, tc.body, nil), tc.status, tc.s1Code) + }) + } +} + +func TestDecisionRoutesOfflineAndEndOfLifeUseNativeShape(t *testing.T) { + for _, llm := range []bool{false, true} { + name := "direct" + if llm { + name = "llm" + } + t.Run(name+" offline", func(t *testing.T) { + mutate := func(e *config.ModelFunctionDetails) { + if llm { + llmDecision(e) + } + e.OfflineMessage = "down for maintenance" + } + mux := openAIMux(t, decisionMappings(true, true, mutate), "http://nvcf.invalid", "http://llm.invalid") + + rec := postDecision(t, mux, decisionsPath, `{"model":"`+decisionModel+`"}`, nil) + assert.Equal(t, "provider_unavailable", requireOpenAIErrorBody(t, rec, http.StatusServiceUnavailable).Error.Code) + assert.Equal(t, "10800", rec.Header().Get("Retry-After")) + assert.Contains(t, rec.Body.String(), "down for maintenance") + + rec = postDecision(t, mux, systemOnePath, `{"model":"`+decisionModel+`"}`, nil) + requireSystemOneErrorBody(t, rec, http.StatusServiceUnavailable, "provider_unavailable") + assert.Equal(t, "10800", rec.Header().Get("Retry-After")) + }) + t.Run(name+" end of life", func(t *testing.T) { + mutate := func(e *config.ModelFunctionDetails) { + if llm { + llmDecision(e) + } + e.EOL = time.Now().Add(-24 * time.Hour) + } + mux := openAIMux(t, decisionMappings(true, true, mutate), "http://nvcf.invalid", "http://llm.invalid") + + requireOpenAIErrorBody(t, postDecision(t, mux, decisionsPath, `{"model":"`+decisionModel+`"}`, nil), http.StatusGone) + requireSystemOneErrorBody(t, postDecision(t, mux, systemOnePath, `{"model":"`+decisionModel+`"}`, nil), http.StatusGone, "not_found") + }) + } +} + +// With usePexec the director rewrites the URL path to the pexec path before it +// proxies, so the error shape must come from the route the client called. +func TestDecisionRoutesProxyFailureKeepsNativeShapeAfterPexecPathRewrite(t *testing.T) { + down := httptest.NewServer(http.NotFoundHandler()) + downURL := down.URL + down.Close() + + for _, usePexec := range []bool{true, false} { + mutate := func(e *config.ModelFunctionDetails) { e.UsePexec = usePexec } + mux := openAIMux(t, decisionMappings(true, true, mutate), downURL, "http://llm.invalid") + + name := "pexec" + if !usePexec { + name = "no pexec" + } + t.Run(name+" systemone", func(t *testing.T) { + rec := postDecision(t, mux, systemOnePath, `{"model":"`+decisionModel+`"}`, nil) + requireSystemOneErrorBody(t, rec, http.StatusBadGateway, "provider_unavailable") + }) + t.Run(name+" decisions", func(t *testing.T) { + rec := postDecision(t, mux, decisionsPath, `{"model":"`+decisionModel+`"}`, nil) + requireOpenAIErrorBody(t, rec, http.StatusBadGateway) + assert.NotContains(t, rec.Header().Get("Content-Type"), "problem+json") + }) + } +} + +func TestDecisionRoutesClientCancelUsesNativeShape(t *testing.T) { + backend := captureServer(t, make(chan capturedRequest, 1), nil) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + req := openAIRequest(t, systemOnePath, `{"model":"`+decisionModel+`"}`).WithContext(ctx) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + requireSystemOneErrorBody(t, rec, 499, "internal_error") + + req = openAIRequest(t, decisionsPath, `{"model":"`+decisionModel+`"}`).WithContext(ctx) + rec = httptest.NewRecorder() + mux.ServeHTTP(rec, req) + requireOpenAIErrorBody(t, rec, 499) +} + +func TestDecisionRoutesPassBackendErrorsThroughUnchanged(t *testing.T) { + for _, path := range []string{decisionsPath, systemOnePath} { + t.Run(path+" validation body", func(t *testing.T) { + backend := captureServer(t, make(chan capturedRequest, 1), func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-System-One-Error-Code", "invalid_request") + w.WriteHeader(http.StatusUnprocessableEntity) + _, _ = w.Write([]byte(fastAPIValidationErr)) + }) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, path, `{"model":"`+decisionModel+`"}`, nil) + + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code) + assert.Equal(t, fastAPIValidationErr, rec.Body.String()) + assert.Equal(t, "invalid_request", rec.Header().Get("X-System-One-Error-Code")) + }) + t.Run(path+" rate limit with Retry-After", func(t *testing.T) { + backend := captureServer(t, make(chan capturedRequest, 1), func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "7") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"detail":"slow down"}`)) + }) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, path, `{"model":"`+decisionModel+`"}`, nil) + + assert.Equal(t, http.StatusTooManyRequests, rec.Code) + assert.Equal(t, `{"detail":"slow down"}`, rec.Body.String()) + assert.Equal(t, "7", rec.Header().Get("Retry-After")) + }) + } +} + +func TestDecisionRoutesAreNotReachableForChatOnlyModels(t *testing.T) { + llmRequests := make(chan capturedRequest, 1) + llmBackend := captureServer(t, llmRequests, nil) + mux := openAIMux(t, llmMappings(llmModelEntry()), "http://nvcf.invalid", llmBackend.URL) + + for _, path := range []string{decisionsPath, systemOnePath} { + rec := postDecision(t, mux, path, `{"model":"`+publicModel+`"}`, nil) + assert.Equal(t, http.StatusNotFound, rec.Code, path) + assert.Equal(t, "application/json", rec.Header().Get("Content-Type"), path) + } + assert.Empty(t, llmRequests) +} + +func TestDecisionRoutesRejectNonPost(t *testing.T) { + mux := openAIMux(t, decisionMappings(true, true, nil), "http://nvcf.invalid", "http://llm.invalid") + for _, path := range []string{decisionsPath, systemOnePath} { + req := httptest.NewRequest(http.MethodGet, path, bytes.NewBufferString("")) + req.Host = openAIHost + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code, path) + } +} + +// With usePexec the invocation service receives the pexec path, so a function +// serving both routes reaches only its default path. The image header and the +// body still pass through. +func TestDecisionRoutesUsePexecRewritesPathAndKeepsHeaderAndBody(t *testing.T) { + for _, path := range []string{decisionsPath, systemOnePath} { + t.Run(path, func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := openAIMux(t, decisionMappings(true, true, func(e *config.ModelFunctionDetails) { e.UsePexec = true }), backend.URL, "http://llm.invalid") + + body := `{"model":"` + decisionModel + `","state":{}}` + rec := postDecision(t, mux, path, body, map[string]string{imagesHeader: "true"}) + + require.Equal(t, http.StatusOK, rec.Code) + received := awaitRequest(t, requests) + assert.Equal(t, "/v2/nvcf/pexec/functions/"+decisionFunctionID, received.path) + assert.Equal(t, "true", received.headers.Get(imagesHeader)) + assert.Equal(t, body, received.body) + }) + } +} + +func TestDecisionRoutesSetEndpointTypeSpanAttribute(t *testing.T) { + for _, tc := range []struct { + path, want string + llm bool + }{ + {decisionsPath, "decisions", false}, + {systemOnePath, "systemone", false}, + {decisionsPath, "decisions", true}, + {systemOnePath, "systemone", true}, + } { + name := tc.path + if tc.llm { + name += " llm" + } + t.Run(name, func(t *testing.T) { + tp, exp := newTestTracerProvider() + defer tp.Shutdown(context.Background()) + backend := captureServer(t, make(chan capturedRequest, 1), nil) + var mutate func(*config.ModelFunctionDetails) + if tc.llm { + mutate = llmDecision + } + mappings := decisionMappings(true, true, mutate) + llm, err := NewLLMGatewayDirector(backend.URL, http.DefaultTransport) + require.NoError(t, err) + vanity, err := NewVanityDirector(backend.URL, http.DefaultTransport) + require.NoError(t, err) + director, err := NewModelDirector(mappings, regexp.MustCompile("^$"), vanity, llm, nil) + require.NoError(t, err) + + req := openAIRequest(t, tc.path, `{"model":"`+decisionModel+`"}`) + ctx, span := tp.Tracer("test").Start(req.Context(), "decision-test") + rec := httptest.NewRecorder() + if tc.path == decisionsPath { + director.ServeDecisions(rec, req.WithContext(ctx)) + } else { + director.ServeSystemOne(rec, req.WithContext(ctx)) + } + span.End() + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + got, ok := spanAttr(exp.GetSpans()[0], attribute.Key("endpoint.type")) + require.True(t, ok) + require.Equal(t, tc.want, got.AsString()) + }) + } +} + +func TestDecisionRoutesRelayBackend429BodiesUnchangedWithCustomMessage(t *testing.T) { + bodies := map[string]string{ + decisionsPath: `{"error":{"message":"slow down","type":"rate_limit","param":null,"code":"rate_limited"}}`, + systemOnePath: `{"error":{"code":"rate_limit_exceeded","message":"slow down","request_id":"r1"},"request_id":"r1"}`, + } + for path, native := range bodies { + for name, body := range map[string]string{"native": native, "fastapi": fastAPIValidationErr} { + t.Run(path+" "+name, func(t *testing.T) { + backend := captureServer(t, make(chan capturedRequest, 1), func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "7") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(body)) + }) + mux := openAIMux(t, decisionMappings(true, true, func(e *config.ModelFunctionDetails) { + e.TooManyRequestsMessage = "Upgrade your plan." + }), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, path, `{"model":"`+decisionModel+`"}`, nil) + + assert.Equal(t, http.StatusTooManyRequests, rec.Code) + assert.Equal(t, body, rec.Body.String()) + assert.Equal(t, "7", rec.Header().Get("Retry-After")) + }) + } + } +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/h2.go b/src/invocation-plane-services/vanity-gateway/gateway/h2.go index bee5d218f6..4501d96ab2 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/h2.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/h2.go @@ -184,6 +184,8 @@ func registerModelAPIs(hostRouter *middleware.HostRouter, mappings *config.Gatew r.Method(http.MethodPost, "/v1/completions", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeCompletions))) r.Method(http.MethodPost, "/v1/embeddings", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeEmbeddings))) r.Method(http.MethodPost, "/v1/messages", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeMessages))) + r.Method(http.MethodPost, decisionsPath, chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeDecisions))) + r.Method(http.MethodPost, systemOnePath, chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeSystemOne))) r.Method(http.MethodPost, "/v1/responses", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeResponses))) r.Method(http.MethodPost, "/v1/images/generations", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeImageGenerations))) r.Method(http.MethodPost, "/v1/images/edits", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeImageEdits))) diff --git a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go index ea6f4782a4..c5def24893 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go @@ -99,11 +99,7 @@ func stripNVCFRoutingHeaders(request *http.Request) { func (d *LLMGatewayDirector) ServeProxy(target LLMGatewayRequest, writer http.ResponseWriter, request *http.Request) error { span := trace.SpanFromContext(request.Context()) - endpointType := traceAttrValueEndpointLLMGateway - if request.URL.Path == "/v1/messages" { - endpointType = traceAttrValueEndpointAnthropic - } - span.SetAttributes(traceAttrEndpointType.String(endpointType)) + span.SetAttributes(traceAttrEndpointType.String(endpointTypeForPath(request.URL.Path, traceAttrValueEndpointLLMGateway))) if writeFunctionStatusError(writer, request, target.OfflineMessage, target.EOL, "") { return nil diff --git a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go index 4ac152a006..4f971850a9 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go @@ -636,7 +636,7 @@ func TestOpenAIDirector_LLMModelIsNotReachableOnUnmappedEndpoints(t *testing.T) mux := openAIMux(t, llmMappings(llmModelEntry()), "http://nvcf.invalid", llmBackend.URL) - for _, path := range []string{"/v1/responses", "/v1/embeddings", "/v1/completions", "/v1/messages"} { + for _, path := range []string{"/v1/responses", "/v1/embeddings", "/v1/completions", "/v1/messages", "/v1/decisions", "/v1/systemone"} { t.Run(path, func(t *testing.T) { rec := httptest.NewRecorder() mux.ServeHTTP(rec, openAIRequest(t, path, `{"model":"`+publicModel+`","input":"hi"}`)) diff --git a/src/invocation-plane-services/vanity-gateway/gateway/models_protocol_test.go b/src/invocation-plane-services/vanity-gateway/gateway/models_protocol_test.go index d99a922455..1e1e16d5e4 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/models_protocol_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/models_protocol_test.go @@ -253,3 +253,79 @@ func TestListModelsForAnthropicClientsUsesMessagesExpiry(t *testing.T) { require.Equal(t, http.StatusOK, getModels(t, mux, "/v1/models/shared/openai-expired", true).Code) require.Equal(t, http.StatusGone, getModels(t, mux, "/v1/models/shared/messages-expired", true).Code) } + +// decisionDiscoveryMux serves decision-only, shared and expired decision models. +func decisionDiscoveryMux(t *testing.T) http.Handler { + t.Helper() + entry := func(name string) config.ModelFunctionDetails { + return config.ModelFunctionDetails{ModelName: name, FunctionID: "func-" + name} + } + expired := entry("decide/expired") + expired.EOL = time.Now().Add(-time.Hour) + offline := entry("decide/offline") + offline.OfflineMessage = "down" + mappings := &config.GatewayConfig{} + mappings.OpenAI.Host = openAIHost + mappings.OpenAI.ChatCompletions = map[string]config.ModelFunctionDetails{"shared": entry("decide/shared")} + mappings.OpenAI.Decisions = map[string]config.ModelFunctionDetails{ + "d": entry("decide/d-only"), + "shared": entry("decide/shared"), + "expired": expired, + "offline": offline, + } + mappings.TypeSafeSystemOne = map[string]config.ModelFunctionDetails{ + "s": entry("decide/s-only"), + "shared": entry("decide/shared"), + } + mux, err := buildChiMux(mappings, Config{ + NvcfApiEndpoint: "http://nvcf.invalid", + LLMGatewayEndpoint: "http://llm.invalid", + PrivateModelNameRegexPattern: "^$", + }) + require.NoError(t, err) + return mux +} + +func TestListModelsIncludesDecisionModelsWithSupportedEndpoints(t *testing.T) { + rec := getModels(t, decisionDiscoveryMux(t), "/v1/models", false) + require.Equal(t, http.StatusOK, rec.Code) + var models ModelListResponse + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &models)) + got := map[string][]string{} + for _, m := range models.Data { + got[m.Id] = m.SupportedEndpoints + } + require.Equal(t, map[string][]string{ + "decide/d-only": {"/v1/decisions"}, + "decide/s-only": {"/v1/systemone"}, + "decide/shared": {"/v1/chat/completions", "/v1/decisions", "/v1/systemone"}, + "decide/offline": {"/v1/decisions"}, + }, got, "the expired model is filtered out") +} + +func TestGetModelDescribesDecisionModels(t *testing.T) { + mux := decisionDiscoveryMux(t) + for model, want := range map[string][]string{ + "decide/d-only": {"/v1/decisions"}, + "decide/s-only": {"/v1/systemone"}, + "decide/shared": {"/v1/chat/completions", "/v1/decisions", "/v1/systemone"}, + } { + rec := getModels(t, mux, "/v1/models/"+model, false) + require.Equal(t, http.StatusOK, rec.Code, model) + var info ModelInfo + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &info)) + require.Equal(t, model, info.Id) + require.Equal(t, want, info.SupportedEndpoints, model) + } + require.Equal(t, http.StatusGone, getModels(t, mux, "/v1/models/decide/expired", false).Code) + require.Equal(t, http.StatusServiceUnavailable, getModels(t, mux, "/v1/models/decide/offline", false).Code) + require.Equal(t, http.StatusNotFound, getModels(t, mux, "/v1/models/decide/missing", false).Code) +} + +func TestAnthropicModelListExcludesDecisionModels(t *testing.T) { + rec := getModels(t, decisionDiscoveryMux(t), "/v1/models", true) + require.Equal(t, http.StatusOK, rec.Code) + var page anthropicModelPage + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &page)) + require.Empty(t, page.Data) +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/native_errors.go b/src/invocation-plane-services/vanity-gateway/gateway/native_errors.go new file mode 100644 index 0000000000..4c0271a487 --- /dev/null +++ b/src/invocation-plane-services/vanity-gateway/gateway/native_errors.go @@ -0,0 +1,161 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + "ai-api-gateway-service/middleware" + "context" + "net/http" + + "github.com/goccy/go-json" +) + +// nativeProtocol names the API whose error shape a response must use. +type nativeProtocol string + +const ( + nativeProtocolNone nativeProtocol = "" + nativeProtocolMessages nativeProtocol = "messages" + nativeProtocolDecisions nativeProtocol = "decisions" + nativeProtocolSystemOne nativeProtocol = "systemone" +) + +const ( + decisionsPath = "/v1/decisions" + systemOnePath = "/v1/systemone" + messagesPath = "/v1/messages" +) + +// System One error codes. The gateway also uses them as the code on the +// OpenAI-shaped decisions errors, except where openAIErrorCode says otherwise. +const ( + errKindInvalidJSON = "invalid_json" + errKindInvalidRequest = "invalid_request" + errKindNotFound = "not_found" + errKindRequestTooLarge = "request_too_large" + errKindInternal = "internal_error" + errKindProviderUnavailable = "provider_unavailable" +) + +const nativeProtocolKey contextKey = tooManyRequestsKey + 1 + +func protocolForPath(urlPath string) nativeProtocol { + switch urlPath { + case messagesPath: + return nativeProtocolMessages + case decisionsPath: + return nativeProtocolDecisions + case systemOnePath: + return nativeProtocolSystemOne + } + return nativeProtocolNone +} + +// withNativeProtocol records the protocol of the route the client called. +// applyExecTarget rewrites URL.Path when usePexec is set, so the path alone +// cannot name the protocol once a request is being proxied. +func withNativeProtocol(request *http.Request, protocol nativeProtocol) *http.Request { + return request.WithContext(context.WithValue(request.Context(), nativeProtocolKey, protocol)) +} + +// requestProtocol returns the protocol stored at route entry, falling back to +// the request path for requests that never went through a route entry. +func requestProtocol(request *http.Request) nativeProtocol { + if protocol, ok := request.Context().Value(nativeProtocolKey).(nativeProtocol); ok { + return protocol + } + return protocolForPath(request.URL.Path) +} + +// endpointTypeForPath is the endpoint.type span value for a request path. +// fallback applies to the OpenAI routes and to the LLM Gateway hop. +func endpointTypeForPath(urlPath, fallback string) string { + switch urlPath { + case messagesPath: + return traceAttrValueEndpointAnthropic + case decisionsPath: + return traceAttrValueEndpointDecisions + case systemOnePath: + return traceAttrValueEndpointSystemOne + } + return fallback +} + +// writeNativeError writes a gateway-generated error in the shape of the route +// the client called and reports whether it did. It writes nothing for the +// OpenAI chat-style routes, which keep their own error handling. +func writeNativeError(writer http.ResponseWriter, request *http.Request, status int, kind, message string) bool { + switch requestProtocol(request) { + case nativeProtocolMessages: + writeAnthropicError(writer, status, message) + case nativeProtocolDecisions: + writeDecisionsError(writer, status, kind, message) + case nativeProtocolSystemOne: + writeSystemOneError(writer, status, kind, message) + default: + return false + } + return true +} + +func openAIErrorCode(status int, kind string) string { + switch status { + case http.StatusNotFound: + return "model_not_found" + case http.StatusGone: + return "model_expired" + } + return kind +} + +func writeDecisionsError(writer http.ResponseWriter, status int, kind, message string) { + errorType := "invalid_request_error" + if status >= http.StatusInternalServerError { + errorType = "server_error" + } + writer.Header().Set(contentTypeHeader, contentTypeApplicationJSON) + writer.WriteHeader(status) + _ = json.NewEncoder(writer).Encode(OpenAIError{Error: CustomOpenAIError{ + Message: message, + Type: errorType, + Code: openAIErrorCode(status, kind), + }}) +} + +type systemOneErrorDetail struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type systemOneError struct { + Error systemOneErrorDetail `json:"error"` + RequestID string `json:"request_id"` +} + +// writeSystemOneError writes the System One error body. Gateway-written errors +// carry no X-System-One-* headers: those mark errors the model server relayed. +func writeSystemOneError(writer http.ResponseWriter, status int, kind, message string) { + requestID := writer.Header().Get(middleware.RequestIDHeader) + writer.Header().Set(contentTypeHeader, contentTypeApplicationJSON) + writer.WriteHeader(status) + _ = json.NewEncoder(writer).Encode(systemOneError{ + Error: systemOneErrorDetail{Code: kind, Message: message, RequestID: requestID}, + RequestID: requestID, + }) +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go index 24063321c6..922ae732e5 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go @@ -74,6 +74,8 @@ type ModelDirector struct { embeddings ModelMapping responses ModelMapping messages ModelMapping + decisions ModelMapping + systemOne ModelMapping imageGenerations ModelMapping imageEdits ModelMapping imageVariations ModelMapping @@ -350,6 +352,16 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp return nil, err } + decisions, err := buildModelMapping(convertIntoModelNameToFunctionIdAndVersionIdMappingV2(mapping.OpenAI.Decisions, "openai.decisions"), privateModelMatcher) + if err != nil { + return nil, err + } + + systemOne, err := buildModelMapping(convertIntoModelNameToFunctionIdAndVersionIdMappingV2(mapping.TypeSafeSystemOne, "typesafeSystemOne"), privateModelMatcher) + if err != nil { + return nil, err + } + imageGenerations, err := buildModelMapping(convertIntoModelNameToFunctionIdAndVersionIdMappingV2(mapping.OpenAI.ImageGenerations, "openai.imageGenerations"), privateModelMatcher) if err != nil { return nil, err @@ -373,6 +385,8 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, messages.modelNameToModelInfo, + decisions.modelNameToModelInfo, + systemOne.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo, @@ -381,9 +395,18 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp seen[id] = info } } - // Annotate aliases that support Messages so shared discovery does not - // imply that a Messages-only model accepts OpenAI requests. - for id := range messages.modelNameToModelInfo { + // Annotate aliases that support a native protocol (Messages, decisions or + // System One) so shared discovery does not imply that such a model accepts + // OpenAI chat requests. + nativeIDs := make(map[string]struct{}) + for _, models := range []map[string]ModelInfo{ + messages.modelNameToModelInfo, decisions.modelNameToModelInfo, systemOne.modelNameToModelInfo, + } { + for id := range models { + nativeIDs[id] = struct{}{} + } + } + for id := range nativeIDs { info := seen[id] for _, endpoint := range []struct { path string @@ -394,6 +417,8 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp {"/v1/embeddings", embeddings.modelNameToModelInfo}, {"/v1/responses", responses.modelNameToModelInfo}, {"/v1/messages", messages.modelNameToModelInfo}, + {decisionsPath, decisions.modelNameToModelInfo}, + {systemOnePath, systemOne.modelNameToModelInfo}, {"/v1/images/generations", imageGenerations.modelNameToModelInfo}, {"/v1/images/edits", imageEdits.modelNameToModelInfo}, {"/v1/images/variations", imageVariations.modelNameToModelInfo}, @@ -403,7 +428,7 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp } } seen[id] = info - for _, models := range []map[string]ModelInfo{chatCompletions.modelNameToModelInfo, completions.modelNameToModelInfo, embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, messages.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo} { + for _, models := range []map[string]ModelInfo{chatCompletions.modelNameToModelInfo, completions.modelNameToModelInfo, embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, messages.modelNameToModelInfo, decisions.modelNameToModelInfo, systemOne.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo} { if _, ok := models[id]; ok { models[id] = info } @@ -418,6 +443,7 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo, messages.modelNameToModelInfo, + decisions.modelNameToModelInfo, systemOne.modelNameToModelInfo, } { if _, ok := models[id]; ok { models[id] = info @@ -435,6 +461,8 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp embeddings: embeddings, responses: responses, messages: messages, + decisions: decisions, + systemOne: systemOne, imageGenerations: imageGenerations, imageEdits: imageEdits, imageVariations: imageVariations, @@ -496,6 +524,8 @@ func (d *ModelDirector) getFilteredModels() []ModelInfo { d.embeddings.modelNameToNVCFUrl, d.responses.modelNameToNVCFUrl, d.messages.modelNameToNVCFUrl, + d.decisions.modelNameToNVCFUrl, + d.systemOne.modelNameToNVCFUrl, d.imageGenerations.modelNameToNVCFUrl, d.imageEdits.modelNameToNVCFUrl, d.imageVariations.modelNameToNVCFUrl, @@ -589,6 +619,21 @@ func (d *ModelDirector) ServeMessages(writer http.ResponseWriter, request *http. d.proxyModelMappedRequest(writer, request, d.messages.modelNameToNVCFUrl, shadowSamplingEndpointMessages) } +// ServeDecisions passes POST /v1/decisions through to the model's function. +// The body is not parsed beyond model, and errors the gateway writes use the +// OpenAI shape. +func (d *ModelDirector) ServeDecisions(writer http.ResponseWriter, request *http.Request) { + request = withNativeProtocol(request, nativeProtocolDecisions) + d.proxyModelMappedRequest(writer, request, d.decisions.modelNameToNVCFUrl, shadowSamplingEndpointNone) +} + +// ServeSystemOne passes POST /v1/systemone through to the model's function. +// Errors the gateway writes use the System One shape. +func (d *ModelDirector) ServeSystemOne(writer http.ResponseWriter, request *http.Request) { + request = withNativeProtocol(request, nativeProtocolSystemOne) + d.proxyModelMappedRequest(writer, request, d.systemOne.modelNameToNVCFUrl, shadowSamplingEndpointNone) +} + func (d *ModelDirector) ServeImageGenerations(writer http.ResponseWriter, request *http.Request) { d.proxyModelMappedRequest(writer, request, d.imageGenerations.modelNameToNVCFUrl, shadowSamplingEndpointNone) } @@ -651,6 +696,8 @@ func (d *ModelDirector) GetModel(writer http.ResponseWriter, request *http.Reque d.chatCompletions.modelNameToModelInfo, d.responses.modelNameToModelInfo, d.messages.modelNameToModelInfo, + d.decisions.modelNameToModelInfo, + d.systemOne.modelNameToModelInfo, d.imageGenerations.modelNameToModelInfo, d.imageEdits.modelNameToModelInfo, d.imageVariations.modelNameToModelInfo, @@ -669,6 +716,8 @@ func (d *ModelDirector) GetModel(writer http.ResponseWriter, request *http.Reque d.chatCompletions.modelNameToNVCFUrl, d.responses.modelNameToNVCFUrl, d.messages.modelNameToNVCFUrl, + d.decisions.modelNameToNVCFUrl, + d.systemOne.modelNameToNVCFUrl, d.imageGenerations.modelNameToNVCFUrl, d.imageEdits.modelNameToNVCFUrl, d.imageVariations.modelNameToNVCFUrl, @@ -689,11 +738,7 @@ func (d *ModelDirector) GetModel(writer http.ResponseWriter, request *http.Reque func (d *ModelDirector) proxyModelMappedRequest(writer http.ResponseWriter, request *http.Request, modelToNVCFUrl map[string]FunctionInfo, endpoint shadowSamplingEndpoint) { span := trace.SpanFromContext(request.Context()) - endpointType := traceAttrValueEndpointOpenAI - if request.URL.Path == "/v1/messages" { - endpointType = traceAttrValueEndpointAnthropic - } - span.SetAttributes(traceAttrEndpointType.String(endpointType)) + span.SetAttributes(traceAttrEndpointType.String(endpointTypeForPath(request.URL.Path, traceAttrValueEndpointOpenAI))) setShadowSpanAttribute(span, request) resolved, handled := d.resolveModelMappedRequest(writer, request, modelToNVCFUrl) @@ -733,19 +778,9 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re body, err := extractOpenAIRequestBody(request) if err != nil { markGatewayRejected(writer, request) - if request.URL.Path == "/v1/messages" { - status, message := http.StatusInternalServerError, "failed to read Messages request" - if errors.Is(err, errInvalidJSONBody) || errors.Is(err, errModelFieldMissing) { - status, message = http.StatusBadRequest, "invalid Messages request JSON" - } - var maxBytesErr *http.MaxBytesError - if errors.As(err, &maxBytesErr) { - status, message = http.StatusRequestEntityTooLarge, "request body is too large" - } - if errors.Is(err, errModelFieldMissing) { - message = "model field is required" - } - writeAnthropicError(writer, status, message) + if protocol := requestProtocol(request); protocol != nativeProtocolNone { + status, kind, message := requestReadFailure(protocol, err) + writeNativeError(writer, request, status, kind, message) return resolvedOpenAIRequest{}, true } var maxBytesErr *http.MaxBytesError @@ -796,9 +831,7 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re if !ok { _ = request.Body.Close() markGatewayRejected(writer, request) - if request.URL.Path == "/v1/messages" { - writeAnthropicError(writer, http.StatusNotFound, "model not found") - } else { + if !writeNativeError(writer, request, http.StatusNotFound, errKindNotFound, "model not found") { http.NotFound(writer, request) } return resolvedOpenAIRequest{}, true @@ -806,33 +839,22 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re middleware.SetPreUpstreamModel(request.Context(), nvcfUrl.publicModelName) middleware.AddOpenAIRequestMetricAttributes(request.Context(), body.Model, nvcfUrl.functionId) - statusHandled := false - if request.URL.Path == "/v1/messages" { - if nvcfUrl.offlineMessage != "" { - markGatewayRejected(writer, request) - writer.Header().Set("Retry-After", "10800") - writeAnthropicError(writer, http.StatusServiceUnavailable, nvcfUrl.offlineMessage) - statusHandled = true - } else if isModelExpired(nvcfUrl.eol) { - markGatewayRejected(writer, request) - writeAnthropicError(writer, http.StatusGone, "model has reached end of life") - statusHandled = true - } - } else { - statusHandled = writeFunctionStatusError(writer, request, nvcfUrl.offlineMessage, nvcfUrl.eol, body.Model) - } + statusHandled := writeFunctionStatusError(writer, request, nvcfUrl.offlineMessage, nvcfUrl.eol, body.Model) if statusHandled { _ = request.Body.Close() return resolvedOpenAIRequest{}, true } - if nvcfUrl.tooManyRequestsMessage != "" { + // Decision backend 429 bodies pass through byte for byte. + if nvcfUrl.tooManyRequestsMessage != "" && !isDecisionRoute(request) { ctx := context.WithValue(request.Context(), tooManyRequestsKey, nvcfUrl.tooManyRequestsMessage) request = request.WithContext(ctx) } // setPollingHeaderIfNotPresent checks the Accept header, so we have to override it first - if body.Stream { + // Decision routes are non-streaming: stream is ignored and the body is + // forwarded as sent. + if body.Stream && !isDecisionRoute(request) { request.Header.Set("Accept", "text/event-stream") } @@ -1241,3 +1263,32 @@ type OpenAIDirector = ModelDirector func NewOpenAIDirectorV2(mapping *config.GatewayConfig, privateModelMatcher *regexp.Regexp, vanityDirector *VanityDirector, llmGatewayDirector *LLMGatewayDirector, shadower *TrafficShadower) (*ModelDirector, error) { return NewModelDirector(mapping, privateModelMatcher, vanityDirector, llmGatewayDirector, shadower) } + +func isDecisionRoute(request *http.Request) bool { + protocol := requestProtocol(request) + return protocol == nativeProtocolDecisions || protocol == nativeProtocolSystemOne +} + +// requestReadFailure maps a failure to read or parse the request body to the +// status, error code and message of the route's native error. +func requestReadFailure(protocol nativeProtocol, err error) (int, string, string) { + status, kind := http.StatusInternalServerError, errKindInternal + message := "failed to read request" + if protocol == nativeProtocolMessages { + message = "failed to read Messages request" + } + if errors.Is(err, errInvalidJSONBody) || errors.Is(err, errModelFieldMissing) { + status, kind, message = http.StatusBadRequest, errKindInvalidJSON, "request body is not valid JSON" + if protocol == nativeProtocolMessages { + message = "invalid Messages request JSON" + } + } + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + status, kind, message = http.StatusRequestEntityTooLarge, errKindRequestTooLarge, "request body is too large" + } + if errors.Is(err, errModelFieldMissing) { + kind, message = errKindInvalidRequest, "model field is required" + } + return status, kind, message +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go b/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go index 153b2c7f12..6f55fde283 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go @@ -67,6 +67,11 @@ func TestGatewayGeneratedErrorsCarryRequestID(t *testing.T) { {"messages unknown model", "/v1/messages", `{"model":"unknown"}`, nil, http.StatusNotFound}, {"messages body too large", "/v1/messages", tooLarge, nil, http.StatusRequestEntityTooLarge}, {"messages reserved shadow header", "/v1/messages", `{"model":"` + publicModel + `"}`, map[string]string{shadowHeader: "true"}, http.StatusBadRequest}, + {"decisions malformed json", "/v1/decisions", `{`, nil, http.StatusBadRequest}, + {"decisions unknown model", "/v1/decisions", `{"model":"unknown"}`, nil, http.StatusNotFound}, + {"systemone malformed json", "/v1/systemone", `{`, nil, http.StatusBadRequest}, + {"systemone unknown model", "/v1/systemone", `{"model":"unknown"}`, nil, http.StatusNotFound}, + {"systemone body too large", "/v1/systemone", tooLarge, nil, http.StatusRequestEntityTooLarge}, {"chat malformed json", "/v1/chat/completions", `{`, nil, http.StatusBadRequest}, {"chat unknown model", "/v1/chat/completions", `{"model":"unknown"}`, nil, http.StatusNotFound}, } diff --git a/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go b/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go index 0f01b89bc8..bf6e99c093 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go @@ -45,6 +45,8 @@ const ( traceAttrValueEndpointAnthropic = "anthropic" traceAttrValueEndpointOpenAI = "openai" traceAttrValueEndpointLLMGateway = "llm_gateway" + traceAttrValueEndpointDecisions = "decisions" + traceAttrValueEndpointSystemOne = "systemone" shadowDroppedReasonBodyReadError = "body_read_error" shadowDroppedReasonBodyRewriteError = "body_rewrite_error" diff --git a/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go b/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go index 874282a587..8af1287a2a 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go @@ -51,6 +51,11 @@ const statusClientClosedRequest = 499 func writeFunctionStatusError(writer http.ResponseWriter, request *http.Request, offlineMessage string, eol time.Time, name string) bool { if offlineMessage != "" { markGatewayRejected(writer, request) + if requestProtocol(request) != nativeProtocolNone { + writer.Header().Set("Retry-After", "10800") + writeNativeError(writer, request, http.StatusServiceUnavailable, errKindProviderUnavailable, offlineMessage) + return true + } writer.Header().Set("Content-Type", "application/problem+json") writer.Header().Set("Retry-After", "10800") writer.WriteHeader(http.StatusServiceUnavailable) @@ -63,6 +68,11 @@ func writeFunctionStatusError(writer http.ResponseWriter, request *http.Request, return true } if isModelExpired(eol) { + if requestProtocol(request) != nativeProtocolNone { + markGatewayRejected(writer, request) + writeNativeError(writer, request, http.StatusGone, errKindNotFound, "model has reached end of life") + return true + } var detail string if name != "" { detail = fmt.Sprintf("The model '%s' has reached its end of life on %s and is no longer available.", name, eol.Format(time.RFC3339)) @@ -146,8 +156,7 @@ func writeProxyError(writer http.ResponseWriter, request *http.Request, err erro zap.String(string(middleware.GatewayProxyOutcomeMetricAttribute), string(middleware.GatewayProxyOutcomeClientCanceled)), zap.Error(err), ) - if request.URL.Path == "/v1/messages" { - writeAnthropicError(writer, statusClientClosedRequest, "client closed the request") + if writeNativeError(writer, request, statusClientClosedRequest, errKindInternal, "client closed the request") { return } writer.Header().Set("Content-Type", "application/problem+json") @@ -170,8 +179,7 @@ func writeBadGatewayProblem(writer http.ResponseWriter, request *http.Request, e zap.String(string(middleware.GatewayProxyOutcomeMetricAttribute), string(middleware.GatewayProxyOutcomeProxyError)), zap.Error(err), ) - if request.URL.Path == "/v1/messages" { - writeAnthropicError(writer, http.StatusBadGateway, "upstream request failed") + if writeNativeError(writer, request, http.StatusBadGateway, errKindProviderUnavailable, "upstream request failed") { return } writer.Header().Set("Content-Type", "application/problem+json") diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go index 1cdc9c3421..9c2f2222c6 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go @@ -418,7 +418,7 @@ const ( ) // llmGatewaySections are the native protocol sections the LLM Gateway serves. -var llmGatewaySections = []string{"chatCompletions", "responses", "embeddings", "anthropicMessages"} +var llmGatewaySections = []string{"chatCompletions", "responses", "embeddings", "anthropicMessages", "decisions", "typesafeSystemOne"} type V2Config struct { OpenAI struct { @@ -430,8 +430,10 @@ type V2Config struct { ImageGenerations map[string]ModelFunctionDetails `json:"imageGenerations"` ImageEdits map[string]ModelFunctionDetails `json:"imageEdits"` ImageVariations map[string]ModelFunctionDetails `json:"imageVariations"` + Decisions map[string]ModelFunctionDetails `json:"decisions"` } `json:"openai"` AnthropicMessages map[string]ModelFunctionDetails `json:"anthropicMessages"` + TypeSafeSystemOne map[string]ModelFunctionDetails `json:"typesafeSystemOne"` Vanity map[string]VanityEntry `json:"vanity"` } @@ -603,7 +605,11 @@ func validateSamplingMethods(location string, fieldName string, sectionName stri return nil } -const anthropicMessagesSection = "anthropicMessages" +const ( + anthropicMessagesSection = "anthropicMessages" + decisionsSection = "openai.decisions" + typeSafeSystemOneSection = "typesafeSystemOne" +) func supportsRequestBodySampling(sectionName string) bool { return sectionName == "openai.chatCompletions" || sectionName == "openai.responses" || @@ -630,6 +636,8 @@ func (c *GatewayConfig) modelSections() map[string]map[string]ModelFunctionDetai "openai.imageGenerations": c.OpenAI.ImageGenerations, "openai.imageEdits": c.OpenAI.ImageEdits, "openai.imageVariations": c.OpenAI.ImageVariations, + decisionsSection: c.OpenAI.Decisions, + typeSafeSystemOneSection: c.TypeSafeSystemOne, } } diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go index daa02ba098..198b5c58fd 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go @@ -1659,9 +1659,11 @@ func TestGatewayConfigValidateScopesRequestBodyShadowSampling(t *testing.T) { }) } for section, location := range map[string]string{ - "completions": "openai.completions", - "embeddings": "openai.embeddings", - "imageGenerations": "openai.imageGenerations", + "completions": "openai.completions", + "embeddings": "openai.embeddings", + "imageGenerations": "openai.imageGenerations", + "decisions": "openai.decisions", + "typesafeSystemOne": "typesafeSystemOne", } { for _, perTarget := range []bool{false, true} { form := "legacy" @@ -1797,6 +1799,10 @@ func gatewayConfigWithShadow( cfg.OpenAI.ImageGenerations = entries case "anthropicMessages": cfg.AnthropicMessages = entries + case "decisions": + cfg.OpenAI.Decisions = entries + case "typesafeSystemOne": + cfg.TypeSafeSystemOne = entries } return cfg } @@ -1818,6 +1824,10 @@ func llmModelConfig(section string, entry ModelFunctionDetails) *GatewayConfig { cfg.OpenAI.Completions = entries case "imageGenerations": cfg.OpenAI.ImageGenerations = entries + case "decisions": + cfg.OpenAI.Decisions = entries + case "typesafeSystemOne": + cfg.TypeSafeSystemOne = entries } return cfg } @@ -1979,3 +1989,45 @@ func TestAnthropicMessagesConfigNamespace(t *testing.T) { cfg.AnthropicMessages["native"] = entry require.ErrorContains(t, cfg.Validate(), "anthropicMessages.native: modelName is required") } + +func TestDecisionSectionsAcceptRandomAndPerBearerKeySampling(t *testing.T) { + methods := samplingMethods(ShadowSamplingMethodPerBearerKey, ShadowSamplingMethodRandom) + for _, section := range []string{"decisions", "typesafeSystemOne"} { + t.Run(section, func(t *testing.T) { + require.NoError(t, gatewayConfigWithShadow(section, methods, nil, true).Validate()) + }) + } +} + +func TestDecisionConfigNamespaces(t *testing.T) { + var cfg GatewayConfig + require.NoError(t, json.Unmarshal([]byte(`{ + "v2config": { + "openai": {"host": "api.example.com", "decisions": { + "dec": {"modelName": "dec-model", "functionID": "dec-function", "modelInfoFields": {"family": "decision"}} + }}, + "typesafeSystemOne": { + "s1": {"modelName": "s1-model", "functionID": "s1-function", "functionType": "LLM"} + } + } + }`), &cfg)) + require.NoError(t, cfg.Validate()) + require.True(t, cfg.HasLLMGatewayRoute()) + require.Equal(t, "dec-model", cfg.OpenAI.Decisions["dec"].ModelName) + require.Equal(t, "s1-model", cfg.TypeSafeSystemOne["s1"].ModelName) + // Shadow targets must stay inside the same section. + entry := cfg.TypeSafeSystemOne["s1"] + entry.ShadowModelName = "dec-model" + cfg.TypeSafeSystemOne["s1"] = entry + require.ErrorContains(t, cfg.Validate(), "typesafeSystemOne.s1: shadow target must reference another model in typesafeSystemOne") +} + +func TestDecisionSectionsRejectLLMOnlyFieldsWithUsePexec(t *testing.T) { + for _, section := range []string{"decisions", "typesafeSystemOne"} { + t.Run(section, func(t *testing.T) { + entry := llmModel() + entry.UsePexec = true + require.ErrorContains(t, llmModelConfig(section, entry).Validate(), "usePexec is unsupported") + }) + } +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/model_info.go b/src/invocation-plane-services/vanity-gateway/gateway_config/model_info.go index 9be348202a..4f078d0b48 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/model_info.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/model_info.go @@ -34,6 +34,7 @@ import ( var modelInfoSectionOrder = []string{ "openai.chatCompletions", "openai.completions", "openai.embeddings", "openai.responses", "openai.imageGenerations", "openai.imageEdits", "openai.imageVariations", anthropicMessagesSection, + decisionsSection, typeSafeSystemOneSection, } func validateModelInfoFields(model string, fields map[string]json.RawMessage) error { diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/model_info_test.go b/src/invocation-plane-services/vanity-gateway/gateway_config/model_info_test.go index ab4ed727b6..794892799a 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/model_info_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/model_info_test.go @@ -100,6 +100,8 @@ func TestModelInfoReservedFields(t *testing.T) { cfg.OpenAI.ImageEdits = map[string]ModelFunctionDetails{} cfg.OpenAI.ImageVariations = map[string]ModelFunctionDetails{} cfg.AnthropicMessages = map[string]ModelFunctionDetails{} + cfg.OpenAI.Decisions = map[string]ModelFunctionDetails{} + cfg.TypeSafeSystemOne = map[string]ModelFunctionDetails{} cfg.modelSections()[section]["example"] = entry err := cfg.Validate() require.ErrorContains(t, err, "example/model") @@ -215,3 +217,23 @@ func TestModelInfoYAMLEmptyCompatibility(t *testing.T) { require.Empty(t, cfg.ModelInfoFieldsByName()) } } + +func TestModelInfoFieldsIncludeDecisionSections(t *testing.T) { + var cfg GatewayConfig + require.NoError(t, unmarshalGatewayConfig([]byte(`v2config: + openai: + decisions: + entry: + modelName: example/decision + modelInfoFields: {from_decisions: true, winner: decisions} + typesafeSystemOne: + entry: + modelName: example/decision + modelInfoFields: {from_system_one: true, winner: systemone} +`), &cfg)) + require.NoError(t, cfg.Validate()) + fields := cfg.ModelInfoFieldsByName()["example/decision"] + require.Equal(t, "true", string(fields["from_decisions"])) + require.Equal(t, "true", string(fields["from_system_one"])) + require.Equal(t, `"decisions"`, string(fields["winner"])) +} diff --git a/src/libraries/rust/stargate/crates/mock-dynamo/src/main.rs b/src/libraries/rust/stargate/crates/mock-dynamo/src/main.rs index dfd613a4b4..2700824de0 100644 --- a/src/libraries/rust/stargate/crates/mock-dynamo/src/main.rs +++ b/src/libraries/rust/stargate/crates/mock-dynamo/src/main.rs @@ -16,6 +16,7 @@ mod engine_driver; mod kv_cache; mod openai; +mod sha256; mod stats_stream; // mock-dynamo is an integration fixture, so its runtime image intentionally // exposes deterministic controls used by deployed behavior tests. @@ -27,6 +28,7 @@ use std::time::Duration; use anyhow::Result; use axum::Router; +use axum::extract::DefaultBodyLimit; use axum::routing::{get, post, put}; use clap::Parser; use tokio::net::TcpListener; @@ -569,6 +571,7 @@ async fn main() -> Result<()> { .route("/v1/models", get(openai::list_models)) .route("/v1/responses", post(openai::responses)) .route("/v1/embeddings", post(openai::embeddings)) + .merge(decision_routes()) .route("/pylon/v1/stats/stream", get(stats_stream::stats_stream)) .route("/kv-cache/stats", get(openai::kv_cache_stats)) .route( @@ -589,9 +592,26 @@ async fn main() -> Result<()> { info!("send POST to http://{actual_http_addr}/v1/chat/completions"); info!("send POST to http://{actual_http_addr}/v1/responses"); info!("send POST to http://{actual_http_addr}/v1/embeddings"); + info!("send POST to http://{actual_http_addr}/v1/decisions"); + info!("send POST to http://{actual_http_addr}/v1/systemone"); Ok(axum::serve(listener, app).await?) } +/// Decision requests can carry base64 images, so these routes accept bodies +/// well above axum's 2 MB default. +fn decision_routes() -> Router { + const MAX_DECISION_BODY_BYTES: usize = 16 << 20; + Router::new() + .route( + "/v1/decisions", + post(openai::decisions).layer(DefaultBodyLimit::max(MAX_DECISION_BODY_BYTES)), + ) + .route( + "/v1/systemone", + post(openai::systemone).layer(DefaultBodyLimit::max(MAX_DECISION_BODY_BYTES)), + ) +} + #[cfg(test)] mod tests; diff --git a/src/libraries/rust/stargate/crates/mock-dynamo/src/openai.rs b/src/libraries/rust/stargate/crates/mock-dynamo/src/openai.rs index 5e44d3d727..75999883fb 100644 --- a/src/libraries/rust/stargate/crates/mock-dynamo/src/openai.rs +++ b/src/libraries/rust/stargate/crates/mock-dynamo/src/openai.rs @@ -14,7 +14,8 @@ // limitations under the License. use axum::Json; -use axum::extract::State; +use axum::body::Bytes; +use axum::extract::{RawQuery, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::sse::{Event, KeepAlive, Sse}; use axum::response::{IntoResponse, Response}; @@ -481,6 +482,340 @@ pub(crate) async fn embeddings( .into_response() } +const DECISION_ECHO_HEADERS: [&str; 5] = [ + "typesafe-danger-zone-images", + "x-model", + "x-input-tokens", + "x-request-id", + "x-vendor-test", +]; +const DECISION_MOCK_PROBABILITY: f64 = 0.99; +const DECISION_MOCK_RETRY_AFTER_SECONDS: &str = "7"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum DecisionRoute { + Decisions, + Systemone, +} + +/// Canned OpenAI Decisions response. The body is not validated, so tests can +/// send large or loosely shaped payloads. +pub(crate) async fn decisions( + State(state): State, + headers: HeaderMap, + RawQuery(query): RawQuery, + body: Bytes, +) -> Response { + decision_response(state, headers, query, body, DecisionRoute::Decisions).await +} + +/// Canned System One response. +pub(crate) async fn systemone( + State(state): State, + headers: HeaderMap, + RawQuery(query): RawQuery, + body: Bytes, +) -> Response { + decision_response(state, headers, query, body, DecisionRoute::Systemone).await +} + +async fn decision_response( + state: AppState, + headers: HeaderMap, + query: Option, + body: Bytes, + route: DecisionRoute, +) -> Response { + let request: serde_json::Value = serde_json::from_slice(&body).unwrap_or_default(); + let model = request + .get("model") + .and_then(serde_json::Value::as_str) + .map_or_else(|| state.model_name.clone(), str::to_string); + let endpoint = match route { + DecisionRoute::Decisions => TestEndpoint::Decisions, + DecisionRoute::Systemone => TestEndpoint::Systemone, + }; + state.record_request(&headers, endpoint, &model).await; + let _request_slot = state.acquire_request_slot().await; + let input_tokens = optional_header(&headers, "x-input-tokens") + .and_then(|value| value.parse::().ok()) + .unwrap_or_else(|| (body.len() / 4).max(1)); + let output_tokens = 1; + let request_id = optional_header(&headers, "x-request-id") + .unwrap_or_else(|| format!("decision-mock-{}", rand_id())); + info!( + request_id = %request_id, + model = %model, + input_tokens = input_tokens, + body_len = body.len(), + "received decision request" + ); + state.emit_counters(&request_id, &model, input_tokens, output_tokens, true); + + let mock_error = query + .as_deref() + .into_iter() + .flat_map(|query| query.split('&')) + .find_map(|pair| pair.strip_prefix("mock_error=")); + match mock_error { + Some("429") => { + let mut response = decision_error(route, StatusCode::TOO_MANY_REQUESTS); + response.headers_mut().insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static(DECISION_MOCK_RETRY_AFTER_SECONDS), + ); + return response; + } + Some("400") => return decision_error(route, StatusCode::BAD_REQUEST), + _ => {} + } + + let echo: serde_json::Map = DECISION_ECHO_HEADERS + .iter() + .map(|name| ((*name).to_string(), optional_header(&headers, name).into())) + .chain([ + ("body_len".to_string(), body.len().into()), + ( + "body_sha256".to_string(), + crate::sha256::sha256_hex(&body).into(), + ), + ]) + .collect(); + let payload = match route { + DecisionRoute::Decisions => { + let answers: Vec<_> = request + .get("questions") + .and_then(serde_json::Value::as_array) + .map(|questions| { + questions + .iter() + .enumerate() + .map(|(index, question)| decisions_answer(index, question)) + .collect() + }) + .unwrap_or_default(); + serde_json::json!({ + "model": model, + "answers": answers, + "usage": { + "input_tokens": input_tokens, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": output_tokens, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": input_tokens + output_tokens, + }, + "echo": echo, + }) + } + DecisionRoute::Systemone => { + let answers: serde_json::Map = request + .get("questions") + .and_then(serde_json::Value::as_object) + .map(|questions| { + questions + .iter() + .map(|(name, question)| (name.clone(), systemone_answer(question))) + .collect() + }) + .unwrap_or_default(); + serde_json::json!({ + "model": model, + "answers": answers, + "usage": {"input_tokens": input_tokens, "output_tokens": output_tokens}, + "echo": echo, + }) + } + }; + let mut response = Json(payload).into_response(); + if let Some(value) = headers.get("typesafe-danger-zone-images") { + response + .headers_mut() + .insert("x-mock-echo-typesafe-danger-zone-images", value.clone()); + } + response +} + +fn question_type(question: &serde_json::Value) -> &str { + question + .get("type") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() +} + +/// Probabilities for `count` options: the first gets the mock probability and +/// the rest share the remainder. +fn option_probabilities(count: usize) -> Vec { + let rest = 0.01 / count.saturating_sub(1).max(1) as f64; + (0..count) + .map(|index| { + if index == 0 { + DECISION_MOCK_PROBABILITY + } else { + rest + } + }) + .collect() +} + +fn mock_box() -> serde_json::Value { + serde_json::json!({"top": 0.1, "bottom": 0.9, "left": 0.1, "right": 0.9}) +} + +/// Canned answer for one Decisions question. Questions are read loosely so an +/// unexpected field never fails the request; unknown types get the predicate +/// answer. +fn decisions_answer(index: usize, question: &serde_json::Value) -> serde_json::Value { + let kind = question_type(question); + let kind = if kind.is_empty() { "predicate" } else { kind }; + let name = question + .get("name") + .and_then(serde_json::Value::as_str) + .map_or_else(|| format!("q{index}"), str::to_string); + let options = |key: &str| -> Vec { + question + .get(key) + .and_then(serde_json::Value::as_array) + .cloned() + .unwrap_or_default() + }; + match kind { + "choice" => { + let values: Vec<_> = options("choices") + .into_iter() + .map(|choice| choice.get("value").cloned().unwrap_or(choice)) + .collect(); + let probabilities = option_probabilities(values.len()); + serde_json::json!({ + "type": kind, + "name": name, + "choice": values.first().cloned().unwrap_or(serde_json::Value::Null), + "probabilities": values.iter().zip(&probabilities) + .map(|(value, probability)| serde_json::json!({"value": value, "probability": probability})) + .collect::>(), + "confidence": DECISION_MOCK_PROBABILITY, + }) + } + "score" => { + let labels: Vec<_> = options("levels") + .into_iter() + .map(|level| level.get("label").cloned().unwrap_or(level)) + .collect(); + let probabilities = option_probabilities(labels.len()); + serde_json::json!({ + "type": kind, + "name": name, + "score": 0, + "probabilities": labels.iter().zip(&probabilities).enumerate() + .map(|(value, (label, probability))| serde_json::json!({"value": value, "label": label, "probability": probability})) + .collect::>(), + "confidence": DECISION_MOCK_PROBABILITY, + }) + } + "box" => serde_json::json!({ + "type": kind, + "name": name, + "box": mock_box(), + "confidence": DECISION_MOCK_PROBABILITY, + }), + _ => serde_json::json!({ + "type": kind, + "name": name, + "probability": DECISION_MOCK_PROBABILITY, + }), + } +} + +/// Canned answer for one System One question, keyed by the caller. Unknown +/// types get the noul answer. +fn systemone_answer(question: &serde_json::Value) -> serde_json::Value { + let criteria = question.get("criteria"); + match question_type(question) { + "choice" => { + let keys: Vec = match criteria { + Some(serde_json::Value::Object(map)) => map.keys().cloned().collect(), + Some(serde_json::Value::Array(items)) => items + .iter() + .filter_map(|item| item.as_str().map(str::to_string)) + .collect(), + _ => Vec::new(), + }; + let probabilities = option_probabilities(keys.len()); + serde_json::json!({ + "type": "choice", + "choice": keys.first().cloned().unwrap_or_default(), + "confidence": DECISION_MOCK_PROBABILITY, + "probabilities": keys.iter().zip(&probabilities) + .map(|(key, probability)| (key.clone(), serde_json::json!(probability))) + .collect::>(), + }) + } + "score" => { + let labels: Vec = criteria + .and_then(serde_json::Value::as_array) + .cloned() + .unwrap_or_default(); + let probabilities = option_probabilities(labels.len()); + serde_json::json!({ + "type": "score", + "score": 0, + "confidence": DECISION_MOCK_PROBABILITY, + "legend": labels.iter().enumerate() + .map(|(index, label)| (index.to_string(), label.clone())) + .collect::>(), + "probabilities": probabilities.iter().enumerate() + .map(|(index, probability)| (index.to_string(), serde_json::json!(probability))) + .collect::>(), + }) + } + "box" => serde_json::json!({ + "type": "box", + "box": mock_box(), + "confidence": DECISION_MOCK_PROBABILITY, + }), + _ => serde_json::json!({"type": "noul", "noul": DECISION_MOCK_PROBABILITY}), + } +} + +/// Error bodies follow each route's native shape: OpenAI style for decisions, +/// FastAPI style for System One. +fn decision_error(route: DecisionRoute, status: StatusCode) -> Response { + let rate_limited = status == StatusCode::TOO_MANY_REQUESTS; + match route { + DecisionRoute::Decisions if rate_limited => error_response( + status, + serde_json::json!({ + "message": "Rate limit exceeded", + "type": "rate_limit_error", + "param": null, + "code": "rate_limit_exceeded", + }), + ), + DecisionRoute::Decisions => error_response( + status, + serde_json::json!({ + "message": "Invalid question", + "type": "invalid_request_error", + "param": "questions", + "code": "invalid_question", + }), + ), + DecisionRoute::Systemone if rate_limited => ( + status, + Json(serde_json::json!({"detail": "Rate limit exceeded"})), + ) + .into_response(), + DecisionRoute::Systemone => ( + status, + Json(serde_json::json!({"detail": [{ + "loc": ["body", "questions"], + "msg": "Invalid question", + "type": "value_error", + }]})), + ) + .into_response(), + } +} + pub(crate) async fn health(State(state): State) -> &'static str { if !state.health_delay.is_zero() { tokio::time::sleep(state.health_delay).await; diff --git a/src/libraries/rust/stargate/crates/mock-dynamo/src/sha256.rs b/src/libraries/rust/stargate/crates/mock-dynamo/src/sha256.rs new file mode 100644 index 0000000000..0c3ccfb446 --- /dev/null +++ b/src/libraries/rust/stargate/crates/mock-dynamo/src/sha256.rs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Minimal SHA-256 so the decision echo can report a body digest without a +//! new dependency. Not for security use. + +const K: [u32; 64] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +]; + +/// Returns the lowercase hex SHA-256 digest of `data`. +pub(crate) fn sha256_hex(data: &[u8]) -> String { + let mut state: [u32; 8] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, + 0x5be0cd19, + ]; + let mut tail = data[data.len() - data.len() % 64..].to_vec(); + tail.push(0x80); + while tail.len() % 64 != 56 { + tail.push(0); + } + tail.extend_from_slice(&((data.len() as u64) * 8).to_be_bytes()); + for block in data[..data.len() - data.len() % 64] + .chunks_exact(64) + .chain(tail.chunks_exact(64)) + { + compress(&mut state, block); + } + state.iter().map(|word| format!("{word:08x}")).collect() +} + +fn compress(state: &mut [u32; 8], block: &[u8]) { + let mut w = [0u32; 64]; + for (index, chunk) in block.chunks_exact(4).enumerate() { + w[index] = u32::from_be_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + } + for index in 16..64 { + let s0 = + w[index - 15].rotate_right(7) ^ w[index - 15].rotate_right(18) ^ (w[index - 15] >> 3); + let s1 = + w[index - 2].rotate_right(17) ^ w[index - 2].rotate_right(19) ^ (w[index - 2] >> 10); + w[index] = w[index - 16] + .wrapping_add(s0) + .wrapping_add(w[index - 7]) + .wrapping_add(s1); + } + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h] = *state; + for index in 0..64 { + let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); + let ch = (e & f) ^ (!e & g); + let t1 = h + .wrapping_add(s1) + .wrapping_add(ch) + .wrapping_add(K[index]) + .wrapping_add(w[index]); + let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); + let maj = (a & b) ^ (a & c) ^ (b & c); + let t2 = s0.wrapping_add(maj); + h = g; + g = f; + f = e; + e = d.wrapping_add(t1); + d = c; + c = b; + b = a; + a = t1.wrapping_add(t2); + } + for (word, add) in state.iter_mut().zip([a, b, c, d, e, f, g, h]) { + *word = word.wrapping_add(add); + } +} diff --git a/src/libraries/rust/stargate/crates/mock-dynamo/src/test_control.rs b/src/libraries/rust/stargate/crates/mock-dynamo/src/test_control.rs index 9fcebb7178..146c775069 100644 --- a/src/libraries/rust/stargate/crates/mock-dynamo/src/test_control.rs +++ b/src/libraries/rust/stargate/crates/mock-dynamo/src/test_control.rs @@ -33,6 +33,8 @@ pub(crate) enum TestEndpoint { ChatCompletions, Responses, Embeddings, + Decisions, + Systemone, } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] diff --git a/src/libraries/rust/stargate/crates/mock-dynamo/src/tests.rs b/src/libraries/rust/stargate/crates/mock-dynamo/src/tests.rs index a5f030714d..e186ab6134 100644 --- a/src/libraries/rust/stargate/crates/mock-dynamo/src/tests.rs +++ b/src/libraries/rust/stargate/crates/mock-dynamo/src/tests.rs @@ -725,6 +725,274 @@ async fn embeddings_endpoint_returns_json_without_stream() { server.abort(); } +const DECISION_ECHO_HEADERS: &str = "connection: close\r\nx-input-tokens: 11\r\nx-request-id: req-decision\r\nx-model: decision-model\r\ntypesafe-danger-zone-images: true\r\nx-vendor-test: v"; + +async fn spawn_decision_app( + state: AppState, +) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + spawn_test_app(decision_routes().with_state(state)).await +} + +fn response_body_json(response: &str) -> serde_json::Value { + let (_, body) = response + .split_once("\r\n\r\n") + .expect("response should have a body"); + serde_json::from_str(body).expect("response body should be json") +} + +#[tokio::test] +async fn decisions_endpoint_returns_answers_and_echoes_request_details() { + let state = test_state(); + let observed_control = state.test_control.clone(); + let (addr, server) = spawn_decision_app(state).await; + let request_body = serde_json::json!({ + "model": "decision-model", + "input": "is it a cat?", + "questions": [{"type": "predicate", "name": "cat", "text": "cat?"}], + }) + .to_string(); + let response = json_response( + addr, + "POST", + "/v1/decisions", + DECISION_ECHO_HEADERS, + &request_body, + ) + .await; + + assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}"); + assert!(response.contains("x-mock-echo-typesafe-danger-zone-images: true")); + let body = response_body_json(&response); + assert_eq!(body["model"], "decision-model"); + assert_eq!(body["answers"][0]["type"], "predicate"); + assert_eq!(body["answers"][0]["name"], "cat"); + assert_eq!(body["answers"][0]["probability"], 0.99); + assert_eq!(body["usage"]["input_tokens"], 11); + assert_eq!(body["usage"]["total_tokens"], 12); + assert_eq!(body["usage"]["input_tokens_details"]["cached_tokens"], 0); + assert_eq!( + body["usage"]["output_tokens_details"]["reasoning_tokens"], + 0 + ); + assert_eq!(body["echo"]["typesafe-danger-zone-images"], "true"); + assert_eq!(body["echo"]["x-vendor-test"], "v"); + assert_eq!(body["echo"]["x-model"], "decision-model"); + assert_eq!(body["echo"]["x-input-tokens"], "11"); + assert_eq!(body["echo"]["x-request-id"], "req-decision"); + assert_eq!(body["echo"]["body_len"], request_body.len()); + let snapshot = observed_control.snapshot().await; + assert_eq!( + snapshot.counter( + TestEndpoint::Decisions, + "decision-model", + TestRequestClass::ApiGateway + ), + 1 + ); + server.abort(); +} + +#[tokio::test] +async fn systemone_endpoint_returns_answers_and_echoes_request_details() { + let (addr, server) = spawn_decision_app(test_state()).await; + let request_body = serde_json::json!({ + "model": "decision-model", + "state": "frame", + "questions": {"cat": {"type": "noul", "text": "cat?"}}, + }) + .to_string(); + let response = json_response( + addr, + "POST", + "/v1/systemone", + DECISION_ECHO_HEADERS, + &request_body, + ) + .await; + + assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}"); + assert!(response.contains("x-mock-echo-typesafe-danger-zone-images: true")); + let body = response_body_json(&response); + assert_eq!(body["model"], "decision-model"); + assert_eq!(body["answers"]["cat"]["type"], "noul"); + assert_eq!(body["answers"]["cat"]["noul"], 0.99); + assert_eq!(body["usage"]["input_tokens"], 11); + assert_eq!(body["echo"]["typesafe-danger-zone-images"], "true"); + assert_eq!(body["echo"]["x-vendor-test"], "v"); + assert_eq!(body["echo"]["body_len"], request_body.len()); + server.abort(); +} + +async fn decision_answers(path: &str, questions: serde_json::Value) -> serde_json::Value { + let (addr, server) = spawn_decision_app(test_state()).await; + let request_body = serde_json::json!({"model": "m", "input": "x", "questions": questions}); + let response = json_response( + addr, + "POST", + path, + "connection: close", + &request_body.to_string(), + ) + .await; + server.abort(); + assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}"); + response_body_json(&response)["answers"].clone() +} + +#[tokio::test] +async fn decisions_answers_follow_each_question_type() { + let answers = decision_answers( + "/v1/decisions", + serde_json::json!([ + {"type": "predicate", "name": "p", "text": "t"}, + {"type": "choice", "name": "c", "choices": [true, false]}, + {"type": "choice", "name": "o", "choices": [{"value": "red", "description": "d"}, "blue"]}, + {"type": "score", "name": "s", "levels": [{"label": "low", "description": "d"}, "high"]}, + {"type": "box", "name": "b", "text": "find it"}, + {"type": "future", "name": "u"} + ]), + ) + .await; + + assert_eq!( + answers, + serde_json::json!([ + {"type": "predicate", "name": "p", "probability": 0.99}, + {"type": "choice", "name": "c", "choice": true, + "probabilities": [{"value": true, "probability": 0.99}, {"value": false, "probability": 0.01}], + "confidence": 0.99}, + {"type": "choice", "name": "o", "choice": "red", + "probabilities": [{"value": "red", "probability": 0.99}, {"value": "blue", "probability": 0.01}], + "confidence": 0.99}, + {"type": "score", "name": "s", "score": 0, + "probabilities": [ + {"value": 0, "label": "low", "probability": 0.99}, + {"value": 1, "label": "high", "probability": 0.01}], + "confidence": 0.99}, + {"type": "box", "name": "b", + "box": {"top": 0.1, "bottom": 0.9, "left": 0.1, "right": 0.9}, "confidence": 0.99}, + {"type": "future", "name": "u", "probability": 0.99} + ]) + ); +} + +#[tokio::test] +async fn systemone_answers_follow_each_question_type() { + let answers = decision_answers( + "/v1/systemone", + serde_json::json!({ + "n": {"type": "noul", "text": "t"}, + "c": {"type": "choice", "criteria": {"alpha": "first", "beta": "second"}}, + "s": {"type": "score", "criteria": ["bad", "good"]}, + "b": {"type": "box", "text": "find it"}, + "u": {"type": "future"} + }), + ) + .await; + + assert_eq!( + answers, + serde_json::json!({ + "n": {"type": "noul", "noul": 0.99}, + "c": {"type": "choice", "choice": "alpha", "confidence": 0.99, + "probabilities": {"alpha": 0.99, "beta": 0.01}}, + "s": {"type": "score", "score": 0, "confidence": 0.99, + "legend": {"0": "bad", "1": "good"}, + "probabilities": {"0": 0.99, "1": 0.01}}, + "b": {"type": "box", "box": {"top": 0.1, "bottom": 0.9, "left": 0.1, "right": 0.9}, + "confidence": 0.99}, + "u": {"type": "noul", "noul": 0.99} + }) + ); +} + +#[tokio::test] +async fn decision_echo_reports_the_sha256_of_the_received_body() { + let (addr, server) = spawn_decision_app(test_state()).await; + let mut digests = Vec::new(); + for body in ["abc", ""] { + let response = + json_response(addr, "POST", "/v1/decisions", "connection: close", body).await; + digests.push(response_body_json(&response)["echo"]["body_sha256"].clone()); + } + server.abort(); + assert_eq!( + digests, + [ + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ] + ); +} + +#[tokio::test] +async fn decision_endpoints_serve_rate_limit_and_validation_errors_on_request() { + let (addr, server) = spawn_decision_app(test_state()).await; + let request_body = r#"{"model":"decision-model","input":"x","questions":[]}"#; + for path in ["/v1/decisions", "/v1/systemone"] { + let limited = json_response( + addr, + "POST", + &format!("{path}?mock_error=429"), + "connection: close", + request_body, + ) + .await; + assert!(limited.starts_with("HTTP/1.1 429"), "{path}: {limited}"); + assert!(limited.contains("retry-after: 7"), "{path}: {limited}"); + assert!( + !limited + .to_ascii_lowercase() + .contains("x-stargate-upstream-retryable"), + "{path}: a 429 must not be marked retryable" + ); + let body = response_body_json(&limited); + if path == "/v1/decisions" { + assert_eq!(body["error"]["type"], "rate_limit_error"); + } else { + assert!(body["detail"].is_string(), "{body}"); + } + + let invalid = json_response( + addr, + "POST", + &format!("{path}?mock_error=400"), + "connection: close", + request_body, + ) + .await; + assert!(invalid.starts_with("HTTP/1.1 400"), "{path}: {invalid}"); + let body = response_body_json(&invalid); + if path == "/v1/decisions" { + assert_eq!(body["error"]["type"], "invalid_request_error"); + } else { + assert!(body["detail"].is_array(), "{body}"); + } + } + server.abort(); +} + +#[tokio::test] +async fn decision_endpoints_accept_base64_image_bodies_above_the_default_limit() { + let (addr, server) = spawn_decision_app(test_state()).await; + let image = "QUJD".repeat(13 * 1024 * 1024 / 4); + let request_body = serde_json::json!({ + "model": "decision-model", + "input": [{"type": "input_image", "image_url": format!("data:image/png;base64,{image}")}], + "questions": [{"type": "predicate", "name": "cat"}], + }) + .to_string(); + for path in ["/v1/decisions", "/v1/systemone"] { + let response = json_response(addr, "POST", path, "connection: close", &request_body).await; + assert!(response.starts_with("HTTP/1.1 200 OK"), "{path}"); + assert!( + response.contains(&format!(r#""body_len":{}"#, request_body.len())), + "{path}" + ); + } + server.abort(); +} + #[test] fn output_token_selection_supports_overrides_and_bounded_distributions() { let mut headers = HeaderMap::new(); diff --git a/src/libraries/rust/stargate/crates/proto/proto/llm_gateway.proto b/src/libraries/rust/stargate/crates/proto/proto/llm_gateway.proto index 7bcb1deedf..757b855076 100644 --- a/src/libraries/rust/stargate/crates/proto/proto/llm_gateway.proto +++ b/src/libraries/rust/stargate/crates/proto/proto/llm_gateway.proto @@ -19,6 +19,29 @@ package llm_gateway; service LlmGateway { rpc AuthLlmWorker (AuthLlmWorkerRequest) returns (AuthLlmWorkerResponse); + rpc AuthLlmInvocation (AuthLlmInvokeRequest) returns (AuthLlmInvokeResponse); +} + +message AuthLlmInvokeRequest { + string clientAuthorizationToken = 1; + string routingKey = 2; // This is just functionId +} + +message AuthLlmInvokeResponse { + string routingKey = 1; + string clientAuthSubject = 2; + map authContext = 3; + map modelSpecs = 4; + optional uint32 priority = 5; + + message ModelSpec { + reserved 3; + reserved "tokenizer"; + + repeated string uris = 1; + optional string tokenRateLimit = 2; + optional string routingMethod = 4; + } } message AuthLlmWorkerRequest { diff --git a/src/libraries/rust/stargate/crates/pylon-lib/src/quic_http_tunnel/tests.rs b/src/libraries/rust/stargate/crates/pylon-lib/src/quic_http_tunnel/tests.rs index 629d697f0d..2c6866d4e3 100644 --- a/src/libraries/rust/stargate/crates/pylon-lib/src/quic_http_tunnel/tests.rs +++ b/src/libraries/rust/stargate/crates/pylon-lib/src/quic_http_tunnel/tests.rs @@ -459,10 +459,16 @@ fn pylon_request_header_filter_strips_tunnel_headers_case_insensitively() &retry )); } - assert!(should_forward_header( - &HeaderName::from_bytes(b"X-Request-Id")?, - &retry - )); + for name in [ + "X-Request-Id", + "TypeSafe-Danger-Zone-Images", + "X-Vendor-Test", + ] { + assert!( + should_forward_header(&HeaderName::from_bytes(name.as_bytes())?, &retry), + "{name} must reach the backend" + ); + } Ok(()) } diff --git a/src/libraries/rust/stargate/crates/stargate/src/bin/stargate-dev-auth.rs b/src/libraries/rust/stargate/crates/stargate/src/bin/stargate-dev-auth.rs index 761ab3ccb5..93a0095f54 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/bin/stargate-dev-auth.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/bin/stargate-dev-auth.rs @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::BTreeMap; use std::fs; use std::net::SocketAddr; use std::path::PathBuf; @@ -25,8 +26,11 @@ use axum::{Router, extract::State}; use clap::Parser; use prometheus::{Encoder, IntCounterVec, Opts, Registry, TextEncoder}; use serde::Deserialize; +use stargate_proto::gateway_pb::auth_llm_invoke_response::ModelSpec; use stargate_proto::gateway_pb::llm_gateway_server::{LlmGateway, LlmGatewayServer}; -use stargate_proto::gateway_pb::{AuthLlmWorkerRequest, AuthLlmWorkerResponse}; +use stargate_proto::gateway_pb::{ + AuthLlmInvokeRequest, AuthLlmInvokeResponse, AuthLlmWorkerRequest, AuthLlmWorkerResponse, +}; use stargate_runtime::wait_for_termination_signal; use tonic::transport::Server; use tonic::{Request, Response, Status}; @@ -34,6 +38,7 @@ use tracing::{info, warn}; use tracing_subscriber::EnvFilter; const WORKER_RPC: &str = "auth_llm_worker"; +const INVOCATION_RPC: &str = "auth_llm_invocation"; const OK_RESULT: &str = "ok"; #[derive(Debug, Parser)] @@ -58,6 +63,33 @@ struct Config { service_token: String, worker_token: String, routing_key: String, + /// Enables the invocation RPC used by the LLM API gateway. Without it the + /// RPC stays unimplemented. + #[serde(default)] + invocation: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct InvocationConfig { + client_token: String, + nca_id: String, + #[serde(default = "default_client_auth_subject")] + client_auth_subject: String, + /// Per-model endpoint allow-lists, as the function API reports them. + #[serde(default)] + model_specs: BTreeMap, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ModelSpecConfig { + #[serde(default)] + uris: Vec, +} + +fn default_client_auth_subject() -> String { + "stargate-dev-client".to_string() } struct DevAuth { @@ -135,6 +167,46 @@ impl LlmGateway for DevAuth { routing_key: self.config.routing_key.clone(), })) } + + #[tracing::instrument(name = "stargate_dev_auth.auth_llm_invocation", skip_all)] + async fn auth_llm_invocation( + &self, + request: Request, + ) -> Result, Status> { + let Some(invocation) = &self.config.invocation else { + return Err(Status::unimplemented("invocation auth is not configured")); + }; + self.authorize_service(&request, INVOCATION_RPC)?; + + let request = request.into_inner(); + if request.client_authorization_token != invocation.client_token { + return Err(self.reject( + INVOCATION_RPC, + "client_token", + Status::unauthenticated("unknown client token"), + )); + } + self.record_success(INVOCATION_RPC); + Ok(Response::new(AuthLlmInvokeResponse { + routing_key: request.routing_key, + client_auth_subject: invocation.client_auth_subject.clone(), + auth_context: [("ncaId".to_string(), invocation.nca_id.clone())].into(), + model_specs: invocation + .model_specs + .iter() + .map(|(model, spec)| { + ( + model.clone(), + ModelSpec { + uris: spec.uris.clone(), + ..ModelSpec::default() + }, + ) + }) + .collect(), + priority: None, + })) + } } #[derive(Clone)] @@ -155,10 +227,13 @@ impl AuthMetrics { )?; registry.register(Box::new(requests_total.clone()))?; - for result in [OK_RESULT, "service_token", "worker_token"] { - requests_total - .with_label_values(&[WORKER_RPC, result]) - .inc_by(0); + for (rpc, rejection) in [ + (WORKER_RPC, "worker_token"), + (INVOCATION_RPC, "client_token"), + ] { + for result in [OK_RESULT, "service_token", rejection] { + requests_total.with_label_values(&[rpc, result]).inc_by(0); + } } Ok(Self { @@ -261,9 +336,87 @@ mod tests { service_token: service.to_string(), worker_token: worker.to_string(), routing_key: routing_key.to_string(), + invocation: None, } } + fn invocation_auth() -> DevAuth { + let config: Config = serde_json::from_str( + r#"{ + "serviceToken": "service-token", + "workerToken": "worker-token", + "routingKey": "mockdc-usw2-a", + "invocation": { + "clientToken": "client-token", + "ncaId": "nca-1", + "modelSpecs": {"m": {"uris": ["/v1/decisions", "/v1/systemone"]}} + } + }"#, + ) + .expect("invocation config should parse"); + DevAuth::new(config).expect("valid test config") + } + + fn invoke_request(token: &str) -> Request { + Request::new(AuthLlmInvokeRequest { + client_authorization_token: token.to_string(), + routing_key: "function-1".to_string(), + }) + } + + #[tokio::test] + async fn invocation_returns_model_specs_for_the_configured_client() { + let auth = invocation_auth(); + let mut request = invoke_request("client-token"); + authorize(&mut request); + + let response = auth + .auth_llm_invocation(request) + .await + .unwrap() + .into_inner(); + + assert_eq!(response.routing_key, "function-1"); + assert_eq!(response.client_auth_subject, "stargate-dev-client"); + assert_eq!(response.auth_context["ncaId"], "nca-1"); + assert_eq!( + response.model_specs["m"].uris, + ["/v1/decisions", "/v1/systemone"] + ); + } + + #[tokio::test] + async fn invocation_rejects_missing_service_token_and_unknown_client_token() { + let auth = invocation_auth(); + assert_eq!( + auth.auth_llm_invocation(invoke_request("client-token")) + .await + .unwrap_err() + .code(), + Code::Unauthenticated + ); + let mut request = invoke_request("other"); + authorize(&mut request); + assert_eq!( + auth.auth_llm_invocation(request).await.unwrap_err().code(), + Code::Unauthenticated + ); + } + + #[tokio::test] + async fn invocation_stays_unimplemented_without_configuration() { + let mut request = invoke_request("client-token"); + authorize(&mut request); + assert_eq!( + auth() + .auth_llm_invocation(request) + .await + .unwrap_err() + .code(), + Code::Unimplemented + ); + } + fn auth() -> DevAuth { DevAuth::new(config("service-token", "worker-token", "mockdc-usw2-a")) .expect("valid test config") @@ -330,6 +483,9 @@ mod tests { "nvcf_stargate_dev_auth_requests_total{result=\"ok\",rpc=\"auth_llm_worker\"} 0", "nvcf_stargate_dev_auth_requests_total{result=\"service_token\",rpc=\"auth_llm_worker\"} 0", "nvcf_stargate_dev_auth_requests_total{result=\"worker_token\",rpc=\"auth_llm_worker\"} 0", + "nvcf_stargate_dev_auth_requests_total{result=\"ok\",rpc=\"auth_llm_invocation\"} 0", + "nvcf_stargate_dev_auth_requests_total{result=\"service_token\",rpc=\"auth_llm_invocation\"} 0", + "nvcf_stargate_dev_auth_requests_total{result=\"client_token\",rpc=\"auth_llm_invocation\"} 0", ] { assert!(body.contains(series), "missing {series}"); } diff --git a/src/libraries/rust/stargate/crates/stargate/src/http_proxy.rs b/src/libraries/rust/stargate/crates/stargate/src/http_proxy.rs index 0aa3a38bd9..f8ebaada5b 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/http_proxy.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/http_proxy.rs @@ -112,6 +112,14 @@ impl OpenAiProxyEndpoint { path: "/v1/embeddings", name: "embeddings", }; + const DECISIONS: Self = Self { + path: "/v1/decisions", + name: "decisions", + }; + const SYSTEMONE: Self = Self { + path: "/v1/systemone", + name: "systemone", + }; } #[derive(Clone, Debug)] @@ -188,6 +196,14 @@ pub fn make_router(app: ProxyAppState) -> Router { OpenAiProxyEndpoint::MESSAGES.path, post(|State(app), req| proxy_openai_request(app, req, OpenAiProxyEndpoint::MESSAGES)), ) + .route( + OpenAiProxyEndpoint::DECISIONS.path, + post(|State(app), req| proxy_openai_request(app, req, OpenAiProxyEndpoint::DECISIONS)), + ) + .route( + OpenAiProxyEndpoint::SYSTEMONE.path, + post(|State(app), req| proxy_openai_request(app, req, OpenAiProxyEndpoint::SYSTEMONE)), + ) .with_state(app) } diff --git a/src/libraries/rust/stargate/crates/stargate/tests/suite/proxy_contract.rs b/src/libraries/rust/stargate/crates/stargate/tests/suite/proxy_contract.rs index b5a3a82220..7b6acce030 100644 --- a/src/libraries/rust/stargate/crates/stargate/tests/suite/proxy_contract.rs +++ b/src/libraries/rust/stargate/crates/stargate/tests/suite/proxy_contract.rs @@ -1671,6 +1671,14 @@ async fn terminal_queue_mismatch_is_sanitized_across_tunnel_protocols_and_endpoi "/v1/embeddings", serde_json::json!({"model": model, "input": "hello"}), ), + ( + "/v1/decisions", + serde_json::json!({"model": model, "input": "hello", "questions": []}), + ), + ( + "/v1/systemone", + serde_json::json!({"model": model, "state": "hello", "questions": {}}), + ), ] { let body = Bytes::from(body.to_string()); let request_body = reqwest::Body::wrap_stream(futures::stream::once(async move { @@ -3848,6 +3856,371 @@ async fn exercise_native_messages_contract(protocol: TunnelTransportProtocol, re backend_task.abort(); } +const DECISION_ROUTES: [(&str, &str); 2] = [ + ("/v1/decisions", "decisions-model"), + ("/v1/systemone", "systemone-model"), +]; +const DECISION_OK: &str = + r#"{"model":"m","answers":[{"type":"predicate","name":"q","probability":0.99}]}"#; +const DECISION_RATE_LIMITED: &str = + r#"{"error":{"message":"slow down","type":"rate_limit_error","code":"rate_limited"}}"#; +const DECISION_FASTAPI_400: &str = + r#"{"detail":[{"loc":["body","questions"],"msg":"field required","type":"missing"}]}"#; +const DECISION_OPENAI_400: &str = r#"{"error":{"message":"bad question","type":"invalid_request_error","param":"questions","code":"invalid_question"}}"#; +const DECISION_MAX_BODY: usize = 32 << 20; + +type DecisionCapture = (String, axum::http::HeaderMap, Bytes); + +struct DecisionBackend { + addr: std::net::SocketAddr, + hits: Arc, + captures: tokio::sync::mpsc::UnboundedReceiver, + task: tokio::task::JoinHandle<()>, +} + +async fn start_decision_backend(path: &'static str) -> DecisionBackend { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let hits = Arc::new(AtomicUsize::new(0)); + let (capture_tx, captures) = tokio::sync::mpsc::unbounded_channel(); + let handler_hits = hits.clone(); + let app = Router::new() + .route( + path, + post(move |req: Request| { + let capture_tx = capture_tx.clone(); + let hits = handler_hits.clone(); + async move { + let (parts, body) = req.into_parts(); + let body = axum::body::to_bytes(body, DECISION_MAX_BODY).await.unwrap(); + hits.fetch_add(1, Ordering::SeqCst); + let query = parts.uri.query().unwrap_or_default().to_string(); + capture_tx + .send((parts.uri.to_string(), parts.headers, body)) + .unwrap(); + let mut response = + Response::builder().header("content-type", "application/json"); + let (status, text) = if query.contains("mode=rate_limit") { + response = response.header("retry-after", "7"); + (StatusCode::TOO_MANY_REQUESTS, DECISION_RATE_LIMITED) + } else if query.contains("mode=fastapi_400") { + (StatusCode::BAD_REQUEST, DECISION_FASTAPI_400) + } else if query.contains("mode=openai_400") { + (StatusCode::BAD_REQUEST, DECISION_OPENAI_400) + } else { + (StatusCode::OK, DECISION_OK) + }; + response.status(status).body(Body::from(text)).unwrap() + } + }), + ) + .route("/health", get(|| async { "ok" })); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + DecisionBackend { + addr, + hits, + captures, + task, + } +} + +async fn register_http_backend( + fixture: &mut ProxyFixture, + id: &str, + model: &str, + addr: std::net::SocketAddr, + protocol: TunnelTransportProtocol, + reverse: bool, +) { + let runtime_state = active_runtime(model); + if reverse { + fixture.register( + reverse_registration_config_for_protocol( + fixture.grpc_addr, + id, + id, + format!("http://{addr}"), + protocol, + runtime_state, + ), + "reverse registration failed", + ); + } else { + let mut config = + QuicHttpTunnelConfig::new("127.0.0.1:0".parse().unwrap(), format!("http://{addr}")); + config.tunnel_protocol = protocol; + config.forwarding.runtime_state = runtime_state.clone(); + let tunnel = start_quic_http_tunnel(config).await.unwrap(); + fixture.register( + active_registration_config_with_state( + fixture.grpc_addr, + id, + id, + format!("quic://{}", tunnel.listen_addr()), + format!("http://{addr}"), + runtime_state, + ), + "direct registration failed", + ); + fixture.own_tunnel(tunnel); + } + fixture.wait_for_clusters(model, 1).await; +} + +fn large_image_body(model: &str, target_bytes: usize) -> Vec { + let image = "QUJD".repeat(target_bytes / 4); + serde_json::to_vec(&serde_json::json!({ + "model": model, + "input": [{"type": "input_image", "image_url": format!("data:image/png;base64,{image}")}], + "questions": [{"type": "predicate", "name": "q", "text": "is it a cat?"}], + })) + .unwrap() +} + +#[tokio::test] +async fn decision_routes_pass_requests_and_backend_responses_through() { + for protocol in [ + TunnelTransportProtocol::RawQuic, + TunnelTransportProtocol::Http3, + TunnelTransportProtocol::WebTransport, + ] { + for reverse in [false, true] { + for (path, model) in DECISION_ROUTES { + exercise_decision_route_contract(protocol, reverse, path, model).await; + } + } + } +} + +async fn exercise_decision_route_contract( + protocol: TunnelTransportProtocol, + reverse: bool, + path: &'static str, + model: &str, +) { + let case = if reverse { + TunnelTestCase::reverse(protocol) + } else { + TunnelTestCase::direct(protocol) + }; + let label = format!("{path} {protocol:?} reverse={reverse}"); + let id = format!( + "decision-{}-{reverse}-{}", + case.protocol_label(), + &path[4..] + ); + let (mut fixture, _) = ProxyFixture::start_for_tunnel_case(&id, case).await; + let mut backend = start_decision_backend(path).await; + register_http_backend(&mut fixture, &id, model, backend.addr, protocol, reverse).await; + let client = reqwest::Client::new(); + + // Vendor headers reach the backend and a small body is byte-identical. + let small = serde_json::to_vec(&serde_json::json!({ + "model": model, "input": "hello", "questions": [{"type": "predicate", "name": "q"}], + })) + .unwrap(); + let response = proxy_request(&client, fixture.http_addr, path, model, "req-decision-ok") + .header("typesafe-danger-zone-images", "true") + .header("x-vendor-test", "v") + .body(small.clone()) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{label}"); + assert_eq!(response.text().await.unwrap(), DECISION_OK, "{label}"); + let (captured_path, headers, body) = backend.captures.recv().await.unwrap(); + assert_eq!(captured_path, path, "{label}"); + assert_eq!(body.as_ref(), small, "{label}"); + assert_eq!(headers["typesafe-danger-zone-images"], "true", "{label}"); + assert_eq!(headers["x-vendor-test"], "v", "{label}"); + + // A base64 image sized request arrives unchanged. + let large = large_image_body(model, 13 * 1024 * 1024); + let response = proxy_request( + &client, + fixture.http_addr, + path, + model, + "req-decision-large", + ) + .header("typesafe-danger-zone-images", "true") + .body(large.clone()) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{label}"); + let (_, headers, body) = backend.captures.recv().await.unwrap(); + assert_eq!(body.len(), large.len(), "{label}"); + assert!(body.as_ref() == large.as_slice(), "{label}: body changed"); + assert_eq!(headers["typesafe-danger-zone-images"], "true", "{label}"); + + // Backend errors reach the client unchanged and are not retried. + for (mode, status, retry_after, expected) in [ + ( + "rate_limit", + StatusCode::TOO_MANY_REQUESTS, + Some("7"), + DECISION_RATE_LIMITED, + ), + ( + "fastapi_400", + StatusCode::BAD_REQUEST, + None, + DECISION_FASTAPI_400, + ), + ( + "openai_400", + StatusCode::BAD_REQUEST, + None, + DECISION_OPENAI_400, + ), + ] { + let response = proxy_request( + &client, + fixture.http_addr, + &format!("{path}?mode={mode}"), + model, + "req-decision-error", + ) + .body(small.clone()) + .send() + .await + .unwrap(); + assert_eq!(response.status(), status, "{label} {mode}"); + assert_eq!( + response_header(&response, "retry-after"), + retry_after, + "{label} {mode}" + ); + assert_eq!(response.text().await.unwrap(), expected, "{label} {mode}"); + backend.captures.recv().await.unwrap(); + } + assert_eq!( + backend.hits.load(Ordering::SeqCst), + 5, + "{label}: each request must reach the backend exactly once" + ); + + fixture.shutdown().await; + backend.task.abort(); +} + +#[tokio::test] +async fn decision_routes_enforce_required_headers_without_upstream_call() { + let mut fixture = ProxyFixture::start("test-sg-decision-headers").await; + for (path, model) in DECISION_ROUTES { + let backend = start_decision_backend(path).await; + register_http_backend( + &mut fixture, + &format!("decision-headers-{}", &path[4..]), + model, + backend.addr, + TunnelTransportProtocol::RawQuic, + false, + ) + .await; + for missing in ["x-model", "x-input-tokens", "x-request-id"] { + let mut request = reqwest::Client::new() + .post(format!("http://{}{path}", fixture.http_addr)) + .header("content-type", "application/json"); + for (name, value) in [ + ("x-model", model), + ("x-input-tokens", "1"), + ("x-request-id", "req-decision-headers"), + ] { + if name != missing { + request = request.header(name, value); + } + } + let response = request + .body(format!(r#"{{"model":"{model}","input":"hi"}}"#)) + .send() + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "{path} without {missing}" + ); + } + assert_eq!( + backend.hits.load(Ordering::SeqCst), + 0, + "{path}: upstream must not run when a required header is missing" + ); + backend.task.abort(); + } + fixture.shutdown().await; +} + +#[tokio::test] +async fn decision_routes_reject_content_length_over_replay_limit_with_413() { + let mut fixture = ProxyFixture::start("test-sg-decision-413").await; + for (path, model) in DECISION_ROUTES { + let backend = start_decision_backend(path).await; + register_http_backend( + &mut fixture, + &format!("decision-413-{}", &path[4..]), + model, + backend.addr, + TunnelTransportProtocol::RawQuic, + false, + ) + .await; + // Send only the head of a request that declares a body over the default + // 64 MiB replay limit; the proxy must answer before any body is sent. + let declared = 64 * 1024 * 1024 + 1; + let mut stream = tokio::net::TcpStream::connect(fixture.http_addr) + .await + .unwrap(); + let head = format!( + "POST {path} HTTP/1.1\r\nHost: {}\r\nx-model: {model}\r\nx-request-id: req-413\r\nx-input-tokens: 1\r\ncontent-type: application/json\r\ncontent-length: {declared}\r\n\r\n", + fixture.http_addr + ); + tokio::io::AsyncWriteExt::write_all(&mut stream, head.as_bytes()) + .await + .unwrap(); + let mut buf = vec![0u8; 256]; + let read = tokio::time::timeout( + Duration::from_secs(5), + tokio::io::AsyncReadExt::read(&mut stream, &mut buf), + ) + .await + .expect("413 response timed out") + .unwrap(); + let text = String::from_utf8_lossy(&buf[..read]); + assert!(text.starts_with("HTTP/1.1 413"), "{path}: got {text}"); + assert_eq!(backend.hits.load(Ordering::SeqCst), 0, "{path}"); + backend.task.abort(); + } + fixture.shutdown().await; +} + +#[tokio::test] +async fn decision_routes_unknown_model_carries_stargate_error_code() { + let (_, http_addr, handle) = start_stargate("test-sg-decision-unknown").await; + for (path, _) in DECISION_ROUTES { + let response = proxy_request( + &reqwest::Client::new(), + http_addr, + path, + "nonexistent", + "req-decision-unknown", + ) + .body(r#"{"model":"nonexistent","input":"hi"}"#) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{path}"); + assert_eq!( + response_header(&response, "x-stargate-error-code"), + Some("no_eligible_candidates"), + "{path}" + ); + } + finish_stargate(handle).await; +} + #[tokio::test] async fn last_cluster_hint_classifies_only_flagged_affinity_requests() { let flagged_model = "last-cluster-on-model"; diff --git a/src/libraries/rust/stargate/docs/api-gateway-contract.md b/src/libraries/rust/stargate/docs/api-gateway-contract.md index a3aa801251..01073a69b9 100644 --- a/src/libraries/rust/stargate/docs/api-gateway-contract.md +++ b/src/libraries/rust/stargate/docs/api-gateway-contract.md @@ -119,6 +119,8 @@ Supported: POST /v1/chat/completions POST /v1/responses POST /v1/embeddings +POST /v1/decisions +POST /v1/systemone ``` Required headers: @@ -195,6 +197,8 @@ Body rules: - `/v1/chat/completions` and `/v1/responses` must be valid JSON with `"stream": true`. - `/v1/embeddings` must be valid JSON and does not need `stream`. +- `/v1/decisions` and `/v1/systemone` (beta) must be valid JSON and do not + need `stream`. Responses are not streamed. ## Responses diff --git a/src/libraries/rust/stargate/docs/diagrams/system-dfd.puml b/src/libraries/rust/stargate/docs/diagrams/system-dfd.puml index cafcef4ff1..ef42509a7d 100644 --- a/src/libraries/rust/stargate/docs/diagrams/system-dfd.puml +++ b/src/libraries/rust/stargate/docs/diagrams/system-dfd.puml @@ -42,7 +42,7 @@ rectangle "stargate" #f0f7e8 { rectangle "Control Plane gRPC\nRegisterInferenceServer\nWatchStargates" as ControlPlane rectangle "Model Discovery gRPC\nListModels" as ModelDiscovery rectangle "Forwarding Dispatch\ngRPC: :authority check\nQUIC: SNI check" as Forwarding - rectangle "HTTP Proxy\n/v1/chat/completions\n/v1/responses\n/v1/embeddings" as HttpProxy + rectangle "HTTP Proxy\n/v1/chat/completions\n/v1/responses\n/v1/embeddings\n/v1/decisions\n/v1/systemone" as HttpProxy rectangle "QUIC Tunnel Client\n+ Reverse Listener" as Quic rectangle "Load Balancer Router" as Router database "Routing State\nkeyed by RoutingTargetKey\n(conn + RTT gated)" as State @@ -59,7 +59,7 @@ rectangle "pylon" #e8f4fd { } rectangle "Dynamo Instance" #fdf5e6 { - rectangle "Upstream HTTP\n/v1/chat/completions\n/v1/responses\n/v1/embeddings\n/pylon/v1/stats/stream\n/health" as Upstream + rectangle "Upstream HTTP\n/v1/chat/completions\n/v1/responses\n/v1/embeddings\n/v1/decisions\n/v1/systemone\n/pylon/v1/stats/stream\n/health" as Upstream } cloud "Metrics / OTEL\nCollectors" as Obs diff --git a/src/libraries/rust/stargate/scripts/e2e-last-cluster.sh b/src/libraries/rust/stargate/scripts/e2e-last-cluster.sh index 657a6e6667..caafa5843e 100755 --- a/src/libraries/rust/stargate/scripts/e2e-last-cluster.sh +++ b/src/libraries/rust/stargate/scripts/e2e-last-cluster.sh @@ -18,7 +18,7 @@ set -euo pipefail usage() { cat <<'EOF' -Usage: scripts/e2e-last-cluster.sh [--no-build] +Usage: scripts/e2e-last-cluster.sh [--no-build] [--decisions-only] Runs the LLM API gateway against a real Stargate, two Pylons, and two mock-dynamo backends on loopback, and reproduces the overflow cache-thrash @@ -34,12 +34,21 @@ scenario for last_cluster_affinity: The scenario runs for wait-and-widen and pulsar-wait-and-widen, each with the flag on (STARGATE_LAST_CLUSTER_ENABLED in the gateway and last_cluster_affinity in Stargate) and off, against a fresh set of processes. +After those cases it runs a decision-route case: POST /v1/decisions and +POST /v1/systemone through the gateway, Stargate, a Pylon, and mock-dynamo. The +gateway authenticates against stargate-dev-auth, which declares both routes in +the model's uris. The case checks that the 200 body, the TypeSafe-Danger-Zone-Images +vendor header, and a 13 MB base64 body reach the backend unchanged, and that a +backend 429 (with Retry-After) and a backend 400 come back unchanged and are not +retried. + The script exits non-zero when any expectation fails. Options: --no-build Reuse the binaries in target/release. To run without the gateway source or go, copy a prebuilt llm-api-gateway there first. + --decisions-only Run only the decision-route case. Environment: E2E_PORT_BASE First loopback port, default 27100. Uses 32 ports. @@ -51,12 +60,15 @@ EOF } build=1 -case "${1:-}" in - "") ;; - --no-build) build=0 ;; - -h|--help) usage; exit 0 ;; - *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; -esac +decisions_only=0 +for arg in "$@"; do + case "${arg}" in + --no-build) build=0 ;; + --decisions-only) decisions_only=1 ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: ${arg}" >&2; usage >&2; exit 2 ;; + esac +done for tool in curl jq; do command -v "${tool}" >/dev/null || { echo "${tool} is required" >&2; exit 2; } @@ -76,6 +88,13 @@ routing_key="e2e" worker_token="e2e-worker-token" service_token="e2e-service-token" session_id="e2e-last-cluster-session" +# The decision-route case sends this client token to the gateway. The gateway +# presents the service token to stargate-dev-auth, which answers the +# invocation RPC and declares the routes each model serves. +client_token="e2e-client-token" +nca_id="e2e-nca" +# 1 makes start_stack point the gateway at stargate-dev-auth. +gateway_auth=0 follow_up_count=10 # Each mock-dynamo token takes token_delay_ms. Holders generate # holder_tokens tokens, so they keep A full for about 3 seconds. @@ -229,11 +248,33 @@ start_stack() { EOF cat >"${run_dir}/worker-auth.json" <"${run_dir}/stargate-secrets.json" <"${run_dir}/gateway-secrets.json" <"${file}" +} + +# POSTs a decision request through the gateway and prints the status code. +# Response headers go to ${run_dir}/decision-headers.txt and the body to +# ${run_dir}/decision-body.txt. +decision_request() { + local route="$1" query="$2" body_file="$3" + curl -s -o "${run_dir}/decision-body.txt" -D "${run_dir}/decision-headers.txt" \ + -w '%{http_code}' --max-time 60 \ + "http://127.0.0.1:${gw_http}/v1/${route}${query}" \ + -H 'content-type: application/json' \ + -H "authorization: Bearer ${client_token}" \ + -H 'TypeSafe-Danger-Zone-Images: true' \ + -H 'X-Vendor-Test: v' \ + --data-binary "@${body_file}" +} + +expect_decision() { + local description="$1" expected_status="$2" status="$3" + [[ "${status}" == "${expected_status}" ]] || + fail "${description}: expected ${expected_status}, got ${status}: $(head -c 300 "${run_dir}/decision-body.txt")" +} + +sha256_stdin() { + if command -v sha256sum >/dev/null; then sha256sum; else shasum -a 256; fi | cut -d' ' -f1 +} + +# The body the gateway should forward. It decodes the top-level JSON object, +# sets model to the routed model, and re-marshals with sorted top-level keys and +# compact separators. Member values are kept as raw JSON, so nested key order is +# unchanged. jq builds the same bytes for this ASCII body, so the comparison is +# on raw bytes, not on a canonical form. +expected_forwarded_body() { + jq -c --arg model "${model}" '.model = $model | to_entries | sort_by(.key) | from_entries' "$1" | + tr -d '\n' +} + +expected_forwarded_digest() { + expected_forwarded_body "$1" | sha256_stdin +} + +# The canned error body mock-dynamo serves for ?mock_error=. +expected_decision_error() { + local route="$1" status="$2" + case "${route}:${status}" in + decisions:429) echo '{"error":{"message":"Rate limit exceeded","type":"rate_limit_error","param":null,"code":"rate_limit_exceeded"}}' ;; + decisions:400) echo '{"error":{"message":"Invalid question","type":"invalid_request_error","param":"questions","code":"invalid_question"}}' ;; + systemone:429) echo '{"detail":"Rate limit exceeded"}' ;; + systemone:400) echo '{"detail":[{"loc":["body","questions"],"msg":"Invalid question","type":"value_error"}]}' ;; + *) fail "no expected body for ${route}:${status}" ;; + esac +} + +# Fails unless the last response body equals the expected JSON, key order aside. +expect_decision_json() { + local description="$1" expected="$2" + diff <(jq -S . <<<"${expected}") <(jq -S . "${run_dir}/decision-body.txt") >"${run_dir}/diff.txt" || + fail "${description} differs from the canned response: $(head -c 600 "${run_dir}/diff.txt")" +} + +# Checks the whole 200 response of the last request, which sent body_file. The +# gateway-assigned request id and input token estimate vary, so they are +# checked for shape and consistency and removed before the exact comparison. +expect_decision_success() { + local route="$1" body_file="$2" answers usage_extra expected_digest body_bytes + if [[ "${route}" == decisions ]]; then + answers='[{"type":"predicate","name":"cat","probability":0.99}]' + usage_extra='"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens_details":{"reasoning_tokens":0},' + else + answers='{"cat":{"type":"noul","noul":0.99}}' + usage_extra='' + fi + expected_digest=$(expected_forwarded_digest "${body_file}") + body_bytes=$(expected_forwarded_body "${body_file}" | wc -c | tr -d ' ') + jq -e '(.echo["x-request-id"] | length) > 0 + and (.echo["x-input-tokens"] | tonumber) == .usage.input_tokens + and (.usage.total_tokens == null + or .usage.total_tokens == (.usage.input_tokens + .usage.output_tokens))' \ + "${run_dir}/decision-body.txt" >/dev/null || + fail "${route}: request id or token usage inconsistent: $(head -c 400 "${run_dir}/decision-body.txt")" + diff <(jq -S -n --arg model "${model}" --arg digest "${expected_digest}" --argjson len "${body_bytes}" \ + --argjson answers "${answers}" '{ + model: $model, + answers: $answers, + usage: {'"${usage_extra}"'output_tokens: 1}, + echo: { + "typesafe-danger-zone-images": "true", + "x-model": $model, + "x-vendor-test": "v", + body_len: $len, + body_sha256: $digest + }}') \ + <(jq -S 'del(.echo["x-request-id"], .echo["x-input-tokens"], .usage.input_tokens, .usage.total_tokens)' \ + "${run_dir}/decision-body.txt") >"${run_dir}/diff.txt" || + fail "${route}: 200 body differs from the canned response or the forwarded body changed: $(head -c 600 "${run_dir}/diff.txt")" +} + +run_decision_case() { + run_dir="${work_dir}/decisions" + mkdir -p "${run_dir}" + echo "=== decision routes through gateway, Stargate, Pylon, mock-dynamo ===" + gateway_auth=1 + start_stack wait-and-widen false + + local route small large status before after + for route in decisions systemone; do + small="${run_dir}/${route}-small.json" + large="${run_dir}/${route}-large.json" + write_decision_body "${route}" 64 "${small}" + write_decision_body "${route}" $((13 * 1024 * 1024)) "${large}" + + before=$(mock_endpoint_total "${route}") + status=$(decision_request "${route}" "" "${small}") + expect_decision "${route} 200" 200 "${status}" + expect_decision_success "${route}" "${small}" + grep -qi '^x-mock-echo-typesafe-danger-zone-images: true' "${run_dir}/decision-headers.txt" || + fail "${route}: TypeSafe-Danger-Zone-Images did not reach the backend" + + status=$(decision_request "${route}" "" "${large}") + expect_decision "${route} 13 MB body" 200 "${status}" + expect_decision_success "${route}" "${large}" + + status=$(decision_request "${route}" "?mock_error=429" "${small}") + expect_decision "${route} backend 429" 429 "${status}" + tr -d '\r' <"${run_dir}/decision-headers.txt" | grep -qix 'retry-after: 7' || + fail "${route}: 429 Retry-After was not 7: $(grep -i '^retry-after' "${run_dir}/decision-headers.txt")" + expect_decision_json "${route} 429 body" "$(expected_decision_error "${route}" 429)" + + status=$(decision_request "${route}" "?mock_error=400" "${small}") + expect_decision "${route} backend 400" 400 "${status}" + expect_decision_json "${route} 400 body" "$(expected_decision_error "${route}" 400)" + + # Four requests, one backend attempt each: the 429 was not retried. + after=$(mock_endpoint_total "${route}") + [[ $((after - before)) -eq 4 ]] || + fail "${route}: expected 4 backend requests, saw $((after - before))" + echo "PASS: ${route}" + done + gateway_auth=0 + cleanup +} + if [[ ${build} -eq 1 ]]; then mkdir -p "${release_dir}" echo "building llm-api-gateway, stargate, pylon, and mock-dynamo" @@ -370,9 +583,12 @@ for binary in llm-api-gateway stargate stargate-dev-auth pylon mock-dynamo; do [[ -x "${release_dir}/${binary}" ]] || fail "missing ${release_dir}/${binary}; run without --no-build" done -for algorithm in wait-and-widen pulsar-wait-and-widen; do - for flag in true false; do - run_case "${algorithm}" "${flag}" +if [[ ${decisions_only} -eq 0 ]]; then + for algorithm in wait-and-widen pulsar-wait-and-widen; do + for flag in true false; do + run_case "${algorithm}" "${flag}" + done done -done +fi +run_decision_case echo "all cases passed" diff --git a/tests/bdd/AGENTS.md b/tests/bdd/AGENTS.md index 46a39b7e73..b05cb5dc51 100644 --- a/tests/bdd/AGENTS.md +++ b/tests/bdd/AGENTS.md @@ -69,6 +69,14 @@ logic into `dsl/`. OpenAI-compatible sample that replies with the request it received when `x_load_tester_echo` is true. Do not point it at the `:local` sample, which returns a fixed reply other scenarios assert. +- The decision route scenarios (`@llm-decision-routes`) use + `SAMPLE_LLM_DECISIONS_IMAGE`, an in-repo sample image that serves + `/v1/decisions` and `/v1/systemone` with a header echo and the `mock_error` + query controls. It defaults to the `:local` sample tag, so existing scenarios + keep their images. The scenarios sit after the field-fidelity scenario under + a `# Beta decision routes` comment, which `TestLLMRoutingFeaturesRejectIncompleteAndMisroutedResponses` + uses to stop its slice; their wiring test is + `TestDecisionRouteScenariosRejectBrokenGateways`. - The BYOO feature reads its task sample from `SAMPLE_BYOO_TASK_IMAGE` and its function sample from `SAMPLE_BYOO_FUNCTION_IMAGE`. Tag every BYOO scenario that installs, upgrades, or removes a release with `@stack-change` diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index 210224bf76..3875ae5d7a 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -198,6 +198,7 @@ These steps use the in-process HTTP client (`harness.HTTPClient`), not the comma | `When I poll {string} {string} for up to {int} seconds until the HTTP response body contains {string}` | Repeats the request every 5 seconds until the body contains the text or the deadline passes. Used for eventually consistent reads. | | `Then the HTTP status should be {int}` | Last response status. The failure message includes a truncated body with API keys (`nvapi-...`) and JWTs masked. HTTP failure messages mask them everywhere they print a body or an expected value. | | `Then the HTTP status should be {int} or {int}` | Last response status equals either value. | +| `Then the HTTP response header {string} should equal {string}` | Compares the first value of a response header, matched case-insensitively by name, with the interpolated text. Fails when the header is absent. Used to check a passthrough keeps a backend header such as `Retry-After`. | | `Then the HTTP response body should contain {string}` | Substring match. The interpolated value must be non-empty. | | `Then the HTTP response body should not contain {string}` | Negative substring match. The interpolated value must be non-empty. | | `Then the HTTP response JSON field {string} should equal {string}` | Dotted path (object keys, array indexes) compared as text. A path continues into a string that holds a JSON object or array, such as a request an echo sample returns as its reply. | diff --git a/tests/bdd/README.md b/tests/bdd/README.md index 0067e74241..4f34679fd9 100644 --- a/tests/bdd/README.md +++ b/tests/bdd/README.md @@ -64,6 +64,25 @@ The Vanity Gateway shadow sampling scenario deploys two LLM functions from the same image, so it must also honor the `LOAD_TESTER_DEFAULT_*` container settings, and the cluster needs two free 1x GPU instances. It serves OpenAI routes on `vanity-openai.localhost`. +The decision route scenarios (`@llm-decision-routes`) cover `POST /v1/decisions` +and `POST /v1/systemone` through the LLM gateway and through the Vanity Gateway. +They deploy the in-repo sample from `SAMPLE_LLM_DECISIONS_IMAGE`, which must +serve both routes. The released `5.1.0` sample does not, so until a release +includes them build the image from this checkout and make the cluster able to +pull it. The default is +`nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local`: + +```sh +docker build --platform linux/amd64 \ + -t nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local \ + examples/function-samples/openai-compatible-sample +k3d image import -c ncp-local \ + nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local +``` + +Or push a tag and set `SAMPLE_LLM_DECISIONS_IMAGE` to it. The scenarios need the +LLM gateway, Stargate, Pylon and Vanity Gateway builds that serve the routes, +and the Vanity scenario needs two free 1x GPU instances. Each `-run` argument is anchored with `^...$` so the live entry point runs without also matching its `...FeatureFileWiresToSteps` wiring @@ -91,6 +110,7 @@ NGC_API_KEY= SAMPLE_NGC_ORG= SAMPLE_NGC_TEAM= \ SAMPLE_HELM_TASK_CHART= \ SAMPLE_HELM_TASK_CHART_WITHOUT_RESOURCES= \ SAMPLE_LLM_ECHO_IMAGE= \ + SAMPLE_LLM_DECISIONS_IMAGE= \ go test -run '^TestSingleClusterHelmfile$' -timeout 90m -v # Single-cluster HA Helmfile feature: installs the control plane with diff --git a/tests/bdd/decision_routes_test.go b/tests/bdd/decision_routes_test.go new file mode 100644 index 0000000000..54125ace49 --- /dev/null +++ b/tests/bdd/decision_routes_test.go @@ -0,0 +1,382 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bdd_tmp + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cucumber/godog" + + "nvcf-bdd/harness" + "nvcf-bdd/steps" +) + +const ( + decisionLLMHost = "llm.localhost:8080" + decisionVanityHost = "vanity-openai.localhost:8080" + decisionsRoute = "/v1/decisions" + systemOneRoute = "/v1/systemone" +) + +// decisionRoutesFake stands in for the LLM gateway, the Vanity Gateway and +// the sample backend on the decision routes. A model serves a route only when +// the scenario's configuration lists it there. Each broken mode reproduces one +// gateway failure the scenarios must reject. +type decisionRoutesFake struct { + broken string + seen map[string]bool + inner harness.HTTPClient +} + +func newDecisionRoutesFake(broken string, inner harness.HTTPClient) *decisionRoutesFake { + return &decisionRoutesFake{broken: broken, seen: map[string]bool{}, inner: inner} +} + +// decisionRoutesByModel lists the routes each model name serves, by host. +var decisionRoutesByModel = map[string]map[string][]string{ + decisionLLMHost: { + "function-1/bdd-decision-both": {decisionsRoute, systemOneRoute}, + "function-1/bdd-decision-only": {decisionsRoute}, + }, + decisionVanityHost: { + "bdd-decision-direct": {decisionsRoute, systemOneRoute}, + "bdd-decision-llm": {decisionsRoute, systemOneRoute}, + "bdd-decision-decisions-only": {decisionsRoute}, + "bdd-decision-systemone-only": {systemOneRoute}, + }, +} + +func isDecisionRoutePath(path string) bool { + return path == decisionsRoute || path == systemOneRoute || path == "/v1/models" || strings.HasPrefix(path, "/v1/models/") +} + +func (f *decisionRoutesFake) mark(mode string) bool { + if f.broken != mode { + return false + } + f.seen[mode] = true + return true +} + +func (f *decisionRoutesFake) Do(ctx context.Context, method, rawURL string, headers map[string]string, body string) (harness.HTTPResponse, error) { + u, err := url.Parse(rawURL) + if err != nil { + return harness.HTTPResponse{}, err + } + known := decisionRoutesByModel[u.Host] + if known == nil || !isDecisionRoutePath(u.Path) { + return f.inner.Do(ctx, method, rawURL, headers, body) + } + if headers["Authorization"] != "Bearer wiring-function-api-key" { + return harness.HTTPResponse{Status: 401}, nil + } + if strings.HasPrefix(u.Path, "/v1/models") { + return f.models(u, method, known) + } + if method != "POST" || headers["Content-Type"] != "application/json" { + return harness.HTTPResponse{Status: 400}, nil + } + var payload struct { + Model string `json:"model"` + Questions json.RawMessage `json:"questions"` + } + if err := json.Unmarshal([]byte(body), &payload); err != nil { + return harness.HTTPResponse{}, err + } + if !f.serves(u, payload.Model, known) { + return f.notFound(u, payload.Model), nil + } + return f.backend(u, headers, payload.Questions), nil +} + +// serves reports whether the model is enabled for the called route. The +// systemone-not-isolated and vanity-section-leak modes enable a model that the +// configuration lists only on the other route. +func (f *decisionRoutesFake) serves(u *url.URL, model string, known map[string][]string) bool { + for _, route := range known[model] { + if route == u.Path { + return true + } + } + switch { + case u.Host == decisionLLMHost && model == "function-1/bdd-decision-only" && u.Path == systemOneRoute: + return f.mark("systemone-not-isolated") + case u.Host == decisionVanityHost && model == "bdd-decision-decisions-only" && u.Path == systemOneRoute: + return f.mark("vanity-section-leak") + } + return false +} + +func (f *decisionRoutesFake) notFound(u *url.URL, model string) harness.HTTPResponse { + message := "model " + model + " is not enabled for " + u.Path + if u.Path == decisionsRoute || f.mark("systemone-404-openai-shape") { + return harness.HTTPResponse{Status: 404, Body: mustJSON(map[string]any{ + "error": map[string]any{"message": message, "type": "invalid_request_error"}, + })} + } + topLevelID := "req-404" + if f.mark("request-id-mismatch") { + topLevelID = "req-other" + } + return harness.HTTPResponse{Status: 404, Body: mustJSON(map[string]any{ + "error": map[string]any{"code": "not_found", "message": message, "request_id": "req-404"}, + "request_id": topLevelID, + })} +} + +// backend answers the way the sample does: canned answers, a header echo and +// the mock_error controls. The gateway under test passes all of it through. +func (f *decisionRoutesFake) backend(u *url.URL, headers map[string]string, questions json.RawMessage) harness.HTTPResponse { + systemOne := u.Path == systemOneRoute + switch u.Query().Get("mock_error") { + case "429": + return f.mockError(systemOne, http.StatusTooManyRequests, "mock rate limit", "rate_limit_error") + case "400": + return f.mockError(systemOne, http.StatusBadRequest, "mock validation error", "invalid_request_error") + } + echo := map[string]any{} + for _, name := range []string{"typesafe-danger-zone-images", "x-vendor-test"} { + for header, value := range headers { + if !strings.EqualFold(header, name) { + continue + } + if name == "x-vendor-test" && f.mark("vendor-header-dropped") { + continue + } + echo[name] = value + } + } + response := harness.HTTPResponse{Status: 200, Header: http.Header{}} + if value, ok := echo["typesafe-danger-zone-images"]; ok { + response.Header.Set("x-mock-echo-typesafe-danger-zone-images", value.(string)) + } + if systemOne { + response.Body = mustJSON(map[string]any{"model": "m", "answers": systemOneAnswers(questions), "echo": echo}) + return response + } + answers := decisionAnswers(questions) + if f.mark("answers-reordered") { + for i, j := 0, len(answers)-1; i < j; i, j = i+1, j-1 { + answers[i], answers[j] = answers[j], answers[i] + } + } + response.Body = mustJSON(map[string]any{"model": "m", "answers": answers, "echo": echo}) + return response +} + +func (f *decisionRoutesFake) mockError(systemOne bool, status int, message, errorType string) harness.HTTPResponse { + response := harness.HTTPResponse{Status: status, Header: http.Header{}} + if status == http.StatusTooManyRequests && !f.mark("retry-after-lost") { + response.Header.Set("Retry-After", "7") + } + switch { + case f.mark("error-body-rewritten"): + response.Body = mustJSON(map[string]any{"error": map[string]any{"message": "upstream error", "type": "server_error"}}) + case systemOne: + response.Body = mustJSON(map[string]any{"detail": []any{map[string]any{"loc": []string{"body"}, "msg": message, "type": "value_error"}}}) + default: + response.Body = mustJSON(map[string]any{"error": map[string]any{"message": message, "type": errorType, "code": http.StatusText(status)}}) + } + return response +} + +func (f *decisionRoutesFake) models(u *url.URL, method string, known map[string][]string) (harness.HTTPResponse, error) { + if method != "GET" { + return harness.HTTPResponse{Status: 405}, nil + } + if u.Path == "/v1/models" { + data := []any{} + for _, id := range []string{"bdd-decision-direct", "bdd-decision-llm", "bdd-decision-decisions-only", "bdd-decision-systemone-only"} { + data = append(data, map[string]any{"id": id, "object": "model"}) + } + return harness.HTTPResponse{Status: 200, Body: mustJSON(map[string]any{"object": "list", "data": data})}, nil + } + id := strings.TrimPrefix(u.Path, "/v1/models/") + routes, ok := known[id] + if !ok { + return harness.HTTPResponse{Status: 404}, nil + } + info := map[string]any{"id": id, "object": "model"} + if !f.mark("models-without-endpoints") { + info["supported_endpoints"] = routes + } + return harness.HTTPResponse{Status: 200, Body: mustJSON(info)}, nil +} + +// attach answers the scenarios' warm-up invocations on the runner. They go +// through the NVCF CLI, which prints the function's reply. +func (f *decisionRoutesFake) attach(runner *fakeRunner) { + previous := runner.onRun + runner.onRun = func(command string) error { + if strings.Contains(command, " function invoke ") && strings.Contains(command, "-ready") && + (strings.Contains(command, "--model-name bdd-decision") || strings.Contains(command, "bdd-decision-direct-ready")) { + runner.results[command] = harness.Result{Stdout: "Function invocation completed!\n\nResponse:\n" + + `{"answers":[{"type":"predicate","probability":0.99}]}` + "\n"} + } + if previous != nil { + return previous(command) + } + return nil + } +} + +func decisionAnswers(questions json.RawMessage) []any { + var list []struct { + Type string `json:"type"` + Name string `json:"name"` + Choices []any `json:"choices"` + } + _ = json.Unmarshal(questions, &list) + answers := []any{} + for _, question := range list { + answer := map[string]any{"type": question.Type, "name": question.Name} + switch question.Type { + case "predicate": + answer["probability"] = 0.99 + case "choice": + answer["choice"] = question.Choices[0] + case "score": + answer["score"] = 0.3 + } + answers = append(answers, answer) + } + return answers +} + +func systemOneAnswers(questions json.RawMessage) map[string]any { + var byName map[string]struct { + Type string `json:"type"` + Choices []any `json:"choices"` + } + _ = json.Unmarshal(questions, &byName) + answers := map[string]any{} + for name, question := range byName { + answer := map[string]any{"type": question.Type} + switch question.Type { + case "noul": + answer["noul"] = 0.99 + case "choice": + answer["choice"] = question.Choices[0] + case "score": + answer["score"] = 0.3 + } + answers[name] = answer + } + return answers +} + +func mustJSON(value any) string { + data, err := json.Marshal(value) + if err != nil { + panic(err) + } + return string(data) +} + +// decisionScenarios returns the two decision route scenarios from the +// single-cluster feature, without the scenarios around them. +func decisionScenarios(t *testing.T) string { + t.Helper() + data, err := os.ReadFile(mustResolveFeaturePath(t, "single-cluster-helmfile.feature")) + if err != nil { + t.Fatal(err) + } + _, scenarios, found := strings.Cut(string(data), "\n # Beta decision routes") + if !found { + t.Fatal("missing decision route scenarios") + } + return " # Beta decision routes" + scenarios +} + +func TestDecisionRouteScenariosRejectBrokenGateways(t *testing.T) { + scenarios := decisionScenarios(t) + for _, tag := range []string{"@llm-function-type @llm-decision-routes\n", "@vanity-decision-routes\n"} { + if !strings.Contains(scenarios, tag) { + t.Fatalf("decision scenarios lost tag line %q", tag) + } + } + brokenModes := []string{ + "", "answers-reordered", "vendor-header-dropped", "systemone-not-isolated", + "systemone-404-openai-shape", "request-id-mismatch", "retry-after-lost", + "error-body-rewritten", "vanity-section-leak", "models-without-endpoints", + } + for _, broken := range brokenModes { + t.Run(broken, func(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("REPO_ROOT", "/repo-root-placeholder") + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + t.Setenv("SAMPLE_LLM_DECISIONS_IMAGE", "nvcr.io/test-org/test-team/openai-compatible-decisions:test") + seedFunctionAPIKey(t, home) + runner := newFakeRunner(map[string]harness.Result{}) + runner.onRun = func(command string) error { + if strings.Contains(command, " status --json") { + runner.results[command] = harness.Result{Stdout: `{"currentFunction":{"hasFunction":true,"functionId":"function-1","versionId":"version-1"}}`} + } + return nil + } + suite := newWiringSuite(t, runner) + envFile := filepath.Join(suite.Config.RepoRoot, "deploy", "stacks", "self-managed", "environments", "local-bdd.yaml") + if err := os.MkdirAll(filepath.Dir(envFile), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(envFile, []byte("addons: {}\n"), 0o644); err != nil { + t.Fatal(err) + } + fake := newDecisionRoutesFake(broken, &llmRoutingHTTPFake{seen: map[string]bool{}}) + newVanityShadowFake("", fake).attach(runner) + fake.attach(runner) + suite.HTTP = fake + sc := steps.NewScenarioContext(suite) + var out strings.Builder + status := godog.TestSuite{ + Name: "decision-routes-feature", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { steps.RegisterAll(ctx, sc) }, + Options: &godog.Options{ + Format: "pretty", Output: &out, Strict: true, + FeatureContents: []godog.Feature{{Name: "decisions", Contents: []byte("Feature: Decision routes\n" + scenarios)}}, + }, + }.Run() + if broken == "" { + if status != 0 { + t.Fatalf("feature failed:\n%s", out.String()) + } + if !commandRanThatContainsAll(runner.runs, "function create --name bdd-llm-decision-routes", + "--image nvcr.io/test-org/test-team/openai-compatible-decisions:test", "--function-type LLM", + "name=bdd-decision-only,uris=/v1/decisions") { + t.Error("LLM gateway function was not created from SAMPLE_LLM_DECISIONS_IMAGE with both models") + } + if !commandRanThatContainsAll(runner.runs, "function create --name bdd-vanity-decision-direct", + "--image nvcr.io/test-org/test-team/openai-compatible-decisions:test", "--inference-url /v1/decisions") { + t.Error("direct Vanity function was not created from SAMPLE_LLM_DECISIONS_IMAGE") + } + return + } + if !fake.seen[broken] || status == 0 { + t.Fatalf("feature did not reject broken %s gateway (exercised %v):\n%s", broken, fake.seen[broken], out.String()) + } + }) + } +} diff --git a/tests/bdd/features/single-cluster-helmfile.feature b/tests/bdd/features/single-cluster-helmfile.feature index 312b9d8857..a6df968a22 100644 --- a/tests/bdd/features/single-cluster-helmfile.feature +++ b/tests/bdd/features/single-cluster-helmfile.feature @@ -964,3 +964,405 @@ Feature: Install a local single-cluster NVCF stack with Helmfile And the HTTP response JSON field "choices.0.message.content.reasoning_format" should be absent And I successfully undeploy the function selected by NVCF CLI + + # Beta decision routes, POST /v1/decisions and POST /v1/systemone, through + # the LLM gateway. Both routes are passthrough, so the sample backend's + # canned answers and header echo prove what reached it. + # SAMPLE_LLM_DECISIONS_IMAGE must be an image of the in-repo + # openai-compatible sample that serves both routes. One function declares + # two models: bdd-decision-both on both routes and bdd-decision-only on + # /v1/decisions alone. A model is enabled for a route only when its uris + # list that route. Depends on the earlier install and registration + # scenarios. + @llm-function-type @llm-decision-routes + Scenario: LLM gateway serves the decisions and systemone routes and keeps each model on its declared routes + Given environment variable "SAMPLE_LLM_DECISIONS_IMAGE" is set + And I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + + When I successfully create function "bdd-llm-decision-routes" from image "${SAMPLE_LLM_DECISIONS_IMAGE}" with CLI options: + | option | value | + | --function-type | LLM | + | --inference-url | /v1/chat/completions | + | --inference-port | 8000 | + | --health-uri | /health | + | --health-port | 8000 | + | --health-timeout | PT30S | + | --llm-model | name=bdd-decision-both,uris=/v1/decisions\|/v1/systemone,routingMethod=round_robin | + | --llm-model | name=bdd-decision-only,uris=/v1/decisions,routingMethod=round_robin | + And I export the function selected by NVCF CLI to environment variables "BDD_DECISION_FUNCTION_ID" and "BDD_DECISION_VERSION_ID" + And function "${BDD_DECISION_FUNCTION_ID}" version "${BDD_DECISION_VERSION_ID}" is deleted at scenario teardown + And function "${BDD_DECISION_FUNCTION_ID}" version "${BDD_DECISION_VERSION_ID}" is undeployed at scenario teardown + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + And I successfully generate a function API key with CLI options: + | option | value | + | --description | bdd-llm-decision-routes | + | --scopes | invoke_function,list_functions,queue_details,list_functions_details | + + # The first gateway call proves the route is ready before the raw checks. + When I successfully invoke model "bdd-decision-both" at "/v1/decisions" with timeout "120" seconds: + """ + {"input":"bdd-decision-ready","questions":[{"type":"predicate","text":"Is it ready?"}]} + """ + Then the command output should contain "answers" + + # Decisions: answers keep the question order, and the vendor headers + # reach the backend, which echoes them. + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "TypeSafe-Danger-Zone-Images" is "true" + And the HTTP request header "X-Vendor-Test" is "bdd-vendor-value" + When I send a "POST" request to "http://llm.localhost:8080/v1/decisions" with timeout 120 seconds and body: + """ + {"model":"${BDD_DECISION_FUNCTION_ID}/bdd-decision-both","input":"bdd-decision-input","questions":[{"type":"predicate","name":"red","text":"Is it red?"},{"type":"choice","name":"animal","text":"Which animal?","choices":["cat","dog"]},{"type":"score","name":"quality","text":"How good is it?"}]} + """ + Then the HTTP status should be 200 + And the HTTP response JSON field "answers.0.type" should equal "predicate" + And the HTTP response JSON field "answers.0.probability" should equal "0.99" + And the HTTP response JSON field "answers.1.type" should equal "choice" + And the HTTP response JSON field "answers.1.choice" should equal "cat" + And the HTTP response JSON field "answers.2.type" should equal "score" + And the HTTP response JSON field "echo.typesafe-danger-zone-images" should equal "true" + And the HTTP response JSON field "echo.x-vendor-test" should equal "bdd-vendor-value" + And the HTTP response header "x-mock-echo-typesafe-danger-zone-images" should equal "true" + + # System One: answers are keyed by question name. + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "TypeSafe-Danger-Zone-Images" is "true" + And the HTTP request header "X-Vendor-Test" is "bdd-vendor-value" + When I send a "POST" request to "http://llm.localhost:8080/v1/systemone" with timeout 120 seconds and body: + """ + {"model":"${BDD_DECISION_FUNCTION_ID}/bdd-decision-both","state":{"text":"bdd-systemone-state"},"questions":{"seen":{"type":"noul","text":"Is it seen?"},"animal":{"type":"choice","choices":["cat","dog"]},"quality":{"type":"score"}}} + """ + Then the HTTP status should be 200 + And the HTTP response JSON field "answers.seen.type" should equal "noul" + And the HTTP response JSON field "answers.seen.noul" should equal "0.99" + And the HTTP response JSON field "answers.animal.choice" should equal "cat" + And the HTTP response JSON field "answers.quality.type" should equal "score" + And the HTTP response JSON field "echo.typesafe-danger-zone-images" should equal "true" + And the HTTP response JSON field "echo.x-vendor-test" should equal "bdd-vendor-value" + And the HTTP response header "x-mock-echo-typesafe-danger-zone-images" should equal "true" + + # Route isolation: bdd-decision-only declares /v1/decisions alone, so + # the declared route works and /v1/systemone is a 404 in the System One + # error shape (error.code, error.message, error.request_id and a + # top-level request_id that match). + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + When I send a "POST" request to "http://llm.localhost:8080/v1/decisions" with timeout 120 seconds and body: + """ + {"model":"${BDD_DECISION_FUNCTION_ID}/bdd-decision-only","input":"bdd-decision-only","questions":[{"type":"predicate","text":"Is it declared?"}]} + """ + Then the HTTP status should be 200 + And the HTTP response JSON field "answers.0.type" should equal "predicate" + + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + When I send a "POST" request to "http://llm.localhost:8080/v1/systemone" with timeout 120 seconds and body: + """ + {"model":"${BDD_DECISION_FUNCTION_ID}/bdd-decision-only","state":{"text":"bdd"},"questions":{"seen":{"type":"noul"}}} + """ + Then the HTTP status should be 404 + And the HTTP response JSON field "error.code" should equal "not_found" + And the HTTP response body should contain "is not enabled for /v1/systemone" + And I export the HTTP response JSON field "error.request_id" to environment variable "BDD_SYSTEMONE_ERROR_REQUEST_ID" + And the HTTP response JSON field "request_id" should equal "${BDD_SYSTEMONE_ERROR_REQUEST_ID}" + + # A model that is not declared at all fails the same way on /v1/decisions, + # in the OpenAI error shape. + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + When I send a "POST" request to "http://llm.localhost:8080/v1/decisions" with timeout 120 seconds and body: + """ + {"model":"${BDD_DECISION_FUNCTION_ID}/not-a-model","input":"bdd-unlisted","questions":[{"type":"predicate","text":"Is it listed?"}]} + """ + Then the HTTP status should be 404 + And the HTTP response JSON field "error.type" should equal "invalid_request_error" + + # Backend errors pass through unchanged: status, Retry-After and body. + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + When I send a "POST" request to "http://llm.localhost:8080/v1/decisions?mock_error=429" with timeout 120 seconds and body: + """ + {"model":"${BDD_DECISION_FUNCTION_ID}/bdd-decision-both","input":"bdd-429","questions":[{"type":"predicate","text":"Is it limited?"}]} + """ + Then the HTTP status should be 429 + And the HTTP response header "Retry-After" should equal "7" + And the HTTP response JSON field "error.type" should equal "rate_limit_error" + And the HTTP response JSON field "error.message" should equal "mock rate limit" + + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + When I send a "POST" request to "http://llm.localhost:8080/v1/systemone?mock_error=429" with timeout 120 seconds and body: + """ + {"model":"${BDD_DECISION_FUNCTION_ID}/bdd-decision-both","state":{"text":"bdd-429"},"questions":{"seen":{"type":"noul"}}} + """ + Then the HTTP status should be 429 + And the HTTP response header "Retry-After" should equal "7" + And the HTTP response JSON field "detail.0.msg" should equal "mock rate limit" + + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + When I send a "POST" request to "http://llm.localhost:8080/v1/decisions?mock_error=400" with timeout 120 seconds and body: + """ + {"model":"${BDD_DECISION_FUNCTION_ID}/bdd-decision-both","input":"bdd-400","questions":[{"type":"predicate","text":"Is it valid?"}]} + """ + Then the HTTP status should be 400 + And the HTTP response JSON field "error.type" should equal "invalid_request_error" + And the HTTP response JSON field "error.message" should equal "mock validation error" + + # The System One backend error is FastAPI style, not the gateway's + # System One error shape, so the body reached the client unchanged. + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + When I send a "POST" request to "http://llm.localhost:8080/v1/systemone?mock_error=400" with timeout 120 seconds and body: + """ + {"model":"${BDD_DECISION_FUNCTION_ID}/bdd-decision-both","state":{"text":"bdd-400"},"questions":{"seen":{"type":"noul"}}} + """ + Then the HTTP status should be 400 + And the HTTP response JSON field "detail.0.msg" should equal "mock validation error" + And the HTTP response JSON field "detail.0.type" should equal "value_error" + And the HTTP response body should not contain "request_id" + + # The same routes through the Vanity Gateway, on the OpenAI host + # vanity-openai.localhost. Two functions from SAMPLE_LLM_DECISIONS_IMAGE: + # a default-type function that the gateway calls directly with function + # ID headers (usePexec false, the path is kept), and an LLM-type function + # that it reaches through the LLM gateway director (functionType LLM). + # Mappings: decisions under v2config.openai and typesafeSystemOne under + # v2config. bdd-decision-direct and bdd-decision-llm are in both sections; + # bdd-decision-decisions-only and bdd-decision-systemone-only are in one. + # The Vanity Gateway build must serve these sections before this runs. + # Depends on the earlier install and registration scenarios. + @llm-function-type @llm-decision-routes @vanity-decision-routes + Scenario: Vanity Gateway serves the decisions and systemone routes directly and through the LLM gateway + Given environment variable "SAMPLE_LLM_DECISIONS_IMAGE" is set + And I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + + When I successfully create function "bdd-vanity-decision-direct" from image "${SAMPLE_LLM_DECISIONS_IMAGE}" with CLI options: + | option | value | + | --inference-url | /v1/decisions | + | --inference-port | 8000 | + | --health-uri | /health | + | --health-port | 8000 | + | --health-timeout | PT30S | + And I export the function selected by NVCF CLI to environment variables "BDD_DECISION_DIRECT_FUNCTION_ID" and "BDD_DECISION_DIRECT_VERSION_ID" + And function "${BDD_DECISION_DIRECT_FUNCTION_ID}" version "${BDD_DECISION_DIRECT_VERSION_ID}" is deleted at scenario teardown + And function "${BDD_DECISION_DIRECT_FUNCTION_ID}" version "${BDD_DECISION_DIRECT_VERSION_ID}" is undeployed at scenario teardown + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + And I successfully generate a function API key with CLI options: + | option | value | + | --description | bdd-vanity-decision-routes | + | --scopes | invoke_function,list_functions,queue_details,list_functions_details | + + # The first call proves the direct function answers before the gateway + # is involved. + When I successfully invoke the function selected by NVCF CLI over HTTP with timeout "120" seconds and poll duration "5" seconds: + """ + {"input":"bdd-decision-direct-ready","questions":[{"type":"predicate","text":"Is it ready?"}]} + """ + Then the command output should contain "answers" + + When I successfully create function "bdd-vanity-decision-llm" from image "${SAMPLE_LLM_DECISIONS_IMAGE}" with CLI options: + | option | value | + | --function-type | LLM | + | --inference-url | /v1/chat/completions | + | --inference-port | 8000 | + | --health-uri | /health | + | --health-port | 8000 | + | --health-timeout | PT30S | + | --llm-model | name=bdd-decision-llm,uris=/v1/decisions\|/v1/systemone,routingMethod=round_robin | + And I export the function selected by NVCF CLI to environment variables "BDD_DECISION_LLM_FUNCTION_ID" and "BDD_DECISION_LLM_VERSION_ID" + And function "${BDD_DECISION_LLM_FUNCTION_ID}" version "${BDD_DECISION_LLM_VERSION_ID}" is deleted at scenario teardown + And function "${BDD_DECISION_LLM_FUNCTION_ID}" version "${BDD_DECISION_LLM_VERSION_ID}" is undeployed at scenario teardown + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + + # The LLM gateway needs a route to the new function before the first + # request through the Vanity Gateway. + When I successfully invoke model "bdd-decision-llm" at "/v1/decisions" with timeout "120" seconds: + """ + {"input":"bdd-decision-llm-ready","questions":[{"type":"predicate","text":"Is it ready?"}]} + """ + Then the command output should contain "answers" + + When I set yaml file "deploy/stacks/self-managed/environments/local-bdd.yaml" key "addons.vanityGateway.mappingConfig.v2config.openai" to: + """ + host: vanity-openai.localhost:8080 + decisions: + bdd_decision_direct: + modelName: bdd-decision-direct + functionID: ${BDD_DECISION_DIRECT_FUNCTION_ID} + functionVersionID: ${BDD_DECISION_DIRECT_VERSION_ID} + usePexec: false + bdd_decision_llm: + modelName: bdd-decision-llm + functionID: ${BDD_DECISION_LLM_FUNCTION_ID} + functionType: LLM + bdd_decision_decisions_only: + modelName: bdd-decision-decisions-only + functionID: ${BDD_DECISION_DIRECT_FUNCTION_ID} + functionVersionID: ${BDD_DECISION_DIRECT_VERSION_ID} + usePexec: false + """ + And I set yaml file "deploy/stacks/self-managed/environments/local-bdd.yaml" key "addons.vanityGateway.mappingConfig.v2config.typesafeSystemOne" to: + """ + bdd_decision_direct: + modelName: bdd-decision-direct + functionID: ${BDD_DECISION_DIRECT_FUNCTION_ID} + functionVersionID: ${BDD_DECISION_DIRECT_VERSION_ID} + usePexec: false + bdd_decision_llm: + modelName: bdd-decision-llm + functionID: ${BDD_DECISION_LLM_FUNCTION_ID} + functionType: LLM + bdd_decision_systemone_only: + modelName: bdd-decision-systemone-only + functionID: ${BDD_DECISION_DIRECT_FUNCTION_ID} + functionVersionID: ${BDD_DECISION_DIRECT_VERSION_ID} + usePexec: false + """ + And I set yaml file "deploy/stacks/self-managed/environments/local-bdd.yaml" key "ingress.gatewayApi.routes.vanityGateway.hostnames" to: + """ + - vanity.localhost + - vanity-openai.localhost + """ + When I successfully run command "k3d kubeconfig merge ncp-local --output ${REPO_ROOT}/tests/bdd/out/ncp-local-vanity-kubeconfig.yaml --overwrite --kubeconfig-switch-context=false" + And I successfully run command "make -C deploy/stacks/self-managed apply HELMFILE_ENV=local-bdd HELMFILE_SELECTOR=name=ingress KUBECONFIG_FILE=${REPO_ROOT}/tests/bdd/out/ncp-local-vanity-kubeconfig.yaml" + And I successfully run command "make -C deploy/stacks/self-managed apply HELMFILE_ENV=local-bdd HELMFILE_SELECTOR=name=vanity-gateway KUBECONFIG_FILE=${REPO_ROOT}/tests/bdd/out/ncp-local-vanity-kubeconfig.yaml" + And I successfully run command "kubectl --context k3d-ncp-local rollout restart deployment/vanity-gateway --namespace nvcf" + Then deployment "vanity-gateway" in namespace "nvcf" using context "k3d-ncp-local" should complete rollout within "5m" + And Kubernetes resource "httproute/vanity-gateway" in namespace "envoy-gateway-system" using context "k3d-ncp-local" should contain: + """ + spec: + hostnames: + - vanity.localhost + - vanity-openai.localhost + """ + And these Gateway API routes should be accepted and resolved using context "k3d-ncp-local" within "2m": + | kind | name | namespace | parent | + | HTTPRoute | vanity-gateway | envoy-gateway-system | shared-gw | + + # Direct models: the gateway keeps the path and selects the function with + # function ID headers, so each route reaches the function as called. + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "TypeSafe-Danger-Zone-Images" is "true" + And the HTTP request header "X-Vendor-Test" is "bdd-vendor-value" + When I send a "POST" request to "http://vanity-openai.localhost:8080/v1/decisions" with timeout 120 seconds and body: + """ + {"model":"bdd-decision-direct","input":"bdd-vanity-direct","questions":[{"type":"predicate","name":"red","text":"Is it red?"},{"type":"choice","name":"animal","text":"Which animal?","choices":["cat","dog"]}]} + """ + Then the HTTP status should be 200 + And the HTTP response JSON field "answers.0.type" should equal "predicate" + And the HTTP response JSON field "answers.1.choice" should equal "cat" + And the HTTP response JSON field "echo.typesafe-danger-zone-images" should equal "true" + And the HTTP response JSON field "echo.x-vendor-test" should equal "bdd-vendor-value" + + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "TypeSafe-Danger-Zone-Images" is "true" + And the HTTP request header "X-Vendor-Test" is "bdd-vendor-value" + When I send a "POST" request to "http://vanity-openai.localhost:8080/v1/systemone" with timeout 120 seconds and body: + """ + {"model":"bdd-decision-direct","state":{"text":"bdd-vanity-direct"},"questions":{"seen":{"type":"noul"},"animal":{"type":"choice","choices":["cat","dog"]}}} + """ + Then the HTTP status should be 200 + And the HTTP response JSON field "answers.seen.noul" should equal "0.99" + And the HTTP response JSON field "answers.animal.choice" should equal "cat" + And the HTTP response JSON field "echo.typesafe-danger-zone-images" should equal "true" + And the HTTP response JSON field "echo.x-vendor-test" should equal "bdd-vendor-value" + + # LLM models: the gateway sends the request through the LLM gateway. + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "TypeSafe-Danger-Zone-Images" is "true" + And the HTTP request header "X-Vendor-Test" is "bdd-vendor-value" + When I send a "POST" request to "http://vanity-openai.localhost:8080/v1/decisions" with timeout 120 seconds and body: + """ + {"model":"bdd-decision-llm","input":"bdd-vanity-llm","questions":[{"type":"predicate","name":"red","text":"Is it red?"},{"type":"choice","name":"animal","text":"Which animal?","choices":["cat","dog"]}]} + """ + Then the HTTP status should be 200 + And the HTTP response JSON field "answers.0.type" should equal "predicate" + And the HTTP response JSON field "answers.1.choice" should equal "cat" + And the HTTP response JSON field "echo.typesafe-danger-zone-images" should equal "true" + And the HTTP response JSON field "echo.x-vendor-test" should equal "bdd-vendor-value" + + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "TypeSafe-Danger-Zone-Images" is "true" + And the HTTP request header "X-Vendor-Test" is "bdd-vendor-value" + When I send a "POST" request to "http://vanity-openai.localhost:8080/v1/systemone" with timeout 120 seconds and body: + """ + {"model":"bdd-decision-llm","state":{"text":"bdd-vanity-llm"},"questions":{"seen":{"type":"noul"},"animal":{"type":"choice","choices":["cat","dog"]}}} + """ + Then the HTTP status should be 200 + And the HTTP response JSON field "answers.seen.noul" should equal "0.99" + And the HTTP response JSON field "answers.animal.choice" should equal "cat" + And the HTTP response JSON field "echo.typesafe-danger-zone-images" should equal "true" + And the HTTP response JSON field "echo.x-vendor-test" should equal "bdd-vendor-value" + + # A model listed only in the other section is a 404 in the native error + # shape of the route that was called, and never reaches a function. + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + When I send a "POST" request to "http://vanity-openai.localhost:8080/v1/systemone" with timeout 120 seconds and body: + """ + {"model":"bdd-decision-decisions-only","state":{"text":"bdd"},"questions":{"seen":{"type":"noul"}}} + """ + Then the HTTP status should be 404 + And the HTTP response JSON field "error.code" should equal "not_found" + And I export the HTTP response JSON field "error.request_id" to environment variable "BDD_VANITY_SYSTEMONE_ERROR_REQUEST_ID" + And the HTTP response JSON field "request_id" should equal "${BDD_VANITY_SYSTEMONE_ERROR_REQUEST_ID}" + + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + When I send a "POST" request to "http://vanity-openai.localhost:8080/v1/decisions" with timeout 120 seconds and body: + """ + {"model":"bdd-decision-systemone-only","input":"bdd","questions":[{"type":"predicate","text":"Is it listed?"}]} + """ + Then the HTTP status should be 404 + And the HTTP response JSON field "error.type" should equal "invalid_request_error" + + # Model discovery lists each decision model with the routes it serves. + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + When I send a "GET" request to "http://vanity-openai.localhost:8080/v1/models" + Then the HTTP status should be 200 + And the HTTP response body should contain "bdd-decision-direct" + And the HTTP response body should contain "bdd-decision-decisions-only" + And the HTTP response body should contain "bdd-decision-systemone-only" + + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + When I send a "GET" request to "http://vanity-openai.localhost:8080/v1/models/bdd-decision-direct" + Then the HTTP status should be 200 + And the HTTP response JSON field "supported_endpoints.0" should equal "/v1/decisions" + And the HTTP response JSON field "supported_endpoints.1" should equal "/v1/systemone" + + Given the HTTP request header "Authorization" is the NVCF CLI function API key with prefix "Bearer " + When I send a "GET" request to "http://vanity-openai.localhost:8080/v1/models/bdd-decision-decisions-only" + Then the HTTP status should be 200 + And the HTTP response JSON field "supported_endpoints.0" should equal "/v1/decisions" + And the HTTP response body should not contain "/v1/systemone" diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index fbeb5d6810..a2644eefa2 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -495,6 +495,7 @@ func TestSingleClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { t.Setenv("SAMPLE_HELM_TASK_CHART", "https://charts.example.test/task-helmchart-test.tgz") t.Setenv("SAMPLE_HELM_TASK_CHART_WITHOUT_RESOURCES", "https://charts.example.test/task-helmchart-test-missing-resources.tgz") t.Setenv("SAMPLE_LLM_ECHO_IMAGE", "nvcr.io/test-org/test-team/openai-compatible-echo:test") + t.Setenv("SAMPLE_LLM_DECISIONS_IMAGE", "nvcr.io/test-org/test-team/openai-compatible-decisions:test") t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") t.Setenv("REPO_ROOT", "/repo-root-placeholder") runner := newFakeRunner(map[string]harness.Result{ @@ -571,8 +572,10 @@ func TestSingleClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { writeHelmfileRegisterValues(t, suite.Config.RepoRoot) seedFunctionAPIKey(t, home) - vanityShadow := newVanityShadowFake("", &llmRoutingHTTPFake{seen: map[string]bool{}}) + decisionRoutes := newDecisionRoutesFake("", &llmRoutingHTTPFake{seen: map[string]bool{}}) + vanityShadow := newVanityShadowFake("", decisionRoutes) vanityShadow.attach(runner) + decisionRoutes.attach(runner) suite.HTTP = vanityShadow sc := steps.NewScenarioContext(suite) featurePath := mustResolveFeaturePath(t, "single-cluster-helmfile.feature") @@ -689,10 +692,16 @@ func TestSingleClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { "--function-type LLM") { t.Fatal("LLM field-fidelity function was not created from SAMPLE_LLM_ECHO_IMAGE") } + if !commandRanThatContainsAll(suite.Runner.(*fakeRunner).runs, + "function create --name bdd-llm-decision-routes", + "--image nvcr.io/test-org/test-team/openai-compatible-decisions:test", + "--function-type LLM") { + t.Fatal("decision route function was not created from SAMPLE_LLM_DECISIONS_IMAGE") + } if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, "function delete --deployment-only") { t.Fatal("function deployment cleanup was never invoked") } - assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 8) + assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 11) for _, key := range []string{"shadow-warmup", "promptCacheKey", "firstMessageHash", "continuation", "failing-shadow"} { if !vanityShadow.seen[key] { t.Errorf("vanity shadow scenario never exercised %s", key) @@ -3078,9 +3087,22 @@ func TestSingleClusterHelmfile(t *testing.T) { if os.Getenv(sampleLLMEchoImageEnv) == "" { t.Setenv(sampleLLMEchoImageEnv, "nvcr.io/"+os.Getenv("SAMPLE_NGC_ORG")+"/"+os.Getenv("SAMPLE_NGC_TEAM")+"/"+sampleLLMEchoDefaultImage) } + if os.Getenv(sampleLLMDecisionsImageEnv) == "" { + t.Setenv(sampleLLMDecisionsImageEnv, "nvcr.io/"+os.Getenv("SAMPLE_NGC_ORG")+"/"+os.Getenv("SAMPLE_NGC_TEAM")+"/"+sampleLLMDecisionsDefaultImage) + } runLiveFeature(t, "single-cluster-helmfile.feature") } +// sampleLLMDecisionsImageEnv selects the OpenAI-compatible sample the decision +// route scenarios deploy. The image must serve /v1/decisions and /v1/systemone, +// which the released sample does not until a release includes them, so it +// defaults to the :local tag an operator builds from the same checkout and +// imports into the cluster. +const ( + sampleLLMDecisionsImageEnv = "SAMPLE_LLM_DECISIONS_IMAGE" + sampleLLMDecisionsDefaultImage = "nvcf-openai-compatible-sample:local" +) + // sampleLLMEchoImageEnv selects the OpenAI-compatible sample the LLM // field-fidelity scenario deploys. It defaults to the released in-repo sample // under the sample registry org and team. @@ -4027,6 +4049,8 @@ func TestLLMRoutingFeaturesRejectIncompleteAndMisroutedResponses(t *testing.T) { if !found { t.Fatal("missing LLM scenario") } + // The decision route scenarios have their own wiring test. + scenario, _, _ = strings.Cut(scenario, "\n # Beta decision routes") brokenModes := []string{"", "chat-stream", "responses", "classic", "pexec", "unlisted", "long-nonstream", "long-stream", "long-nonstream-too-fast", "long-stream-too-fast"} exercised := []string{"chat-stream", "responses", "classic", "pexec", "unlisted", "long-nonstream", "long-stream"} // Field fidelity runs only in the single-cluster feature. @@ -4119,6 +4143,9 @@ func (f *vanityShadowFake) Do(ctx context.Context, method, rawURL string, header if u.Host != "vanity-openai.localhost:8080" { return f.inner.Do(ctx, method, rawURL, headers, body) } + if isDecisionRoutePath(u.Path) { + return f.inner.Do(ctx, method, rawURL, headers, body) + } if method != "POST" || headers["Authorization"] != "Bearer wiring-function-api-key" || headers["Content-Type"] != "application/json" { return harness.HTTPResponse{Status: 401}, nil } diff --git a/tests/bdd/harness/http.go b/tests/bdd/harness/http.go index 7e762c44c1..6cdceff107 100644 --- a/tests/bdd/harness/http.go +++ b/tests/bdd/harness/http.go @@ -28,9 +28,11 @@ import ( const httpTimeout = 30 * time.Second -// HTTPResponse records status, body, and duration through the last body byte. +// HTTPResponse records status, headers, body, and duration through the last +// body byte. type HTTPResponse struct { Status int + Header http.Header Body string Elapsed time.Duration } @@ -84,5 +86,5 @@ func (c *netHTTPClient) Do(ctx context.Context, method, url string, headers map[ if err != nil { return HTTPResponse{}, fmt.Errorf("read %s %s response: %w", method, url, err) } - return HTTPResponse{Status: resp.StatusCode, Body: string(data), Elapsed: time.Since(started)}, nil + return HTTPResponse{Status: resp.StatusCode, Header: resp.Header, Body: string(data), Elapsed: time.Since(started)}, nil } diff --git a/tests/bdd/harness/http_test.go b/tests/bdd/harness/http_test.go index 8af31adbc3..07db87224d 100644 --- a/tests/bdd/harness/http_test.go +++ b/tests/bdd/harness/http_test.go @@ -51,6 +51,22 @@ func TestNetHTTPClientSendsHeadersAndBody(t *testing.T) { } } +func TestNetHTTPClientRecordsResponseHeaders(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "7") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + resp, err := NewHTTPClient().Do(context.Background(), "POST", server.URL, nil, "") + if err != nil { + t.Fatal(err) + } + if got := resp.Header.Get("retry-after"); got != "7" { + t.Fatalf("Retry-After = %q, want 7", got) + } +} + func TestNetHTTPClientDoesNotFollowRedirects(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/elsewhere", http.StatusMovedPermanently) diff --git a/tests/bdd/steps/http_steps.go b/tests/bdd/steps/http_steps.go index f4ce62b0bb..14ad8eb3a2 100644 --- a/tests/bdd/steps/http_steps.go +++ b/tests/bdd/steps/http_steps.go @@ -51,6 +51,7 @@ func registerHTTPSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { ctx.Step(`^the HTTP response body should end with "([^"]*)"$`, sc.theHTTPBodyShouldEndWith) ctx.Step(`^the HTTP status should be (\d+)$`, sc.theHTTPStatusShouldBe) ctx.Step(`^the HTTP status should be (\d+) or (\d+)$`, sc.theHTTPStatusShouldBeOneOf) + ctx.Step(`^the HTTP response header "([^"]*)" should equal "([^"]*)"$`, sc.theHTTPHeaderShouldEqual) ctx.Step(`^the HTTP response body should contain "([^"]*)"$`, sc.theHTTPBodyShouldContain) ctx.Step(`^the HTTP response body should not contain "([^"]*)"$`, sc.theHTTPBodyShouldNotContain) ctx.Step(`^the HTTP response JSON field "([^"]*)" should equal "([^"]*)"$`, sc.theHTTPJSONFieldShouldEqual) @@ -181,6 +182,25 @@ func (sc *ScenarioContext) theHTTPStatusShouldBeOneOf(first, second int) error { return nil } +func (sc *ScenarioContext) theHTTPHeaderShouldEqual(name, value string) error { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("http response header name must be non-empty") + } + resp, err := sc.requireHTTP() + if err != nil { + return err + } + want := dsl.Interpolate(value) + got := resp.Header.Values(name) + if len(got) == 0 { + return fmt.Errorf("http response header %q is absent", name) + } + if got[0] != want { + return fmt.Errorf("http response header %q = %q, want %q", name, dsl.RedactSecrets(got[0]), dsl.RedactSecrets(want)) + } + return nil +} + func (sc *ScenarioContext) theHTTPBodyShouldContain(text string) error { resp, err := sc.requireHTTP() if err != nil { diff --git a/tests/bdd/steps/http_steps_test.go b/tests/bdd/steps/http_steps_test.go index 6a75803858..3e72f61b2f 100644 --- a/tests/bdd/steps/http_steps_test.go +++ b/tests/bdd/steps/http_steps_test.go @@ -20,6 +20,7 @@ package steps import ( "context" "errors" + "net/http" "os" "path/filepath" "strings" @@ -97,6 +98,28 @@ func TestHTTPStatusAssertion(t *testing.T) { } } +func TestHTTPHeaderAssertion(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 429, Header: http.Header{"Retry-After": {"7"}}}) + if err := sc.theHTTPHeaderShouldEqual("Retry-After", "7"); err == nil { + t.Fatal("assertion must fail before any request") + } + if err := sc.iSendHTTPRequest(context.Background(), "POST", "http://x/"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPHeaderShouldEqual("retry-after", "7"); err != nil { + t.Fatalf("matching header: %v", err) + } + if err := sc.theHTTPHeaderShouldEqual("Retry-After", "8"); err == nil || !strings.Contains(err.Error(), `"7"`) { + t.Fatalf("expected mismatch naming the actual value, got %v", err) + } + if err := sc.theHTTPHeaderShouldEqual("X-Missing", "7"); err == nil || !strings.Contains(err.Error(), "absent") { + t.Fatalf("expected absent header failure, got %v", err) + } + if err := sc.theHTTPHeaderShouldEqual(" ", "7"); err == nil { + t.Fatal("blank header name accepted") + } +} + func TestHTTPStatusOneOfAssertion(t *testing.T) { sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 403, Body: "forbidden"}) if err := sc.iSendHTTPRequest(context.Background(), "POST", "http://x/"); err != nil { diff --git a/tools/ci/helm-validate-values/vanity-gateway.yaml b/tools/ci/helm-validate-values/vanity-gateway.yaml index c134361b78..a5c143c30e 100644 --- a/tools/ci/helm-validate-values/vanity-gateway.yaml +++ b/tools/ci/helm-validate-values/vanity-gateway.yaml @@ -17,9 +17,19 @@ vanityGateway: functionType: LLM customHeaders: X-Provider-Feature: enabled + decisions: + example_decision-model: + modelName: example/decision-model + functionID: 00000000-0000-0000-0000-000000000003 + functionType: LLM anthropicMessages: example_native-model: modelName: example/native-model functionID: 00000000-0000-0000-0000-000000000002 functionType: LLM + typesafeSystemOne: + example_decision-model: + modelName: example/decision-model + functionID: 00000000-0000-0000-0000-000000000003 + functionType: LLM vanity: {}