From 6aa8432f95b91ccac9153f8a2d34834e1cf31d48 Mon Sep 17 00:00:00 2001 From: Sachin Prabhakar Date: Tue, 27 Oct 2026 19:23:10 -0700 Subject: [PATCH 1/5] feat(vanity-gateway): add decisions and systemone config sections Add v2config.openai.decisions and v2config.typesafeSystemOne as model mapping sections that accept functionType LLM. Request body sampling stays unsupported for them, so promptCacheKey and firstMessageHash are rejected. The values schema, CI values and chart-render checks cover both sections. Relates to #2922 Signed-off-by: Sachin Prabhakar --- .../values.schema.json | 52 +++++++++++++++++ .../verify-llm-gateway-routing.sh | 52 ++++++++++++++++- .../chart-render/verify-shadow-schema.sh | 33 +++++++++-- .../gateway_config/gateway_config.go | 12 +++- .../gateway_config/gateway_config_test.go | 58 ++++++++++++++++++- .../gateway_config/model_info.go | 1 + .../gateway_config/model_info_test.go | 22 +++++++ .../helm-validate-values/vanity-gateway.yaml | 10 ++++ 8 files changed, 230 insertions(+), 10 deletions(-) diff --git a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json index 93782c2d7e..def02ac087 100644 --- a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json +++ b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json @@ -145,6 +145,9 @@ "anthropicMessages": { "$ref": "#/definitions/openAIStickyEndpointRoutes" }, + "typesafeSystemOne": { + "$ref": "#/definitions/openAILLMEndpointRoutes" + }, "vanity": { "$ref": "#/definitions/vanityRoutes" } @@ -273,6 +276,29 @@ "required": [ "embeddings" ] + }, + { + "properties": { + "decisions": { + "not": { + "additionalProperties": { + "not": { + "properties": { + "functionType": { + "const": "LLM" + } + }, + "required": [ + "functionType" + ] + } + } + } + } + }, + "required": [ + "decisions" + ] } ] } @@ -303,6 +329,29 @@ "required": [ "anthropicMessages" ] + }, + { + "properties": { + "typesafeSystemOne": { + "not": { + "additionalProperties": { + "not": { + "properties": { + "functionType": { + "const": "LLM" + } + }, + "required": [ + "functionType" + ] + } + } + } + } + }, + "required": [ + "typesafeSystemOne" + ] } ] } @@ -357,6 +406,9 @@ "embeddings": { "$ref": "#/definitions/openAILLMEndpointRoutes" }, + "decisions": { + "$ref": "#/definitions/openAILLMEndpointRoutes" + }, "responses": { "$ref": "#/definitions/openAIStickyEndpointRoutes" }, diff --git a/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh b/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh index 9b853b15b5..ecb89cc12c 100755 --- a/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh +++ b/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh @@ -76,11 +76,21 @@ assert_renders "a model without functionType renders with an empty endpoint" write_values "${ENDPOINT}" chatCompletions " functionType: llmGateway" assert_rejected "value must be 'LLM'" "an unrecognized functionType is rejected" -for section in responses embeddings; do +for section in responses embeddings decisions; do write_values "${ENDPOINT}" "${section}" "${LLM}" assert_renders "an LLM model renders in ${section}" done +write_values "" decisions "${LLM}" +assert_rejected "/vanityGateway/config/llmGatewayEndpoint" "an LLM decisions model requires an endpoint" + +write_values "${ENDPOINT}" decisions "${LLM} + usePexec: true" +assert_rejected "a_model/usePexec" "usePexec is rejected on an LLM decisions model" + +write_values "" decisions "" +assert_renders "a decisions model without functionType renders with an empty endpoint" + for section in completions imageGenerations imageEdits imageVariations; do write_values "${ENDPOINT}" "${section}" "${LLM}" assert_rejected "openai/${section}/a_model" "functionType is rejected in ${section}" @@ -156,6 +166,46 @@ write_messages_values "${ENDPOINT}" "${MESSAGES_LLM} sessionTimeout: 900" assert_rejected "anthropicMessages/a_model/sessionTimeout" "sessionTimeout is rejected on an LLM Messages model" +# typesafeSystemOne is a top-level sibling of anthropicMessages. +write_system_one_values() { + local endpoint="$1" extra="$2" + cat > "${VALUES}" < "${VALUES}" < Date: Tue, 27 Oct 2026 19:27:24 -0700 Subject: [PATCH 2/5] feat(vanity-gateway): serve /v1/decisions and /v1/systemone Register both routes behind the shared request size limit and resolve the vanity model name in each route's own section. The body is forwarded unchanged and stream is ignored, so Accept is not set to event-stream. Gateway-generated errors now use the shape of the route the client called: the OpenAI shape on /v1/decisions and the System One shape on /v1/systemone, with request_id equal to the X-Request-Id response header. One writeNativeError helper covers body errors, unknown model (now JSON instead of plain text), offline and end-of-life, client cancel and proxy failure. The protocol is stored in the request context at route entry because usePexec rewrites the URL path before proxying. Backend errors still pass through unchanged. Relates to #2922 Signed-off-by: Sachin Prabhakar --- .../vanity-gateway/gateway/BUILD.bazel | 2 + .../gateway/decision_routes_test.go | 424 ++++++++++++++++++ .../vanity-gateway/gateway/h2.go | 2 + .../gateway/llm_gateway_director.go | 6 +- .../gateway/llm_gateway_director_test.go | 2 +- .../vanity-gateway/gateway/native_errors.go | 161 +++++++ .../vanity-gateway/gateway/openai_director.go | 104 +++-- .../vanity-gateway/gateway/request_id_test.go | 5 + .../vanity-gateway/gateway/tracing_attrs.go | 2 + .../vanity-gateway/gateway/vanity_director.go | 16 +- 10 files changed, 677 insertions(+), 47 deletions(-) create mode 100644 src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go create mode 100644 src/invocation-plane-services/vanity-gateway/gateway/native_errors.go diff --git a/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel b/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel index 3e1f418c84..a1c2470d47 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel +++ b/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "h2.go", "health.go", "llm_gateway_director.go", + "native_errors.go", "openai_director.go", "shadow.go", "shadow_metrics.go", @@ -62,6 +63,7 @@ go_test( name = "gateway_test", srcs = [ "client_metrics_test.go", + "decision_routes_test.go", "error_origin_test.go", "gateway_test.go", "h2_test.go", diff --git a/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go b/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go new file mode 100644 index 0000000000..264cf0448e --- /dev/null +++ b/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go @@ -0,0 +1,424 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + config "ai-api-gateway-service/gateway_config" + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + decisionModel = "acme/decision-model" + imagesHeader = "TypeSafe-Danger-Zone-Images" + decisionFunctionID = "decision-func-1" + fastAPIValidationErr = `{"detail":[{"loc":["body","questions"],"msg":"field required","type":"missing"}]}` +) + +type systemOneErrorBody struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` + RequestID string `json:"request_id"` +} + +// decisionMappings enables decisionModel on the listed routes only. +func decisionMappings(decisions, systemOne bool, mutate func(*config.ModelFunctionDetails)) *config.GatewayConfig { + entry := config.ModelFunctionDetails{ModelName: decisionModel, FunctionID: decisionFunctionID} + if mutate != nil { + mutate(&entry) + } + mappings := &config.GatewayConfig{} + mappings.OpenAI.Host = openAIHost + if decisions { + mappings.OpenAI.Decisions = map[string]config.ModelFunctionDetails{"d": entry} + } + if systemOne { + mappings.TypeSafeSystemOne = map[string]config.ModelFunctionDetails{"s": entry} + } + return mappings +} + +func llmDecision(entry *config.ModelFunctionDetails) { entry.FunctionType = config.FunctionTypeLLM } + +func postDecision(t *testing.T, mux http.Handler, path, body string, headers map[string]string) *httptest.ResponseRecorder { + t.Helper() + req := openAIRequest(t, path, body) + req.Header.Set("Content-Type", "application/json") + for name, value := range headers { + req.Header.Set(name, value) + } + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + return rec +} + +func requireOpenAIErrorBody(t *testing.T, rec *httptest.ResponseRecorder, status int) OpenAIError { + t.Helper() + require.Equal(t, status, rec.Code, rec.Body.String()) + require.Equal(t, "application/json", rec.Header().Get("Content-Type")) + var payload OpenAIError + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &payload), rec.Body.String()) + require.NotEmpty(t, payload.Error.Message) + require.NotEmpty(t, payload.Error.Code) + return payload +} + +func requireSystemOneErrorBody(t *testing.T, rec *httptest.ResponseRecorder, status int, code string) systemOneErrorBody { + t.Helper() + require.Equal(t, status, rec.Code, rec.Body.String()) + require.Equal(t, "application/json", rec.Header().Get("Content-Type")) + var payload systemOneErrorBody + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &payload), rec.Body.String()) + require.Equal(t, code, payload.Error.Code) + require.NotEmpty(t, payload.Error.Message) + requestID := rec.Header().Get("X-Request-Id") + require.NotEmpty(t, requestID) + require.Equal(t, requestID, payload.RequestID, "top-level request_id must equal the X-Request-Id header") + require.Equal(t, requestID, payload.Error.RequestID, "error.request_id must equal the X-Request-Id header") + return payload +} + +func TestDecisionRoutesForwardDirectWithoutTranslation(t *testing.T) { + for _, path := range []string{decisionsPath, systemOnePath} { + t.Run(path, func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"answers":{"q1":{"yes":0.9}}}`)) + }) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + + body := `{"model":"` + decisionModel + `","input":"is it red?","questions":[{"type":"predicate","text":"red?"}],"extra":{"keep":true}}` + rec := postDecision(t, mux, path, body, map[string]string{imagesHeader: "true"}) + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.JSONEq(t, `{"answers":{"q1":{"yes":0.9}}}`, rec.Body.String()) + received := awaitRequest(t, requests) + assert.Equal(t, path, received.path, "the route is kept when usePexec is false") + assert.Equal(t, decisionFunctionID, received.headers.Get("function-id")) + assert.Equal(t, "true", received.headers.Get(imagesHeader), "vendor image header must pass unchanged") + assert.Equal(t, body, received.body, "the body must pass through unchanged") + }) + } +} + +func TestDecisionRoutesForwardThroughLLMGatewayWithImageHeader(t *testing.T) { + for _, path := range []string{decisionsPath, systemOnePath} { + t.Run(path, func(t *testing.T) { + nvcfRequests := make(chan capturedRequest, 1) + nvcfBackend := captureServer(t, nvcfRequests, nil) + llmRequests := make(chan capturedRequest, 1) + llmBackend := captureServer(t, llmRequests, nil) + mux := openAIMux(t, decisionMappings(true, true, llmDecision), nvcfBackend.URL, llmBackend.URL) + + rec := postDecision(t, mux, path, `{"model":"`+decisionModel+`","input":"x"}`, map[string]string{imagesHeader: "true"}) + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + received := awaitRequest(t, llmRequests) + assert.Equal(t, path, received.path) + assert.Contains(t, received.body, `"model":"`+decisionFunctionID+"/"+decisionModel+`"`) + assert.Equal(t, "true", received.headers.Get(imagesHeader)) + assert.Empty(t, received.headers.Get("function-id")) + assert.Empty(t, nvcfRequests) + }) + } +} + +func TestDecisionRoutesIgnoreStreamTrueForAccept(t *testing.T) { + for _, tc := range []struct { + path string + wantAccept string + }{ + {decisionsPath, ""}, + {systemOnePath, ""}, + } { + t.Run(tc.path, func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, tc.path, `{"model":"`+decisionModel+`","stream":true}`, nil) + + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, tc.wantAccept, awaitRequest(t, requests).headers.Get("Accept")) + }) + } + + t.Run("control: chat completions still sets Accept for stream true", func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mappings := decisionMappings(true, true, nil) + mappings.OpenAI.ChatCompletions = mappings.OpenAI.Decisions + mux := openAIMux(t, mappings, backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, "/v1/chat/completions", `{"model":"`+decisionModel+`","stream":true}`, nil) + + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "text/event-stream", awaitRequest(t, requests).headers.Get("Accept")) + }) +} + +func TestDecisionRoutesAcceptLargeImageBodies(t *testing.T) { + image := strings.Repeat("A", 8*1024*1024) + body := `{"model":"` + decisionModel + `","state":{"image":"data:image/png;base64,` + image + `"}}` + for _, path := range []string{decisionsPath, systemOnePath} { + t.Run(path, func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, path, body, map[string]string{imagesHeader: "true"}) + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Equal(t, len(body), len(awaitRequest(t, requests).body)) + }) + } +} + +func TestDecisionRoutesUnknownModelReturnsNativeJSON404(t *testing.T) { + mux := openAIMux(t, decisionMappings(true, true, nil), "http://nvcf.invalid", "http://llm.invalid") + + rec := postDecision(t, mux, decisionsPath, `{"model":"nope"}`, nil) + assert.Equal(t, "model_not_found", requireOpenAIErrorBody(t, rec, http.StatusNotFound).Error.Code) + assert.Equal(t, "gateway_rejected", rec.Header().Get("NVCF-Error-Source")) + + rec = postDecision(t, mux, systemOnePath, `{"model":"nope"}`, nil) + requireSystemOneErrorBody(t, rec, http.StatusNotFound, "not_found") + assert.Equal(t, "gateway_rejected", rec.Header().Get("NVCF-Error-Source")) +} + +func TestDecisionModelEnabledForOneRouteIsNotFoundOnTheOther(t *testing.T) { + t.Run("decisions only", func(t *testing.T) { + requests := make(chan capturedRequest, 2) + backend := captureServer(t, requests, nil) + mux := openAIMux(t, decisionMappings(true, false, nil), backend.URL, "http://llm.invalid") + + requireSystemOneErrorBody(t, postDecision(t, mux, systemOnePath, `{"model":"`+decisionModel+`"}`, nil), http.StatusNotFound, "not_found") + assert.Empty(t, requests, "the other route must not reach the backend") + require.Equal(t, http.StatusOK, postDecision(t, mux, decisionsPath, `{"model":"`+decisionModel+`"}`, nil).Code) + }) + t.Run("systemone only", func(t *testing.T) { + requests := make(chan capturedRequest, 2) + backend := captureServer(t, requests, nil) + mux := openAIMux(t, decisionMappings(false, true, nil), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, decisionsPath, `{"model":"`+decisionModel+`"}`, nil) + requireOpenAIErrorBody(t, rec, http.StatusNotFound) + assert.Empty(t, requests) + require.Equal(t, http.StatusOK, postDecision(t, mux, systemOnePath, `{"model":"`+decisionModel+`"}`, nil).Code) + }) +} + +func TestDecisionRoutesLocalErrorsUseNativeShape(t *testing.T) { + tooLarge := `{"model":"` + strings.Repeat("a", maxRequestSize) + `"}` + for _, tc := range []struct { + name, body string + status int + s1Code string + }{ + {"malformed json", `{`, http.StatusBadRequest, "invalid_json"}, + {"model wrong type", `{"model":1}`, http.StatusBadRequest, "invalid_json"}, + {"missing model", `{"input":"x"}`, http.StatusBadRequest, "invalid_request"}, + {"body too large", tooLarge, http.StatusRequestEntityTooLarge, "request_too_large"}, + } { + t.Run("decisions "+tc.name, func(t *testing.T) { + mux := openAIMux(t, decisionMappings(true, true, nil), "http://nvcf.invalid", "http://llm.invalid") + rec := postDecision(t, mux, decisionsPath, tc.body, nil) + assert.Equal(t, tc.s1Code, requireOpenAIErrorBody(t, rec, tc.status).Error.Code) + }) + t.Run("systemone "+tc.name, func(t *testing.T) { + mux := openAIMux(t, decisionMappings(true, true, nil), "http://nvcf.invalid", "http://llm.invalid") + requireSystemOneErrorBody(t, postDecision(t, mux, systemOnePath, tc.body, nil), tc.status, tc.s1Code) + }) + } +} + +func TestDecisionRoutesOfflineAndEndOfLifeUseNativeShape(t *testing.T) { + for _, llm := range []bool{false, true} { + name := "direct" + if llm { + name = "llm" + } + t.Run(name+" offline", func(t *testing.T) { + mutate := func(e *config.ModelFunctionDetails) { + if llm { + llmDecision(e) + } + e.OfflineMessage = "down for maintenance" + } + mux := openAIMux(t, decisionMappings(true, true, mutate), "http://nvcf.invalid", "http://llm.invalid") + + rec := postDecision(t, mux, decisionsPath, `{"model":"`+decisionModel+`"}`, nil) + assert.Equal(t, "provider_unavailable", requireOpenAIErrorBody(t, rec, http.StatusServiceUnavailable).Error.Code) + assert.Equal(t, "10800", rec.Header().Get("Retry-After")) + assert.Contains(t, rec.Body.String(), "down for maintenance") + + rec = postDecision(t, mux, systemOnePath, `{"model":"`+decisionModel+`"}`, nil) + requireSystemOneErrorBody(t, rec, http.StatusServiceUnavailable, "provider_unavailable") + assert.Equal(t, "10800", rec.Header().Get("Retry-After")) + }) + t.Run(name+" end of life", func(t *testing.T) { + mutate := func(e *config.ModelFunctionDetails) { + if llm { + llmDecision(e) + } + e.EOL = time.Now().Add(-24 * time.Hour) + } + mux := openAIMux(t, decisionMappings(true, true, mutate), "http://nvcf.invalid", "http://llm.invalid") + + requireOpenAIErrorBody(t, postDecision(t, mux, decisionsPath, `{"model":"`+decisionModel+`"}`, nil), http.StatusGone) + requireSystemOneErrorBody(t, postDecision(t, mux, systemOnePath, `{"model":"`+decisionModel+`"}`, nil), http.StatusGone, "not_found") + }) + } +} + +// With usePexec the director rewrites the URL path to the pexec path before it +// proxies, so the error shape must come from the route the client called. +func TestDecisionRoutesProxyFailureKeepsNativeShapeAfterPexecPathRewrite(t *testing.T) { + down := httptest.NewServer(http.NotFoundHandler()) + downURL := down.URL + down.Close() + + for _, usePexec := range []bool{true, false} { + mutate := func(e *config.ModelFunctionDetails) { e.UsePexec = usePexec } + mux := openAIMux(t, decisionMappings(true, true, mutate), downURL, "http://llm.invalid") + + name := "pexec" + if !usePexec { + name = "no pexec" + } + t.Run(name+" systemone", func(t *testing.T) { + rec := postDecision(t, mux, systemOnePath, `{"model":"`+decisionModel+`"}`, nil) + requireSystemOneErrorBody(t, rec, http.StatusBadGateway, "provider_unavailable") + }) + t.Run(name+" decisions", func(t *testing.T) { + rec := postDecision(t, mux, decisionsPath, `{"model":"`+decisionModel+`"}`, nil) + requireOpenAIErrorBody(t, rec, http.StatusBadGateway) + assert.NotContains(t, rec.Header().Get("Content-Type"), "problem+json") + }) + } +} + +func TestDecisionRoutesClientCancelUsesNativeShape(t *testing.T) { + backend := captureServer(t, make(chan capturedRequest, 1), nil) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + req := openAIRequest(t, systemOnePath, `{"model":"`+decisionModel+`"}`).WithContext(ctx) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + requireSystemOneErrorBody(t, rec, 499, "internal_error") + + req = openAIRequest(t, decisionsPath, `{"model":"`+decisionModel+`"}`).WithContext(ctx) + rec = httptest.NewRecorder() + mux.ServeHTTP(rec, req) + requireOpenAIErrorBody(t, rec, 499) +} + +func TestDecisionRoutesPassBackendErrorsThroughUnchanged(t *testing.T) { + for _, path := range []string{decisionsPath, systemOnePath} { + t.Run(path+" validation body", func(t *testing.T) { + backend := captureServer(t, make(chan capturedRequest, 1), func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-System-One-Error-Code", "invalid_request") + w.WriteHeader(http.StatusUnprocessableEntity) + _, _ = w.Write([]byte(fastAPIValidationErr)) + }) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, path, `{"model":"`+decisionModel+`"}`, nil) + + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code) + assert.Equal(t, fastAPIValidationErr, rec.Body.String()) + assert.Equal(t, "invalid_request", rec.Header().Get("X-System-One-Error-Code")) + }) + t.Run(path+" rate limit with Retry-After", func(t *testing.T) { + backend := captureServer(t, make(chan capturedRequest, 1), func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "7") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"detail":"slow down"}`)) + }) + mux := openAIMux(t, decisionMappings(true, true, nil), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, path, `{"model":"`+decisionModel+`"}`, nil) + + assert.Equal(t, http.StatusTooManyRequests, rec.Code) + assert.Equal(t, `{"detail":"slow down"}`, rec.Body.String()) + assert.Equal(t, "7", rec.Header().Get("Retry-After")) + }) + } +} + +func TestDecisionRoutesAreNotReachableForChatOnlyModels(t *testing.T) { + llmRequests := make(chan capturedRequest, 1) + llmBackend := captureServer(t, llmRequests, nil) + mux := openAIMux(t, llmMappings(llmModelEntry()), "http://nvcf.invalid", llmBackend.URL) + + for _, path := range []string{decisionsPath, systemOnePath} { + rec := postDecision(t, mux, path, `{"model":"`+publicModel+`"}`, nil) + assert.Equal(t, http.StatusNotFound, rec.Code, path) + assert.Equal(t, "application/json", rec.Header().Get("Content-Type"), path) + } + assert.Empty(t, llmRequests) +} + +func TestDecisionRoutesRejectNonPost(t *testing.T) { + mux := openAIMux(t, decisionMappings(true, true, nil), "http://nvcf.invalid", "http://llm.invalid") + for _, path := range []string{decisionsPath, systemOnePath} { + req := httptest.NewRequest(http.MethodGet, path, bytes.NewBufferString("")) + req.Host = openAIHost + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code, path) + } +} + +// With usePexec the invocation service receives the pexec path, so a function +// serving both routes reaches only its default path. The image header and the +// body still pass through. +func TestDecisionRoutesUsePexecRewritesPathAndKeepsHeaderAndBody(t *testing.T) { + for _, path := range []string{decisionsPath, systemOnePath} { + t.Run(path, func(t *testing.T) { + requests := make(chan capturedRequest, 1) + backend := captureServer(t, requests, nil) + mux := openAIMux(t, decisionMappings(true, true, func(e *config.ModelFunctionDetails) { e.UsePexec = true }), backend.URL, "http://llm.invalid") + + body := `{"model":"` + decisionModel + `","state":{}}` + rec := postDecision(t, mux, path, body, map[string]string{imagesHeader: "true"}) + + require.Equal(t, http.StatusOK, rec.Code) + received := awaitRequest(t, requests) + assert.Equal(t, "/v2/nvcf/pexec/functions/"+decisionFunctionID, received.path) + assert.Equal(t, "true", received.headers.Get(imagesHeader)) + assert.Equal(t, body, received.body) + }) + } +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/h2.go b/src/invocation-plane-services/vanity-gateway/gateway/h2.go index bee5d218f6..4501d96ab2 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/h2.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/h2.go @@ -184,6 +184,8 @@ func registerModelAPIs(hostRouter *middleware.HostRouter, mappings *config.Gatew r.Method(http.MethodPost, "/v1/completions", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeCompletions))) r.Method(http.MethodPost, "/v1/embeddings", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeEmbeddings))) r.Method(http.MethodPost, "/v1/messages", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeMessages))) + r.Method(http.MethodPost, decisionsPath, chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeDecisions))) + r.Method(http.MethodPost, systemOnePath, chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeSystemOne))) r.Method(http.MethodPost, "/v1/responses", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeResponses))) r.Method(http.MethodPost, "/v1/images/generations", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeImageGenerations))) r.Method(http.MethodPost, "/v1/images/edits", chimiddleware.RequestSize(maxRequestSize)(http.HandlerFunc(modelDirector.ServeImageEdits))) diff --git a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go index ea6f4782a4..c5def24893 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director.go @@ -99,11 +99,7 @@ func stripNVCFRoutingHeaders(request *http.Request) { func (d *LLMGatewayDirector) ServeProxy(target LLMGatewayRequest, writer http.ResponseWriter, request *http.Request) error { span := trace.SpanFromContext(request.Context()) - endpointType := traceAttrValueEndpointLLMGateway - if request.URL.Path == "/v1/messages" { - endpointType = traceAttrValueEndpointAnthropic - } - span.SetAttributes(traceAttrEndpointType.String(endpointType)) + span.SetAttributes(traceAttrEndpointType.String(endpointTypeForPath(request.URL.Path, traceAttrValueEndpointLLMGateway))) if writeFunctionStatusError(writer, request, target.OfflineMessage, target.EOL, "") { return nil diff --git a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go index 4ac152a006..4f971850a9 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/llm_gateway_director_test.go @@ -636,7 +636,7 @@ func TestOpenAIDirector_LLMModelIsNotReachableOnUnmappedEndpoints(t *testing.T) mux := openAIMux(t, llmMappings(llmModelEntry()), "http://nvcf.invalid", llmBackend.URL) - for _, path := range []string{"/v1/responses", "/v1/embeddings", "/v1/completions", "/v1/messages"} { + for _, path := range []string{"/v1/responses", "/v1/embeddings", "/v1/completions", "/v1/messages", "/v1/decisions", "/v1/systemone"} { t.Run(path, func(t *testing.T) { rec := httptest.NewRecorder() mux.ServeHTTP(rec, openAIRequest(t, path, `{"model":"`+publicModel+`","input":"hi"}`)) diff --git a/src/invocation-plane-services/vanity-gateway/gateway/native_errors.go b/src/invocation-plane-services/vanity-gateway/gateway/native_errors.go new file mode 100644 index 0000000000..4c0271a487 --- /dev/null +++ b/src/invocation-plane-services/vanity-gateway/gateway/native_errors.go @@ -0,0 +1,161 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + "ai-api-gateway-service/middleware" + "context" + "net/http" + + "github.com/goccy/go-json" +) + +// nativeProtocol names the API whose error shape a response must use. +type nativeProtocol string + +const ( + nativeProtocolNone nativeProtocol = "" + nativeProtocolMessages nativeProtocol = "messages" + nativeProtocolDecisions nativeProtocol = "decisions" + nativeProtocolSystemOne nativeProtocol = "systemone" +) + +const ( + decisionsPath = "/v1/decisions" + systemOnePath = "/v1/systemone" + messagesPath = "/v1/messages" +) + +// System One error codes. The gateway also uses them as the code on the +// OpenAI-shaped decisions errors, except where openAIErrorCode says otherwise. +const ( + errKindInvalidJSON = "invalid_json" + errKindInvalidRequest = "invalid_request" + errKindNotFound = "not_found" + errKindRequestTooLarge = "request_too_large" + errKindInternal = "internal_error" + errKindProviderUnavailable = "provider_unavailable" +) + +const nativeProtocolKey contextKey = tooManyRequestsKey + 1 + +func protocolForPath(urlPath string) nativeProtocol { + switch urlPath { + case messagesPath: + return nativeProtocolMessages + case decisionsPath: + return nativeProtocolDecisions + case systemOnePath: + return nativeProtocolSystemOne + } + return nativeProtocolNone +} + +// withNativeProtocol records the protocol of the route the client called. +// applyExecTarget rewrites URL.Path when usePexec is set, so the path alone +// cannot name the protocol once a request is being proxied. +func withNativeProtocol(request *http.Request, protocol nativeProtocol) *http.Request { + return request.WithContext(context.WithValue(request.Context(), nativeProtocolKey, protocol)) +} + +// requestProtocol returns the protocol stored at route entry, falling back to +// the request path for requests that never went through a route entry. +func requestProtocol(request *http.Request) nativeProtocol { + if protocol, ok := request.Context().Value(nativeProtocolKey).(nativeProtocol); ok { + return protocol + } + return protocolForPath(request.URL.Path) +} + +// endpointTypeForPath is the endpoint.type span value for a request path. +// fallback applies to the OpenAI routes and to the LLM Gateway hop. +func endpointTypeForPath(urlPath, fallback string) string { + switch urlPath { + case messagesPath: + return traceAttrValueEndpointAnthropic + case decisionsPath: + return traceAttrValueEndpointDecisions + case systemOnePath: + return traceAttrValueEndpointSystemOne + } + return fallback +} + +// writeNativeError writes a gateway-generated error in the shape of the route +// the client called and reports whether it did. It writes nothing for the +// OpenAI chat-style routes, which keep their own error handling. +func writeNativeError(writer http.ResponseWriter, request *http.Request, status int, kind, message string) bool { + switch requestProtocol(request) { + case nativeProtocolMessages: + writeAnthropicError(writer, status, message) + case nativeProtocolDecisions: + writeDecisionsError(writer, status, kind, message) + case nativeProtocolSystemOne: + writeSystemOneError(writer, status, kind, message) + default: + return false + } + return true +} + +func openAIErrorCode(status int, kind string) string { + switch status { + case http.StatusNotFound: + return "model_not_found" + case http.StatusGone: + return "model_expired" + } + return kind +} + +func writeDecisionsError(writer http.ResponseWriter, status int, kind, message string) { + errorType := "invalid_request_error" + if status >= http.StatusInternalServerError { + errorType = "server_error" + } + writer.Header().Set(contentTypeHeader, contentTypeApplicationJSON) + writer.WriteHeader(status) + _ = json.NewEncoder(writer).Encode(OpenAIError{Error: CustomOpenAIError{ + Message: message, + Type: errorType, + Code: openAIErrorCode(status, kind), + }}) +} + +type systemOneErrorDetail struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type systemOneError struct { + Error systemOneErrorDetail `json:"error"` + RequestID string `json:"request_id"` +} + +// writeSystemOneError writes the System One error body. Gateway-written errors +// carry no X-System-One-* headers: those mark errors the model server relayed. +func writeSystemOneError(writer http.ResponseWriter, status int, kind, message string) { + requestID := writer.Header().Get(middleware.RequestIDHeader) + writer.Header().Set(contentTypeHeader, contentTypeApplicationJSON) + writer.WriteHeader(status) + _ = json.NewEncoder(writer).Encode(systemOneError{ + Error: systemOneErrorDetail{Code: kind, Message: message, RequestID: requestID}, + RequestID: requestID, + }) +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go index 24063321c6..23227eca01 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go @@ -74,6 +74,8 @@ type ModelDirector struct { embeddings ModelMapping responses ModelMapping messages ModelMapping + decisions ModelMapping + systemOne ModelMapping imageGenerations ModelMapping imageEdits ModelMapping imageVariations ModelMapping @@ -350,6 +352,16 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp return nil, err } + decisions, err := buildModelMapping(convertIntoModelNameToFunctionIdAndVersionIdMappingV2(mapping.OpenAI.Decisions, "openai.decisions"), privateModelMatcher) + if err != nil { + return nil, err + } + + systemOne, err := buildModelMapping(convertIntoModelNameToFunctionIdAndVersionIdMappingV2(mapping.TypeSafeSystemOne, "typesafeSystemOne"), privateModelMatcher) + if err != nil { + return nil, err + } + imageGenerations, err := buildModelMapping(convertIntoModelNameToFunctionIdAndVersionIdMappingV2(mapping.OpenAI.ImageGenerations, "openai.imageGenerations"), privateModelMatcher) if err != nil { return nil, err @@ -435,6 +447,8 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp embeddings: embeddings, responses: responses, messages: messages, + decisions: decisions, + systemOne: systemOne, imageGenerations: imageGenerations, imageEdits: imageEdits, imageVariations: imageVariations, @@ -589,6 +603,21 @@ func (d *ModelDirector) ServeMessages(writer http.ResponseWriter, request *http. d.proxyModelMappedRequest(writer, request, d.messages.modelNameToNVCFUrl, shadowSamplingEndpointMessages) } +// ServeDecisions passes POST /v1/decisions through to the model's function. +// The body is not parsed beyond model, and errors the gateway writes use the +// OpenAI shape. +func (d *ModelDirector) ServeDecisions(writer http.ResponseWriter, request *http.Request) { + request = withNativeProtocol(request, nativeProtocolDecisions) + d.proxyModelMappedRequest(writer, request, d.decisions.modelNameToNVCFUrl, shadowSamplingEndpointNone) +} + +// ServeSystemOne passes POST /v1/systemone through to the model's function. +// Errors the gateway writes use the System One shape. +func (d *ModelDirector) ServeSystemOne(writer http.ResponseWriter, request *http.Request) { + request = withNativeProtocol(request, nativeProtocolSystemOne) + d.proxyModelMappedRequest(writer, request, d.systemOne.modelNameToNVCFUrl, shadowSamplingEndpointNone) +} + func (d *ModelDirector) ServeImageGenerations(writer http.ResponseWriter, request *http.Request) { d.proxyModelMappedRequest(writer, request, d.imageGenerations.modelNameToNVCFUrl, shadowSamplingEndpointNone) } @@ -689,11 +718,7 @@ func (d *ModelDirector) GetModel(writer http.ResponseWriter, request *http.Reque func (d *ModelDirector) proxyModelMappedRequest(writer http.ResponseWriter, request *http.Request, modelToNVCFUrl map[string]FunctionInfo, endpoint shadowSamplingEndpoint) { span := trace.SpanFromContext(request.Context()) - endpointType := traceAttrValueEndpointOpenAI - if request.URL.Path == "/v1/messages" { - endpointType = traceAttrValueEndpointAnthropic - } - span.SetAttributes(traceAttrEndpointType.String(endpointType)) + span.SetAttributes(traceAttrEndpointType.String(endpointTypeForPath(request.URL.Path, traceAttrValueEndpointOpenAI))) setShadowSpanAttribute(span, request) resolved, handled := d.resolveModelMappedRequest(writer, request, modelToNVCFUrl) @@ -733,19 +758,9 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re body, err := extractOpenAIRequestBody(request) if err != nil { markGatewayRejected(writer, request) - if request.URL.Path == "/v1/messages" { - status, message := http.StatusInternalServerError, "failed to read Messages request" - if errors.Is(err, errInvalidJSONBody) || errors.Is(err, errModelFieldMissing) { - status, message = http.StatusBadRequest, "invalid Messages request JSON" - } - var maxBytesErr *http.MaxBytesError - if errors.As(err, &maxBytesErr) { - status, message = http.StatusRequestEntityTooLarge, "request body is too large" - } - if errors.Is(err, errModelFieldMissing) { - message = "model field is required" - } - writeAnthropicError(writer, status, message) + if protocol := requestProtocol(request); protocol != nativeProtocolNone { + status, kind, message := requestReadFailure(protocol, err) + writeNativeError(writer, request, status, kind, message) return resolvedOpenAIRequest{}, true } var maxBytesErr *http.MaxBytesError @@ -796,9 +811,7 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re if !ok { _ = request.Body.Close() markGatewayRejected(writer, request) - if request.URL.Path == "/v1/messages" { - writeAnthropicError(writer, http.StatusNotFound, "model not found") - } else { + if !writeNativeError(writer, request, http.StatusNotFound, errKindNotFound, "model not found") { http.NotFound(writer, request) } return resolvedOpenAIRequest{}, true @@ -806,21 +819,7 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re middleware.SetPreUpstreamModel(request.Context(), nvcfUrl.publicModelName) middleware.AddOpenAIRequestMetricAttributes(request.Context(), body.Model, nvcfUrl.functionId) - statusHandled := false - if request.URL.Path == "/v1/messages" { - if nvcfUrl.offlineMessage != "" { - markGatewayRejected(writer, request) - writer.Header().Set("Retry-After", "10800") - writeAnthropicError(writer, http.StatusServiceUnavailable, nvcfUrl.offlineMessage) - statusHandled = true - } else if isModelExpired(nvcfUrl.eol) { - markGatewayRejected(writer, request) - writeAnthropicError(writer, http.StatusGone, "model has reached end of life") - statusHandled = true - } - } else { - statusHandled = writeFunctionStatusError(writer, request, nvcfUrl.offlineMessage, nvcfUrl.eol, body.Model) - } + statusHandled := writeFunctionStatusError(writer, request, nvcfUrl.offlineMessage, nvcfUrl.eol, body.Model) if statusHandled { _ = request.Body.Close() return resolvedOpenAIRequest{}, true @@ -832,7 +831,9 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re } // setPollingHeaderIfNotPresent checks the Accept header, so we have to override it first - if body.Stream { + // Decision routes are non-streaming: stream is ignored and the body is + // forwarded as sent. + if body.Stream && !isDecisionRoute(request) { request.Header.Set("Accept", "text/event-stream") } @@ -1241,3 +1242,32 @@ type OpenAIDirector = ModelDirector func NewOpenAIDirectorV2(mapping *config.GatewayConfig, privateModelMatcher *regexp.Regexp, vanityDirector *VanityDirector, llmGatewayDirector *LLMGatewayDirector, shadower *TrafficShadower) (*ModelDirector, error) { return NewModelDirector(mapping, privateModelMatcher, vanityDirector, llmGatewayDirector, shadower) } + +func isDecisionRoute(request *http.Request) bool { + protocol := requestProtocol(request) + return protocol == nativeProtocolDecisions || protocol == nativeProtocolSystemOne +} + +// requestReadFailure maps a failure to read or parse the request body to the +// status, error code and message of the route's native error. +func requestReadFailure(protocol nativeProtocol, err error) (int, string, string) { + status, kind := http.StatusInternalServerError, errKindInternal + message := "failed to read request" + if protocol == nativeProtocolMessages { + message = "failed to read Messages request" + } + if errors.Is(err, errInvalidJSONBody) || errors.Is(err, errModelFieldMissing) { + status, kind, message = http.StatusBadRequest, errKindInvalidJSON, "request body is not valid JSON" + if protocol == nativeProtocolMessages { + message = "invalid Messages request JSON" + } + } + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + status, kind, message = http.StatusRequestEntityTooLarge, errKindRequestTooLarge, "request body is too large" + } + if errors.Is(err, errModelFieldMissing) { + kind, message = errKindInvalidRequest, "model field is required" + } + return status, kind, message +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go b/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go index 153b2c7f12..6f55fde283 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go @@ -67,6 +67,11 @@ func TestGatewayGeneratedErrorsCarryRequestID(t *testing.T) { {"messages unknown model", "/v1/messages", `{"model":"unknown"}`, nil, http.StatusNotFound}, {"messages body too large", "/v1/messages", tooLarge, nil, http.StatusRequestEntityTooLarge}, {"messages reserved shadow header", "/v1/messages", `{"model":"` + publicModel + `"}`, map[string]string{shadowHeader: "true"}, http.StatusBadRequest}, + {"decisions malformed json", "/v1/decisions", `{`, nil, http.StatusBadRequest}, + {"decisions unknown model", "/v1/decisions", `{"model":"unknown"}`, nil, http.StatusNotFound}, + {"systemone malformed json", "/v1/systemone", `{`, nil, http.StatusBadRequest}, + {"systemone unknown model", "/v1/systemone", `{"model":"unknown"}`, nil, http.StatusNotFound}, + {"systemone body too large", "/v1/systemone", tooLarge, nil, http.StatusRequestEntityTooLarge}, {"chat malformed json", "/v1/chat/completions", `{`, nil, http.StatusBadRequest}, {"chat unknown model", "/v1/chat/completions", `{"model":"unknown"}`, nil, http.StatusNotFound}, } diff --git a/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go b/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go index 0f01b89bc8..bf6e99c093 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go @@ -45,6 +45,8 @@ const ( traceAttrValueEndpointAnthropic = "anthropic" traceAttrValueEndpointOpenAI = "openai" traceAttrValueEndpointLLMGateway = "llm_gateway" + traceAttrValueEndpointDecisions = "decisions" + traceAttrValueEndpointSystemOne = "systemone" shadowDroppedReasonBodyReadError = "body_read_error" shadowDroppedReasonBodyRewriteError = "body_rewrite_error" diff --git a/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go b/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go index 874282a587..8af1287a2a 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go @@ -51,6 +51,11 @@ const statusClientClosedRequest = 499 func writeFunctionStatusError(writer http.ResponseWriter, request *http.Request, offlineMessage string, eol time.Time, name string) bool { if offlineMessage != "" { markGatewayRejected(writer, request) + if requestProtocol(request) != nativeProtocolNone { + writer.Header().Set("Retry-After", "10800") + writeNativeError(writer, request, http.StatusServiceUnavailable, errKindProviderUnavailable, offlineMessage) + return true + } writer.Header().Set("Content-Type", "application/problem+json") writer.Header().Set("Retry-After", "10800") writer.WriteHeader(http.StatusServiceUnavailable) @@ -63,6 +68,11 @@ func writeFunctionStatusError(writer http.ResponseWriter, request *http.Request, return true } if isModelExpired(eol) { + if requestProtocol(request) != nativeProtocolNone { + markGatewayRejected(writer, request) + writeNativeError(writer, request, http.StatusGone, errKindNotFound, "model has reached end of life") + return true + } var detail string if name != "" { detail = fmt.Sprintf("The model '%s' has reached its end of life on %s and is no longer available.", name, eol.Format(time.RFC3339)) @@ -146,8 +156,7 @@ func writeProxyError(writer http.ResponseWriter, request *http.Request, err erro zap.String(string(middleware.GatewayProxyOutcomeMetricAttribute), string(middleware.GatewayProxyOutcomeClientCanceled)), zap.Error(err), ) - if request.URL.Path == "/v1/messages" { - writeAnthropicError(writer, statusClientClosedRequest, "client closed the request") + if writeNativeError(writer, request, statusClientClosedRequest, errKindInternal, "client closed the request") { return } writer.Header().Set("Content-Type", "application/problem+json") @@ -170,8 +179,7 @@ func writeBadGatewayProblem(writer http.ResponseWriter, request *http.Request, e zap.String(string(middleware.GatewayProxyOutcomeMetricAttribute), string(middleware.GatewayProxyOutcomeProxyError)), zap.Error(err), ) - if request.URL.Path == "/v1/messages" { - writeAnthropicError(writer, http.StatusBadGateway, "upstream request failed") + if writeNativeError(writer, request, http.StatusBadGateway, errKindProviderUnavailable, "upstream request failed") { return } writer.Header().Set("Content-Type", "application/problem+json") From a589d43229bc9ba6e52d3befacaa6f547d5b458b Mon Sep 17 00:00:00 2001 From: Sachin Prabhakar Date: Tue, 27 Oct 2026 19:33:54 -0700 Subject: [PATCH 3/5] feat(vanity-gateway): list decision models in /v1/models Include models enabled on /v1/decisions or /v1/systemone in the shared model list and in /v1/models/{model}, with supported_endpoints naming the routes each model serves. Expiry and offline state follow the decision mappings too. Decision models stay out of the Anthropic list. Span endpoint.type reports decisions and systemone for both the direct and the LLM gateway path. Relates to #2922 Signed-off-by: Sachin Prabhakar --- .../gateway/decision_routes_test.go | 50 ++++++++++++ .../gateway/models_protocol_test.go | 76 +++++++++++++++++++ .../vanity-gateway/gateway/openai_director.go | 28 ++++++- 3 files changed, 150 insertions(+), 4 deletions(-) diff --git a/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go b/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go index 264cf0448e..d428b5ac50 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go @@ -24,12 +24,14 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "regexp" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" ) const ( @@ -422,3 +424,51 @@ func TestDecisionRoutesUsePexecRewritesPathAndKeepsHeaderAndBody(t *testing.T) { }) } } + +func TestDecisionRoutesSetEndpointTypeSpanAttribute(t *testing.T) { + for _, tc := range []struct { + path, want string + llm bool + }{ + {decisionsPath, "decisions", false}, + {systemOnePath, "systemone", false}, + {decisionsPath, "decisions", true}, + {systemOnePath, "systemone", true}, + } { + name := tc.path + if tc.llm { + name += " llm" + } + t.Run(name, func(t *testing.T) { + tp, exp := newTestTracerProvider() + defer tp.Shutdown(context.Background()) + backend := captureServer(t, make(chan capturedRequest, 1), nil) + var mutate func(*config.ModelFunctionDetails) + if tc.llm { + mutate = llmDecision + } + mappings := decisionMappings(true, true, mutate) + llm, err := NewLLMGatewayDirector(backend.URL, http.DefaultTransport) + require.NoError(t, err) + vanity, err := NewVanityDirector(backend.URL, http.DefaultTransport) + require.NoError(t, err) + director, err := NewModelDirector(mappings, regexp.MustCompile("^$"), vanity, llm, nil) + require.NoError(t, err) + + req := openAIRequest(t, tc.path, `{"model":"`+decisionModel+`"}`) + ctx, span := tp.Tracer("test").Start(req.Context(), "decision-test") + rec := httptest.NewRecorder() + if tc.path == decisionsPath { + director.ServeDecisions(rec, req.WithContext(ctx)) + } else { + director.ServeSystemOne(rec, req.WithContext(ctx)) + } + span.End() + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + got, ok := spanAttr(exp.GetSpans()[0], attribute.Key("endpoint.type")) + require.True(t, ok) + require.Equal(t, tc.want, got.AsString()) + }) + } +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/models_protocol_test.go b/src/invocation-plane-services/vanity-gateway/gateway/models_protocol_test.go index d99a922455..1e1e16d5e4 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/models_protocol_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/models_protocol_test.go @@ -253,3 +253,79 @@ func TestListModelsForAnthropicClientsUsesMessagesExpiry(t *testing.T) { require.Equal(t, http.StatusOK, getModels(t, mux, "/v1/models/shared/openai-expired", true).Code) require.Equal(t, http.StatusGone, getModels(t, mux, "/v1/models/shared/messages-expired", true).Code) } + +// decisionDiscoveryMux serves decision-only, shared and expired decision models. +func decisionDiscoveryMux(t *testing.T) http.Handler { + t.Helper() + entry := func(name string) config.ModelFunctionDetails { + return config.ModelFunctionDetails{ModelName: name, FunctionID: "func-" + name} + } + expired := entry("decide/expired") + expired.EOL = time.Now().Add(-time.Hour) + offline := entry("decide/offline") + offline.OfflineMessage = "down" + mappings := &config.GatewayConfig{} + mappings.OpenAI.Host = openAIHost + mappings.OpenAI.ChatCompletions = map[string]config.ModelFunctionDetails{"shared": entry("decide/shared")} + mappings.OpenAI.Decisions = map[string]config.ModelFunctionDetails{ + "d": entry("decide/d-only"), + "shared": entry("decide/shared"), + "expired": expired, + "offline": offline, + } + mappings.TypeSafeSystemOne = map[string]config.ModelFunctionDetails{ + "s": entry("decide/s-only"), + "shared": entry("decide/shared"), + } + mux, err := buildChiMux(mappings, Config{ + NvcfApiEndpoint: "http://nvcf.invalid", + LLMGatewayEndpoint: "http://llm.invalid", + PrivateModelNameRegexPattern: "^$", + }) + require.NoError(t, err) + return mux +} + +func TestListModelsIncludesDecisionModelsWithSupportedEndpoints(t *testing.T) { + rec := getModels(t, decisionDiscoveryMux(t), "/v1/models", false) + require.Equal(t, http.StatusOK, rec.Code) + var models ModelListResponse + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &models)) + got := map[string][]string{} + for _, m := range models.Data { + got[m.Id] = m.SupportedEndpoints + } + require.Equal(t, map[string][]string{ + "decide/d-only": {"/v1/decisions"}, + "decide/s-only": {"/v1/systemone"}, + "decide/shared": {"/v1/chat/completions", "/v1/decisions", "/v1/systemone"}, + "decide/offline": {"/v1/decisions"}, + }, got, "the expired model is filtered out") +} + +func TestGetModelDescribesDecisionModels(t *testing.T) { + mux := decisionDiscoveryMux(t) + for model, want := range map[string][]string{ + "decide/d-only": {"/v1/decisions"}, + "decide/s-only": {"/v1/systemone"}, + "decide/shared": {"/v1/chat/completions", "/v1/decisions", "/v1/systemone"}, + } { + rec := getModels(t, mux, "/v1/models/"+model, false) + require.Equal(t, http.StatusOK, rec.Code, model) + var info ModelInfo + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &info)) + require.Equal(t, model, info.Id) + require.Equal(t, want, info.SupportedEndpoints, model) + } + require.Equal(t, http.StatusGone, getModels(t, mux, "/v1/models/decide/expired", false).Code) + require.Equal(t, http.StatusServiceUnavailable, getModels(t, mux, "/v1/models/decide/offline", false).Code) + require.Equal(t, http.StatusNotFound, getModels(t, mux, "/v1/models/decide/missing", false).Code) +} + +func TestAnthropicModelListExcludesDecisionModels(t *testing.T) { + rec := getModels(t, decisionDiscoveryMux(t), "/v1/models", true) + require.Equal(t, http.StatusOK, rec.Code) + var page anthropicModelPage + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &page)) + require.Empty(t, page.Data) +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go index 23227eca01..ae5dba3b5b 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go @@ -385,6 +385,8 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, messages.modelNameToModelInfo, + decisions.modelNameToModelInfo, + systemOne.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo, @@ -393,9 +395,18 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp seen[id] = info } } - // Annotate aliases that support Messages so shared discovery does not - // imply that a Messages-only model accepts OpenAI requests. - for id := range messages.modelNameToModelInfo { + // Annotate aliases that support a native protocol (Messages, decisions or + // System One) so shared discovery does not imply that such a model accepts + // OpenAI chat requests. + nativeIDs := make(map[string]struct{}) + for _, models := range []map[string]ModelInfo{ + messages.modelNameToModelInfo, decisions.modelNameToModelInfo, systemOne.modelNameToModelInfo, + } { + for id := range models { + nativeIDs[id] = struct{}{} + } + } + for id := range nativeIDs { info := seen[id] for _, endpoint := range []struct { path string @@ -406,6 +417,8 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp {"/v1/embeddings", embeddings.modelNameToModelInfo}, {"/v1/responses", responses.modelNameToModelInfo}, {"/v1/messages", messages.modelNameToModelInfo}, + {decisionsPath, decisions.modelNameToModelInfo}, + {systemOnePath, systemOne.modelNameToModelInfo}, {"/v1/images/generations", imageGenerations.modelNameToModelInfo}, {"/v1/images/edits", imageEdits.modelNameToModelInfo}, {"/v1/images/variations", imageVariations.modelNameToModelInfo}, @@ -415,7 +428,7 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp } } seen[id] = info - for _, models := range []map[string]ModelInfo{chatCompletions.modelNameToModelInfo, completions.modelNameToModelInfo, embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, messages.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo} { + for _, models := range []map[string]ModelInfo{chatCompletions.modelNameToModelInfo, completions.modelNameToModelInfo, embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, messages.modelNameToModelInfo, decisions.modelNameToModelInfo, systemOne.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo} { if _, ok := models[id]; ok { models[id] = info } @@ -430,6 +443,7 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo, messages.modelNameToModelInfo, + decisions.modelNameToModelInfo, systemOne.modelNameToModelInfo, } { if _, ok := models[id]; ok { models[id] = info @@ -510,6 +524,8 @@ func (d *ModelDirector) getFilteredModels() []ModelInfo { d.embeddings.modelNameToNVCFUrl, d.responses.modelNameToNVCFUrl, d.messages.modelNameToNVCFUrl, + d.decisions.modelNameToNVCFUrl, + d.systemOne.modelNameToNVCFUrl, d.imageGenerations.modelNameToNVCFUrl, d.imageEdits.modelNameToNVCFUrl, d.imageVariations.modelNameToNVCFUrl, @@ -680,6 +696,8 @@ func (d *ModelDirector) GetModel(writer http.ResponseWriter, request *http.Reque d.chatCompletions.modelNameToModelInfo, d.responses.modelNameToModelInfo, d.messages.modelNameToModelInfo, + d.decisions.modelNameToModelInfo, + d.systemOne.modelNameToModelInfo, d.imageGenerations.modelNameToModelInfo, d.imageEdits.modelNameToModelInfo, d.imageVariations.modelNameToModelInfo, @@ -698,6 +716,8 @@ func (d *ModelDirector) GetModel(writer http.ResponseWriter, request *http.Reque d.chatCompletions.modelNameToNVCFUrl, d.responses.modelNameToNVCFUrl, d.messages.modelNameToNVCFUrl, + d.decisions.modelNameToNVCFUrl, + d.systemOne.modelNameToNVCFUrl, d.imageGenerations.modelNameToNVCFUrl, d.imageEdits.modelNameToNVCFUrl, d.imageVariations.modelNameToNVCFUrl, From f9f54c426ddcfe19e4b34d8d283659e43e0834ed Mon Sep 17 00:00:00 2001 From: Sachin Prabhakar Date: Tue, 27 Oct 2026 19:34:44 -0700 Subject: [PATCH 4/5] docs(vanity-gateway): document decision model routes Describe the beta /v1/decisions and /v1/systemone routes, their mapping sections, the example mapping, error shapes and the usePexec caveat: a usePexec function reaches only its default path, so a function serving both routes should set usePexec to false. Bodies pass through unchanged. Relates to #2922 Signed-off-by: Sachin Prabhakar --- deploy/helm/vanity-gateway/README.md | 52 +++++++- .../vanity-gateway/README.md | 112 +++++++++++++++++- 2 files changed, 153 insertions(+), 11 deletions(-) diff --git a/deploy/helm/vanity-gateway/README.md b/deploy/helm/vanity-gateway/README.md index be2153ab9a..4121e57d53 100644 --- a/deploy/helm/vanity-gateway/README.md +++ b/deploy/helm/vanity-gateway/README.md @@ -106,14 +106,18 @@ or the pod is killed mid-drain. ## Route mapping -`vanityGateway.mappingConfig.v2config` has three sections: +`vanityGateway.mappingConfig.v2config` has four sections: - `openai`: per-endpoint model routes, keyed by endpoint (`chatCompletions`, - `completions`, `embeddings`, `responses`, and the image endpoints). Each route - requires `modelName` and `functionID`. + `completions`, `embeddings`, `responses`, `decisions`, and the image + endpoints). Each route requires `modelName` and `functionID`. `decisions` + serves the beta `POST /v1/decisions` route. - `anthropicMessages`: native Anthropic Messages model routes, served on `POST /v1/messages`. Routes use the same fields as `openai.chatCompletions` and share the API host set by `openai.host`. +- `typesafeSystemOne`: beta TypeSafe System One model routes, served on + `POST /v1/systemone`. A top-level sibling of `anthropicMessages`, with the + same fields as `openai.decisions`. - `vanity`: host-based routes, each requiring a `host` and a `paths` map. Each path requires `path` and `functionID`. @@ -123,13 +127,14 @@ across all routes. A model may set `functionType: LLM` to be served by the LLM Gateway instead of the invocation service, supported in `openai.chatCompletions`, -`openai.responses`, `openai.embeddings`, and `anthropicMessages`. Callers still send the public `modelName`; the gateway +`openai.responses`, `openai.embeddings`, `openai.decisions`, `anthropicMessages`, and +`typesafeSystemOne`. Callers still send the public `modelName`; the gateway rewrites the request model to `functionID/modelName` before forwarding. The values schema enforces what the gateway checks at startup, so a values file that would fail the container fails the render instead: -- `functionType` is rejected outside those four endpoints +- `functionType` is rejected outside those endpoints - `usePexec`, `outgoingPathOverride`, and `sessionTimeout` are rejected on such a model, since the LLM Gateway ignores them. An explicit `false`, `""`, or `0` is accepted, because that is what an absent key produces @@ -162,6 +167,43 @@ Messages. Upgrade Pylon workers, the Stargate router, and the LLM Gateway before enabling Messages routes. See `docs/overview/llm-gateway.md` for the request, accounting, and Claude Code details. +Decision models (beta) are mapped per route. A model is enabled for a route only +when listed in that route's section, and a function that serves both formats is +listed in both: + +```yaml +vanityGateway: + mappingConfig: + v2config: + openai: + host: api.example.com + decisions: + example_decision-model: + modelName: example/decision-model + functionID: + functionVersionID: + example_decision-model-llm: + modelName: example/decision-model-llm + functionID: + functionType: LLM + typesafeSystemOne: + example_decision-model: + modelName: example/decision-model + functionID: + functionVersionID: +``` + +Request and response bodies pass through unchanged, and responses are not +streamed. With `usePexec: true` the invocation service normally sends the request +to the default invocation path registered for the function, whichever route the +client called, so a `usePexec: true` function serves only its default path. +A function that serves both routes should set `usePexec: false`. Models with +`functionType: LLM` need LLM Gateway and Stargate builds that serve these routes; +enable them only after those builds run in the environment. Request body +sampling (`promptCacheKey`, `firstMessageHash`) and `promptCacheKeyHeaders` are +rejected in the decision sections. See the service README for error shapes and +`/v1/models` behavior. + Shadow traffic is supported. Each shadow target is resolved from the same model table and routed by its own `functionType`, so a shadow of an LLM model reaches the LLM Gateway, and an LLM model may shadow a model served by the invocation diff --git a/src/invocation-plane-services/vanity-gateway/README.md b/src/invocation-plane-services/vanity-gateway/README.md index dc99be9951..14f0236db9 100644 --- a/src/invocation-plane-services/vanity-gateway/README.md +++ b/src/invocation-plane-services/vanity-gateway/README.md @@ -22,11 +22,14 @@ Use the [release workflow](../../../docs/dev/github-release-process.md) for sour ## About The Vanity Gateway fronts invocation APIs with route mappings from a YAML config -file. It supports three request shapes: +file. It supports four request shapes: - OpenAI-compatible `/v1/*` endpoints that map public model names to functions. - The native Anthropic Messages endpoint, `POST /v1/messages`, configured under `v2config.anthropicMessages`. +- Beta decision model endpoints, `POST /v1/decisions` and + `POST /v1/systemone`, configured under `v2config.openai.decisions` and + `v2config.typesafeSystemOne`. - Vanity URL routes that forward caller-defined paths to function executions and polling. The gateway is meant to run against an upstream invocation service with pexec @@ -156,6 +159,7 @@ v2config: completions: {} embeddings: {} responses: {} + decisions: {} imageGenerations: {} imageEdits: {} imageVariations: {} @@ -208,7 +212,8 @@ v2config: Shadow traffic is supported for mapping sections with JSON request bodies: `chatCompletions`, `completions`, `embeddings`, `responses`, `imageGenerations`, -and `anthropicMessages`. Shadow targets must be model names in the same mapping +`anthropicMessages`, `decisions`, and `typesafeSystemOne`. The decision sections +support `random` and `perBearerKey` sampling only. Shadow targets must be model names in the same mapping section as the primary model. The gateway rewrites the request `model` field to the shadow target and marks the replay with `NVCF-Shadow: true` so shadow requests do not recursively shadow. @@ -352,8 +357,8 @@ added to logs, traces, or metric labels. Set `functionType: LLM` on a model entry to serve it from the LLM Gateway instead of the invocation service. Supported in `openai.chatCompletions`, -`openai.responses`, `openai.embeddings`, and `anthropicMessages`, the four -routes the LLM Gateway serves. +`openai.responses`, `openai.embeddings`, `anthropicMessages`, `openai.decisions`, +and `typesafeSystemOne`, the routes the LLM Gateway serves. ```yaml v2config: @@ -383,7 +388,7 @@ forwarded: the mapping is what decides which function a caller reaches. Vanity Gateway passes `Authorization`, `NVCF-POLL-SECONDS`, and the caller's other headers to the LLM Gateway. -Config validation rejects `functionType: LLM` outside the four supported +Config validation rejects `functionType: LLM` outside the supported sections, and rejects `usePexec`, `outgoingPathOverride`, and `sessionTimeout` on those entries because the LLM Gateway ignores them. An `X-Priority` entry in `customHeaders` is rejected too: the LLM Gateway answers `400 Bad Request` for @@ -508,7 +513,8 @@ YAML merge defaults and explicit overrides are supported. For a repeated `modelName`, disjoint metadata keys merge. A conflicting key keeps its first value, including null, in this section order: `chatCompletions`, `completions`, `embeddings`, `responses`, `imageGenerations`, -`imageEdits`, `imageVariations`, then `anthropicMessages`. Within each section, +`imageEdits`, `imageVariations`, `anthropicMessages`, `decisions`, then +`typesafeSystemOne`. Within each section, YAML entry order decides precedence. Programmatically constructed configs without source order use alphabetical entry keys. Nested objects are whole values; their contents do not merge. Both OpenAI discovery endpoints use the @@ -583,6 +589,100 @@ The Messages path requires compatible LLM Gateway, Stargate router, and Pylon worker builds. See `docs/overview/llm-gateway.md` for upgrade order, token accounting, session affinity, and Claude Code configuration. +## Invoking Decision Models (Beta) + +Decision models answer typed questions about an input and return +probabilities instead of generated text. Two beta routes pass these requests +through on the shared API host set by `openai.host`: + +- `POST /v1/decisions` serves the OpenAI Decisions format. Map models under + `v2config.openai.decisions`. +- `POST /v1/systemone` serves the TypeSafe System One format. Map models under + `v2config.typesafeSystemOne`, a top-level sibling of `anthropicMessages`. + +Entries use the same fields and validation as `openai.chatCompletions`. A model +is enabled for a route only when it is listed in that route's section. A +function that serves both formats is listed in both sections. A request for a +model that is not listed in the section of the route it called returns `404`. + +```yaml +v2config: + openai: + host: api.example.com + decisions: + example_decision-model: + modelName: example/decision-model + functionID: 00000000-0000-0000-0000-000000000001 + functionVersionID: 11111111-1111-1111-1111-111111111111 + example_decision-model-llm: + modelName: example/decision-model-llm + functionID: 00000000-0000-0000-0000-000000000002 + functionType: LLM + typesafeSystemOne: + example_decision-model: + modelName: example/decision-model + functionID: 00000000-0000-0000-0000-000000000001 + functionVersionID: 11111111-1111-1111-1111-111111111111 +``` + +In this example `example/decision-model` is one function that serves both +formats, so it appears in both sections. `example/decision-model-llm` is served +through the LLM Gateway on `/v1/decisions` only. + +The gateway reads only `model` from the body and never translates between the +two formats, so the backend must implement each route it is configured for. +Request and response bodies pass through unchanged, including backend 4xx and +5xx responses. Responses are not streamed: `stream` is ignored and the gateway +does not ask for an event stream. Vendor headers such as +`TypeSafe-Danger-Zone-Images: true` reach the backend unchanged. Request bodies +may be up to 25 MB. + +```shell +source .env +curl http://localhost:10081/v1/decisions \ +-H "Host: api.example.com" \ +-H "Content-Type: application/json" \ +-H "Authorization: Bearer ${API_TOKEN}" \ +-d '{ +"model": "example/decision-model", +"input": "The sky is clear.", +"questions": [{"type": "predicate", "text": "Is it raining?"}] +}' +``` + +Direct models: with `usePexec: false` and no `outgoingPathOverride`, the +gateway keeps the path, `/v1/decisions` or `/v1/systemone`, and selects the +function with function ID headers. With `usePexec: true` the gateway calls the +pexec path of the function, and the invocation service then normally sends the request to +the default invocation path registered for that function, whichever route the +client called. A `usePexec: true` function therefore serves only its default path. A +function that serves both routes should use `usePexec: false`. + +Errors generated by the Vanity Gateway use the native shape of the route: + +- `/v1/decisions` uses the OpenAI error shape. +- `/v1/systemone` uses + `{"error":{"code":"...","message":"...","request_id":"..."},"request_id":"..."}`, + where `request_id` has the same value as the `X-Request-Id` response header. + The codes are `invalid_json` and `invalid_request` for `400`, `not_found` for + `404` and `410`, `request_too_large` for `413`, `provider_unavailable` for + `502` and `503`, and `internal_error`. The gateway does not set + `X-System-One-*` headers. + +Malformed JSON and a missing `model` return `400`, an unknown model returns +`404`, a body over the size limit returns `413`, an offline model returns `503` +with `Retry-After`, an end-of-life model returns `410`, and a failed upstream +request returns `502`. Upstream status codes and bodies pass through unchanged. + +`/v1/models` and `/v1/models/{model}` list decision models and add +`supported_endpoints`, for example `["/v1/decisions"]`. + +Shadow request body sampling (`promptCacheKey` and `firstMessageHash`) and +`promptCacheKeyHeaders` are not valid in the decision sections. + +A model with `functionType: LLM` needs LLM Gateway and Stargate builds that +serve these routes. Enable it only after those builds run in the environment. + ## Invoking Vanity URL Routes ```shell From a22b866e24bc709e8e9f9933947b4b720eb002b4 Mon Sep 17 00:00:00 2001 From: Sachin Prabhakar Date: Tue, 27 Oct 2026 19:59:16 -0700 Subject: [PATCH 5/5] fix(vanity-gateway): relay decision backend 429 bodies unchanged The tooManyRequestsMessage customization rewrote error.message in backend 429 bodies on the decision routes. Skip it there so these bodies pass through byte for byte. Relates to #2922 Signed-off-by: Sachin Prabhakar --- .../gateway/decision_routes_test.go | 27 +++++++++++++++++++ .../vanity-gateway/gateway/openai_director.go | 3 ++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go b/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go index d428b5ac50..dc00146356 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/decision_routes_test.go @@ -472,3 +472,30 @@ func TestDecisionRoutesSetEndpointTypeSpanAttribute(t *testing.T) { }) } } + +func TestDecisionRoutesRelayBackend429BodiesUnchangedWithCustomMessage(t *testing.T) { + bodies := map[string]string{ + decisionsPath: `{"error":{"message":"slow down","type":"rate_limit","param":null,"code":"rate_limited"}}`, + systemOnePath: `{"error":{"code":"rate_limit_exceeded","message":"slow down","request_id":"r1"},"request_id":"r1"}`, + } + for path, native := range bodies { + for name, body := range map[string]string{"native": native, "fastapi": fastAPIValidationErr} { + t.Run(path+" "+name, func(t *testing.T) { + backend := captureServer(t, make(chan capturedRequest, 1), func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "7") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(body)) + }) + mux := openAIMux(t, decisionMappings(true, true, func(e *config.ModelFunctionDetails) { + e.TooManyRequestsMessage = "Upgrade your plan." + }), backend.URL, "http://llm.invalid") + + rec := postDecision(t, mux, path, `{"model":"`+decisionModel+`"}`, nil) + + assert.Equal(t, http.StatusTooManyRequests, rec.Code) + assert.Equal(t, body, rec.Body.String()) + assert.Equal(t, "7", rec.Header().Get("Retry-After")) + }) + } + } +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go index ae5dba3b5b..922ae732e5 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go @@ -845,7 +845,8 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re return resolvedOpenAIRequest{}, true } - if nvcfUrl.tooManyRequestsMessage != "" { + // Decision backend 429 bodies pass through byte for byte. + if nvcfUrl.tooManyRequestsMessage != "" && !isDecisionRoute(request) { ctx := context.WithValue(request.Context(), tooManyRequestsKey, nvcfUrl.tooManyRequestsMessage) request = request.WithContext(ctx) }