diff --git a/deploy/helm/vanity-gateway/README.md b/deploy/helm/vanity-gateway/README.md index be2153ab9a..4121e57d53 100644 --- a/deploy/helm/vanity-gateway/README.md +++ b/deploy/helm/vanity-gateway/README.md @@ -106,14 +106,18 @@ or the pod is killed mid-drain. ## Route mapping -`vanityGateway.mappingConfig.v2config` has three sections: +`vanityGateway.mappingConfig.v2config` has four sections: - `openai`: per-endpoint model routes, keyed by endpoint (`chatCompletions`, - `completions`, `embeddings`, `responses`, and the image endpoints). Each route - requires `modelName` and `functionID`. + `completions`, `embeddings`, `responses`, `decisions`, and the image + endpoints). Each route requires `modelName` and `functionID`. `decisions` + serves the beta `POST /v1/decisions` route. - `anthropicMessages`: native Anthropic Messages model routes, served on `POST /v1/messages`. Routes use the same fields as `openai.chatCompletions` and share the API host set by `openai.host`. +- `typesafeSystemOne`: beta TypeSafe System One model routes, served on + `POST /v1/systemone`. A top-level sibling of `anthropicMessages`, with the + same fields as `openai.decisions`. - `vanity`: host-based routes, each requiring a `host` and a `paths` map. Each path requires `path` and `functionID`. @@ -123,13 +127,14 @@ across all routes. A model may set `functionType: LLM` to be served by the LLM Gateway instead of the invocation service, supported in `openai.chatCompletions`, -`openai.responses`, `openai.embeddings`, and `anthropicMessages`. Callers still send the public `modelName`; the gateway +`openai.responses`, `openai.embeddings`, `openai.decisions`, `anthropicMessages`, and +`typesafeSystemOne`. Callers still send the public `modelName`; the gateway rewrites the request model to `functionID/modelName` before forwarding. The values schema enforces what the gateway checks at startup, so a values file that would fail the container fails the render instead: -- `functionType` is rejected outside those four endpoints +- `functionType` is rejected outside those endpoints - `usePexec`, `outgoingPathOverride`, and `sessionTimeout` are rejected on such a model, since the LLM Gateway ignores them. An explicit `false`, `""`, or `0` is accepted, because that is what an absent key produces @@ -162,6 +167,43 @@ Messages. Upgrade Pylon workers, the Stargate router, and the LLM Gateway before enabling Messages routes. See `docs/overview/llm-gateway.md` for the request, accounting, and Claude Code details. +Decision models (beta) are mapped per route. A model is enabled for a route only +when listed in that route's section, and a function that serves both formats is +listed in both: + +```yaml +vanityGateway: + mappingConfig: + v2config: + openai: + host: api.example.com + decisions: + example_decision-model: + modelName: example/decision-model + functionID: + functionVersionID: + example_decision-model-llm: + modelName: example/decision-model-llm + functionID: + functionType: LLM + typesafeSystemOne: + example_decision-model: + modelName: example/decision-model + functionID: + functionVersionID: +``` + +Request and response bodies pass through unchanged, and responses are not +streamed. With `usePexec: true` the invocation service normally sends the request +to the default invocation path registered for the function, whichever route the +client called, so a `usePexec: true` function serves only its default path. +A function that serves both routes should set `usePexec: false`. Models with +`functionType: LLM` need LLM Gateway and Stargate builds that serve these routes; +enable them only after those builds run in the environment. Request body +sampling (`promptCacheKey`, `firstMessageHash`) and `promptCacheKeyHeaders` are +rejected in the decision sections. See the service README for error shapes and +`/v1/models` behavior. + Shadow traffic is supported. Each shadow target is resolved from the same model table and routed by its own `functionType`, so a shadow of an LLM model reaches the LLM Gateway, and an LLM model may shadow a model served by the invocation diff --git a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json index 93782c2d7e..def02ac087 100644 --- a/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json +++ b/deploy/helm/vanity-gateway/helm-nvcf-vanity-gateway/values.schema.json @@ -145,6 +145,9 @@ "anthropicMessages": { "$ref": "#/definitions/openAIStickyEndpointRoutes" }, + "typesafeSystemOne": { + "$ref": "#/definitions/openAILLMEndpointRoutes" + }, "vanity": { "$ref": "#/definitions/vanityRoutes" } @@ -273,6 +276,29 @@ "required": [ "embeddings" ] + }, + { + "properties": { + "decisions": { + "not": { + "additionalProperties": { + "not": { + "properties": { + "functionType": { + "const": "LLM" + } + }, + "required": [ + "functionType" + ] + } + } + } + } + }, + "required": [ + "decisions" + ] } ] } @@ -303,6 +329,29 @@ "required": [ "anthropicMessages" ] + }, + { + "properties": { + "typesafeSystemOne": { + "not": { + "additionalProperties": { + "not": { + "properties": { + "functionType": { + "const": "LLM" + } + }, + "required": [ + "functionType" + ] + } + } + } + } + }, + "required": [ + "typesafeSystemOne" + ] } ] } @@ -357,6 +406,9 @@ "embeddings": { "$ref": "#/definitions/openAILLMEndpointRoutes" }, + "decisions": { + "$ref": "#/definitions/openAILLMEndpointRoutes" + }, "responses": { "$ref": "#/definitions/openAIStickyEndpointRoutes" }, diff --git a/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh b/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh index 9b853b15b5..ecb89cc12c 100755 --- a/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh +++ b/deploy/helm/vanity-gateway/tests/chart-render/verify-llm-gateway-routing.sh @@ -76,11 +76,21 @@ assert_renders "a model without functionType renders with an empty endpoint" write_values "${ENDPOINT}" chatCompletions " functionType: llmGateway" assert_rejected "value must be 'LLM'" "an unrecognized functionType is rejected" -for section in responses embeddings; do +for section in responses embeddings decisions; do write_values "${ENDPOINT}" "${section}" "${LLM}" assert_renders "an LLM model renders in ${section}" done +write_values "" decisions "${LLM}" +assert_rejected "/vanityGateway/config/llmGatewayEndpoint" "an LLM decisions model requires an endpoint" + +write_values "${ENDPOINT}" decisions "${LLM} + usePexec: true" +assert_rejected "a_model/usePexec" "usePexec is rejected on an LLM decisions model" + +write_values "" decisions "" +assert_renders "a decisions model without functionType renders with an empty endpoint" + for section in completions imageGenerations imageEdits imageVariations; do write_values "${ENDPOINT}" "${section}" "${LLM}" assert_rejected "openai/${section}/a_model" "functionType is rejected in ${section}" @@ -156,6 +166,46 @@ write_messages_values "${ENDPOINT}" "${MESSAGES_LLM} sessionTimeout: 900" assert_rejected "anthropicMessages/a_model/sessionTimeout" "sessionTimeout is rejected on an LLM Messages model" +# typesafeSystemOne is a top-level sibling of anthropicMessages. +write_system_one_values() { + local endpoint="$1" extra="$2" + cat > "${VALUES}" < "${VALUES}" <= http.StatusInternalServerError { + errorType = "server_error" + } + writer.Header().Set(contentTypeHeader, contentTypeApplicationJSON) + writer.WriteHeader(status) + _ = json.NewEncoder(writer).Encode(OpenAIError{Error: CustomOpenAIError{ + Message: message, + Type: errorType, + Code: openAIErrorCode(status, kind), + }}) +} + +type systemOneErrorDetail struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type systemOneError struct { + Error systemOneErrorDetail `json:"error"` + RequestID string `json:"request_id"` +} + +// writeSystemOneError writes the System One error body. Gateway-written errors +// carry no X-System-One-* headers: those mark errors the model server relayed. +func writeSystemOneError(writer http.ResponseWriter, status int, kind, message string) { + requestID := writer.Header().Get(middleware.RequestIDHeader) + writer.Header().Set(contentTypeHeader, contentTypeApplicationJSON) + writer.WriteHeader(status) + _ = json.NewEncoder(writer).Encode(systemOneError{ + Error: systemOneErrorDetail{Code: kind, Message: message, RequestID: requestID}, + RequestID: requestID, + }) +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go index 24063321c6..922ae732e5 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go @@ -74,6 +74,8 @@ type ModelDirector struct { embeddings ModelMapping responses ModelMapping messages ModelMapping + decisions ModelMapping + systemOne ModelMapping imageGenerations ModelMapping imageEdits ModelMapping imageVariations ModelMapping @@ -350,6 +352,16 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp return nil, err } + decisions, err := buildModelMapping(convertIntoModelNameToFunctionIdAndVersionIdMappingV2(mapping.OpenAI.Decisions, "openai.decisions"), privateModelMatcher) + if err != nil { + return nil, err + } + + systemOne, err := buildModelMapping(convertIntoModelNameToFunctionIdAndVersionIdMappingV2(mapping.TypeSafeSystemOne, "typesafeSystemOne"), privateModelMatcher) + if err != nil { + return nil, err + } + imageGenerations, err := buildModelMapping(convertIntoModelNameToFunctionIdAndVersionIdMappingV2(mapping.OpenAI.ImageGenerations, "openai.imageGenerations"), privateModelMatcher) if err != nil { return nil, err @@ -373,6 +385,8 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, messages.modelNameToModelInfo, + decisions.modelNameToModelInfo, + systemOne.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo, @@ -381,9 +395,18 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp seen[id] = info } } - // Annotate aliases that support Messages so shared discovery does not - // imply that a Messages-only model accepts OpenAI requests. - for id := range messages.modelNameToModelInfo { + // Annotate aliases that support a native protocol (Messages, decisions or + // System One) so shared discovery does not imply that such a model accepts + // OpenAI chat requests. + nativeIDs := make(map[string]struct{}) + for _, models := range []map[string]ModelInfo{ + messages.modelNameToModelInfo, decisions.modelNameToModelInfo, systemOne.modelNameToModelInfo, + } { + for id := range models { + nativeIDs[id] = struct{}{} + } + } + for id := range nativeIDs { info := seen[id] for _, endpoint := range []struct { path string @@ -394,6 +417,8 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp {"/v1/embeddings", embeddings.modelNameToModelInfo}, {"/v1/responses", responses.modelNameToModelInfo}, {"/v1/messages", messages.modelNameToModelInfo}, + {decisionsPath, decisions.modelNameToModelInfo}, + {systemOnePath, systemOne.modelNameToModelInfo}, {"/v1/images/generations", imageGenerations.modelNameToModelInfo}, {"/v1/images/edits", imageEdits.modelNameToModelInfo}, {"/v1/images/variations", imageVariations.modelNameToModelInfo}, @@ -403,7 +428,7 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp } } seen[id] = info - for _, models := range []map[string]ModelInfo{chatCompletions.modelNameToModelInfo, completions.modelNameToModelInfo, embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, messages.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo} { + for _, models := range []map[string]ModelInfo{chatCompletions.modelNameToModelInfo, completions.modelNameToModelInfo, embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, messages.modelNameToModelInfo, decisions.modelNameToModelInfo, systemOne.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo} { if _, ok := models[id]; ok { models[id] = info } @@ -418,6 +443,7 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp embeddings.modelNameToModelInfo, responses.modelNameToModelInfo, imageGenerations.modelNameToModelInfo, imageEdits.modelNameToModelInfo, imageVariations.modelNameToModelInfo, messages.modelNameToModelInfo, + decisions.modelNameToModelInfo, systemOne.modelNameToModelInfo, } { if _, ok := models[id]; ok { models[id] = info @@ -435,6 +461,8 @@ func NewModelDirector(mapping *config.GatewayConfig, privateModelMatcher *regexp embeddings: embeddings, responses: responses, messages: messages, + decisions: decisions, + systemOne: systemOne, imageGenerations: imageGenerations, imageEdits: imageEdits, imageVariations: imageVariations, @@ -496,6 +524,8 @@ func (d *ModelDirector) getFilteredModels() []ModelInfo { d.embeddings.modelNameToNVCFUrl, d.responses.modelNameToNVCFUrl, d.messages.modelNameToNVCFUrl, + d.decisions.modelNameToNVCFUrl, + d.systemOne.modelNameToNVCFUrl, d.imageGenerations.modelNameToNVCFUrl, d.imageEdits.modelNameToNVCFUrl, d.imageVariations.modelNameToNVCFUrl, @@ -589,6 +619,21 @@ func (d *ModelDirector) ServeMessages(writer http.ResponseWriter, request *http. d.proxyModelMappedRequest(writer, request, d.messages.modelNameToNVCFUrl, shadowSamplingEndpointMessages) } +// ServeDecisions passes POST /v1/decisions through to the model's function. +// The body is not parsed beyond model, and errors the gateway writes use the +// OpenAI shape. +func (d *ModelDirector) ServeDecisions(writer http.ResponseWriter, request *http.Request) { + request = withNativeProtocol(request, nativeProtocolDecisions) + d.proxyModelMappedRequest(writer, request, d.decisions.modelNameToNVCFUrl, shadowSamplingEndpointNone) +} + +// ServeSystemOne passes POST /v1/systemone through to the model's function. +// Errors the gateway writes use the System One shape. +func (d *ModelDirector) ServeSystemOne(writer http.ResponseWriter, request *http.Request) { + request = withNativeProtocol(request, nativeProtocolSystemOne) + d.proxyModelMappedRequest(writer, request, d.systemOne.modelNameToNVCFUrl, shadowSamplingEndpointNone) +} + func (d *ModelDirector) ServeImageGenerations(writer http.ResponseWriter, request *http.Request) { d.proxyModelMappedRequest(writer, request, d.imageGenerations.modelNameToNVCFUrl, shadowSamplingEndpointNone) } @@ -651,6 +696,8 @@ func (d *ModelDirector) GetModel(writer http.ResponseWriter, request *http.Reque d.chatCompletions.modelNameToModelInfo, d.responses.modelNameToModelInfo, d.messages.modelNameToModelInfo, + d.decisions.modelNameToModelInfo, + d.systemOne.modelNameToModelInfo, d.imageGenerations.modelNameToModelInfo, d.imageEdits.modelNameToModelInfo, d.imageVariations.modelNameToModelInfo, @@ -669,6 +716,8 @@ func (d *ModelDirector) GetModel(writer http.ResponseWriter, request *http.Reque d.chatCompletions.modelNameToNVCFUrl, d.responses.modelNameToNVCFUrl, d.messages.modelNameToNVCFUrl, + d.decisions.modelNameToNVCFUrl, + d.systemOne.modelNameToNVCFUrl, d.imageGenerations.modelNameToNVCFUrl, d.imageEdits.modelNameToNVCFUrl, d.imageVariations.modelNameToNVCFUrl, @@ -689,11 +738,7 @@ func (d *ModelDirector) GetModel(writer http.ResponseWriter, request *http.Reque func (d *ModelDirector) proxyModelMappedRequest(writer http.ResponseWriter, request *http.Request, modelToNVCFUrl map[string]FunctionInfo, endpoint shadowSamplingEndpoint) { span := trace.SpanFromContext(request.Context()) - endpointType := traceAttrValueEndpointOpenAI - if request.URL.Path == "/v1/messages" { - endpointType = traceAttrValueEndpointAnthropic - } - span.SetAttributes(traceAttrEndpointType.String(endpointType)) + span.SetAttributes(traceAttrEndpointType.String(endpointTypeForPath(request.URL.Path, traceAttrValueEndpointOpenAI))) setShadowSpanAttribute(span, request) resolved, handled := d.resolveModelMappedRequest(writer, request, modelToNVCFUrl) @@ -733,19 +778,9 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re body, err := extractOpenAIRequestBody(request) if err != nil { markGatewayRejected(writer, request) - if request.URL.Path == "/v1/messages" { - status, message := http.StatusInternalServerError, "failed to read Messages request" - if errors.Is(err, errInvalidJSONBody) || errors.Is(err, errModelFieldMissing) { - status, message = http.StatusBadRequest, "invalid Messages request JSON" - } - var maxBytesErr *http.MaxBytesError - if errors.As(err, &maxBytesErr) { - status, message = http.StatusRequestEntityTooLarge, "request body is too large" - } - if errors.Is(err, errModelFieldMissing) { - message = "model field is required" - } - writeAnthropicError(writer, status, message) + if protocol := requestProtocol(request); protocol != nativeProtocolNone { + status, kind, message := requestReadFailure(protocol, err) + writeNativeError(writer, request, status, kind, message) return resolvedOpenAIRequest{}, true } var maxBytesErr *http.MaxBytesError @@ -796,9 +831,7 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re if !ok { _ = request.Body.Close() markGatewayRejected(writer, request) - if request.URL.Path == "/v1/messages" { - writeAnthropicError(writer, http.StatusNotFound, "model not found") - } else { + if !writeNativeError(writer, request, http.StatusNotFound, errKindNotFound, "model not found") { http.NotFound(writer, request) } return resolvedOpenAIRequest{}, true @@ -806,33 +839,22 @@ func (d *ModelDirector) resolveModelMappedRequest(writer http.ResponseWriter, re middleware.SetPreUpstreamModel(request.Context(), nvcfUrl.publicModelName) middleware.AddOpenAIRequestMetricAttributes(request.Context(), body.Model, nvcfUrl.functionId) - statusHandled := false - if request.URL.Path == "/v1/messages" { - if nvcfUrl.offlineMessage != "" { - markGatewayRejected(writer, request) - writer.Header().Set("Retry-After", "10800") - writeAnthropicError(writer, http.StatusServiceUnavailable, nvcfUrl.offlineMessage) - statusHandled = true - } else if isModelExpired(nvcfUrl.eol) { - markGatewayRejected(writer, request) - writeAnthropicError(writer, http.StatusGone, "model has reached end of life") - statusHandled = true - } - } else { - statusHandled = writeFunctionStatusError(writer, request, nvcfUrl.offlineMessage, nvcfUrl.eol, body.Model) - } + statusHandled := writeFunctionStatusError(writer, request, nvcfUrl.offlineMessage, nvcfUrl.eol, body.Model) if statusHandled { _ = request.Body.Close() return resolvedOpenAIRequest{}, true } - if nvcfUrl.tooManyRequestsMessage != "" { + // Decision backend 429 bodies pass through byte for byte. + if nvcfUrl.tooManyRequestsMessage != "" && !isDecisionRoute(request) { ctx := context.WithValue(request.Context(), tooManyRequestsKey, nvcfUrl.tooManyRequestsMessage) request = request.WithContext(ctx) } // setPollingHeaderIfNotPresent checks the Accept header, so we have to override it first - if body.Stream { + // Decision routes are non-streaming: stream is ignored and the body is + // forwarded as sent. + if body.Stream && !isDecisionRoute(request) { request.Header.Set("Accept", "text/event-stream") } @@ -1241,3 +1263,32 @@ type OpenAIDirector = ModelDirector func NewOpenAIDirectorV2(mapping *config.GatewayConfig, privateModelMatcher *regexp.Regexp, vanityDirector *VanityDirector, llmGatewayDirector *LLMGatewayDirector, shadower *TrafficShadower) (*ModelDirector, error) { return NewModelDirector(mapping, privateModelMatcher, vanityDirector, llmGatewayDirector, shadower) } + +func isDecisionRoute(request *http.Request) bool { + protocol := requestProtocol(request) + return protocol == nativeProtocolDecisions || protocol == nativeProtocolSystemOne +} + +// requestReadFailure maps a failure to read or parse the request body to the +// status, error code and message of the route's native error. +func requestReadFailure(protocol nativeProtocol, err error) (int, string, string) { + status, kind := http.StatusInternalServerError, errKindInternal + message := "failed to read request" + if protocol == nativeProtocolMessages { + message = "failed to read Messages request" + } + if errors.Is(err, errInvalidJSONBody) || errors.Is(err, errModelFieldMissing) { + status, kind, message = http.StatusBadRequest, errKindInvalidJSON, "request body is not valid JSON" + if protocol == nativeProtocolMessages { + message = "invalid Messages request JSON" + } + } + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + status, kind, message = http.StatusRequestEntityTooLarge, errKindRequestTooLarge, "request body is too large" + } + if errors.Is(err, errModelFieldMissing) { + kind, message = errKindInvalidRequest, "model field is required" + } + return status, kind, message +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go b/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go index 153b2c7f12..6f55fde283 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/request_id_test.go @@ -67,6 +67,11 @@ func TestGatewayGeneratedErrorsCarryRequestID(t *testing.T) { {"messages unknown model", "/v1/messages", `{"model":"unknown"}`, nil, http.StatusNotFound}, {"messages body too large", "/v1/messages", tooLarge, nil, http.StatusRequestEntityTooLarge}, {"messages reserved shadow header", "/v1/messages", `{"model":"` + publicModel + `"}`, map[string]string{shadowHeader: "true"}, http.StatusBadRequest}, + {"decisions malformed json", "/v1/decisions", `{`, nil, http.StatusBadRequest}, + {"decisions unknown model", "/v1/decisions", `{"model":"unknown"}`, nil, http.StatusNotFound}, + {"systemone malformed json", "/v1/systemone", `{`, nil, http.StatusBadRequest}, + {"systemone unknown model", "/v1/systemone", `{"model":"unknown"}`, nil, http.StatusNotFound}, + {"systemone body too large", "/v1/systemone", tooLarge, nil, http.StatusRequestEntityTooLarge}, {"chat malformed json", "/v1/chat/completions", `{`, nil, http.StatusBadRequest}, {"chat unknown model", "/v1/chat/completions", `{"model":"unknown"}`, nil, http.StatusNotFound}, } diff --git a/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go b/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go index 0f01b89bc8..bf6e99c093 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/tracing_attrs.go @@ -45,6 +45,8 @@ const ( traceAttrValueEndpointAnthropic = "anthropic" traceAttrValueEndpointOpenAI = "openai" traceAttrValueEndpointLLMGateway = "llm_gateway" + traceAttrValueEndpointDecisions = "decisions" + traceAttrValueEndpointSystemOne = "systemone" shadowDroppedReasonBodyReadError = "body_read_error" shadowDroppedReasonBodyRewriteError = "body_rewrite_error" diff --git a/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go b/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go index 874282a587..8af1287a2a 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/vanity_director.go @@ -51,6 +51,11 @@ const statusClientClosedRequest = 499 func writeFunctionStatusError(writer http.ResponseWriter, request *http.Request, offlineMessage string, eol time.Time, name string) bool { if offlineMessage != "" { markGatewayRejected(writer, request) + if requestProtocol(request) != nativeProtocolNone { + writer.Header().Set("Retry-After", "10800") + writeNativeError(writer, request, http.StatusServiceUnavailable, errKindProviderUnavailable, offlineMessage) + return true + } writer.Header().Set("Content-Type", "application/problem+json") writer.Header().Set("Retry-After", "10800") writer.WriteHeader(http.StatusServiceUnavailable) @@ -63,6 +68,11 @@ func writeFunctionStatusError(writer http.ResponseWriter, request *http.Request, return true } if isModelExpired(eol) { + if requestProtocol(request) != nativeProtocolNone { + markGatewayRejected(writer, request) + writeNativeError(writer, request, http.StatusGone, errKindNotFound, "model has reached end of life") + return true + } var detail string if name != "" { detail = fmt.Sprintf("The model '%s' has reached its end of life on %s and is no longer available.", name, eol.Format(time.RFC3339)) @@ -146,8 +156,7 @@ func writeProxyError(writer http.ResponseWriter, request *http.Request, err erro zap.String(string(middleware.GatewayProxyOutcomeMetricAttribute), string(middleware.GatewayProxyOutcomeClientCanceled)), zap.Error(err), ) - if request.URL.Path == "/v1/messages" { - writeAnthropicError(writer, statusClientClosedRequest, "client closed the request") + if writeNativeError(writer, request, statusClientClosedRequest, errKindInternal, "client closed the request") { return } writer.Header().Set("Content-Type", "application/problem+json") @@ -170,8 +179,7 @@ func writeBadGatewayProblem(writer http.ResponseWriter, request *http.Request, e zap.String(string(middleware.GatewayProxyOutcomeMetricAttribute), string(middleware.GatewayProxyOutcomeProxyError)), zap.Error(err), ) - if request.URL.Path == "/v1/messages" { - writeAnthropicError(writer, http.StatusBadGateway, "upstream request failed") + if writeNativeError(writer, request, http.StatusBadGateway, errKindProviderUnavailable, "upstream request failed") { return } writer.Header().Set("Content-Type", "application/problem+json") diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go index 1cdc9c3421..9c2f2222c6 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config.go @@ -418,7 +418,7 @@ const ( ) // llmGatewaySections are the native protocol sections the LLM Gateway serves. -var llmGatewaySections = []string{"chatCompletions", "responses", "embeddings", "anthropicMessages"} +var llmGatewaySections = []string{"chatCompletions", "responses", "embeddings", "anthropicMessages", "decisions", "typesafeSystemOne"} type V2Config struct { OpenAI struct { @@ -430,8 +430,10 @@ type V2Config struct { ImageGenerations map[string]ModelFunctionDetails `json:"imageGenerations"` ImageEdits map[string]ModelFunctionDetails `json:"imageEdits"` ImageVariations map[string]ModelFunctionDetails `json:"imageVariations"` + Decisions map[string]ModelFunctionDetails `json:"decisions"` } `json:"openai"` AnthropicMessages map[string]ModelFunctionDetails `json:"anthropicMessages"` + TypeSafeSystemOne map[string]ModelFunctionDetails `json:"typesafeSystemOne"` Vanity map[string]VanityEntry `json:"vanity"` } @@ -603,7 +605,11 @@ func validateSamplingMethods(location string, fieldName string, sectionName stri return nil } -const anthropicMessagesSection = "anthropicMessages" +const ( + anthropicMessagesSection = "anthropicMessages" + decisionsSection = "openai.decisions" + typeSafeSystemOneSection = "typesafeSystemOne" +) func supportsRequestBodySampling(sectionName string) bool { return sectionName == "openai.chatCompletions" || sectionName == "openai.responses" || @@ -630,6 +636,8 @@ func (c *GatewayConfig) modelSections() map[string]map[string]ModelFunctionDetai "openai.imageGenerations": c.OpenAI.ImageGenerations, "openai.imageEdits": c.OpenAI.ImageEdits, "openai.imageVariations": c.OpenAI.ImageVariations, + decisionsSection: c.OpenAI.Decisions, + typeSafeSystemOneSection: c.TypeSafeSystemOne, } } diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go index daa02ba098..198b5c58fd 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/gateway_config_test.go @@ -1659,9 +1659,11 @@ func TestGatewayConfigValidateScopesRequestBodyShadowSampling(t *testing.T) { }) } for section, location := range map[string]string{ - "completions": "openai.completions", - "embeddings": "openai.embeddings", - "imageGenerations": "openai.imageGenerations", + "completions": "openai.completions", + "embeddings": "openai.embeddings", + "imageGenerations": "openai.imageGenerations", + "decisions": "openai.decisions", + "typesafeSystemOne": "typesafeSystemOne", } { for _, perTarget := range []bool{false, true} { form := "legacy" @@ -1797,6 +1799,10 @@ func gatewayConfigWithShadow( cfg.OpenAI.ImageGenerations = entries case "anthropicMessages": cfg.AnthropicMessages = entries + case "decisions": + cfg.OpenAI.Decisions = entries + case "typesafeSystemOne": + cfg.TypeSafeSystemOne = entries } return cfg } @@ -1818,6 +1824,10 @@ func llmModelConfig(section string, entry ModelFunctionDetails) *GatewayConfig { cfg.OpenAI.Completions = entries case "imageGenerations": cfg.OpenAI.ImageGenerations = entries + case "decisions": + cfg.OpenAI.Decisions = entries + case "typesafeSystemOne": + cfg.TypeSafeSystemOne = entries } return cfg } @@ -1979,3 +1989,45 @@ func TestAnthropicMessagesConfigNamespace(t *testing.T) { cfg.AnthropicMessages["native"] = entry require.ErrorContains(t, cfg.Validate(), "anthropicMessages.native: modelName is required") } + +func TestDecisionSectionsAcceptRandomAndPerBearerKeySampling(t *testing.T) { + methods := samplingMethods(ShadowSamplingMethodPerBearerKey, ShadowSamplingMethodRandom) + for _, section := range []string{"decisions", "typesafeSystemOne"} { + t.Run(section, func(t *testing.T) { + require.NoError(t, gatewayConfigWithShadow(section, methods, nil, true).Validate()) + }) + } +} + +func TestDecisionConfigNamespaces(t *testing.T) { + var cfg GatewayConfig + require.NoError(t, json.Unmarshal([]byte(`{ + "v2config": { + "openai": {"host": "api.example.com", "decisions": { + "dec": {"modelName": "dec-model", "functionID": "dec-function", "modelInfoFields": {"family": "decision"}} + }}, + "typesafeSystemOne": { + "s1": {"modelName": "s1-model", "functionID": "s1-function", "functionType": "LLM"} + } + } + }`), &cfg)) + require.NoError(t, cfg.Validate()) + require.True(t, cfg.HasLLMGatewayRoute()) + require.Equal(t, "dec-model", cfg.OpenAI.Decisions["dec"].ModelName) + require.Equal(t, "s1-model", cfg.TypeSafeSystemOne["s1"].ModelName) + // Shadow targets must stay inside the same section. + entry := cfg.TypeSafeSystemOne["s1"] + entry.ShadowModelName = "dec-model" + cfg.TypeSafeSystemOne["s1"] = entry + require.ErrorContains(t, cfg.Validate(), "typesafeSystemOne.s1: shadow target must reference another model in typesafeSystemOne") +} + +func TestDecisionSectionsRejectLLMOnlyFieldsWithUsePexec(t *testing.T) { + for _, section := range []string{"decisions", "typesafeSystemOne"} { + t.Run(section, func(t *testing.T) { + entry := llmModel() + entry.UsePexec = true + require.ErrorContains(t, llmModelConfig(section, entry).Validate(), "usePexec is unsupported") + }) + } +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/model_info.go b/src/invocation-plane-services/vanity-gateway/gateway_config/model_info.go index 9be348202a..4f078d0b48 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/model_info.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/model_info.go @@ -34,6 +34,7 @@ import ( var modelInfoSectionOrder = []string{ "openai.chatCompletions", "openai.completions", "openai.embeddings", "openai.responses", "openai.imageGenerations", "openai.imageEdits", "openai.imageVariations", anthropicMessagesSection, + decisionsSection, typeSafeSystemOneSection, } func validateModelInfoFields(model string, fields map[string]json.RawMessage) error { diff --git a/src/invocation-plane-services/vanity-gateway/gateway_config/model_info_test.go b/src/invocation-plane-services/vanity-gateway/gateway_config/model_info_test.go index ab4ed727b6..794892799a 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway_config/model_info_test.go +++ b/src/invocation-plane-services/vanity-gateway/gateway_config/model_info_test.go @@ -100,6 +100,8 @@ func TestModelInfoReservedFields(t *testing.T) { cfg.OpenAI.ImageEdits = map[string]ModelFunctionDetails{} cfg.OpenAI.ImageVariations = map[string]ModelFunctionDetails{} cfg.AnthropicMessages = map[string]ModelFunctionDetails{} + cfg.OpenAI.Decisions = map[string]ModelFunctionDetails{} + cfg.TypeSafeSystemOne = map[string]ModelFunctionDetails{} cfg.modelSections()[section]["example"] = entry err := cfg.Validate() require.ErrorContains(t, err, "example/model") @@ -215,3 +217,23 @@ func TestModelInfoYAMLEmptyCompatibility(t *testing.T) { require.Empty(t, cfg.ModelInfoFieldsByName()) } } + +func TestModelInfoFieldsIncludeDecisionSections(t *testing.T) { + var cfg GatewayConfig + require.NoError(t, unmarshalGatewayConfig([]byte(`v2config: + openai: + decisions: + entry: + modelName: example/decision + modelInfoFields: {from_decisions: true, winner: decisions} + typesafeSystemOne: + entry: + modelName: example/decision + modelInfoFields: {from_system_one: true, winner: systemone} +`), &cfg)) + require.NoError(t, cfg.Validate()) + fields := cfg.ModelInfoFieldsByName()["example/decision"] + require.Equal(t, "true", string(fields["from_decisions"])) + require.Equal(t, "true", string(fields["from_system_one"])) + require.Equal(t, `"decisions"`, string(fields["winner"])) +} diff --git a/tools/ci/helm-validate-values/vanity-gateway.yaml b/tools/ci/helm-validate-values/vanity-gateway.yaml index c134361b78..a5c143c30e 100644 --- a/tools/ci/helm-validate-values/vanity-gateway.yaml +++ b/tools/ci/helm-validate-values/vanity-gateway.yaml @@ -17,9 +17,19 @@ vanityGateway: functionType: LLM customHeaders: X-Provider-Feature: enabled + decisions: + example_decision-model: + modelName: example/decision-model + functionID: 00000000-0000-0000-0000-000000000003 + functionType: LLM anthropicMessages: example_native-model: modelName: example/native-model functionID: 00000000-0000-0000-0000-000000000002 functionType: LLM + typesafeSystemOne: + example_decision-model: + modelName: example/decision-model + functionID: 00000000-0000-0000-0000-000000000003 + functionType: LLM vanity: {}