From e068c3d02ff8664c1142d5f7c51a05a82f75ad3d Mon Sep 17 00:00:00 2001 From: zeng gang Date: Fri, 16 Oct 2026 11:17:44 +0800 Subject: [PATCH 01/29] fix(openai): preserve overload retries after queued and metadata events --- backend/internal/service/openai_capacity_shed_test.go | 8 ++++++++ backend/internal/service/openai_gateway_passthrough.go | 4 +++- backend/internal/service/openai_ws_http_bridge_test.go | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/internal/service/openai_capacity_shed_test.go b/backend/internal/service/openai_capacity_shed_test.go index 7d2f8c8dd28a..086f7380e3c7 100644 --- a/backend/internal/service/openai_capacity_shed_test.go +++ b/backend/internal/service/openai_capacity_shed_test.go @@ -124,6 +124,8 @@ func TestOpenAIStreamErrorFrameDoesNotStartClientOutput(t *testing.T) { {`{"type":"error","error":{"type":"invalid_request_error","code":"content_policy_violation","message":"blocked"}}`, "error", true}, {`{"type":"response.failed","response":{"error":{"code":"server_is_overloaded"}}}`, "response.failed", false}, {`{"type":"response.created","response":{"id":"resp_1"}}`, "response.created", false}, + {`{"type":"response.queued","response":{"id":"resp_1","status":"queued"}}`, "response.queued", false}, + {`{"type":"response.metadata","response_id":"resp_1","metadata":{"openai_chatgpt_moderation_metadata":{"prompt":{}}}}`, "response.metadata", false}, {`{"type":"response.in_progress","response":{"id":"resp_1"}}`, "response.in_progress", false}, {`{"type":"keepalive"}`, "keepalive", false}, {`{"type":"unknown_event"}`, "unknown_event", true}, @@ -147,6 +149,12 @@ func TestOpenAIStreamMetadataAndKeepaliveBeforeOverloadFailOver(t *testing.T) { "event: response.created", `data: {"type":"response.created","response":{"id":"resp_1","metadata":{"padding":"` + largeMetadata + `"}}}`, "", + "event: response.queued", + `data: {"type":"response.queued","response":{"id":"resp_1","status":"queued"}}`, + "", + "event: response.metadata", + `data: {"type":"response.metadata","response_id":"resp_1","metadata":{"openai_chatgpt_moderation_metadata":{"prompt":{}}}}`, + "", "event: response.output_item.added", `data: {"type":"response.output_item.added","item":{"type":"reasoning","summary":[]}}`, "", diff --git a/backend/internal/service/openai_gateway_passthrough.go b/backend/internal/service/openai_gateway_passthrough.go index 6feadb8fb6d5..170830ae4f80 100644 --- a/backend/internal/service/openai_gateway_passthrough.go +++ b/backend/internal/service/openai_gateway_passthrough.go @@ -1087,7 +1087,9 @@ func openAIStreamClientOutputStarted(c *gin.Context, localStarted bool) bool { // Lifecycle metadata and transport heartbeats are not model output. func openAIStreamEventIsMetadata(eventType string) bool { switch strings.TrimSpace(eventType) { - case "response.created", "response.in_progress", "keepalive": + // Queuing and moderation metadata do not commit model output or disarm the + // first-output timeout; a following overload must remain safe to retry. + case "response.created", "response.queued", "response.in_progress", "response.metadata", "keepalive": return true default: return false diff --git a/backend/internal/service/openai_ws_http_bridge_test.go b/backend/internal/service/openai_ws_http_bridge_test.go index 12f08c655a87..c8c654e6c6ec 100644 --- a/backend/internal/service/openai_ws_http_bridge_test.go +++ b/backend/internal/service/openai_ws_http_bridge_test.go @@ -1192,6 +1192,10 @@ func TestProxyOpenAIWSHTTPBridgeTurnStagesMetadataAndRelaysKeepaliveBeforeCapaci preamble := strings.Join([]string{ `data: {"type":"response.created","response":{"id":"resp_shed"}}`, "", + `data: {"type":"response.queued","response":{"id":"resp_shed","status":"queued"}}`, + "", + `data: {"type":"response.metadata","response_id":"resp_shed","metadata":{"openai_chatgpt_moderation_metadata":{"prompt":{}}}}`, + "", `data: {"type":"response.in_progress","response":{"id":"resp_shed"}}`, "", `data: {"type":"keepalive"}`, From 7fcaa32db4940ff9ca42b961b00c453b496d8d25 Mon Sep 17 00:00:00 2001 From: ahe <53739884+qhmhyp@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:54:50 +0800 Subject: [PATCH 02/29] fix(accounts): persist platform and status filters --- frontend/src/views/admin/AccountsView.vue | 41 +++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/admin/AccountsView.vue b/frontend/src/views/admin/AccountsView.vue index 2502f2db8cfa..6d71337fddcd 100644 --- a/frontend/src/views/admin/AccountsView.vue +++ b/frontend/src/views/admin/AccountsView.vue @@ -496,6 +496,7 @@ import { useI18n } from 'vue-i18n' import { useAppStore } from '@/stores/app' import { useAuthStore } from '@/stores/auth' import { adminAPI } from '@/api/admin' +import { CONCRETE_PLATFORM_OPTIONS } from '@/constants/platforms' import { useTableLoader } from '@/composables/useTableLoader' import { useSwipeSelect, type SwipeSelectVirtualContext } from '@/composables/useSwipeSelect' import { useTableSelection } from '@/composables/useTableSelection' @@ -653,6 +654,31 @@ const HIDDEN_COLUMNS_KEY = 'account-hidden-columns' const HIDDEN_COLUMNS_VERSION_KEY = 'account-hidden-columns-version' const HIDDEN_COLUMNS_CURRENT_VERSION = 'scheduler-score-hidden-by-default' +// Account filter settings +const ACCOUNT_FILTER_STORAGE_KEY = 'account-filter-values' +type AccountFilterValues = { + platform: string + status: string +} +const loadInitialAccountFilterValues = (): AccountFilterValues => { + const fallback: AccountFilterValues = { platform: '', status: '' } + try { + const raw = localStorage.getItem(ACCOUNT_FILTER_STORAGE_KEY) + if (!raw) return fallback + const parsed: unknown = JSON.parse(raw) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return fallback + const values = parsed as Record + return { + platform: typeof values.platform === 'string' && CONCRETE_PLATFORM_OPTIONS.some(option => option.value === values.platform) + ? values.platform : '', + status: typeof values.status === 'string' && ['active', 'inactive', 'error', 'rate_limited', 'temp_unschedulable', 'unschedulable'].includes(values.status) + ? values.status : '' + } + } catch { + return fallback + } +} + // Sorting settings const ACCOUNT_SORT_STORAGE_KEY = 'account-table-sort' type AccountSortOrder = 'asc' | 'desc' @@ -1084,9 +1110,8 @@ const { } = useTableLoader({ fetchFn: adminAPI.accounts.list, initialParams: { - platform: '', + ...loadInitialAccountFilterValues(), type: '', - status: '', privacy_mode: '', group: '', search: '', @@ -1292,7 +1317,19 @@ const refreshUpstreamBillingSortedList = async (force = false) => { useIntervalFn(() => { void refreshUpstreamBillingRates() }, 5 * 60_000, { immediate: false }) +const saveAccountFilterValues = () => { + try { + localStorage.setItem(ACCOUNT_FILTER_STORAGE_KEY, JSON.stringify({ + platform: params.platform, + status: params.status + })) + } catch { + // Filtering must remain available when browser storage is unavailable. + } +} + const debouncedReload = () => { + saveAccountFilterValues() clearSelection() syncAccountListDerivedParams() hasPendingListSync.value = false From b0a02a3304a0a78cdd003ddda1dd23ba02c03084 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:42:02 +0000 Subject: [PATCH 03/29] chore: sync VERSION to 0.2.17 [skip ci] --- backend/cmd/server/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cmd/server/VERSION b/backend/cmd/server/VERSION index 9097948ffe8d..0fcb3bba7e0b 100644 --- a/backend/cmd/server/VERSION +++ b/backend/cmd/server/VERSION @@ -1 +1 @@ -0.2.16 +0.2.17 From 2b1f129385f4475ffddf8a91b44feceef5101fb6 Mon Sep 17 00:00:00 2001 From: feeeei Date: Sat, 17 Oct 2026 00:37:36 +0800 Subject: [PATCH 04/29] =?UTF-8?q?fix(gemini):=20=E6=B5=81=E5=BC=8F?= =?UTF-8?q?=E4=B8=AD=E9=80=94=E5=AE=A2=E6=88=B7=E7=AB=AF=E6=96=AD=E5=BC=80?= =?UTF-8?q?=E6=97=B6=E6=8C=89=E5=B7=B2=E6=94=B6=E9=9B=86=E7=94=A8=E9=87=8F?= =?UTF-8?q?=E8=AE=A1=E8=B4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 原生 /v1beta、Claude 兼容 /v1/messages、Chat 兼容三条流式路径：读取中断由客户端 断开引起（context.Canceled 或请求 context 已取消），且已观测到 token 用量或已回吐 图片时，以已收集结果收尾并返回成功，handler 照常提交 usage；ForwardResult 标记 ClientDisconnect - 读到结束分片后断开按完整用量计费；中途断开按断开前最后一份 usageMetadata 计费， 断开前无任何用量时仍不计费 - 图片数：收到 finishReason 前断开只计已送达的图片（原生按已观测 inlineData，Claude/ Chat 兼容流不转发图片计 0）；收到 finishReason 后断开与正常完成同口径 - Chat 兼容流写失败断开的既有返回同样带上断开标记与结束状态 - 原生路径读错误时不处理未以换行结尾的残行，避免截断的 usageMetadata 覆盖已收集用量 - 上游 context 不解绑：Gemini 兼容链路没有流空闲超时，解绑后上游挂起会让读取无限期阻塞 - 客户端未断开时的上游读错误、非流式客户端断开、Antigravity 非流式转换的读错误 行为不变 --- .../gemini_chat_completions_compat_service.go | 62 +++-- .../service/gemini_messages_compat_service.go | 57 +++- .../service/gemini_stream_client_abort.go | 25 ++ .../gemini_stream_client_abort_test.go | 244 ++++++++++++++++++ 4 files changed, 359 insertions(+), 29 deletions(-) create mode 100644 backend/internal/service/gemini_stream_client_abort.go create mode 100644 backend/internal/service/gemini_stream_client_abort_test.go diff --git a/backend/internal/service/gemini_chat_completions_compat_service.go b/backend/internal/service/gemini_chat_completions_compat_service.go index f85ac0577a2e..61a635b64d23 100644 --- a/backend/internal/service/gemini_chat_completions_compat_service.go +++ b/backend/internal/service/gemini_chat_completions_compat_service.go @@ -250,6 +250,8 @@ func (s *GeminiMessagesCompatService) forwardClaudeBodyAsChatCompletions( var usage *ClaudeUsage var firstTokenMs *int + clientDisconnect := false + streamFinished := false if clientStream { streamRes, err := s.handleChatCompletionsStreamingResponseFromGemini(c, resp, startTime, originalModel, account.Type == AccountTypeOAuth, includeUsage) if err != nil { @@ -257,6 +259,8 @@ func (s *GeminiMessagesCompatService) forwardClaudeBodyAsChatCompletions( } usage = streamRes.usage firstTokenMs = streamRes.firstTokenMs + clientDisconnect = streamRes.clientDisconnect + streamFinished = streamRes.finished } else if useUpstreamStream { collected, usageObj, err := collectGeminiSSE(resp.Body, account.Type == AccountTypeOAuth) if err != nil { @@ -287,6 +291,10 @@ func (s *GeminiMessagesCompatService) forwardClaudeBodyAsChatCompletions( if isImageGenerationModel(originalModel) { imageCount = 1 } + // 客户端在收到结束分片前断开时只计已送达的图片：Chat 流不向客户端转发图片。 + if clientDisconnect && !streamFinished { + imageCount = 0 + } return &ForwardResult{ RequestID: requestID, @@ -301,7 +309,7 @@ func (s *GeminiMessagesCompatService) forwardClaudeBodyAsChatCompletions( ImageCount: imageCount, ImageSize: imageSize, ImageInputSize: imageInputSize, - ClientDisconnect: false, + ClientDisconnect: clientDisconnect, }, nil } @@ -532,6 +540,11 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom var usage ClaudeUsage var firstTokenMs *int firstChunk := true + finishReason := "" + clientDisconnect := false + streamResult := func() *geminiStreamResult { + return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs, clientDisconnect: clientDisconnect, finished: finishReason != ""} + } writeChatChunk := func(chunk apicompat.ChatCompletionsChunk) bool { sse, err := apicompat.ChatChunkToSSE(chunk) @@ -539,6 +552,7 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom return false } if _, err := io.WriteString(c.Writer, sse); err != nil { + clientDisconnect = true return true } return false @@ -571,10 +585,9 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom Usage: apicompat.AnthropicUsage{}, }, }) { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } - finishReason := "" sawToolUse := false nextBlockIndex := 0 openBlockIndex := -1 @@ -643,12 +656,12 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom } if openToolIndex >= 0 { if closeOpenTool() { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } } if openBlockType != "thinking" { if closeOpenBlock() { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } idx := nextBlockIndex nextBlockIndex++ @@ -659,21 +672,21 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom Index: &idx, ContentBlock: &apicompat.AnthropicContentBlock{Type: "thinking"}, }) { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } } if emitAnthropicEvent(&apicompat.AnthropicStreamEvent{ Type: "content_block_delta", Delta: &apicompat.AnthropicDelta{Type: "thinking_delta", Thinking: delta}, }) { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } continue } if text, ok := part["text"].(string); ok && text != "" { if openToolIndex >= 0 { if closeOpenTool() { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } } delta, newSeen := computeGeminiTextDelta(seenText, text) @@ -683,7 +696,7 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom } if openBlockType != "text" { if closeOpenBlock() { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } idx := nextBlockIndex nextBlockIndex++ @@ -697,7 +710,7 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom Text: "", }, }) { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } } if emitAnthropicEvent(&apicompat.AnthropicStreamEvent{ @@ -707,7 +720,7 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom Text: delta, }, }) { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } continue } @@ -718,11 +731,11 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom name = "tool" } if closeOpenBlock() { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } if openToolIndex >= 0 && openToolName != name { if closeOpenTool() { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } } if openToolIndex < 0 { @@ -741,7 +754,7 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom Input: json.RawMessage(`{}`), }, }) { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } } @@ -767,7 +780,7 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom PartialJSON: delta, }, }) { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } } } @@ -781,15 +794,20 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom break } if err != nil { + if geminiStreamAbortedWithBillableOutput(c, err, &usage) { + logger.LegacyPrintf("service.gemini_messages_compat", "[GeminiAPI] Client disconnected during chat completions stream, billing collected usage: %v", err) + clientDisconnect = true + return streamResult(), nil + } return nil, fmt.Errorf("stream read error: %w", err) } } if closeOpenBlock() { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } if closeOpenTool() { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } stopReason := mapGeminiFinishReasonToClaudeStopReason(finishReason) @@ -810,30 +828,30 @@ func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFrom CacheReadInputTokens: usage.CacheReadInputTokens, }, }) { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } if emitAnthropicEvent(&apicompat.AnthropicStreamEvent{Type: "message_stop"}) { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } for _, resEvt := range apicompat.FinalizeAnthropicResponsesStream(anthState) { chunks := apicompat.ResponsesEventToChatChunks(&resEvt, ccState) for _, chunk := range chunks { if disconnected := writeChatChunk(chunk); disconnected { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } } } for _, chunk := range apicompat.FinalizeResponsesChatStream(ccState) { if disconnected := writeChatChunk(chunk); disconnected { - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } } _, _ = io.WriteString(c.Writer, "data: [DONE]\n\n") flusher.Flush() - return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil + return streamResult(), nil } func (s *GeminiMessagesCompatService) writeGeminiChatCompletionsMappedError( diff --git a/backend/internal/service/gemini_messages_compat_service.go b/backend/internal/service/gemini_messages_compat_service.go index 48b095fb3620..6f41c1dfeffa 100644 --- a/backend/internal/service/gemini_messages_compat_service.go +++ b/backend/internal/service/gemini_messages_compat_service.go @@ -1015,6 +1015,8 @@ func (s *GeminiMessagesCompatService) Forward(ctx context.Context, c *gin.Contex var usage *ClaudeUsage var firstTokenMs *int + clientDisconnect := false + streamFinished := false if req.Stream { streamRes, err := s.handleStreamingResponse(c, resp, startTime, originalModel) if err != nil { @@ -1022,6 +1024,8 @@ func (s *GeminiMessagesCompatService) Forward(ctx context.Context, c *gin.Contex } usage = streamRes.usage firstTokenMs = streamRes.firstTokenMs + clientDisconnect = streamRes.clientDisconnect + streamFinished = streamRes.finished } else { if useUpstreamStream { collected, usageObj, err := collectGeminiSSE(resp.Body, true) @@ -1049,6 +1053,10 @@ func (s *GeminiMessagesCompatService) Forward(ctx context.Context, c *gin.Contex imageInputSize := s.extractImageInputSize(body) imageSize := normalizeOpenAIImageSizeTier(imageInputSize) imageCount := resolveGeminiImageCount(c, originalModel, mappedModel) + // 客户端在收到结束分片前断开时只计已送达的图片：Claude 兼容流不向客户端转发图片。 + if clientDisconnect && !streamFinished { + imageCount = 0 + } return &ForwardResult{ RequestID: requestID, @@ -1062,6 +1070,7 @@ func (s *GeminiMessagesCompatService) Forward(ctx context.Context, c *gin.Contex Duration: time.Since(startTime), FirstTokenMs: firstTokenMs, ReasoningEffort: extractGeminiReasoningEffortFromBody(geminiReq), + ClientDisconnect: clientDisconnect, ImageCount: imageCount, ImageSize: imageSize, ImageInputSize: imageInputSize, @@ -1530,6 +1539,8 @@ func (s *GeminiMessagesCompatService) ForwardNative(ctx context.Context, c *gin. var usage *ClaudeUsage var firstTokenMs *int + clientDisconnect := false + streamFinished := false if stream { streamRes, err := s.handleNativeStreamingResponse(c, resp, startTime, isOAuth, account, requestID) @@ -1538,6 +1549,8 @@ func (s *GeminiMessagesCompatService) ForwardNative(ctx context.Context, c *gin. } usage = streamRes.usage firstTokenMs = streamRes.firstTokenMs + clientDisconnect = streamRes.clientDisconnect + streamFinished = streamRes.finished } else { if useUpstreamStream { var best geminiResponseSignal @@ -1572,6 +1585,10 @@ func (s *GeminiMessagesCompatService) ForwardNative(ctx context.Context, c *gin. imageInputSize := s.extractImageInputSize(body) imageSize := normalizeOpenAIImageSizeTier(imageInputSize) imageCount := resolveGeminiImageCount(c, originalModel, mappedModel) + // 客户端在收到结束分片前断开时只计已送达的图片。 + if clientDisconnect && !streamFinished { + imageCount = observedGeminiImageOutputs(c) + } return &ForwardResult{ RequestID: requestID, @@ -1585,6 +1602,7 @@ func (s *GeminiMessagesCompatService) ForwardNative(ctx context.Context, c *gin. ReasoningEffort: extractGeminiReasoningEffortFromBody(body), Duration: time.Since(startTime), FirstTokenMs: firstTokenMs, + ClientDisconnect: clientDisconnect, ImageCount: imageCount, ImageSize: imageSize, ImageInputSize: imageInputSize, @@ -2068,8 +2086,10 @@ func mapGeminiStatusToClaudeErrorType(status string) string { } type geminiStreamResult struct { - usage *ClaudeUsage - firstTokenMs *int + usage *ClaudeUsage + firstTokenMs *int + clientDisconnect bool // 客户端在流式传输中途断开，结果为断开前已收集的部分 + finished bool // 断开前已收到上游 finishReason } func (s *GeminiMessagesCompatService) handleNonStreamingResponse(c *gin.Context, resp *http.Response, originalModel string) (*ClaudeUsage, error) { @@ -2151,6 +2171,15 @@ func (s *GeminiMessagesCompatService) handleStreamingResponse(c *gin.Context, re for { line, err := reader.ReadString('\n') if err != nil && !errors.Is(err, io.EOF) { + if geminiStreamAbortedWithBillableOutput(c, err, &usage) { + logger.LegacyPrintf("service.gemini_messages_compat", "[GeminiAPI] Client disconnected during messages stream, billing collected usage: %v", err) + return &geminiStreamResult{ + usage: &usage, + firstTokenMs: firstTokenMs, + clientDisconnect: true, + finished: finishReason != "", + }, nil + } return nil, fmt.Errorf("stream read error: %w", err) } @@ -2660,8 +2689,10 @@ func mergeCollectedTextParts(response map[string]any, textParts []string) map[st } type geminiNativeStreamResult struct { - usage *ClaudeUsage - firstTokenMs *int + usage *ClaudeUsage + firstTokenMs *int + clientDisconnect bool // 客户端在流式传输中途断开，结果为断开前已收集的部分 + finished bool // 断开前已收到上游 finishReason } func isGeminiInsufficientScope(headers http.Header, body []byte) bool { @@ -2838,10 +2869,13 @@ func (s *GeminiMessagesCompatService) handleNativeStreamingResponse(c *gin.Conte var best geminiResponseSignal sawDataEvent := false fallback := &geminiSSEFallbackBody{} + clientDisconnect := false + finished := false for { line, err := reader.ReadString('\n') - if len(line) > 0 { + // 读错误时 line 是未以换行结尾的残行，不透传也不参与用量统计；EOF 时的末行完整有效。 + if len(line) > 0 && (err == nil || errors.Is(err, io.EOF)) { trimmed := strings.TrimRight(line, "\r\n") if strings.HasPrefix(trimmed, "data:") { payload := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:")) @@ -2874,6 +2908,10 @@ func (s *GeminiMessagesCompatService) handleNativeStreamingResponse(c *gin.Conte observer.ObserveGemini(rawBytes) observeGeminiImageOutputs(c, rawBytes) + if gjson.GetBytes(rawBytes, "candidates.0.finishReason").String() != "" { + finished = true + } + if firstTokenMs == nil { ms := int(time.Since(startTime).Milliseconds()) firstTokenMs = &ms @@ -2901,13 +2939,18 @@ func (s *GeminiMessagesCompatService) handleNativeStreamingResponse(c *gin.Conte break } if err != nil { - return nil, err + if !geminiStreamAbortedWithBillableOutput(c, err, usage) { + return nil, err + } + clientDisconnect = true + logger.LegacyPrintf("service.gemini_messages_compat", "[GeminiAPI] Client disconnected during native stream (account=%d), billing collected usage: %v", account.ID, err) + break } } s.finalizeGeminiSSESignal(c, account, true, upstreamRequestID, best, sawDataEvent, fallback) - return &geminiNativeStreamResult{usage: usage, firstTokenMs: firstTokenMs}, nil + return &geminiNativeStreamResult{usage: usage, firstTokenMs: firstTokenMs, clientDisconnect: clientDisconnect, finished: finished}, nil } // unwrapGeminiResponse 解包 Gemini OAuth 响应中的 response 字段 diff --git a/backend/internal/service/gemini_stream_client_abort.go b/backend/internal/service/gemini_stream_client_abort.go new file mode 100644 index 000000000000..d1dba1b8028e --- /dev/null +++ b/backend/internal/service/gemini_stream_client_abort.go @@ -0,0 +1,25 @@ +package service + +import ( + "context" + "errors" + "io" + + "github.com/gin-gonic/gin" +) + +// geminiStreamAbortedWithBillableOutput 判断流式读取中断是否由客户端断开引起，且断开前 +// 已观测到可计费的输出（token 用量或已回吐的图片）。满足时调用方以已收集的结果收尾并照常计费。 +// usageMetadata 为累计值，按断开前收到的最后一份计费：读到结束分片后断开即完整用量； +// 中途断开时中间分片可能只含输入计数或不含用量，此时按已有计数计费或不计费。 +func geminiStreamAbortedWithBillableOutput(c *gin.Context, err error, usage *ClaudeUsage) bool { + if err == nil || errors.Is(err, io.EOF) { + return false + } + clientGone := errors.Is(err, context.Canceled) || + (c != nil && c.Request != nil && c.Request.Context().Err() != nil) + if !clientGone { + return false + } + return usage.hasObservedTokens() || observedGeminiImageOutputs(c) > 0 +} diff --git a/backend/internal/service/gemini_stream_client_abort_test.go b/backend/internal/service/gemini_stream_client_abort_test.go new file mode 100644 index 000000000000..0825d5152aa5 --- /dev/null +++ b/backend/internal/service/gemini_stream_client_abort_test.go @@ -0,0 +1,244 @@ +//go:build unit + +package service + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +// geminiAbortingStreamBody 先吐出 data，读完后返回 abortErr；onAbort 用于模拟客户端断开时请求 context 被取消。 +type geminiAbortingStreamBody struct { + data *bytes.Reader + abortErr error + onAbort func() +} + +func (b *geminiAbortingStreamBody) Read(p []byte) (int, error) { + n, err := b.data.Read(p) + if err == io.EOF { + if b.onAbort != nil { + b.onAbort() + } + return n, b.abortErr + } + return n, err +} + +func (b *geminiAbortingStreamBody) Close() error { return nil } + +const ( + geminiAbortTextChunk = `data: {"candidates":[{"content":{"parts":[{"text":"hello"}],"role":"model"}}]}` + "\n\n" + geminiAbortFinalChunk = `data: {"candidates":[{"content":{"parts":[{"text":" world"}],"role":"model"},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":5,"totalTokenCount":15}}` + "\n\n" + geminiAbortImageChunk = `data: {"candidates":[{"content":{"parts":[{"inlineData":{"mimeType":"image/png","data":"aGVsbG8="}}],"role":"model"},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":1290,"totalTokenCount":1300}}` + "\n\n" + geminiAbortUsageChunk = `data: {"candidates":[{"content":{"parts":[{"text":"drawing"}],"role":"model"}}],"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":2,"totalTokenCount":12}}` + "\n\n" +) + +// newGeminiAbortingStreamService 返回的上游流在吐完 sse 后中断；clientGone=true 时中断伴随请求 context 取消。 +func newGeminiAbortingStreamService(t *testing.T, path string, reqBody []byte, sse string, clientGone bool) (*GeminiMessagesCompatService, *gin.Context, *httptest.ResponseRecorder) { + t.Helper() + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + reqCtx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + c.Request = httptest.NewRequest(http.MethodPost, path, bytes.NewReader(reqBody)).WithContext(reqCtx) + + body := &geminiAbortingStreamBody{data: bytes.NewReader([]byte(sse)), abortErr: io.ErrUnexpectedEOF} + if clientGone { + body.abortErr = context.Canceled + body.onAbort = cancel + } + svc, httpStub := newGeminiVertexAPIKeyTestService() + httpStub.response = &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: body, + } + return svc, c, rec +} + +func TestGeminiForwardNative_ClientDisconnectAfterFinalChunkBillsCollectedUsage(t *testing.T) { + reqBody := []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`) + svc, c, _ := newGeminiAbortingStreamService(t, "/v1beta/models/gemini-3-flash-preview:streamGenerateContent", reqBody, geminiAbortTextChunk+geminiAbortFinalChunk, true) + + result, err := svc.ForwardNative(context.Background(), c, geminiVertexAPIKeyTestAccount(""), "gemini-3-flash-preview", "streamGenerateContent", true, reqBody) + require.NoError(t, err) + require.NotNil(t, result) + require.True(t, result.ClientDisconnect) + require.Equal(t, 10, result.Usage.InputTokens) + require.Equal(t, 5, result.Usage.OutputTokens) + require.Zero(t, result.ImageCount) +} + +func TestGeminiForwardNative_ClientDisconnectWithoutUsageStaysUnbilled(t *testing.T) { + reqBody := []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`) + svc, c, _ := newGeminiAbortingStreamService(t, "/v1beta/models/gemini-3-flash-preview:streamGenerateContent", reqBody, geminiAbortTextChunk, true) + + result, err := svc.ForwardNative(context.Background(), c, geminiVertexAPIKeyTestAccount(""), "gemini-3-flash-preview", "streamGenerateContent", true, reqBody) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, result) +} + +func TestGeminiForwardNative_UpstreamReadErrorWithLiveClientStaysError(t *testing.T) { + reqBody := []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`) + svc, c, _ := newGeminiAbortingStreamService(t, "/v1beta/models/gemini-3-flash-preview:streamGenerateContent", reqBody, geminiAbortTextChunk+geminiAbortFinalChunk, false) + + result, err := svc.ForwardNative(context.Background(), c, geminiVertexAPIKeyTestAccount(""), "gemini-3-flash-preview", "streamGenerateContent", true, reqBody) + require.ErrorIs(t, err, io.ErrUnexpectedEOF) + require.Nil(t, result) +} + +func TestGeminiForwardNative_ClientDisconnectBillsOnlyDeliveredImages(t *testing.T) { + reqBody := []byte(`{"contents":[{"role":"user","parts":[{"text":"draw a cat"}]}]}`) + + t.Run("image delivered before disconnect", func(t *testing.T) { + svc, c, _ := newGeminiAbortingStreamService(t, "/v1beta/models/gemini-3-pro-image:streamGenerateContent", reqBody, geminiAbortImageChunk, true) + result, err := svc.ForwardNative(context.Background(), c, geminiVertexAPIKeyTestAccount(""), "gemini-3-pro-image", "streamGenerateContent", true, reqBody) + require.NoError(t, err) + require.True(t, result.ClientDisconnect) + require.Equal(t, 1, result.ImageCount) + }) + + t.Run("disconnect before the image arrives", func(t *testing.T) { + svc, c, _ := newGeminiAbortingStreamService(t, "/v1beta/models/gemini-3-pro-image:streamGenerateContent", reqBody, geminiAbortUsageChunk, true) + result, err := svc.ForwardNative(context.Background(), c, geminiVertexAPIKeyTestAccount(""), "gemini-3-pro-image", "streamGenerateContent", true, reqBody) + require.NoError(t, err) + require.True(t, result.ClientDisconnect) + require.Zero(t, result.ImageCount) + require.Equal(t, 2, result.Usage.OutputTokens) + }) +} + +func TestGeminiCompatForward_ClientDisconnectAfterFinalChunkBillsCollectedUsage(t *testing.T) { + reqBody := []byte(`{"model":"gemini-3-flash-preview","max_tokens":16,"stream":true,"messages":[{"role":"user","content":"hi"}]}`) + + t.Run("client gone", func(t *testing.T) { + svc, c, _ := newGeminiAbortingStreamService(t, "/v1/messages", reqBody, geminiAbortTextChunk+geminiAbortFinalChunk, true) + result, err := svc.Forward(context.Background(), c, geminiVertexAPIKeyTestAccount(""), reqBody) + require.NoError(t, err) + require.True(t, result.ClientDisconnect) + require.Equal(t, 10, result.Usage.InputTokens) + require.Equal(t, 5, result.Usage.OutputTokens) + }) + + t.Run("upstream read error with live client", func(t *testing.T) { + svc, c, _ := newGeminiAbortingStreamService(t, "/v1/messages", reqBody, geminiAbortTextChunk+geminiAbortFinalChunk, false) + result, err := svc.Forward(context.Background(), c, geminiVertexAPIKeyTestAccount(""), reqBody) + require.ErrorIs(t, err, io.ErrUnexpectedEOF) + require.Nil(t, result) + }) +} + +func TestGeminiForwardAsChatCompletions_ClientDisconnectAfterFinalChunkBillsCollectedUsage(t *testing.T) { + reqBody := []byte(`{"model":"gemini-3-flash-preview","stream":true,"messages":[{"role":"user","content":"hi"}]}`) + + t.Run("client gone", func(t *testing.T) { + svc, c, _ := newGeminiAbortingStreamService(t, "/v1/chat/completions", reqBody, geminiAbortTextChunk+geminiAbortFinalChunk, true) + result, err := svc.ForwardAsChatCompletions(context.Background(), c, geminiVertexAPIKeyTestAccount(""), reqBody) + require.NoError(t, err) + require.True(t, result.ClientDisconnect) + require.Equal(t, 10, result.Usage.InputTokens) + require.Equal(t, 5, result.Usage.OutputTokens) + }) + + t.Run("client gone before usage", func(t *testing.T) { + svc, c, _ := newGeminiAbortingStreamService(t, "/v1/chat/completions", reqBody, geminiAbortTextChunk, true) + result, err := svc.ForwardAsChatCompletions(context.Background(), c, geminiVertexAPIKeyTestAccount(""), reqBody) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, result) + }) +} + +func TestGeminiForwardNative_ClientDisconnectAfterFinishKeepsImageFallback(t *testing.T) { + // 结束分片已到达但图片不以 inlineData 回吐时，断开与正常完成同口径按模型名兜底。 + finishWithoutInline := `data: {"candidates":[{"content":{"parts":[{"fileData":{"mimeType":"image/png","fileUri":"gs://bucket/cat.png"}}],"role":"model"},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":1290,"totalTokenCount":1300}}` + "\n\n" + reqBody := []byte(`{"contents":[{"role":"user","parts":[{"text":"draw a cat"}]}]}`) + svc, c, _ := newGeminiAbortingStreamService(t, "/v1beta/models/gemini-3-pro-image:streamGenerateContent", reqBody, finishWithoutInline, true) + + result, err := svc.ForwardNative(context.Background(), c, geminiVertexAPIKeyTestAccount(""), "gemini-3-pro-image", "streamGenerateContent", true, reqBody) + require.NoError(t, err) + require.True(t, result.ClientDisconnect) + require.Equal(t, 1, result.ImageCount) +} + +func TestGeminiForwardNative_ClientDisconnectIgnoresTruncatedTrailingLine(t *testing.T) { + complete := `data: {"candidates":[{"content":{"parts":[{"text":"hello"}],"role":"model"}}],"usageMetadata":{"promptTokenCount":1000,"candidatesTokenCount":300,"totalTokenCount":1300}}` + "\n\n" + truncated := `data: {"candidates":[{"content":{"parts":[{"text":" more"}],"role":"model"}}],"usageMetadata":{` + reqBody := []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`) + svc, c, _ := newGeminiAbortingStreamService(t, "/v1beta/models/gemini-3-flash-preview:streamGenerateContent", reqBody, complete+truncated, true) + + result, err := svc.ForwardNative(context.Background(), c, geminiVertexAPIKeyTestAccount(""), "gemini-3-flash-preview", "streamGenerateContent", true, reqBody) + require.NoError(t, err) + require.True(t, result.ClientDisconnect) + require.Equal(t, 1000, result.Usage.InputTokens) + require.Equal(t, 300, result.Usage.OutputTokens) +} + +func TestGeminiForwardAsChatCompletions_ClientDisconnectImageCount(t *testing.T) { + t.Run("disconnect before finish bills no image", func(t *testing.T) { + reqBody := []byte(`{"model":"gemini-3-pro-image","stream":true,"messages":[{"role":"user","content":"draw a cat"}]}`) + svc, c, _ := newGeminiAbortingStreamService(t, "/v1/chat/completions", reqBody, geminiAbortUsageChunk, true) + result, err := svc.ForwardAsChatCompletions(context.Background(), c, geminiVertexAPIKeyTestAccount(""), reqBody) + require.NoError(t, err) + require.True(t, result.ClientDisconnect) + require.Zero(t, result.ImageCount) + require.Equal(t, 2, result.Usage.OutputTokens) + }) + + t.Run("disconnect after finish keeps completed-request image count", func(t *testing.T) { + reqBody := []byte(`{"model":"gemini-3-pro-image","stream":true,"messages":[{"role":"user","content":"draw a cat"}]}`) + svc, c, _ := newGeminiAbortingStreamService(t, "/v1/chat/completions", reqBody, geminiAbortImageChunk, true) + result, err := svc.ForwardAsChatCompletions(context.Background(), c, geminiVertexAPIKeyTestAccount(""), reqBody) + require.NoError(t, err) + require.True(t, result.ClientDisconnect) + require.Equal(t, 1, result.ImageCount) + }) +} + +// geminiFailingResponseWriter 模拟客户端已断开：所有写入失败。 +type geminiFailingResponseWriter struct { + header http.Header +} + +func (w *geminiFailingResponseWriter) Header() http.Header { return w.header } +func (w *geminiFailingResponseWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe } +func (w *geminiFailingResponseWriter) WriteHeader(int) {} +func (w *geminiFailingResponseWriter) Flush() {} + +func TestGeminiForwardAsChatCompletions_WriteFailureOnImageModelBillsNoImage(t *testing.T) { + gin.SetMode(gin.TestMode) + reqBody := []byte(`{"model":"gemini-3-pro-image","stream":true,"messages":[{"role":"user","content":"draw a cat"}]}`) + c, _ := gin.CreateTestContext(&geminiFailingResponseWriter{header: http.Header{}}) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(reqBody)) + svc, httpStub := newGeminiVertexAPIKeyTestService() + httpStub.response = &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(bytes.NewReader([]byte(geminiAbortUsageChunk))), + } + + result, err := svc.ForwardAsChatCompletions(context.Background(), c, geminiVertexAPIKeyTestAccount(""), reqBody) + require.NoError(t, err) + require.True(t, result.ClientDisconnect) + require.Zero(t, result.ImageCount) +} + +func TestGeminiCompatForward_ClientDisconnectBeforeFinishBillsNoImage(t *testing.T) { + inlineWithoutFinish := `data: {"candidates":[{"content":{"parts":[{"inlineData":{"mimeType":"image/png","data":"aGVsbG8="}}],"role":"model"}}],"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":1290,"totalTokenCount":1300}}` + "\n\n" + reqBody := []byte(`{"model":"gemini-3-pro-image","max_tokens":16,"stream":true,"messages":[{"role":"user","content":"draw a cat"}]}`) + svc, c, _ := newGeminiAbortingStreamService(t, "/v1/messages", reqBody, inlineWithoutFinish, true) + + result, err := svc.Forward(context.Background(), c, geminiVertexAPIKeyTestAccount(""), reqBody) + require.NoError(t, err) + require.True(t, result.ClientDisconnect) + require.Zero(t, result.ImageCount) + require.Equal(t, 1290, result.Usage.OutputTokens) +} From 1cdecf4a88f016115392033bdbe0662d7ff92e81 Mon Sep 17 00:00:00 2001 From: haruka <1628615876@qq.com> Date: Fri, 16 Oct 2026 20:13:58 -0400 Subject: [PATCH 05/29] =?UTF-8?q?fix(billing):=20Grok=204.7=20Fast=20?= =?UTF-8?q?=E6=8C=89=E5=AE=98=E6=96=B9=20Fast=20=E4=BB=B7=E5=8D=A1?= =?UTF-8?q?=E8=AE=A1=E8=B4=B9=20(#8119)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit grok-4.7-fast 没有专属价格，落入「未知 Grok 文本模型」兜底按 grok-4.6 价卡计费，与普通 grok-4.7 同价（$2 / $0.50 / $6），少收一半。 - 内置兜底价新增 grok-4.7-fast：<200k 为 $4 输入 / $1 缓存读取 / $12 输出 （每百万 token，标准价 2 倍）；>=200k 整次请求为 $6 / $1.50 / $18， 即 Fast 基础价 1.5 倍，阈值口径与其它 xAI 价卡一致（达到即进高档） - grok-4.7-build-fast（Grok Build 侧的 Fast ID，见 #7502）同样命中该价卡 - 本地价格快照新增 grok-4.7-fast 条目，带 xai/ 前缀的写法也能识别 价格来源：https://docs.x.ai/developers/pricing（Grok 4.7 Fast，仅 Cursor 与 Grok Build 提供）。 Co-Authored-By: Claude Opus 5.5 --- backend/internal/service/billing_service.go | 19 +++++++ .../internal/service/billing_service_test.go | 52 +++++++++++++++++++ .../internal/service/pricing_service_test.go | 37 +++++++++++++ .../model_prices_and_context_window.json | 12 +++++ 4 files changed, 120 insertions(+) diff --git a/backend/internal/service/billing_service.go b/backend/internal/service/billing_service.go index 7341fb123dfb..bd639ebfab30 100644 --- a/backend/internal/service/billing_service.go +++ b/backend/internal/service/billing_service.go @@ -978,6 +978,20 @@ func (s *BillingService) initFallbackPricing() { LongContextOutputMultiplier: 2, } + // xAI Grok 4.7 Fast (Cursor and Grok Build only): the same model on faster + // infrastructure at $4 input / $1 cached input / $12 output below 200k; + // long-context rates are $6 / $1.50 / $18 (1.5x of the Fast base card). + s.fallbackPrices["grok-4.7-fast"] = &ModelPricing{ + InputPricePerToken: 4e-6, + OutputPricePerToken: 12e-6, + CacheReadPricePerToken: 1e-6, + SupportsCacheBreakdown: false, + LongContextInputThreshold: 200000, + LongContextThresholdInclusive: true, + LongContextInputMultiplier: 1.5, + LongContextOutputMultiplier: 1.5, + } + // xAI Grok 4.3: $1.25 input / $0.20 cached / $2.50 output below 200k; // long-context rates are $2.50 / $0.40 / $5. s.fallbackPrices["grok-4.3"] = &ModelPricing{ @@ -1274,6 +1288,11 @@ func (s *BillingService) getFallbackPricing(model string) *ModelPricing { return s.fallbackPrices["grok-4.6"] case "grok-4.7", "grok-4.7-latest": return s.fallbackPrices["grok-4.7"] + // Grok 4.7 Fast IDs (grok-4.7-fast, and the Grok Build ID + // grok-4.7-build-fast). Without these cases both fall into the unknown-Grok + // family card and bill at the standard 4.7 rate (#8119). + case "grok-4.7-fast", "grok-4.7-build-fast": + return s.fallbackPrices["grok-4.7-fast"] case "grok-4.5", "grok-4.5-latest": return s.fallbackPrices["grok-4.5"] case "grok-3-mini": diff --git a/backend/internal/service/billing_service_test.go b/backend/internal/service/billing_service_test.go index 45544508ec1c..d02c942cb3fc 100644 --- a/backend/internal/service/billing_service_test.go +++ b/backend/internal/service/billing_service_test.go @@ -1203,6 +1203,58 @@ func TestGetModelPricing_Grok47OfficialFallback(t *testing.T) { } } +func TestGetModelPricing_Grok47FastOfficialFallback(t *testing.T) { + svc := newTestBillingService() + + for _, model := range []string{"grok-4.7-fast", "grok-4.7-build-fast", "GROK-4.7-FAST"} { + model := model + t.Run(model, func(t *testing.T) { + pricing, err := svc.GetModelPricing(model) + require.NoError(t, err) + require.InDelta(t, 4e-6, pricing.InputPricePerToken, 1e-12) + require.InDelta(t, 12e-6, pricing.OutputPricePerToken, 1e-12) + require.InDelta(t, 1e-6, pricing.CacheReadPricePerToken, 1e-12) + require.Equal(t, 200000, pricing.LongContextInputThreshold) + require.True(t, pricing.LongContextThresholdInclusive) + require.InDelta(t, 1.5, pricing.LongContextInputMultiplier, 1e-12) + require.InDelta(t, 1.5, pricing.LongContextOutputMultiplier, 1e-12) + require.False(t, pricing.SupportsCacheBreakdown) + }) + } + require.True(t, svc.HasIdentifiedTokenPricing("grok-4.7-fast")) +} + +// Issue #8119: Grok 4.7 Fast used to inherit the unknown-Grok card and bill at +// the standard grok-4.7 rate. +func TestCalculateCost_Grok47FastUsesFastCard(t *testing.T) { + svc := newTestBillingService() + + short := UsageTokens{InputTokens: 100_000, CacheReadTokens: 50_000, OutputTokens: 10_000} + standard, err := svc.CalculateCost("grok-4.7", short, 1) + require.NoError(t, err) + fast, err := svc.CalculateCost("grok-4.7-fast", short, 1) + require.NoError(t, err) + require.InDelta(t, 0.4, fast.InputCost, 1e-12) + require.InDelta(t, 0.05, fast.CacheReadCost, 1e-12) + require.InDelta(t, 0.12, fast.OutputCost, 1e-12) + require.False(t, fast.LongContextBillingApplied) + // Below 200k the Fast card is exactly twice the standard card. + require.InDelta(t, standard.TotalCost*2, fast.TotalCost, 1e-12) + + // Reaching 200k prompt tokens moves the whole request to $6 / $1.50 / $18. + long := UsageTokens{InputTokens: 150_000, CacheReadTokens: 50_000, OutputTokens: 10_000} + standardLong, err := svc.CalculateCost("grok-4.7", long, 1) + require.NoError(t, err) + fastLong, err := svc.CalculateCost("grok-4.7-fast", long, 1) + require.NoError(t, err) + require.True(t, fastLong.LongContextBillingApplied) + require.InDelta(t, 0.9, fastLong.InputCost, 1e-12) + require.InDelta(t, 0.075, fastLong.CacheReadCost, 1e-12) + require.InDelta(t, 0.18, fastLong.OutputCost, 1e-12) + // Long-context Fast is 1.5x the standard long-context card. + require.InDelta(t, standardLong.TotalCost*1.5, fastLong.TotalCost, 1e-12) +} + func TestGetModelPricing_GrokOfficialFamilyCards(t *testing.T) { svc := newTestBillingService() for _, tc := range []struct { diff --git a/backend/internal/service/pricing_service_test.go b/backend/internal/service/pricing_service_test.go index 08f5171a7c08..d72e2f121414 100644 --- a/backend/internal/service/pricing_service_test.go +++ b/backend/internal/service/pricing_service_test.go @@ -819,6 +819,43 @@ func TestDefaultPricingIncludesGemini4ArgonRates(t *testing.T) { require.Contains(t, pricingSvc.ListModelNamesByProvider("gemini"), "gemini-4-argon") } +func TestDefaultPricingIncludesGrok47FastRates(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "resources", "model-pricing", "model_prices_and_context_window.json")) + require.NoError(t, err) + + pricingSvc := &PricingService{} + pricingData, err := pricingSvc.parsePricingData(data) + require.NoError(t, err) + pricingSvc.pricingData = pricingData + + for _, model := range []string{"grok-4.7-fast", "xai/grok-4.7-fast"} { + t.Run(model, func(t *testing.T) { + pricing := pricingSvc.GetModelPricing(model) + require.NotNil(t, pricing) + require.Equal(t, "xai", pricing.LiteLLMProvider) + require.InDelta(t, 4e-6, pricing.InputCostPerToken, 1e-15) + require.InDelta(t, 12e-6, pricing.OutputCostPerToken, 1e-15) + require.InDelta(t, 1e-6, pricing.CacheReadInputTokenCost, 1e-15) + // Long-context Fast is $6 / $1.50 / $18, i.e. 1.5x the Fast base card. + require.Equal(t, 200000, pricing.LongContextInputTokenThreshold) + require.InDelta(t, 1.5, pricing.LongContextInputCostMultiplier, 1e-12) + require.InDelta(t, 1.5, pricing.LongContextOutputCostMultiplier, 1e-12) + }) + } + // The Fast entry must not shadow the standard grok-4.7 card. + standard := pricingSvc.GetModelPricing("grok-4.7") + require.NotNil(t, standard) + require.InDelta(t, 2e-6, standard.InputCostPerToken, 1e-15) + + billing := NewBillingService(&config.Config{}, pricingSvc) + cost, err := billing.CalculateCost("grok-4.7-fast", UsageTokens{InputTokens: 150_000, CacheReadTokens: 50_000, OutputTokens: 10_000}, 1) + require.NoError(t, err) + require.True(t, cost.LongContextBillingApplied) + require.InDelta(t, 0.9, cost.InputCost, 1e-12) + require.InDelta(t, 0.075, cost.CacheReadCost, 1e-12) + require.InDelta(t, 0.18, cost.OutputCost, 1e-12) +} + func TestDefaultPricingUsesCurrentCodexAutoReviewBaseRates(t *testing.T) { data, err := os.ReadFile(filepath.Join("..", "..", "resources", "model-pricing", "model_prices_and_context_window.json")) require.NoError(t, err) diff --git a/backend/resources/model-pricing/model_prices_and_context_window.json b/backend/resources/model-pricing/model_prices_and_context_window.json index 5e0f608d2f0d..c9a5c3fb48eb 100644 --- a/backend/resources/model-pricing/model_prices_and_context_window.json +++ b/backend/resources/model-pricing/model_prices_and_context_window.json @@ -64,6 +64,18 @@ "output_cost_per_token_above_200k_tokens": 1.2e-05, "supports_prompt_caching": true }, + "grok-4.7-fast": { + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_above_200k_tokens": 1.5e-06, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "supports_prompt_caching": true + }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_batches": 1.25e-06, From 9c41b32695c4aa5d2673f228097762cba14e64f0 Mon Sep 17 00:00:00 2001 From: feeeei Date: Sat, 17 Oct 2026 18:02:18 +0800 Subject: [PATCH 06/29] =?UTF-8?q?fix(gemini):=20prompt=20=E8=A2=AB?= =?UTF-8?q?=E6=8B=A6=E6=88=AA=E5=8F=AA=E6=8A=A5=E7=BC=93=E5=AD=98=E5=91=BD?= =?UTF-8?q?=E4=B8=AD=E6=97=B6=20input=5Ftokens=20=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E4=B8=BA=E8=B4=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Vertex 拦截 prompt（promptFeedback.blockReason，如 PROHIBITED_CONTENT）时 usageMetadata 可能只带 cachedContentTokenCount、缺 promptTokenCount，promptTokenCount − cachedContentTokenCount 得到负的 input_tokens 与负费用 - extractGeminiUsage 与 antigravity 非流式 / 流式（累计用量、message_start）四处换算改为差值下限 0， 缓存命中仍照上游所报计入 cache_read_input_tokens - 补充表测试、原生流式拦截用例与 antigravity 两条路径用例 --- .../pkg/antigravity/response_transformer.go | 5 ++-- .../pkg/antigravity/stream_transformer.go | 7 ++--- .../pkg/antigravity/usage_contract_test.go | 26 +++++++++++++++++++ .../service/gemini_messages_compat_service.go | 5 ++-- .../gemini_messages_compat_service_test.go | 10 +++++++ .../gemini_native_response_signal_test.go | 23 ++++++++++++++++ 6 files changed, 69 insertions(+), 7 deletions(-) diff --git a/backend/internal/pkg/antigravity/response_transformer.go b/backend/internal/pkg/antigravity/response_transformer.go index 0db6822f315b..03e26ac50cd9 100644 --- a/backend/internal/pkg/antigravity/response_transformer.go +++ b/backend/internal/pkg/antigravity/response_transformer.go @@ -277,11 +277,12 @@ func (p *NonStreamingProcessor) buildResponse(geminiResp *GeminiResponse, respon } // 注意：Gemini 的 promptTokenCount 包含 cachedContentTokenCount， - // 但 Claude 的 input_tokens 不包含 cache_read_input_tokens，需要减去 + // 但 Claude 的 input_tokens 不包含 cache_read_input_tokens，需要减去。 + // prompt 被拦截时上游可能只报 cachedContentTokenCount，差值下限为 0。 usage := ClaudeUsage{} if geminiResp.UsageMetadata != nil { cached := geminiResp.UsageMetadata.CachedContentTokenCount - usage.InputTokens = geminiResp.UsageMetadata.PromptTokenCount - cached + usage.InputTokens = max(geminiResp.UsageMetadata.PromptTokenCount-cached, 0) usage.OutputTokens = geminiResp.UsageMetadata.CandidatesTokenCount + geminiResp.UsageMetadata.ThoughtsTokenCount usage.CacheReadInputTokens = cached usage.ImageOutputTokens = geminiResp.UsageMetadata.ImageOutputTokens() diff --git a/backend/internal/pkg/antigravity/stream_transformer.go b/backend/internal/pkg/antigravity/stream_transformer.go index faecf233ecac..6b52ba882f93 100644 --- a/backend/internal/pkg/antigravity/stream_transformer.go +++ b/backend/internal/pkg/antigravity/stream_transformer.go @@ -109,10 +109,11 @@ func (p *StreamingProcessor) ProcessLine(line string) []byte { // 更新 usage // 注意：Gemini 的 promptTokenCount 包含 cachedContentTokenCount， - // 但 Claude 的 input_tokens 不包含 cache_read_input_tokens，需要减去 + // 但 Claude 的 input_tokens 不包含 cache_read_input_tokens，需要减去。 + // prompt 被拦截时上游可能只报 cachedContentTokenCount，差值下限为 0。 if geminiResp.UsageMetadata != nil { cached := geminiResp.UsageMetadata.CachedContentTokenCount - p.inputTokens = geminiResp.UsageMetadata.PromptTokenCount - cached + p.inputTokens = max(geminiResp.UsageMetadata.PromptTokenCount-cached, 0) p.outputTokens = geminiResp.UsageMetadata.CandidatesTokenCount + geminiResp.UsageMetadata.ThoughtsTokenCount p.cacheReadTokens = cached p.imageOutputTokens = geminiResp.UsageMetadata.ImageOutputTokens() @@ -190,7 +191,7 @@ func (p *StreamingProcessor) emitMessageStart(v1Resp *V1InternalResponse) []byte usage := ClaudeUsage{} if v1Resp.Response.UsageMetadata != nil { cached := v1Resp.Response.UsageMetadata.CachedContentTokenCount - usage.InputTokens = v1Resp.Response.UsageMetadata.PromptTokenCount - cached + usage.InputTokens = max(v1Resp.Response.UsageMetadata.PromptTokenCount-cached, 0) usage.OutputTokens = v1Resp.Response.UsageMetadata.CandidatesTokenCount + v1Resp.Response.UsageMetadata.ThoughtsTokenCount usage.CacheReadInputTokens = cached usage.ImageOutputTokens = v1Resp.Response.UsageMetadata.ImageOutputTokens() diff --git a/backend/internal/pkg/antigravity/usage_contract_test.go b/backend/internal/pkg/antigravity/usage_contract_test.go index 832bffbfb2f7..e52431742f2f 100644 --- a/backend/internal/pkg/antigravity/usage_contract_test.go +++ b/backend/internal/pkg/antigravity/usage_contract_test.go @@ -38,3 +38,29 @@ func TestGeminiUsageMapping_NoCacheCreationTokens(t *testing.T) { require.Zero(t, usage.CacheCreationInputTokens) }) } + +// usageMetadata 缺 promptTokenCount、只有 cachedContentTokenCount 时，input_tokens 下限为 0， +// 缓存命中仍计入 cache_read_input_tokens。 +func TestGeminiUsageMapping_MissingPromptTokenCountKeepsInputNonNegative(t *testing.T) { + const geminiBody = `{"candidates":[{"content":{"parts":[{"text":"hi"}],"role":"model"},"finishReason":"STOP"}],` + + `"usageMetadata":{"cachedContentTokenCount":30}}` + + t.Run("non-stream", func(t *testing.T) { + _, usage, err := TransformGeminiToClaude([]byte(geminiBody), "gemini-3.1-pro-preview") + require.NoError(t, err) + require.NotNil(t, usage) + require.Zero(t, usage.InputTokens) + require.Equal(t, 30, usage.CacheReadInputTokens) + }) + + t.Run("stream", func(t *testing.T) { + p := NewStreamingProcessor("gemini-3.1-pro-preview") + out := p.ProcessLine(`data: {"response":` + geminiBody + `}`) + require.True(t, strings.Contains(string(out), `"message_start"`)) + require.NotContains(t, string(out), `"input_tokens":-`, "message_start 的 input_tokens 不应为负") + _, usage := p.Finish() + require.NotNil(t, usage) + require.Zero(t, usage.InputTokens) + require.Equal(t, 30, usage.CacheReadInputTokens) + }) +} diff --git a/backend/internal/service/gemini_messages_compat_service.go b/backend/internal/service/gemini_messages_compat_service.go index 48b095fb3620..0ca68d535495 100644 --- a/backend/internal/service/gemini_messages_compat_service.go +++ b/backend/internal/service/gemini_messages_compat_service.go @@ -3036,9 +3036,10 @@ func extractGeminiUsage(data []byte) *ClaudeUsage { } // 注意：Gemini 的 promptTokenCount 包含 cachedContentTokenCount， - // 但 Claude 的 input_tokens 不包含 cache_read_input_tokens，需要减去 + // 但 Claude 的 input_tokens 不包含 cache_read_input_tokens，需要减去。 + // prompt 被拦截时上游可能只报 cachedContentTokenCount，差值下限为 0。 return &ClaudeUsage{ - InputTokens: prompt - cached, + InputTokens: max(prompt-cached, 0), OutputTokens: cand + thoughts, CacheReadInputTokens: cached, ImageOutputTokens: imageTokens, diff --git a/backend/internal/service/gemini_messages_compat_service_test.go b/backend/internal/service/gemini_messages_compat_service_test.go index 3e6c3539d99f..31995cf99be0 100644 --- a/backend/internal/service/gemini_messages_compat_service_test.go +++ b/backend/internal/service/gemini_messages_compat_service_test.go @@ -947,6 +947,16 @@ func TestExtractGeminiUsage(t *testing.T) { CacheReadInputTokens: 0, }, }, + { + name: "prompt 被拦截只报 cachedContentTokenCount — input 下限为 0", + input: `{"promptFeedback":{"blockReason":"PROHIBITED_CONTENT"},"usageMetadata":{"cachedContentTokenCount":12980}}`, + wantNil: false, + wantUsage: &ClaudeUsage{ + InputTokens: 0, + OutputTokens: 0, + CacheReadInputTokens: 12980, + }, + }, { name: "无 usageMetadata", input: `{"candidates":[]}`, diff --git a/backend/internal/service/gemini_native_response_signal_test.go b/backend/internal/service/gemini_native_response_signal_test.go index 78c061ff089a..1005eca1e7cb 100644 --- a/backend/internal/service/gemini_native_response_signal_test.go +++ b/backend/internal/service/gemini_native_response_signal_test.go @@ -180,6 +180,29 @@ func TestGeminiForwardNative_StreamPromptBlockedMarksContentPolicy(t *testing.T) require.Empty(t, upstreamErrorEventsFromContext(t, c)) } +// Vertex 拦截 prompt 时 usageMetadata 可能只带 cachedContentTokenCount、缺 promptTokenCount： +// input 下限为 0，缓存命中照上游所报计入 cache_read。 +func TestGeminiForwardNative_StreamPromptBlockedWithOnlyCachedTokensKeepsInputNonNegative(t *testing.T) { + gin.SetMode(gin.TestMode) + body := `data: {"promptFeedback":{"blockReason":"PROHIBITED_CONTENT"},"usageMetadata":{"cachedContentTokenCount":12980,"totalTokenCount":0,"trafficType":"ON_DEMAND"},"modelVersion":"gemini-3.5-flash-lite"} + +` + svc := newGeminiSignalService("text/event-stream", body) + c, rec := newGeminiNativeTestContext(t) + + result, err := svc.ForwardNative(context.Background(), c, geminiSignalTestAccount(), + "gemini-3.5-flash-lite", "streamGenerateContent", true, geminiSignalTestRequest()) + require.NoError(t, err) + require.Equal(t, body, rec.Body.String()) + require.Equal(t, 0, result.Usage.InputTokens) + require.Equal(t, 12980, result.Usage.CacheReadInputTokens) + require.Equal(t, 0, result.Usage.OutputTokens) + + streamErrs := GetOpsStreamErrors(c) + require.Len(t, streamErrs, 1) + require.Equal(t, "PROHIBITED_CONTENT", streamErrs[0].Code) +} + func TestGeminiForwardNative_EmptyStreamMarksUpstreamFailure(t *testing.T) { gin.SetMode(gin.TestMode) svc := newGeminiSignalService("text/event-stream", "") From c7bfcd93aa206065238b67757d5d8fae81cf697b Mon Sep 17 00:00:00 2001 From: wucm667 Date: Sun, 18 Oct 2026 04:40:42 +0800 Subject: [PATCH 07/29] fix(tools): normalize parameter constants and annotations --- .../service/gemini_messages_compat_service.go | 82 ++++++- .../gemini_messages_compat_service_test.go | 220 ++++++++++++++++++ 2 files changed, 298 insertions(+), 4 deletions(-) diff --git a/backend/internal/service/gemini_messages_compat_service.go b/backend/internal/service/gemini_messages_compat_service.go index 48b095fb3620..98c0cbf159d0 100644 --- a/backend/internal/service/gemini_messages_compat_service.go +++ b/backend/internal/service/gemini_messages_compat_service.go @@ -3765,12 +3765,24 @@ func isClaudeWebSearchToolMap(tool map[string]any) bool { // cleanToolSchema 清理工具的 JSON Schema，移除 Gemini 不支持的字段 func cleanToolSchema(schema any) any { + return cleanToolSchemaNode(schema, true) +} + +func cleanToolSchemaNode(schema any, schemaNode bool) any { if schema == nil { return nil } switch v := schema.(type) { case map[string]any: + if !schemaNode { + cleaned := make(map[string]any, len(v)) + for key, value := range v { + cleaned[key] = cleanToolSchemaNode(value, false) + } + return cleaned + } + cleaned := make(map[string]any) for key, value := range v { // 跳过不支持的字段 @@ -3778,11 +3790,73 @@ func cleanToolSchema(schema any) any { key == "$defs" || key == "definitions" || key == "additionalProperties" || key == "patternProperties" || key == "minLength" || key == "maxLength" || key == "minItems" || key == "maxItems" || key == "exclusiveMinimum" || - key == "deprecated" { + key == "deprecated" || key == "examples" { continue } - // 递归清理嵌套对象 - cleaned[key] = cleanToolSchema(value) + + switch key { + case "properties", "dependentSchemas": + if propertySchemas, ok := value.(map[string]any); ok { + cleanedProperties := make(map[string]any, len(propertySchemas)) + for propertyName, propertySchema := range propertySchemas { + cleanedProperties[propertyName] = cleanToolSchemaNode(propertySchema, true) + } + cleaned[key] = cleanedProperties + } else { + cleaned[key] = cleanToolSchemaNode(value, false) + } + case "items", "contains", "propertyNames", "if", "then", "else", "not", "contentSchema", + "additionalItems", "unevaluatedItems", "unevaluatedProperties": + cleaned[key] = cleanToolSchemaNode(value, true) + case "dependencies": + if dependencies, ok := value.(map[string]any); ok { + cleanedDependencies := make(map[string]any, len(dependencies)) + for propertyName, dependency := range dependencies { + if dependencySchema, ok := dependency.(map[string]any); ok { + cleanedDependencies[propertyName] = cleanToolSchemaNode(dependencySchema, true) + } else { + cleanedDependencies[propertyName] = cleanToolSchemaNode(dependency, false) + } + } + cleaned[key] = cleanedDependencies + } else { + cleaned[key] = cleanToolSchemaNode(value, false) + } + case "anyOf", "allOf", "oneOf", "prefixItems": + if schemas, ok := value.([]any); ok { + cleanedSchemas := make([]any, len(schemas)) + for i, childSchema := range schemas { + cleanedSchemas[i] = cleanToolSchemaNode(childSchema, true) + } + cleaned[key] = cleanedSchemas + } else { + cleaned[key] = cleanToolSchemaNode(value, false) + } + default: + cleaned[key] = cleanToolSchemaNode(value, false) + } + } + if constValue, exists := v["const"]; exists { + if stringValue, ok := constValue.(string); ok { + if originalEnum, hasEnum := v["enum"].([]any); hasEnum { + intersection := make([]any, 0, 1) + for _, value := range originalEnum { + if enumString, ok := value.(string); ok && enumString == stringValue { + intersection = append(intersection, stringValue) + break + } + } + cleaned["enum"] = intersection + } else { + cleaned["enum"] = []any{stringValue} + } + if _, hasType := cleaned["type"]; !hasType { + cleaned["type"] = "STRING" + } + } + // Non-string constants are unsupported: omit the constant, retain the + // original type, and do not stringify the constant or infer a type. + delete(cleaned, "const") } if enum, ok := cleaned["enum"].([]any); ok { if normalized, ok := normalizeGeminiEnum(enum); ok { @@ -3817,7 +3891,7 @@ func cleanToolSchema(schema any) any { case []any: cleaned := make([]any, len(v)) for i, item := range v { - cleaned[i] = cleanToolSchema(item) + cleaned[i] = cleanToolSchemaNode(item, schemaNode) } return cleaned default: diff --git a/backend/internal/service/gemini_messages_compat_service_test.go b/backend/internal/service/gemini_messages_compat_service_test.go index 3e6c3539d99f..64399e01d5fe 100644 --- a/backend/internal/service/gemini_messages_compat_service_test.go +++ b/backend/internal/service/gemini_messages_compat_service_test.go @@ -509,6 +509,226 @@ func TestCleanToolSchema_DropsEnumWithNonScalarValue(t *testing.T) { require.NotContains(t, cleaned, "enum") } +func TestCleanToolSchema_PreservesSchemaKeywordNamedProperties(t *testing.T) { + schema := map[string]any{ + "type": "object", + "examples": []any{"root example"}, + "properties": map[string]any{ + "const": map[string]any{ + "type": "string", + "const": "property const", + }, + "enum": map[string]any{ + "type": "string", + "const": "property enum", + }, + "examples": map[string]any{ + "type": "string", + }, + "type": map[string]any{ + "type": "string", + }, + }, + } + + cleaned, ok := cleanToolSchema(schema).(map[string]any) + require.True(t, ok) + require.NotContains(t, cleaned, "examples") + + properties, ok := cleaned["properties"].(map[string]any) + require.True(t, ok) + require.Contains(t, properties, "const") + require.Contains(t, properties, "enum") + require.Contains(t, properties, "examples") + require.Contains(t, properties, "type") + + constSchema, ok := properties["const"].(map[string]any) + require.True(t, ok) + require.Equal(t, []any{"property const"}, constSchema["enum"]) + + enumPropertySchema, ok := properties["enum"].(map[string]any) + require.True(t, ok) + require.Equal(t, []any{"property enum"}, enumPropertySchema["enum"]) +} + +func TestCleanToolSchema_NormalizesConstAndExamplesAtNestedSchemaNodes(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "values": map[string]any{ + "type": "array", + "items": map[string]any{ + "type": "string", + "examples": []any{"item example"}, + "anyOf": []any{ + map[string]any{"type": "string", "const": "a"}, + map[string]any{"type": "string", "examples": []any{"branch example"}}, + }, + }, + }, + }, + } + + cleaned, ok := cleanToolSchema(schema).(map[string]any) + require.True(t, ok) + properties, ok := cleaned["properties"].(map[string]any) + require.True(t, ok) + values, ok := properties["values"].(map[string]any) + require.True(t, ok) + items, ok := values["items"].(map[string]any) + require.True(t, ok) + require.NotContains(t, items, "examples") + anyOf, ok := items["anyOf"].([]any) + require.True(t, ok) + require.Len(t, anyOf, 2) + firstBranch, ok := anyOf[0].(map[string]any) + require.True(t, ok) + require.Equal(t, []any{"a"}, firstBranch["enum"]) + secondBranch, ok := anyOf[1].(map[string]any) + require.True(t, ok) + require.NotContains(t, secondBranch, "examples") +} + +func TestCleanToolSchema_TraversesRetainedSchemaValuedKeywords(t *testing.T) { + schema := map[string]any{ + "type": "object", + "additionalItems": map[string]any{ + "type": "string", + "const": "additional", + "examples": []any{"additional example"}, + }, + "unevaluatedItems": map[string]any{ + "type": "string", + "const": "unevaluated item", + "examples": []any{"item example"}, + }, + "unevaluatedProperties": map[string]any{ + "type": "string", + "const": "unevaluated property", + "examples": []any{"property example"}, + }, + "dependencies": map[string]any{ + "schema dependency": map[string]any{ + "type": "string", + "const": "dependency", + "examples": []any{"dependency example"}, + }, + "required dependency": []any{"first", "second"}, + }, + } + + cleaned, ok := cleanToolSchema(schema).(map[string]any) + require.True(t, ok) + expectedEnums := map[string][]any{ + "additionalItems": {"additional"}, + "unevaluatedItems": {"unevaluated item"}, + "unevaluatedProperties": {"unevaluated property"}, + } + for keyword, expectedEnum := range expectedEnums { + child, ok := cleaned[keyword].(map[string]any) + require.True(t, ok) + require.Equal(t, expectedEnum, child["enum"]) + require.NotContains(t, child, "const") + require.NotContains(t, child, "examples") + } + + dependencies, ok := cleaned["dependencies"].(map[string]any) + require.True(t, ok) + dependencySchema, ok := dependencies["schema dependency"].(map[string]any) + require.True(t, ok) + require.Equal(t, []any{"dependency"}, dependencySchema["enum"]) + require.NotContains(t, dependencySchema, "const") + require.NotContains(t, dependencySchema, "examples") + require.Equal(t, []any{"first", "second"}, dependencies["required dependency"]) +} + +func TestCleanToolSchema_IntersectsStringConstWithEnum(t *testing.T) { + tests := []struct { + name string + constValue string + enum []any + expected []any + }{ + {name: "matching", constValue: "browser", enum: []any{"browser", "shell"}, expected: []any{"browser"}}, + {name: "contradictory", constValue: "browser", enum: []any{"shell"}, expected: []any{}}, + {name: "numeric string collision", constValue: "7", enum: []any{float64(7)}, expected: []any{}}, + {name: "empty string", constValue: "", enum: []any{"", "shell"}, expected: []any{""}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + schema := map[string]any{ + "type": "string", + "const": test.constValue, + "enum": test.enum, + } + + cleaned, ok := cleanToolSchema(schema).(map[string]any) + require.True(t, ok) + require.NotContains(t, cleaned, "const") + require.Equal(t, test.expected, cleaned["enum"]) + }) + } + + cleaned, ok := cleanToolSchema(map[string]any{"const": "browser"}).(map[string]any) + require.True(t, ok) + require.Equal(t, "STRING", cleaned["type"]) + require.Equal(t, []any{"browser"}, cleaned["enum"]) +} + +func TestCleanToolSchema_DropsNonStringConstWithoutInferringType(t *testing.T) { + for _, constValue := range []any{true, float64(7), map[string]any{"value": "x"}} { + cleaned, ok := cleanToolSchema(map[string]any{"const": constValue}).(map[string]any) + require.True(t, ok) + require.NotContains(t, cleaned, "const") + require.NotContains(t, cleaned, "type") + require.NotContains(t, cleaned, "enum") + } +} + +func TestCleanToolSchema_DropsNonStringConstAndRetainsExistingTypeAndEnum(t *testing.T) { + schema := map[string]any{ + "type": "integer", + "const": float64(7), + "enum": []any{float64(8)}, + } + + cleaned, ok := cleanToolSchema(schema).(map[string]any) + require.True(t, ok) + require.NotContains(t, cleaned, "const") + require.Equal(t, "INTEGER", cleaned["type"]) + require.Equal(t, []any{"8"}, cleaned["enum"]) +} + +func TestCleanToolSchema_PreservesNestedOneOfCombinator(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "value": map[string]any{ + "oneOf": []any{ + map[string]any{"const": "a"}, + map[string]any{"type": "string", "const": "b"}, + }, + }, + }, + } + + cleaned, ok := cleanToolSchema(schema).(map[string]any) + require.True(t, ok) + properties, ok := cleaned["properties"].(map[string]any) + require.True(t, ok) + valueSchema, ok := properties["value"].(map[string]any) + require.True(t, ok) + oneOf, ok := valueSchema["oneOf"].([]any) + require.True(t, ok) + require.Len(t, oneOf, 2) + firstBranch, ok := oneOf[0].(map[string]any) + require.True(t, ok) + secondBranch, ok := oneOf[1].(map[string]any) + require.True(t, ok) + require.Equal(t, []any{"a"}, firstBranch["enum"]) + require.Equal(t, []any{"b"}, secondBranch["enum"]) +} + func TestConvertClaudeToolsToGeminiTools_PreservesWebSearchAlongsideFunctions(t *testing.T) { tools := []any{ map[string]any{ From 94c7d6b025a4074efe34b444e7ea218c85409dd6 Mon Sep 17 00:00:00 2001 From: feeeei Date: Mon, 19 Oct 2026 12:27:57 +0800 Subject: [PATCH 08/29] =?UTF-8?q?fix(gateway):=20Anthropic=20=E8=B4=A6?= =?UTF-8?q?=E5=8F=B7=20chat/responses=20=E5=85=BC=E5=AE=B9=E6=B5=81?= =?UTF-8?q?=E5=BC=8F=E5=AE=A2=E6=88=B7=E7=AB=AF=E6=96=AD=E5=BC=80=E5=90=8E?= =?UTF-8?q?=E8=AF=BB=E5=AE=8C=E4=B8=8A=E6=B8=B8=E5=86=8D=E8=AE=A1=E8=B4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handleCCStreamingFromAnthropic 与 handleResponsesStreamingResponse 写客户端失败时 立即返回，message_delta 的 output_tokens 未合并，usage 记为 0。改为断开后停止向 下游写入、继续读完上游合并 usage，与原生 /v1/messages 路径一致；ForwardResult 标记 ClientDisconnect。 两条路径改用 anthropicNativeLinePump 读上游，按 gateway.stream_data_interval_timeout 限制逐行读间隔，上游挂住时关闭 body 并按已累计 usage 收尾。 --- .../gateway_forward_as_chat_completions.go | 114 +++++++--- ...y_forward_as_chat_completions_ping_test.go | 5 +- .../service/gateway_forward_as_responses.go | 65 +++--- ...y_forward_compat_client_disconnect_test.go | 204 ++++++++++++++++++ 4 files changed, 328 insertions(+), 60 deletions(-) create mode 100644 backend/internal/service/gateway_forward_compat_client_disconnect_test.go diff --git a/backend/internal/service/gateway_forward_as_chat_completions.go b/backend/internal/service/gateway_forward_as_chat_completions.go index 83d82e6ed0b0..bb96b28a6b28 100644 --- a/backend/internal/service/gateway_forward_as_chat_completions.go +++ b/backend/internal/service/gateway_forward_as_chat_completions.go @@ -381,6 +381,8 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( var usage ClaudeUsage var firstTokenMs *int firstChunk := true + // 客户端断开后停止向下游写入，继续读完上游以合并 message_delta 的最终 usage。 + clientDisconnected := false scanner := bufio.NewScanner(resp.Body) maxLineSize := defaultMaxLineSize @@ -388,21 +390,35 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( maxLineSize = s.cfg.Gateway.MaxLineSize } scanner.Buffer(make([]byte, 0, 64*1024), maxLineSize) + streamInterval := s.anthropicCompatStreamInterval() + pump := newAnthropicNativeLinePump(scanner, streamInterval) + defer pump.stop() + nextLine := func() (string, bool) { + return nextAnthropicCompatLine(pump, resp.Body, streamInterval, "forward_as_cc", requestID, clientDisconnected) + } resultWithUsage := func() *ForwardResult { return &ForwardResult{ - RequestID: requestID, - UpstreamHeaders: resp.Header, - Usage: usage, - Model: originalModel, - UpstreamModel: mappedModel, - ReasoningEffort: reasoningEffort, - Stream: true, - Duration: time.Since(startTime), - FirstTokenMs: firstTokenMs, + RequestID: requestID, + UpstreamHeaders: resp.Header, + Usage: usage, + Model: originalModel, + UpstreamModel: mappedModel, + ReasoningEffort: reasoningEffort, + Stream: true, + Duration: time.Since(startTime), + FirstTokenMs: firstTokenMs, + ClientDisconnect: clientDisconnected, } } + markClientDisconnected := func() { + clientDisconnected = true + logger.L().Info("forward_as_cc stream: client disconnected, continuing to drain upstream for billing", + zap.String("request_id", requestID), + ) + } + writeChunk := func(chunk apicompat.ChatCompletionsChunk) bool { sse, err := apicompat.ChatChunkToSSE(chunk) if err != nil { @@ -417,16 +433,20 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( return false } - processAnthropicEvent := func(event *apicompat.AnthropicStreamEvent) bool { + processAnthropicEvent := func(event *apicompat.AnthropicStreamEvent) { if event == nil { - return false + return } if event.Type == "ping" { + if clientDisconnected { + return + } if _, err := fmt.Fprint(c.Writer, ": ping\n\n"); err != nil { - return true + markClientDisconnected() + return } c.Writer.Flush() - return false + return } if firstChunk { firstChunk = false @@ -442,6 +462,9 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( if event.Type == "message_start" && event.Message != nil { mergeAnthropicUsage(&usage, event.Message.Usage) } + if clientDisconnected { + return + } // Keep the outward Responses/Chat usage on the same normalized buckets used // for billing, including converter handlers that consume event usage. @@ -454,25 +477,29 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( ccChunks := apicompat.ResponsesEventToChatChunks(&resEvt, ccState) for _, chunk := range ccChunks { if disconnected := writeChunk(chunk); disconnected { - return true + markClientDisconnected() + return } } } c.Writer.Flush() - return false } - for scanner.Scan() { - line := scanner.Text() + for { + line, ok := nextLine() + if !ok { + break + } // 与缓冲路径一致：接受 SSE 紧凑格式（冒号后无空格，#4653 同根因）。 if _, ok := extractOpenAISSEEventLine(line); !ok { continue } - if !scanner.Scan() { + dataLine, ok := nextLine() + if !ok { break } - payload, ok := extractOpenAISSEDataLine(scanner.Text()) + payload, ok := extractOpenAISSEDataLine(dataLine) if !ok { continue } @@ -486,18 +513,11 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( // The intermediate Responses converter synthesizes usage even when absent. ccState.IncludeUsage = ccState.IncludeUsage || anthropicChatStreamHasUsage(&event, payload) - if processAnthropicEvent(&event) { - return resultWithUsage(), nil - } + processAnthropicEvent(&event) } - if err := scanner.Err(); err != nil { - if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { - logger.L().Warn("forward_as_cc stream: read error", - zap.Error(err), - zap.String("request_id", requestID), - ) - } + if clientDisconnected { + return resultWithUsage(), nil } // Finalize both state machines @@ -532,6 +552,42 @@ func writeGatewayCCError(c *gin.Context, statusCode int, errType, message string }) } +// anthropicCompatStreamInterval 返回 Anthropic 上游协议转换流的逐行读间隔上限， +// gateway.stream_data_interval_timeout <= 0 时不限。 +func (s *GatewayService) anthropicCompatStreamInterval() time.Duration { + if s.cfg != nil && s.cfg.Gateway.StreamDataIntervalTimeout > 0 { + return time.Duration(s.cfg.Gateway.StreamDataIntervalTimeout) * time.Second + } + return 0 +} + +// nextAnthropicCompatLine 从行泵取下一行；ok=false 表示上游流已结束（EOF、读错误 +// 或读间隔超时），调用方按已累计的 usage 收尾。读间隔超时时关闭 body，解除泵 +// goroutine 阻塞中的读并释放上游连接。 +func nextAnthropicCompatLine(pump *anthropicNativeLinePump, body io.Closer, interval time.Duration, logPrefix, requestID string, clientDisconnected bool) (string, bool) { + line, err := pump.next() + if err == nil { + return line, true + } + switch { + case errors.Is(err, errAnthropicNativeStreamIdle): + _ = body.Close() + logger.L().Warn(logPrefix+" stream: data interval timeout", + zap.String("request_id", requestID), + zap.Duration("interval", interval), + zap.Bool("client_disconnected", clientDisconnected), + ) + case errors.Is(err, io.EOF), errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + // 正常收流或取消：不记日志。 + default: + logger.L().Warn(logPrefix+" stream: read error", + zap.Error(err), + zap.String("request_id", requestID), + ) + } + return "", false +} + // anthropicChatStreamHasUsage distinguishes an explicit zero-valued usage object // from omitted/null usage before the Anthropic→Responses conversion loses that distinction. func anthropicChatStreamHasUsage(event *apicompat.AnthropicStreamEvent, payload string) bool { diff --git a/backend/internal/service/gateway_forward_as_chat_completions_ping_test.go b/backend/internal/service/gateway_forward_as_chat_completions_ping_test.go index df08d20f78ce..7eda0f90d9f4 100644 --- a/backend/internal/service/gateway_forward_as_chat_completions_ping_test.go +++ b/backend/internal/service/gateway_forward_as_chat_completions_ping_test.go @@ -190,15 +190,16 @@ func TestHandleCCStreamingFromAnthropic_PingWriteFailureStopsDownstream(t *testi ) require.NoError(t, err) require.True(t, writer.failed) + require.True(t, result.ClientDisconnect) require.Zero(t, writer.writesAfterError) require.Zero(t, writer.flushesAfterError) require.NotContains(t, writer.Body.String(), "[DONE]") - require.Zero(t, result.Usage.OutputTokens) + require.Equal(t, 7, result.Usage.OutputTokens) + require.NotNil(t, result.FirstTokenMs) if withUsage { require.Equal(t, 10, result.Usage.InputTokens) } else { require.Zero(t, result.Usage.InputTokens) - require.Nil(t, result.FirstTokenMs) } }) } diff --git a/backend/internal/service/gateway_forward_as_responses.go b/backend/internal/service/gateway_forward_as_responses.go index 189c0861923d..fa2d601a0536 100644 --- a/backend/internal/service/gateway_forward_as_responses.go +++ b/backend/internal/service/gateway_forward_as_responses.go @@ -545,6 +545,8 @@ func (s *GatewayService) handleResponsesStreamingResponse( var usage ClaudeUsage var firstTokenMs *int firstChunk := true + // 客户端断开后停止向下游写入，继续读完上游以合并 message_delta 的最终 usage。 + clientDisconnected := false scanner := bufio.NewScanner(resp.Body) maxLineSize := defaultMaxLineSize @@ -552,23 +554,30 @@ func (s *GatewayService) handleResponsesStreamingResponse( maxLineSize = s.cfg.Gateway.MaxLineSize } scanner.Buffer(make([]byte, 0, 64*1024), maxLineSize) + streamInterval := s.anthropicCompatStreamInterval() + pump := newAnthropicNativeLinePump(scanner, streamInterval) + defer pump.stop() + nextLine := func() (string, bool) { + return nextAnthropicCompatLine(pump, resp.Body, streamInterval, "forward_as_responses", requestID, clientDisconnected) + } resultWithUsage := func() *ForwardResult { return &ForwardResult{ - RequestID: requestID, - UpstreamHeaders: resp.Header, - Usage: usage, - Model: originalModel, - UpstreamModel: mappedModel, - ReasoningEffort: reasoningEffort, - Stream: true, - Duration: time.Since(startTime), - FirstTokenMs: firstTokenMs, + RequestID: requestID, + UpstreamHeaders: resp.Header, + Usage: usage, + Model: originalModel, + UpstreamModel: mappedModel, + ReasoningEffort: reasoningEffort, + Stream: true, + Duration: time.Since(startTime), + FirstTokenMs: firstTokenMs, + ClientDisconnect: clientDisconnected, } } // processEvent handles a single parsed Anthropic SSE event. - processEvent := func(event *apicompat.AnthropicStreamEvent) bool { + processEvent := func(event *apicompat.AnthropicStreamEvent) { if firstChunk { firstChunk = false ms := int(time.Since(startTime).Milliseconds()) @@ -583,6 +592,9 @@ func (s *GatewayService) handleResponsesStreamingResponse( if event.Type == "message_start" && event.Message != nil { mergeAnthropicUsage(&usage, event.Message.Usage) } + if clientDisconnected { + return + } // Keep the terminal Responses usage aligned with the normalized billing // buckets. Normalize the converter input too, so message handlers cannot @@ -613,20 +625,23 @@ func (s *GatewayService) handleResponsesStreamingResponse( for _, restored := range payloads { eventType := gjson.GetBytes(restored, "type").String() if _, err := fmt.Fprintf(c.Writer, "event: %s\ndata: %s\n\n", eventType, restored); err != nil { - logger.L().Info("forward_as_responses stream: client disconnected", + clientDisconnected = true + logger.L().Info("forward_as_responses stream: client disconnected, continuing to drain upstream for billing", zap.String("request_id", requestID), ) - return true // client disconnected + return } } } if len(events) > 0 { c.Writer.Flush() } - return false } finalizeStream := func() (*ForwardResult, error) { + if clientDisconnected { + return resultWithUsage(), nil + } if finalEvents := apicompat.FinalizeAnthropicResponsesStream(state); len(finalEvents) > 0 { for _, evt := range finalEvents { sse, err := apicompat.ResponsesEventToSSE(evt) @@ -642,18 +657,21 @@ func (s *GatewayService) handleResponsesStreamingResponse( } // Read Anthropic SSE events - for scanner.Scan() { - line := scanner.Text() + for { + line, ok := nextLine() + if !ok { + break + } eventType, ok := parseAnthropicSSEField(line, "event") if !ok { continue } // Read data line - if !scanner.Scan() { + dataLine, ok := nextLine() + if !ok { break } - dataLine := scanner.Text() payload, ok := parseAnthropicSSEField(dataLine, "data") if !ok { continue @@ -669,18 +687,7 @@ func (s *GatewayService) handleResponsesStreamingResponse( continue } - if processEvent(&event) { - return resultWithUsage(), nil - } - } - - if err := scanner.Err(); err != nil { - if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { - logger.L().Warn("forward_as_responses stream: read error", - zap.Error(err), - zap.String("request_id", requestID), - ) - } + processEvent(&event) } return finalizeStream() diff --git a/backend/internal/service/gateway_forward_compat_client_disconnect_test.go b/backend/internal/service/gateway_forward_compat_client_disconnect_test.go new file mode 100644 index 000000000000..cfa55a0f0ab1 --- /dev/null +++ b/backend/internal/service/gateway_forward_compat_client_disconnect_test.go @@ -0,0 +1,204 @@ +//go:build unit + +package service + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/apicompat" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +// disconnectOnMarkerResponseWriter 在首次写入包含 marker 的数据时返回错误，模拟客户端中途断开。 +type disconnectOnMarkerResponseWriter struct { + *httptest.ResponseRecorder + marker string + failed bool + writesAfterError int + flushesAfterError int +} + +func (writer *disconnectOnMarkerResponseWriter) Write(data []byte) (int, error) { + if writer.failed { + writer.writesAfterError++ + return 0, errors.New("client disconnected") + } + if strings.Contains(string(data), writer.marker) { + writer.failed = true + return 0, errors.New("client disconnected") + } + return writer.ResponseRecorder.Write(data) +} + +func (writer *disconnectOnMarkerResponseWriter) WriteString(data string) (int, error) { + return writer.Write([]byte(data)) +} + +func (writer *disconnectOnMarkerResponseWriter) Flush() { + if writer.failed { + writer.flushesAfterError++ + } + writer.ResponseRecorder.Flush() +} + +func anthropicStreamWithMidStreamText() string { + return strings.Join([]string{ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_disc","type":"message","role":"assistant","content":[],"model":"claude-sonnet-4-6","usage":{"input_tokens":20,"cache_read_input_tokens":11,"cache_creation_input_tokens":4,"output_tokens":1}}}`, + ``, + `event: content_block_start`, + `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"first"}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"second"}}`, + ``, + `event: ping`, + `data: {"type":"ping"}`, + ``, + `event: content_block_stop`, + `data: {"type":"content_block_stop","index":0}`, + ``, + `event: message_delta`, + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":42}}`, + ``, + `event: message_stop`, + `data: {"type":"message_stop"}`, + ``, + }, "\n") +} + +// stallingAnthropicUpstream 写完 stream 后保持连接不关闭，模拟上游挂住。 +func stallingAnthropicUpstream(t *testing.T, stream string) (*http.Response, *io.PipeWriter) { + t.Helper() + pr, pw := io.Pipe() + go func() { + _, _ = io.WriteString(pw, stream) + }() + t.Cleanup(func() { + _ = pw.Close() + _ = pr.Close() + }) + return &http.Response{Header: http.Header{}, Body: pr}, pw +} + +func newCompatStreamIdleTestService(intervalSec int) *GatewayService { + return &GatewayService{cfg: &config.Config{Gateway: config.GatewayConfig{StreamDataIntervalTimeout: intervalSec}}} +} + +func TestAnthropicCompatStreaming_IdleUpstreamEndsWithinInterval(t *testing.T) { + handlers := map[string]func(*GatewayService, *http.Response, *gin.Context) (*ForwardResult, error){ + "chat_completions": func(svc *GatewayService, resp *http.Response, c *gin.Context) (*ForwardResult, error) { + return svc.handleCCStreamingFromAnthropic(resp, c, "claude-sonnet-4-6", "claude-sonnet-4-6", nil, time.Now()) + }, + "responses": func(svc *GatewayService, resp *http.Response, c *gin.Context) (*ForwardResult, error) { + return svc.handleResponsesStreamingResponse(resp, c, "claude-sonnet-4-6", "claude-sonnet-4-6", nil, time.Now(), apicompat.ResponsesClientToolMapping{}) + }, + } + stream := strings.TrimSuffix(anthropicStreamWithMidStreamText(), "event: message_stop\ndata: {\"type\":\"message_stop\"}\n") + for name, handle := range handlers { + for _, disconnect := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/client_disconnected=%t", name, disconnect), func(t *testing.T) { + gin.SetMode(gin.TestMode) + marker := "\x00never-written" + if disconnect { + marker = "second" + } + writer := &disconnectOnMarkerResponseWriter{ResponseRecorder: httptest.NewRecorder(), marker: marker} + c, _ := gin.CreateTestContext(writer) + resp, pw := stallingAnthropicUpstream(t, stream) + + type outcome struct { + result *ForwardResult + err error + } + done := make(chan outcome, 1) + go func() { + result, err := handle(newCompatStreamIdleTestService(1), resp, c) + done <- outcome{result: result, err: err} + }() + var got outcome + select { + case got = <-done: + case <-time.After(5 * time.Second): + t.Fatal("stream did not end after upstream idle interval") + } + + require.NoError(t, got.err) + require.Equal(t, disconnect, got.result.ClientDisconnect) + require.Equal(t, disconnect, writer.failed) + require.Equal(t, 42, got.result.Usage.OutputTokens) + require.Zero(t, writer.writesAfterError) + _, werr := io.WriteString(pw, "event: ping\n") + require.Error(t, werr, "upstream body must be closed after idle timeout") + }) + } + } +} + +func TestHandleCCStreamingFromAnthropic_ClientDisconnectDrainsUpstreamUsage(t *testing.T) { + gin.SetMode(gin.TestMode) + writer := &disconnectOnMarkerResponseWriter{ResponseRecorder: httptest.NewRecorder(), marker: "second"} + c, _ := gin.CreateTestContext(writer) + + result, err := (&GatewayService{}).handleCCStreamingFromAnthropic( + &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(anthropicStreamWithMidStreamText()))}, + c, "claude-sonnet-4-6", "claude-sonnet-4-6", nil, time.Now(), + ) + require.NoError(t, err) + require.NotNil(t, result) + require.True(t, writer.failed) + require.True(t, result.ClientDisconnect) + require.Zero(t, writer.writesAfterError) + require.Zero(t, writer.flushesAfterError) + + body := writer.Body.String() + require.Contains(t, body, `"content":"first"`) + require.NotContains(t, body, "second") + require.NotContains(t, body, "[DONE]") + + require.Equal(t, 20, result.Usage.InputTokens) + require.Equal(t, 42, result.Usage.OutputTokens) + require.Equal(t, 11, result.Usage.CacheReadInputTokens) + require.Equal(t, 4, result.Usage.CacheCreationInputTokens) + require.NotNil(t, result.FirstTokenMs) +} + +func TestHandleResponsesStreamingResponse_ClientDisconnectDrainsUpstreamUsage(t *testing.T) { + gin.SetMode(gin.TestMode) + writer := &disconnectOnMarkerResponseWriter{ResponseRecorder: httptest.NewRecorder(), marker: "second"} + c, _ := gin.CreateTestContext(writer) + + result, err := (&GatewayService{}).handleResponsesStreamingResponse( + &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(anthropicStreamWithMidStreamText()))}, + c, "claude-sonnet-4-6", "claude-sonnet-4-6", nil, time.Now(), apicompat.ResponsesClientToolMapping{}, + ) + require.NoError(t, err) + require.NotNil(t, result) + require.True(t, writer.failed) + require.True(t, result.ClientDisconnect) + require.Zero(t, writer.writesAfterError) + require.Zero(t, writer.flushesAfterError) + + body := writer.Body.String() + require.Contains(t, body, `"delta":"first"`) + require.NotContains(t, body, "second") + require.NotContains(t, body, "response.completed") + + require.Equal(t, 20, result.Usage.InputTokens) + require.Equal(t, 42, result.Usage.OutputTokens) + require.Equal(t, 11, result.Usage.CacheReadInputTokens) + require.Equal(t, 4, result.Usage.CacheCreationInputTokens) + require.NotNil(t, result.FirstTokenMs) +} From ab71e12ec46c532839023c013daeca48e9f3e099 Mon Sep 17 00:00:00 2001 From: shaw Date: Mon, 19 Oct 2026 17:51:18 +0800 Subject: [PATCH 09/29] =?UTF-8?q?fix(images):=20=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E7=94=9F=E5=9B=BE=E8=BD=AC=E5=AD=98=E7=9A=84=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E4=B8=8E=E5=90=8C=E6=AD=A5=E5=9B=9E=E5=A1=AB?= =?UTF-8?q?=E7=AD=96=E7=95=A5=E5=AF=B9=E9=BD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 异步生图任务把上游返回的图片 url 下载后转存对象存储，此前沿用默认 http.Client， 与同步 Images 端点 url 回填（c227863d）的下载策略不一致。 - 下载目的地为 localhost、回环、私网、链路本地、CGNAT 等非公网地址时不下载， 重定向的每一跳同样校验；直连时在 socket 层按实际连接 IP 判断 - 继续支持环境变量代理：经代理时先校验目标主机的解析结果 - 响应声明类型与字节嗅探都不是图片时任务失败，不再按 png 落盘 - 下载客户端进程内共享，复用连接池；超时与大小上限不变 --- backend/internal/service/image_storage.go | 132 ++++++++- .../internal/service/image_storage_test.go | 265 +++++++++++++++++- 2 files changed, 386 insertions(+), 11 deletions(-) diff --git a/backend/internal/service/image_storage.go b/backend/internal/service/image_storage.go index b160846b4716..f0e0d110d948 100644 --- a/backend/internal/service/image_storage.go +++ b/backend/internal/service/image_storage.go @@ -8,10 +8,16 @@ import ( "fmt" "io" "mime" + "net" "net/http" + "net/url" "strconv" "strings" + "sync" + "syscall" "time" + + "github.com/Wei-Shaw/sub2api/internal/util/urlvalidator" ) const defaultImageMaxDownloadBytes int64 = 32 << 20 // 32 MiB @@ -53,8 +59,123 @@ func NewImageResultUploader(storage ImageStorage, prefix string, maxDownloadByte } } -func defaultImageDownloadHTTPClient() *http.Client { - return &http.Client{Timeout: 60 * time.Second} +// defaultImageDownloadHTTPClient 是进程内共享的图片下载客户端，复用连接池。 +var defaultImageDownloadHTTPClient = sync.OnceValue(func() *http.Client { + return newImageDownloadHTTPClient(http.ProxyFromEnvironment) +}) + +// newImageDownloadHTTPClient 返回只允许下载公网目的地的客户端。 +// 图片 url 来自上游响应，是不可信输入：目的地为 localhost、回环、私网、链路本地、 +// 云元数据等地址时一律拒绝，重定向的每一跳都经同一 RoundTrip 校验。 +// 不经代理时在 socket 层校验真实连接 IP（防 DNS rebinding）；经环境代理时由代理解析目标， +// 只能在发出前校验本地解析结果。 +func newImageDownloadHTTPClient(proxy func(*http.Request) (*url.URL, error)) *http.Client { + direct := newImageDownloadBaseTransport(imageDownloadDialControl) + proxied := newImageDownloadBaseTransport(nil) + proxied.Proxy = proxy + return &http.Client{ + Timeout: 60 * time.Second, + Transport: &imageDownloadTransport{direct: direct, proxied: proxied, proxy: proxy, resolvesPrivate: isPrivateOrLoopbackHost}, + } +} + +// newImageDownloadBaseTransport 返回与 http.DefaultTransport 参数一致、但不带代理的 Transport。 +// control 非空时在每个 socket 连接前校验目标地址。 +func newImageDownloadBaseTransport(control func(network, address string, c syscall.RawConn) error) *http.Transport { + dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, Control: control} + return &http.Transport{ + DialContext: dialer.DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } +} + +// imageDownloadDialControl 拒绝连接到非公网 IP。address 是 DNS 解析后的真实连接地址， +// 多个解析结果时 Dialer 会跳过被拒地址继续尝试下一个。 +func imageDownloadDialControl(_, address string, _ syscall.RawConn) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + return err + } + if ip := net.ParseIP(stripIPv6Zone(host)); ip == nil || isPrivateIP(ip) { + return &net.AddrError{Err: "address not allowed for image download", Addr: address} + } + return nil +} + +// imageDownloadTransport 按请求是否走环境代理分派到对应 Transport，并在发出前拒绝非公网目的地。 +type imageDownloadTransport struct { + direct http.RoundTripper + proxied http.RoundTripper + proxy func(*http.Request) (*url.URL, error) + // resolvesPrivate 报告 host 的本地解析结果是否含非公网地址，仅用于经代理的请求。 + resolvesPrivate func(ctx context.Context, host string) (bool, error) +} + +func (t *imageDownloadTransport) RoundTrip(req *http.Request) (*http.Response, error) { + next, err := t.route(req) + if err != nil { + if req.Body != nil { + _ = req.Body.Close() + } + return nil, err + } + return next.RoundTrip(req) +} + +func (t *imageDownloadTransport) route(req *http.Request) (http.RoundTripper, error) { + if req.URL == nil { + return nil, errors.New("image url is empty") + } + host := req.URL.Hostname() + if isBlockedImageDownloadHost(host) { + return nil, fmt.Errorf("image url host is not allowed: %s", host) + } + var proxyURL *url.URL + if t.proxy != nil { + var err error + if proxyURL, err = t.proxy(req); err != nil { + return nil, err + } + } + if proxyURL == nil { + return t.direct, nil + } + if t.resolvesPrivate == nil { + return nil, errors.New("image download proxy host check is not configured") + } + blocked, err := t.resolvesPrivate(req.Context(), host) + if err != nil { + return nil, fmt.Errorf("resolve image url host: %w", err) + } + if blocked { + return nil, fmt.Errorf("image url host is not allowed: %s", host) + } + return t.proxied, nil +} + +// isBlockedImageDownloadHost 报告 host 是否为空、被拒绝的主机名，或非公网 IP 字面量。 +// 域名的解析结果由 socket 层或代理分支再校验。 +func isBlockedImageDownloadHost(host string) bool { + host = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(host)), ".") + if urlvalidator.IsBlockedHost(host) || isBlockedHostname(host) { + return true + } + if ip := net.ParseIP(stripIPv6Zone(host)); ip != nil { + return isPrivateIP(ip) + } + return false +} + +// stripIPv6Zone 去掉 IPv6 字面量的 zone（如 fe80::1%eth0），使其可被 net.ParseIP 解析。 +func stripIPv6Zone(host string) string { + if i := strings.LastIndexByte(host, '%'); i >= 0 { + return host[:i] + } + return host } // Rewrite 将 result（上游生图响应 JSON）里的每张图片转存到对象存储， @@ -215,9 +336,12 @@ func (u *ImageResultUploader) download(ctx context.Context, rawURL string) ([]by if int64(len(data)) > limit { return nil, "", fmt.Errorf("downloaded image exceeds %d bytes", limit) } - contentType := strings.TrimSpace(strings.Split(resp.Header.Get("Content-Type"), ";")[0]) + contentType := strings.ToLower(strings.TrimSpace(strings.Split(resp.Header.Get("Content-Type"), ";")[0])) if !strings.HasPrefix(contentType, "image/") { - contentType = detectImageContentType(data) + // 声明与嗅探都不是图片时拒绝，避免把任意响应体（内部接口的 JSON/HTML 等）转存给租户。 + if contentType = detectedImageContentType(data); contentType == "" { + return nil, "", errors.New("download image: content is not an image") + } } return data, contentType, nil } diff --git a/backend/internal/service/image_storage_test.go b/backend/internal/service/image_storage_test.go index 3a320671ee12..960b95857969 100644 --- a/backend/internal/service/image_storage_test.go +++ b/backend/internal/service/image_storage_test.go @@ -1,12 +1,17 @@ package service import ( + "bytes" "context" "encoding/base64" "encoding/json" "errors" + "io" + "net" "net/http" "net/http/httptest" + "net/url" + "sync/atomic" "testing" "time" @@ -73,18 +78,24 @@ func TestImageResultUploaderRewritesB64JSON(t *testing.T) { } func TestImageResultUploaderRewritesURL(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "image/png") - _, _ = w.Write(pngBytes) - })) - defer upstream.Close() + var requested string + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + requested = req.URL.String() + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"image/png"}}, + Body: io.NopCloser(bytes.NewReader(pngBytes)), + Request: req, + }, nil + })} storage := &fakeImageStorage{} - uploader := NewImageResultUploader(storage, "images/", 0, nil) + uploader := NewImageResultUploader(storage, "images/", 0, client) - result := json.RawMessage(`{"created":1,"data":[{"url":"` + upstream.URL + `/pic.png"}]}`) + result := json.RawMessage(`{"created":1,"data":[{"url":"https://images.example.test/pic.png"}]}`) out, err := uploader.Rewrite(context.Background(), "imgtask_xyz", result) require.NoError(t, err) + require.Equal(t, "https://images.example.test/pic.png", requested) require.Len(t, storage.saved, 1) require.Equal(t, pngBytes, storage.saved[0].data) @@ -244,3 +255,243 @@ func TestImageTaskServiceCompleteOffloadFailureMarksFailed(t *testing.T) { require.Contains(t, string(got.Error), "object storage") require.NotContains(t, string(got.Result), "b64_json", "failed offload must not persist base64 to Redis") } + +func TestImageResultUploaderDefaultClientRejectsPrivateURL(t *testing.T) { + var hits atomic.Int32 + internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + _, _ = w.Write([]byte("GIF89a-payload")) + })) + defer internal.Close() + + storage := &fakeImageStorage{} + uploader := NewImageResultUploader(storage, "images/", 0, nil) + result := json.RawMessage(`{"data":[{"url":"` + internal.URL + `/a.gif"}]}`) + + _, err := uploader.Rewrite(context.Background(), "imgtask_private", result) + require.ErrorContains(t, err, "image url host is not allowed") + require.Zero(t, hits.Load(), "internal service must not be contacted") + require.Empty(t, storage.saved) +} + +func TestImageTaskServiceCompletePrivateURLMarksFailed(t *testing.T) { + var hits atomic.Int32 + internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + _, _ = w.Write(pngBytes) + })) + defer internal.Close() + + store := &imageTaskMemoryStore{} + storage := &fakeImageStorage{} + svc := NewImageTaskServiceWithUploader(store, NewImageResultUploader(storage, "images/", 0, nil), time.Hour, time.Minute) + owner := ImageTaskOwner{UserID: 1, APIKeyID: 2} + created, err := svc.Create(context.Background(), owner) + require.NoError(t, err) + + result := json.RawMessage(`{"data":[{"url":"` + internal.URL + `/a.gif"}]}`) + require.NoError(t, svc.Complete(context.Background(), created.ID, http.StatusOK, result)) + + got, err := svc.Get(context.Background(), owner, created.ID) + require.NoError(t, err) + require.Equal(t, ImageTaskStatusFailed, got.Status) + require.Empty(t, got.ImageURL) + require.NotContains(t, string(got.Result), internal.URL) + require.Zero(t, hits.Load()) + require.Empty(t, storage.saved) +} + +func newCountingImageDownloadTransport(proxyURL *url.URL) (*imageDownloadTransport, *atomic.Int32, *atomic.Int32) { + var directCalls, proxiedCalls atomic.Int32 + respond := func(counter *atomic.Int32) http.RoundTripper { + return roundTripFunc(func(req *http.Request) (*http.Response, error) { + counter.Add(1) + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(pngBytes)), Request: req}, nil + }) + } + return &imageDownloadTransport{ + direct: respond(&directCalls), + proxied: respond(&proxiedCalls), + proxy: func(*http.Request) (*url.URL, error) { return proxyURL, nil }, + resolvesPrivate: func(ctx context.Context, host string) (bool, error) { + if host == "intranet.example.test" { + return true, nil + } + if host == "unresolvable.example.test" { + return false, errors.New("no such host") + } + return isPrivateOrLoopbackHost(ctx, host) + }, + }, &directCalls, &proxiedCalls +} + +func TestImageDownloadTransportRejectsNonPublicHosts(t *testing.T) { + hosts := []string{ + "http://127.0.0.1:18082/a.gif", + "http://localhost/a.png", + "http://LOCALHOST./a.png", + "http://foo.localhost/a.png", + "http://10.0.0.8/a.png", + "http://172.16.3.4/a.png", + "http://192.168.1.1/a.png", + "http://100.64.0.1/a.png", + "http://169.254.169.254/latest/meta-data/", + "http://metadata.google.internal/computeMetadata/v1/", + "http://0.0.0.0/a.png", + "http://[::1]/a.png", + "http://[::ffff:127.0.0.1]/a.png", + "http://[fd00::1]/a.png", + "http://[fe80::1%25eth0]/a.png", + "http:///a.png", + } + for _, proxied := range []bool{false, true} { + for _, rawURL := range hosts { + var proxyURL *url.URL + if proxied { + proxyURL = &url.URL{Scheme: "http", Host: "127.0.0.1:7890"} + } + transport, directCalls, proxiedCalls := newCountingImageDownloadTransport(proxyURL) + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + require.NoError(t, err) + + _, err = transport.RoundTrip(req) + require.Error(t, err, "proxied=%v url=%s", proxied, rawURL) + require.Zero(t, directCalls.Load(), rawURL) + require.Zero(t, proxiedCalls.Load(), rawURL) + } + } +} + +func TestImageDownloadTransportRoutesPublicHosts(t *testing.T) { + transport, directCalls, proxiedCalls := newCountingImageDownloadTransport(nil) + req, err := http.NewRequest(http.MethodGet, "https://203.0.113.10/a.png", nil) + require.NoError(t, err) + resp, err := transport.RoundTrip(req) + require.NoError(t, err) + _ = resp.Body.Close() + require.EqualValues(t, 1, directCalls.Load()) + require.Zero(t, proxiedCalls.Load()) + + transport, directCalls, proxiedCalls = newCountingImageDownloadTransport(&url.URL{Scheme: "http", Host: "127.0.0.1:7890"}) + req, err = http.NewRequest(http.MethodGet, "https://203.0.113.10/a.png", nil) + require.NoError(t, err) + resp, err = transport.RoundTrip(req) + require.NoError(t, err) + _ = resp.Body.Close() + require.Zero(t, directCalls.Load()) + require.EqualValues(t, 1, proxiedCalls.Load(), "a loopback env proxy must still be usable for public targets") +} + +func TestImageDownloadClientRejectsRedirectToPrivateHost(t *testing.T) { + var calls atomic.Int32 + transport := &imageDownloadTransport{ + direct: roundTripFunc(func(req *http.Request) (*http.Response, error) { + calls.Add(1) + return &http.Response{ + StatusCode: http.StatusFound, + Header: http.Header{"Location": []string{"http://169.254.169.254/latest/meta-data/"}}, + Body: io.NopCloser(bytes.NewReader(nil)), + Request: req, + }, nil + }), + } + uploader := NewImageResultUploader(&fakeImageStorage{}, "images/", 0, &http.Client{Transport: transport}) + + _, err := uploader.Rewrite(context.Background(), "imgtask_redirect", json.RawMessage(`{"data":[{"url":"https://images.example.test/a.png"}]}`)) + require.ErrorContains(t, err, "image url host is not allowed: 169.254.169.254") + require.EqualValues(t, 1, calls.Load(), "the private redirect hop must not be sent") +} + +func TestImageDownloadClientDialsOnlyPublicAddresses(t *testing.T) { + var hits atomic.Int32 + internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + _, _ = w.Write(pngBytes) + })) + defer internal.Close() + + client := newImageDownloadHTTPClient(nil) + direct, ok := client.Transport.(*imageDownloadTransport).direct.(*http.Transport) + require.True(t, ok) + require.Nil(t, direct.Proxy, "the direct transport must not pick up an environment proxy") + + // 绕过 RoundTrip 预检直接 dial：IP 字面量与经 DNS 解析到回环的域名都必须在 socket 层被拒。 + conn, err := direct.DialContext(context.Background(), "tcp", internal.Listener.Addr().String()) + if conn != nil { + _ = conn.Close() + } + require.ErrorContains(t, err, "address not allowed for image download") + _, port, err := net.SplitHostPort(internal.Listener.Addr().String()) + require.NoError(t, err) + conn, err = direct.DialContext(context.Background(), "tcp", net.JoinHostPort("localhost", port)) + if conn != nil { + _ = conn.Close() + } + require.ErrorContains(t, err, "address not allowed for image download") + require.Zero(t, hits.Load()) +} + +func newStaticImageDownloadClient(contentType string, body []byte) *http.Client { + return &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + header := http.Header{} + if contentType != "" { + header.Set("Content-Type", contentType) + } + return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(bytes.NewReader(body)), Request: req}, nil + })} +} + +func TestImageResultUploaderDownloadContentTypes(t *testing.T) { + avif := []byte("\x00\x00\x00\x1cftypavif-not-sniffable") + tests := []struct { + name string + contentType string + body []byte + wantType string + wantErr string + }{ + {name: "declared png", contentType: "image/png", body: pngBytes, wantType: "image/png"}, + {name: "octet-stream sniffed png", contentType: "application/octet-stream", body: pngBytes, wantType: "image/png"}, + {name: "missing header sniffed png", body: pngBytes, wantType: "image/png"}, + {name: "declared avif not sniffable", contentType: "Image/AVIF; charset=binary", body: avif, wantType: "image/avif"}, + {name: "json body", contentType: "application/json", body: []byte(`{"ok":true}`), wantErr: "content is not an image"}, + {name: "html body", contentType: "text/html", body: []byte("admin"), wantErr: "content is not an image"}, + {name: "empty body", body: nil, wantErr: "content is not an image"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + storage := &fakeImageStorage{} + uploader := NewImageResultUploader(storage, "images/", 0, newStaticImageDownloadClient(tt.contentType, tt.body)) + _, err := uploader.Rewrite(context.Background(), "imgtask_ct", json.RawMessage(`{"data":[{"url":"https://images.example.test/a"}]}`)) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + require.Empty(t, storage.saved) + return + } + require.NoError(t, err) + require.Len(t, storage.saved, 1) + require.Equal(t, tt.wantType, storage.saved[0].contentType) + }) + } +} + +func TestImageDownloadDialControl(t *testing.T) { + for _, address := range []string{"127.0.0.1:443", "10.1.2.3:80", "100.100.100.200:80", "169.254.169.254:80", "[::1]:443", "[fe80::1%eth0]:443", "[::ffff:192.168.0.1]:443", "bad"} { + require.Error(t, imageDownloadDialControl("tcp", address, nil), address) + } + for _, address := range []string{"203.0.113.10:443", "[2001:db8::1]:443"} { + require.NoError(t, imageDownloadDialControl("tcp", address, nil), address) + } +} + +func TestImageDownloadTransportProxiedChecksResolvedHost(t *testing.T) { + for _, host := range []string{"intranet.example.test", "unresolvable.example.test"} { + transport, directCalls, proxiedCalls := newCountingImageDownloadTransport(&url.URL{Scheme: "http", Host: "127.0.0.1:7890"}) + req, err := http.NewRequest(http.MethodGet, "https://"+host+"/a.png", nil) + require.NoError(t, err) + _, err = transport.RoundTrip(req) + require.Error(t, err, host) + require.Zero(t, directCalls.Load(), host) + require.Zero(t, proxiedCalls.Load(), "a proxied request must fail closed when the target is private or unresolvable: %s", host) + } +} From afbef7c8cca571660d6ca66a211d36ad83ea46ff Mon Sep 17 00:00:00 2001 From: wucm667 Date: Mon, 19 Oct 2026 20:15:37 +0800 Subject: [PATCH 10/29] fix(deepseek): avoid duplicate image payloads --- ...penai_gateway_deepseek_input_image_test.go | 24 ++++++++++++------- .../service/openai_gateway_request_body.go | 20 +++++++--------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/backend/internal/service/openai_gateway_deepseek_input_image_test.go b/backend/internal/service/openai_gateway_deepseek_input_image_test.go index 61f75c193922..b95d75cf3742 100644 --- a/backend/internal/service/openai_gateway_deepseek_input_image_test.go +++ b/backend/internal/service/openai_gateway_deepseek_input_image_test.go @@ -63,7 +63,7 @@ func TestNormalizeDeepSeekResponsesRequestBodyAliasesInputImageURL(t *testing.T) got := normalizeDeepSeekResponsesRequestBody(mapped, body) require.Equal(t, "input_image", gjson.GetBytes(got, "input.0.content.1.type").String()) require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.image_url").String()) - require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.url").String()) + require.False(t, gjson.GetBytes(got, "input.0.content.1.url").Exists()) require.Equal(t, "what is this?", gjson.GetBytes(got, "input.0.content.0.text").String()) require.True(t, gjson.GetBytes(got, "store").Bool(), "mapped openai 账号不应被无状态适配改掉 store") }) @@ -73,7 +73,7 @@ func TestNormalizeDeepSeekResponsesRequestBodyAliasesInputImageURL(t *testing.T) got := normalizeDeepSeekResponsesRequestBody(native, body) require.Equal(t, gjson.String, gjson.GetBytes(got, "input.0.content.1.image_url").Type) require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.image_url").String()) - require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.url").String()) + require.False(t, gjson.GetBytes(got, "input.0.content.1.url").Exists()) }) t.Run("chat_completions_image_url_part_converted", func(t *testing.T) { @@ -81,7 +81,7 @@ func TestNormalizeDeepSeekResponsesRequestBodyAliasesInputImageURL(t *testing.T) got := normalizeDeepSeekResponsesRequestBody(mapped, body) require.Equal(t, "input_image", gjson.GetBytes(got, "input.0.content.1.type").String()) require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.image_url").String()) - require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.url").String()) + require.False(t, gjson.GetBytes(got, "input.0.content.1.url").Exists()) }) t.Run("anthropic_source_becomes_data_uri", func(t *testing.T) { @@ -89,16 +89,24 @@ func TestNormalizeDeepSeekResponsesRequestBodyAliasesInputImageURL(t *testing.T) got := normalizeDeepSeekResponsesRequestBody(native, body) require.Equal(t, "input_image", gjson.GetBytes(got, "input.0.content.1.type").String()) require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.image_url").String()) - require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.url").String()) + require.False(t, gjson.GetBytes(got, "input.0.content.1.url").Exists()) }) - t.Run("lifted_tool_output_image_gets_url", func(t *testing.T) { + t.Run("lifted_tool_output_image_keeps_image_url_only", func(t *testing.T) { body := []byte(`{"model":"deepseek-flash","input":[{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":"` + deepSeekInputImageDataURI + `"}]}]}`) got := normalizeDeepSeekResponsesRequestBody(mapped, body) require.Equal(t, gjson.String, gjson.GetBytes(got, "input.1.output").Type) require.Equal(t, "input_image", gjson.GetBytes(got, "input.2.content.1.type").String()) require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.2.content.1.image_url").String()) - require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.2.content.1.url").String()) + require.False(t, gjson.GetBytes(got, "input.2.content.1.url").Exists()) + }) + + t.Run("dual_field_input_drops_duplicate_url", func(t *testing.T) { + body := deepSeekUserImageBody(`{"type":"input_image","image_url":"` + deepSeekInputImageDataURI + `","url":"` + deepSeekInputImageDataURI + `"}`) + got := normalizeDeepSeekResponsesRequestBody(native, body) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.image_url").String()) + require.False(t, gjson.GetBytes(got, "input.0.content.1.url").Exists()) + require.Equal(t, string(got), string(normalizeDeepSeekResponsesRequestBody(native, got))) }) t.Run("file_id_only_does_not_gain_empty_url", func(t *testing.T) { @@ -144,7 +152,7 @@ func TestForwardResponses_OpenAIMappedDeepSeekAliasesInputImageURL(t *testing.T) require.NotNil(t, result) require.Contains(t, upstream.lastReq.URL.String(), "/responses") require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(upstream.lastBody, "input.0.content.1.image_url").String()) - require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(upstream.lastBody, "input.0.content.1.url").String()) + require.False(t, gjson.GetBytes(upstream.lastBody, "input.0.content.1.url").Exists()) } func TestForwardResponses_NativeDeepSeekAliasesInputImageURL(t *testing.T) { @@ -167,5 +175,5 @@ func TestForwardResponses_NativeDeepSeekAliasesInputImageURL(t *testing.T) { require.NotNil(t, result) require.Equal(t, "https://api.deepseek.com/responses", upstream.lastReq.URL.String()) require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(upstream.lastBody, "input.0.content.1.image_url").String()) - require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(upstream.lastBody, "input.0.content.1.url").String()) + require.False(t, gjson.GetBytes(upstream.lastBody, "input.0.content.1.url").Exists()) } diff --git a/backend/internal/service/openai_gateway_request_body.go b/backend/internal/service/openai_gateway_request_body.go index 52ca1d57b467..38cc743d9df1 100644 --- a/backend/internal/service/openai_gateway_request_body.go +++ b/backend/internal/service/openai_gateway_request_body.go @@ -133,9 +133,8 @@ func deleteOpenAIResponsesNoneReasoningEffortFromObject(account *Account, body m // 强制 store=false 并清除 previous_response_id（DeepSeek / Kimi 官方 // Responses 均不支持服务端状态存储，携带这些字段会被拒绝）。 // -// DeepSeek 另需把 Codex/OpenAI 的 input_image.image_url 写成线上 serde -// 要求的 url 字段；openai 平台但 base_url 指向 api.deepseek.com 的映射 -// 账号同样走这条出站改写（Codex 贴图会 422 missing field url）。 +// DeepSeek 另需把兼容格式的图片输入归一化为官方 image_url 字段；openai +// 平台但 base_url 指向 api.deepseek.com 的映射账号同样走这条出站改写。 // 非 CN / 非 DeepSeek 上游原样返回。 func normalizeDeepSeekResponsesRequestBody(account *Account, body []byte) []byte { if account == nil { @@ -194,10 +193,9 @@ func shouldAliasDeepSeekResponsesInputImages(account *Account) bool { return targetsDeepSeekAPIHost(account) } -// aliasDeepSeekResponsesInputImages 把图片 part 改写成 DeepSeek 能反序列化 -// 的形状：type=input_image，同时带字符串字段 image_url 与 url。 -// 文档与部分 400 提 image_url；线上 serde 要求 url。只发其中一个都会踩坑。 -// 仅有 file_id、没有可用 URL 的 part 原样保留。 +// aliasDeepSeekResponsesInputImages 把图片 part 改写成 DeepSeek 官方形状： +// type=input_image，同时只保留字符串字段 image_url。仅有 file_id、没有可用 +// URL 的 part 原样保留。 func aliasDeepSeekResponsesInputImages(input any) (any, bool) { items, ok := asDeepSeekResponsesSlice(input) if !ok { @@ -279,18 +277,18 @@ func aliasDeepSeekResponsesImagePart(part map[string]any) bool { part["image_url"] = imageURL changed = true } - if current, ok := part["url"].(string); !ok || strings.TrimSpace(current) != imageURL { - part["url"] = imageURL + if _, exists := part["url"]; exists { + delete(part, "url") changed = true } return changed } func extractDeepSeekResponsesImageURL(part map[string]any) string { - if imageURL := deepSeekResponsesURLValue(part["url"]); imageURL != "" { + if imageURL := deepSeekResponsesURLValue(part["image_url"]); imageURL != "" { return imageURL } - if imageURL := deepSeekResponsesURLValue(part["image_url"]); imageURL != "" { + if imageURL := deepSeekResponsesURLValue(part["url"]); imageURL != "" { return imageURL } if imageURL := deepSeekResponsesURLValue(part["image"]); imageURL != "" { From d6120b5d8e05fe4edf54f1575c4dd6e5c7beb856 Mon Sep 17 00:00:00 2001 From: feeeei Date: Mon, 19 Oct 2026 23:31:46 +0800 Subject: [PATCH 11/29] =?UTF-8?q?feat(account):=20Vertex=20=E8=B4=A6?= =?UTF-8?q?=E5=8F=B7=E6=94=AF=E6=8C=81=E9=85=8D=E9=A2=9D=E4=B8=8A=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 AccountTypeSupportsQuota / SupportsAccountQuota(apikey/bedrock/service_account) 作为配额账号类型的唯一判定,累加、超限停调度、额度通知、接口输出与计费仓储统一改用; 池模式等其它特性仍只看 IsAPIKeyOrBedrock - service_account 配额语义与 apikey 一致:配置了任一上限才累加,用满停调度,跨阈值发额度通知 - 计费事务中账号已删除时跳过账号配额计数,与 API Key 对称,用户扣费不再整笔回滚 - 账号额度通知只对配置了上限的账号做检查 - 前端创建/编辑弹窗对 Vertex 开放配额设置,设上限后用量列显示配额进度条; 账号类型判断收敛到 supportsAccountQuota --- .../dto/account_quota_service_account_test.go | 78 +++++++ backend/internal/handler/dto/mappers.go | 4 +- .../internal/repository/usage_billing_repo.go | 5 +- .../usage_billing_repo_integration_test.go | 102 +++++++++ .../usage_billing_repo_unit_test.go | 34 +++ backend/internal/service/account.go | 19 +- .../service/account_quota_schedulable_test.go | 42 ++++ .../account_quota_service_account_test.go | 204 ++++++++++++++++++ .../internal/service/gateway_scheduling.go | 4 +- .../internal/service/gateway_usage_billing.go | 11 +- .../account/AccountCapacityCell.vue | 3 +- .../components/account/AccountUsageCell.vue | 26 ++- .../components/account/CreateAccountModal.vue | 9 +- .../components/account/EditAccountModal.vue | 13 +- .../__tests__/AccountUsageCell.spec.ts | 65 ++++++ .../__tests__/EditAccountModal.spec.ts | 20 ++ .../admin/account/AccountActionMenu.vue | 3 +- frontend/src/utils/accountQuota.ts | 6 + 18 files changed, 624 insertions(+), 24 deletions(-) create mode 100644 backend/internal/handler/dto/account_quota_service_account_test.go create mode 100644 backend/internal/service/account_quota_service_account_test.go create mode 100644 frontend/src/utils/accountQuota.ts diff --git a/backend/internal/handler/dto/account_quota_service_account_test.go b/backend/internal/handler/dto/account_quota_service_account_test.go new file mode 100644 index 000000000000..1f6f1ae5104f --- /dev/null +++ b/backend/internal/handler/dto/account_quota_service_account_test.go @@ -0,0 +1,78 @@ +package dto + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Wei-Shaw/sub2api/internal/service" +) + +// service_account 与 apikey 同语义：设了总额度才输出 quota_limit / quota_used。 +func TestAccountFromServiceShallow_ServiceAccountQuota(t *testing.T) { + t.Run("no limit omits quota fields", func(t *testing.T) { + out := AccountFromServiceShallow(&service.Account{ + ID: 1, + Platform: service.PlatformGemini, + Type: service.AccountTypeServiceAccount, + Extra: map[string]any{"quota_used": 12.5}, + }) + require.Nil(t, out.QuotaUsed) + require.Nil(t, out.QuotaLimit) + require.Nil(t, out.QuotaDailyLimit) + require.Nil(t, out.QuotaWeeklyLimit) + + payload, err := json.Marshal(out) + require.NoError(t, err) + var decoded map[string]any + require.NoError(t, json.Unmarshal(payload, &decoded)) + require.NotContains(t, decoded, "quota_used") + require.NotContains(t, decoded, "quota_limit") + }) + + t.Run("with limits outputs limits and usage", func(t *testing.T) { + out := AccountFromServiceShallow(&service.Account{ + ID: 3, + Platform: service.PlatformGemini, + Type: service.AccountTypeServiceAccount, + Extra: map[string]any{ + "quota_used": 30.0, + "quota_limit": 100.0, + "quota_daily_limit": 10.0, + }, + }) + require.NotNil(t, out.QuotaLimit) + require.InDelta(t, 100.0, *out.QuotaLimit, 1e-9) + require.NotNil(t, out.QuotaUsed) + require.InDelta(t, 30.0, *out.QuotaUsed, 1e-9) + require.NotNil(t, out.QuotaDailyLimit) + require.NotNil(t, out.QuotaDailyUsed) + }) + + t.Run("lite list item keeps quota fields", func(t *testing.T) { + item := AccountListItemFromAccount(AccountFromServiceShallow(&service.Account{ + ID: 4, + Platform: service.PlatformGemini, + Type: service.AccountTypeServiceAccount, + Extra: map[string]any{"quota_used": 7.25, "quota_limit": 290.0}, + })) + payload, err := json.Marshal(item) + require.NoError(t, err) + var decoded map[string]any + require.NoError(t, json.Unmarshal(payload, &decoded)) + require.InDelta(t, 7.25, decoded["quota_used"], 1e-9) + require.InDelta(t, 290.0, decoded["quota_limit"], 1e-9) + }) +} + +// apikey 维持原语义：未设总额度时不输出 quota_used。 +func TestAccountFromServiceShallow_APIKeyWithoutLimitOmitsQuotaUsed(t *testing.T) { + out := AccountFromServiceShallow(&service.Account{ + ID: 5, + Type: service.AccountTypeAPIKey, + Extra: map[string]any{"quota_used": 5.0}, + }) + require.Nil(t, out.QuotaUsed) + require.Nil(t, out.QuotaLimit) +} diff --git a/backend/internal/handler/dto/mappers.go b/backend/internal/handler/dto/mappers.go index d0c7de939ac2..a84d493e52f1 100644 --- a/backend/internal/handler/dto/mappers.go +++ b/backend/internal/handler/dto/mappers.go @@ -341,8 +341,8 @@ func AccountFromServiceShallow(a *service.Account) *Account { } } - // 提取账号配额限制（apikey / bedrock 类型有效） - if a.IsAPIKeyOrBedrock() { + // 提取账号配额限制（apikey / bedrock / service_account 类型有效） + if a.SupportsAccountQuota() { if limit := a.GetQuotaLimit(); limit > 0 { out.QuotaLimit = &limit used := a.GetQuotaUsed() diff --git a/backend/internal/repository/usage_billing_repo.go b/backend/internal/repository/usage_billing_repo.go index bfb1bb0de5f4..01f5f857f5b2 100644 --- a/backend/internal/repository/usage_billing_repo.go +++ b/backend/internal/repository/usage_billing_repo.go @@ -202,9 +202,10 @@ func (r *usageBillingRepository) applyUsageBillingEffects(ctx context.Context, t } } - if cmd.AccountQuotaCost > 0 && (strings.EqualFold(cmd.AccountType, service.AccountTypeAPIKey) || strings.EqualFold(cmd.AccountType, service.AccountTypeBedrock)) { + // 账号已不存在时跳过其配额计数，其余结算项不受影响。 + if cmd.AccountQuotaCost > 0 && service.AccountTypeSupportsQuota(cmd.AccountType) { quotaState, err := incrementUsageBillingAccountQuota(ctx, tx, cmd.AccountID, cmd.AccountQuotaCost) - if err != nil { + if err != nil && !errors.Is(err, service.ErrAccountNotFound) { return err } result.QuotaState = quotaState diff --git a/backend/internal/repository/usage_billing_repo_integration_test.go b/backend/internal/repository/usage_billing_repo_integration_test.go index 9f2f0169dcec..c7f2e6c52859 100644 --- a/backend/internal/repository/usage_billing_repo_integration_test.go +++ b/backend/internal/repository/usage_billing_repo_integration_test.go @@ -217,6 +217,54 @@ func TestUsageBillingRepositoryApply_DeletedAPIKeyStillBillsBalance(t *testing.T require.Equal(t, 1, dedupCount) } +func TestUsageBillingRepositoryApply_DeletedAccountStillBillsBalance(t *testing.T) { + ctx := context.Background() + client := testEntClient(t) + repo := NewUsageBillingRepository(client, integrationDB) + + user := mustCreateUser(t, client, &service.User{ + Email: fmt.Sprintf("usage-billing-deleted-account-user-%d@example.com", time.Now().UnixNano()), + PasswordHash: "hash", + Balance: 100, + }) + apiKey := mustCreateApiKey(t, client, &service.APIKey{ + UserID: user.ID, + Key: "sk-usage-billing-deleted-account-" + uuid.NewString(), + Name: "billing-deleted-account", + }) + account := mustCreateAccount(t, client, &service.Account{ + Name: "usage-billing-deleted-account-" + uuid.NewString(), + Platform: service.PlatformGemini, + Type: service.AccountTypeServiceAccount, + }) + + _, err := integrationDB.ExecContext(ctx, "UPDATE accounts SET deleted_at = NOW() WHERE id = $1", account.ID) + require.NoError(t, err) + + requestID := uuid.NewString() + result, err := repo.Apply(ctx, &service.UsageBillingCommand{ + RequestID: requestID, + APIKeyID: apiKey.ID, + UserID: user.ID, + AccountID: account.ID, + AccountType: service.AccountTypeServiceAccount, + BalanceCost: 1.25, + AccountQuotaCost: 1.25, + }) + require.NoError(t, err) + require.NotNil(t, result) + require.True(t, result.Applied) + require.Nil(t, result.QuotaState) + + var balance float64 + require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT balance FROM users WHERE id = $1", user.ID).Scan(&balance)) + require.InDelta(t, 98.75, balance, 0.000001) + + var dedupCount int + require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM usage_billing_dedup WHERE request_id = $1 AND api_key_id = $2", requestID, apiKey.ID).Scan(&dedupCount)) + require.Equal(t, 1, dedupCount) +} + func TestUsageBillingRepositoryApply_UpdatesAccountQuota(t *testing.T) { ctx := context.Background() client := testEntClient(t) @@ -254,6 +302,60 @@ func TestUsageBillingRepositoryApply_UpdatesAccountQuota(t *testing.T) { require.InDelta(t, 3.5, quotaUsed, 0.000001) } +// service_account 只设总上限：累加 quota_used；日/周用量只在设置了对应 limit 时累加，未跨越上限不入队调度 outbox。 +func TestUsageBillingRepositoryApply_UpdatesServiceAccountQuota(t *testing.T) { + ctx := context.Background() + client := testEntClient(t) + repo := NewUsageBillingRepository(client, integrationDB) + + user := mustCreateUser(t, client, &service.User{ + Email: fmt.Sprintf("usage-billing-sa-user-%d@example.com", time.Now().UnixNano()), + PasswordHash: "hash", + }) + apiKey := mustCreateApiKey(t, client, &service.APIKey{ + UserID: user.ID, + Key: "sk-usage-billing-sa-" + uuid.NewString(), + Name: "billing-sa", + }) + account := mustCreateAccount(t, client, &service.Account{ + Name: "usage-billing-sa-quota-" + uuid.NewString(), + Platform: service.PlatformGemini, + Type: service.AccountTypeServiceAccount, + Extra: map[string]any{ + "quota_limit": 100.0, + }, + }) + + for _, cost := range []float64{2.25, 1.5} { + _, err := repo.Apply(ctx, &service.UsageBillingCommand{ + RequestID: uuid.NewString(), + APIKeyID: apiKey.ID, + UserID: user.ID, + AccountID: account.ID, + AccountType: service.AccountTypeServiceAccount, + AccountQuotaCost: cost, + }) + require.NoError(t, err) + } + + var quotaUsed float64 + var hasDailyUsed, hasWeeklyUsed bool + require.NoError(t, integrationDB.QueryRowContext(ctx, + "SELECT COALESCE((extra->>'quota_used')::numeric, 0), extra ? 'quota_daily_used', extra ? 'quota_weekly_used' FROM accounts WHERE id = $1", + account.ID, + ).Scan("aUsed, &hasDailyUsed, &hasWeeklyUsed)) + require.InDelta(t, 3.75, quotaUsed, 0.000001) + require.False(t, hasDailyUsed) + require.False(t, hasWeeklyUsed) + + var outboxCount int + require.NoError(t, integrationDB.QueryRowContext(ctx, + "SELECT COUNT(*) FROM scheduler_outbox WHERE event_type = $1 AND account_id = $2", + service.SchedulerOutboxEventAccountChanged, account.ID, + ).Scan(&outboxCount)) + require.Zero(t, outboxCount, "service_account under its limit must not enqueue scheduler outbox") +} + func TestUsageBillingRepositoryApply_EnqueuesSchedulerOutboxOnQuotaCrossing(t *testing.T) { ctx := context.Background() client := testEntClient(t) diff --git a/backend/internal/repository/usage_billing_repo_unit_test.go b/backend/internal/repository/usage_billing_repo_unit_test.go index 40394d819c8d..a745eb8d7301 100644 --- a/backend/internal/repository/usage_billing_repo_unit_test.go +++ b/backend/internal/repository/usage_billing_repo_unit_test.go @@ -22,6 +22,7 @@ const ( userExistsForBillingSQL = `(?s)SELECT 1\s+FROM users\s+WHERE id = \$1 AND deleted_at IS NULL` apiKeyQuotaIncrementSQL = `(?s)UPDATE api_keys\s+SET quota_used = quota_used \+ \$1,.*WHERE id = \$2 AND deleted_at IS NULL\s+RETURNING` apiKeyRateLimitIncrementSQL = `(?s)UPDATE api_keys SET\s+usage_5h = .*WHERE id = \$2 AND deleted_at IS NULL` + accountQuotaIncrementSQL = `(?s)UPDATE accounts SET extra = .*WHERE id = \$2 AND deleted_at IS NULL\s+RETURNING` ) func TestDeductUsageBillingBalance_UsesSufficientBalanceGuard(t *testing.T) { @@ -137,6 +138,39 @@ func TestApplyUsageBillingEffects_DeletedAPIKeyStillBillsBalance(t *testing.T) { require.NoError(t, mock.ExpectationsWereMet()) } +func TestApplyUsageBillingEffects_DeletedAccountStillBillsBalance(t *testing.T) { + ctx := context.Background() + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { _ = db.Close() }() + + mock.ExpectBegin() + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + mock.ExpectQuery(conditionalBalanceDeductSQL). + WithArgs(10.0, int64(42)). + WillReturnRows(sqlmock.NewRows([]string{"balance"}).AddRow(90.0)) + mock.ExpectQuery(accountQuotaIncrementSQL). + WithArgs(4.0, int64(9)). + WillReturnRows(sqlmock.NewRows([]string{"quota_used", "quota_limit", "quota_daily_used", "quota_daily_limit", "quota_weekly_used", "quota_weekly_limit"})) + mock.ExpectCommit() + + result := &service.UsageBillingApplyResult{Applied: true} + err = (&usageBillingRepository{}).applyUsageBillingEffects(ctx, tx, &service.UsageBillingCommand{ + UserID: 42, + AccountID: 9, + AccountType: service.AccountTypeServiceAccount, + BalanceCost: 10, + AccountQuotaCost: 4, + }, result) + require.NoError(t, err) + require.NotNil(t, result.NewBalance) + require.InDelta(t, 90.0, *result.NewBalance, 0.000001) + require.Nil(t, result.QuotaState) + require.NoError(t, tx.Commit()) + require.NoError(t, mock.ExpectationsWereMet()) +} + func TestApplyUsageBillingEffects_APIKeyCounterErrorStillFails(t *testing.T) { ctx := context.Background() db, mock, err := sqlmock.New() diff --git a/backend/internal/service/account.go b/backend/internal/service/account.go index 06b94b754e5d..ebc07b8c5cef 100644 --- a/backend/internal/service/account.go +++ b/backend/internal/service/account.go @@ -206,7 +206,7 @@ func (a *Account) IsSchedulable() bool { if a.TempUnschedulableUntil != nil && now.Before(*a.TempUnschedulableUntil) { return false } - if a.IsAPIKeyOrBedrock() && a.IsQuotaExceeded() { + if a.SupportsAccountQuota() && a.IsQuotaExceeded() { return false } return true @@ -1378,11 +1378,26 @@ func (a *Account) IsBedrockAPIKey() bool { return a.IsBedrock() && a.GetCredential("auth_mode") == "apikey" } -// IsAPIKeyOrBedrock 返回账号类型是否支持配额和池模式等特性 +// IsAPIKeyOrBedrock 返回账号类型是否支持池模式等特性（账号配额判定见 SupportsAccountQuota） func (a *Account) IsAPIKeyOrBedrock() bool { return a.Type == AccountTypeAPIKey || a.Type == AccountTypeBedrock } +// AccountTypeSupportsQuota 返回账号类型是否支持账号配额（quota_limit 及日/周额度、quota_used 累加、超限停调度）。 +// 在 apikey/bedrock 之外额外放开 Vertex service_account；池模式等其它特性仍只看 IsAPIKeyOrBedrock。 +func AccountTypeSupportsQuota(accountType string) bool { + switch strings.ToLower(accountType) { + case AccountTypeAPIKey, AccountTypeBedrock, AccountTypeServiceAccount: + return true + } + return false +} + +// SupportsAccountQuota 返回账号是否支持账号配额，类型集合见 AccountTypeSupportsQuota。 +func (a *Account) SupportsAccountQuota() bool { + return AccountTypeSupportsQuota(a.Type) +} + func (a *Account) IsOpenAI() bool { return a.Platform == PlatformOpenAI } diff --git a/backend/internal/service/account_quota_schedulable_test.go b/backend/internal/service/account_quota_schedulable_test.go index 2895b34c889e..dfe34c4fc507 100644 --- a/backend/internal/service/account_quota_schedulable_test.go +++ b/backend/internal/service/account_quota_schedulable_test.go @@ -113,6 +113,48 @@ func TestAccountIsSchedulable_QuotaExceeded(t *testing.T) { }, want: false, }, + { + name: "service_account total quota exceeded", + account: &Account{ + Status: StatusActive, + Schedulable: true, + Platform: PlatformGemini, + Type: AccountTypeServiceAccount, + Extra: map[string]any{ + "quota_limit": 50.0, + "quota_used": 50.0, + }, + }, + want: false, + }, + { + name: "service_account daily quota exceeded", + account: &Account{ + Status: StatusActive, + Schedulable: true, + Platform: PlatformGemini, + Type: AccountTypeServiceAccount, + Extra: map[string]any{ + "quota_daily_limit": 10.0, + "quota_daily_used": 12.0, + "quota_daily_start": now.Add(-1 * time.Hour).Format(time.RFC3339), + }, + }, + want: false, + }, + { + name: "service_account without limit stays schedulable regardless of quota_used", + account: &Account{ + Status: StatusActive, + Schedulable: true, + Platform: PlatformGemini, + Type: AccountTypeServiceAccount, + Extra: map[string]any{ + "quota_used": 9999.0, + }, + }, + want: true, + }, } for _, tt := range tests { diff --git a/backend/internal/service/account_quota_service_account_test.go b/backend/internal/service/account_quota_service_account_test.go new file mode 100644 index 000000000000..6358f3202417 --- /dev/null +++ b/backend/internal/service/account_quota_service_account_test.go @@ -0,0 +1,204 @@ +package service + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAccountSupportsAccountQuota(t *testing.T) { + tests := []struct { + accountType string + want bool + }{ + {AccountTypeAPIKey, true}, + {AccountTypeBedrock, true}, + {AccountTypeServiceAccount, true}, + {AccountTypeOAuth, false}, + {AccountTypeSetupToken, false}, + {AccountTypeUpstream, false}, + } + for _, tt := range tests { + t.Run(tt.accountType, func(t *testing.T) { + account := &Account{Type: tt.accountType} + require.Equal(t, tt.want, account.SupportsAccountQuota()) + }) + } +} + +// 配额专用判定放开 service_account，不能连带放开池模式等其它 IsAPIKeyOrBedrock 特性。 +func TestServiceAccountQuotaDoesNotWidenPoolMode(t *testing.T) { + account := &Account{ + Platform: PlatformGemini, + Type: AccountTypeServiceAccount, + Credentials: map[string]any{"pool_mode": true}, + } + require.True(t, account.SupportsAccountQuota()) + require.False(t, account.IsAPIKeyOrBedrock()) + require.False(t, account.IsPoolMode()) +} + +func TestShouldUpdateAccountQuota(t *testing.T) { + tests := []struct { + name string + account *Account + totalCost float64 + want bool + }{ + { + name: "apikey without limit does not accumulate", + account: &Account{Type: AccountTypeAPIKey, Extra: map[string]any{}}, + totalCost: 1.5, + want: false, + }, + { + name: "apikey with total limit accumulates", + account: &Account{Type: AccountTypeAPIKey, Extra: map[string]any{"quota_limit": 100.0}}, + totalCost: 1.5, + want: true, + }, + { + name: "bedrock with daily limit accumulates", + account: &Account{Type: AccountTypeBedrock, Extra: map[string]any{"quota_daily_limit": 10.0}}, + totalCost: 1.5, + want: true, + }, + { + name: "service_account without limit does not accumulate", + account: &Account{Platform: PlatformGemini, Type: AccountTypeServiceAccount, Extra: map[string]any{"quota_used": 30.0}}, + totalCost: 1.5, + want: false, + }, + { + name: "service_account with limit accumulates", + account: &Account{Platform: PlatformGemini, Type: AccountTypeServiceAccount, Extra: map[string]any{"quota_weekly_limit": 50.0}}, + totalCost: 1.5, + want: true, + }, + { + name: "service_account zero cost does not accumulate", + account: &Account{Platform: PlatformGemini, Type: AccountTypeServiceAccount, Extra: map[string]any{"quota_limit": 100.0}}, + totalCost: 0, + want: false, + }, + { + name: "oauth with limit is not quota eligible", + account: &Account{Type: AccountTypeOAuth, Extra: map[string]any{"quota_limit": 100.0}}, + totalCost: 1.5, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &postUsageBillingParams{ + Cost: &CostBreakdown{TotalCost: tt.totalCost}, + Account: tt.account, + } + require.Equal(t, tt.want, p.shouldUpdateAccountQuota()) + }) + } +} + +func TestBuildUsageBillingCommandAccountQuotaCostForServiceAccount(t *testing.T) { + build := func(account *Account) *UsageBillingCommand { + return buildUsageBillingCommand("req-sa-quota", nil, &postUsageBillingParams{ + Cost: &CostBreakdown{TotalCost: 2, ActualCost: 3}, + User: &User{ID: 7}, + APIKey: &APIKey{ID: 13}, + Account: account, + AccountRateMultiplier: 1.5, + }) + } + + // service_account 设了上限：AccountQuotaCost = TotalCost × 账号倍率 + cmd := build(&Account{ID: 9, Platform: PlatformGemini, Type: AccountTypeServiceAccount, Extra: map[string]any{"quota_limit": 100.0}}) + require.NotNil(t, cmd) + require.Equal(t, AccountTypeServiceAccount, cmd.AccountType) + require.InDelta(t, 3.0, cmd.AccountQuotaCost, 1e-9) + + // service_account 不设上限：与 apikey 一致不累加 + cmd = build(&Account{ID: 10, Platform: PlatformGemini, Type: AccountTypeServiceAccount}) + require.NotNil(t, cmd) + require.Zero(t, cmd.AccountQuotaCost) +} + +type accountQuotaIncrementRepoStub struct { + AccountRepository + calls []float64 +} + +func (s *accountQuotaIncrementRepoStub) IncrementQuotaUsed(_ context.Context, _ int64, amount float64) error { + s.calls = append(s.calls, amount) + return nil +} + +func TestPostUsageBillingLegacyAccumulatesServiceAccountQuota(t *testing.T) { + repo := &accountQuotaIncrementRepoStub{} + postUsageBilling(context.Background(), &postUsageBillingParams{ + Cost: &CostBreakdown{TotalCost: 2}, + User: &User{ID: 7}, + APIKey: &APIKey{ID: 13}, + Account: &Account{ID: 9, Platform: PlatformGemini, Type: AccountTypeServiceAccount, Extra: map[string]any{"quota_limit": 100.0}}, + AccountRateMultiplier: 1.25, + }, &billingDeps{accountRepo: repo}) + + require.Len(t, repo.calls, 1) + require.InDelta(t, 2.5, repo.calls[0], 1e-9) +} + +type settingGetValueCounter struct { + SettingRepository + calls int +} + +func (s *settingGetValueCounter) GetValue(context.Context, string) (string, error) { + s.calls++ + return "false", nil +} + +// 未设任何上限的账号没有可跨越的阈值，不应逐请求读取通知设置。 +func TestNotifyAccountQuotaSkipsAccountWithoutLimit(t *testing.T) { + tests := []struct { + name string + account *Account + wantCalls int + }{ + { + name: "service_account without limit", + account: &Account{ID: 9, Platform: PlatformGemini, Type: AccountTypeServiceAccount}, + wantCalls: 0, + }, + { + name: "service_account with total limit", + account: &Account{ID: 9, Platform: PlatformGemini, Type: AccountTypeServiceAccount, Extra: map[string]any{"quota_limit": 100.0}}, + wantCalls: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + settings := &settingGetValueCounter{} + notifyAccountQuota(&postUsageBillingParams{ + Cost: &CostBreakdown{TotalCost: 2}, + Account: tt.account, + AccountRateMultiplier: 1, + }, &billingDeps{ + balanceNotifyService: NewBalanceNotifyService(NewEmailService(settings, nil), settings, nil), + }, nil) + require.Equal(t, tt.wantCalls, settings.calls) + }) + } +} + +// 账号已删除时计数被跳过、无递增后状态，不应回退到快照做阈值判定。 +func TestNotifyAccountQuotaSkipsWhenQuotaIncrementSkipped(t *testing.T) { + settings := &settingGetValueCounter{} + notifyAccountQuota(&postUsageBillingParams{ + Cost: &CostBreakdown{TotalCost: 2}, + Account: &Account{ID: 9, Platform: PlatformGemini, Type: AccountTypeServiceAccount, Extra: map[string]any{"quota_limit": 100.0}}, + AccountRateMultiplier: 1, + }, &billingDeps{ + balanceNotifyService: NewBalanceNotifyService(NewEmailService(settings, nil), settings, nil), + }, &UsageBillingApplyResult{Applied: true}) + require.Zero(t, settings.calls) +} diff --git a/backend/internal/service/gateway_scheduling.go b/backend/internal/service/gateway_scheduling.go index 059d7e2b2cb5..46df44824555 100644 --- a/backend/internal/service/gateway_scheduling.go +++ b/backend/internal/service/gateway_scheduling.go @@ -1309,9 +1309,9 @@ func (s *GatewayService) withWindowCostPrefetch(ctx context.Context, accounts [] } // isAccountSchedulableForQuota 检查账号是否在配额限制内 -// 适用于配置了 quota_limit 的 apikey 和 bedrock 类型账号 +// 适用于配置了 quota_limit 的 apikey、bedrock 和 service_account 类型账号 func (s *GatewayService) isAccountSchedulableForQuota(account *Account) bool { - if !account.IsAPIKeyOrBedrock() { + if !account.SupportsAccountQuota() { return true } return !account.IsQuotaExceeded() diff --git a/backend/internal/service/gateway_usage_billing.go b/backend/internal/service/gateway_usage_billing.go index 84c604fd0412..426209b2966f 100644 --- a/backend/internal/service/gateway_usage_billing.go +++ b/backend/internal/service/gateway_usage_billing.go @@ -137,7 +137,7 @@ func (p *postUsageBillingParams) shouldUpdateRateLimits() bool { } func (p *postUsageBillingParams) shouldUpdateAccountQuota() bool { - return p.Cost.TotalCost > 0 && p.Account.IsAPIKeyOrBedrock() && p.Account.HasAnyQuotaLimit() + return p.Cost.TotalCost > 0 && p.Account.SupportsAccountQuota() && p.Account.HasAnyQuotaLimit() } // postUsageBilling is the legacy fallback billing path used when the unified @@ -541,11 +541,12 @@ func notifyAccountQuota(p *postUsageBillingParams, deps *billingDeps, result *Us slog.Error("panic in notifyAccountQuota", "recover", r) } }() - if p.Cost.TotalCost <= 0 || p.Account == nil || !p.Account.IsAPIKeyOrBedrock() || deps.balanceNotifyService == nil { + if p.Cost.TotalCost <= 0 || p.Account == nil || !p.Account.SupportsAccountQuota() || !p.Account.HasAnyQuotaLimit() || deps.balanceNotifyService == nil { slog.Debug("notifyAccountQuota: skipped", "total_cost", p.Cost.TotalCost, "account_nil", p.Account == nil, - "is_apikey_or_bedrock", p.Account != nil && p.Account.IsAPIKeyOrBedrock(), + "supports_account_quota", p.Account != nil && p.Account.SupportsAccountQuota(), + "has_quota_limit", p.Account != nil && p.Account.HasAnyQuotaLimit(), "service_nil", deps.balanceNotifyService == nil, ) return @@ -554,6 +555,10 @@ func notifyAccountQuota(p *postUsageBillingParams, deps *billingDeps, result *Us var quotaState *AccountQuotaState if result != nil { quotaState = result.QuotaState + // 本应累加却无递增后状态（账号已删除，计数被跳过）：没有新用量，不做阈值判定。 + if quotaState == nil && p.shouldUpdateAccountQuota() { + return + } } slog.Debug("notifyAccountQuota: calling CheckAccountQuotaAfterIncrement", "account_id", p.Account.ID, diff --git a/frontend/src/components/account/AccountCapacityCell.vue b/frontend/src/components/account/AccountCapacityCell.vue index 3df5b24b4ff1..4e89708d87ff 100644 --- a/frontend/src/components/account/AccountCapacityCell.vue +++ b/frontend/src/components/account/AccountCapacityCell.vue @@ -39,6 +39,7 @@ import { computed } from 'vue' import { useI18n } from 'vue-i18n' import type { Account } from '@/types' +import { supportsAccountQuota } from '@/utils/accountQuota' import CapacityBadge from '@/components/account/CapacityBadge.vue' import QuotaBadge from '@/components/account/QuotaBadge.vue' @@ -176,7 +177,7 @@ const formatCost = (value: number | null | undefined) => { } // ====== 配额 ====== -const isQuotaEligible = computed(() => props.account.type === 'apikey' || props.account.type === 'bedrock') +const isQuotaEligible = computed(() => supportsAccountQuota(props.account.type)) const showDailyQuota = computed(() => isQuotaEligible.value && props.account.quota_daily_limit != null && props.account.quota_daily_limit > 0 diff --git a/frontend/src/components/account/AccountUsageCell.vue b/frontend/src/components/account/AccountUsageCell.vue index 6bf5f12af5d5..a12527b82bcf 100644 --- a/frontend/src/components/account/AccountUsageCell.vue +++ b/frontend/src/components/account/AccountUsageCell.vue @@ -554,6 +554,29 @@
+ +
@@ -684,6 +707,7 @@ import { buildOpenAIUsageRefreshKey } from '@/utils/accountUsageRefresh' import { enqueueUsageRequest } from '@/utils/usageLoadQueue' import { formatCompactNumber } from '@/utils/format' import { isOfficialGeminiBaseUrl } from '@/utils/geminiExplicitCache' +import { supportsAccountQuota } from '@/utils/accountQuota' import UsageProgressBar from './UsageProgressBar.vue' import AccountQuotaInfo from './AccountQuotaInfo.vue' import ClaudeResetCreditsCell from './ClaudeResetCreditsCell.vue' @@ -1572,7 +1596,7 @@ const makeQuotaBar = ( } const hasApiKeyQuota = computed(() => { - if (props.account.type !== 'apikey' && props.account.type !== 'bedrock') return false + if (!supportsAccountQuota(props.account.type)) return false return ( (props.account.quota_daily_limit ?? 0) > 0 || (props.account.quota_weekly_limit ?? 0) > 0 || diff --git a/frontend/src/components/account/CreateAccountModal.vue b/frontend/src/components/account/CreateAccountModal.vue index 93deb2f4c31a..cb013a4dd32f 100644 --- a/frontend/src/components/account/CreateAccountModal.vue +++ b/frontend/src/components/account/CreateAccountModal.vue @@ -2347,9 +2347,9 @@ />
- +
@@ -4203,6 +4203,7 @@ import { parseDateTimeLocalInput } from '@/utils/format' import { createStableObjectKeyResolver } from '@/utils/stableObjectKey' +import { supportsAccountQuota } from '@/utils/accountQuota' import { getAccountExpiryTimestamp } from '@/components/account/accountExpiry' import { VERTEX_LOCATION_OPTIONS } from '@/constants/account' import { @@ -6282,9 +6283,9 @@ const createAccountAndFinish = async ( if (!applyTempUnschedConfig(credentials)) { return } - // Inject quota limits for apikey/bedrock accounts + // Inject quota limits for apikey/bedrock/service_account accounts let finalExtra = withUpstreamRequestIdHeader(extra) - if (type === 'apikey' || type === 'bedrock') { + if (supportsAccountQuota(type)) { const quotaExtra: Record = { ...(finalExtra || {}) } if (editQuotaLimit.value != null && editQuotaLimit.value > 0) { quotaExtra.quota_limit = editQuotaLimit.value diff --git a/frontend/src/components/account/EditAccountModal.vue b/frontend/src/components/account/EditAccountModal.vue index bbddaaf44e8d..288abd793e45 100644 --- a/frontend/src/components/account/EditAccountModal.vue +++ b/frontend/src/components/account/EditAccountModal.vue @@ -2319,9 +2319,9 @@ @update:quotaNotifyTotalThresholdType="quotaNotifyState.total.thresholdType = $event" />
- +
@@ -3316,6 +3316,7 @@ import { } from '@/utils/format' import { extractApiErrorMessage, extractI18nErrorMessage } from '@/utils/apiError' import { createStableObjectKeyResolver } from '@/utils/stableObjectKey' +import { supportsAccountQuota } from '@/utils/accountQuota' import { getAccountExpiryTimestamp } from '@/components/account/accountExpiry' import { allSelectedGroupsEnableLongContextPricing } from '@/components/account/longContextBilling' import { VERTEX_LOCATION_OPTIONS } from '@/constants/account' @@ -4438,8 +4439,8 @@ const syncFormFromAccount = (newAccount: Account | null) => { } } - // Load quota limit for apikey/bedrock accounts (bedrock quota is also loaded in its own branch above) - if (newAccount.type === 'apikey' || newAccount.type === 'bedrock') { + // Load quota limit for apikey/bedrock/service_account accounts (bedrock quota is also loaded in its own branch above) + if (supportsAccountQuota(newAccount.type)) { const quotaVal = extra?.quota_limit as number | undefined editQuotaLimit.value = (quotaVal && quotaVal > 0) ? quotaVal : null const dailyVal = extra?.quota_daily_limit as number | undefined @@ -6016,8 +6017,8 @@ const handleSubmit = async () => { updatePayload.extra = newExtra } - // For apikey/bedrock accounts, handle quota_limit in extra - if (props.account.type === 'apikey' || props.account.type === 'bedrock') { + // For apikey/bedrock/service_account accounts, handle quota_limit in extra + if (supportsAccountQuota(props.account.type)) { const currentExtra = (updatePayload.extra as Record) || (props.account.extra as Record) || {} const newExtra: Record = { ...currentExtra } diff --git a/frontend/src/components/account/__tests__/AccountUsageCell.spec.ts b/frontend/src/components/account/__tests__/AccountUsageCell.spec.ts index d595102f66d1..acbf0884ca46 100644 --- a/frontend/src/components/account/__tests__/AccountUsageCell.spec.ts +++ b/frontend/src/components/account/__tests__/AccountUsageCell.spec.ts @@ -1715,6 +1715,71 @@ describe('AccountUsageCell', () => { expect(getUsage).not.toHaveBeenCalled() }) + it('Vertex 账号不设上限时不显示配额进度条', async () => { + const wrapper = mount(AccountUsageCell, { + props: { + account: makeAccount({ + id: 4011, + platform: 'gemini', + type: 'service_account', + credentials: { tier_id: 'vertex', project_id: 'vertex-proj', location: 'global' }, + extra: {} + }), + todayStats: { + requests: 3, + tokens: 1200, + cost: 0.42, + standard_cost: 0.42, + user_cost: 0.5 + } + }, + global: { + stubs: { + UsageProgressBar: true, + AccountQuotaInfo: true + } + } + }) + + await flushPromises() + + expect(wrapper.text()).toContain('A $0.42') + expect(wrapper.findAll('usage-progress-bar-stub')).toHaveLength(0) + expect(getUsage).not.toHaveBeenCalled() + }) + + it('Vertex 账号设置了配额上限时显示配额进度条', async () => { + const wrapper = mount(AccountUsageCell, { + props: { + account: makeAccount({ + id: 4012, + platform: 'gemini', + type: 'service_account', + credentials: { tier_id: 'vertex', project_id: 'vertex-proj', location: 'global' }, + extra: { quota_daily_start: new Date().toISOString() }, + quota_used: 123.456, + quota_limit: 500, + quota_daily_limit: 50, + quota_daily_used: 12.5 + }), + todayStats: null, + todayStatsLoading: false + }, + global: { + stubs: { + UsageProgressBar: true, + AccountQuotaInfo: true + } + } + }) + + await flushPromises() + + const bars = wrapper.findAll('usage-progress-bar-stub') + expect(bars.map((bar) => bar.attributes('label'))).toEqual(['1d', 'total']) + expect(getUsage).not.toHaveBeenCalled() + }) + it('Gemini 第三方上游 API Key 按普通 Key 账号展示今日统计与额度，不显示 Gemini 徽章', async () => { const wrapper = mount(AccountUsageCell, { props: { diff --git a/frontend/src/components/account/__tests__/EditAccountModal.spec.ts b/frontend/src/components/account/__tests__/EditAccountModal.spec.ts index 7061bc84d639..9e786042f24b 100644 --- a/frontend/src/components/account/__tests__/EditAccountModal.spec.ts +++ b/frontend/src/components/account/__tests__/EditAccountModal.spec.ts @@ -1715,6 +1715,26 @@ describe('EditAccountModal', () => { expect(updateAccountMock).toHaveBeenCalledTimes(1) }) + it('shows quota limit card for Vertex SA account and keeps quota limits on save', async () => { + const account = buildVertexAccount() + account.extra = { quota_limit: 200, quota_daily_limit: 20, quota_used: 35.5 } + updateAccountMock.mockReset() + checkMixedChannelRiskMock.mockReset() + checkMixedChannelRiskMock.mockResolvedValue({ has_risk: false }) + updateAccountMock.mockResolvedValue(account) + + const wrapper = mountModal(account) + expect(wrapper.findComponent({ name: 'QuotaLimitCard' }).exists()).toBe(true) + + await wrapper.get('form#edit-account-form').trigger('submit.prevent') + + expect(updateAccountMock).toHaveBeenCalledTimes(1) + const extra = updateAccountMock.mock.calls[0]?.[1]?.extra + expect(extra?.quota_limit).toBe(200) + expect(extra?.quota_daily_limit).toBe(20) + expect(extra?.quota_weekly_limit).toBeUndefined() + }) + it('blocks Vertex SA save when neither credentials_status nor legacy json indicates existence', async () => { const account = buildVertexAccount() account.credentials = { diff --git a/frontend/src/components/admin/account/AccountActionMenu.vue b/frontend/src/components/admin/account/AccountActionMenu.vue index 0523dcad028b..f376cce12da7 100644 --- a/frontend/src/components/admin/account/AccountActionMenu.vue +++ b/frontend/src/components/admin/account/AccountActionMenu.vue @@ -68,6 +68,7 @@ import { useResizeObserver, useWindowSize } from '@vueuse/core' import { useI18n } from 'vue-i18n' import { Icon } from '@/components/icons' import type { Account } from '@/types' +import { supportsAccountQuota } from '@/utils/accountQuota' const props = defineProps<{ show: boolean; account: Account | null; anchorRect: DOMRect | null }>() const emit = defineEmits(['close', 'test', 'stats', 'schedule', 'duplicate', 'reauth', 'refresh-token', 'recover-state', 'reset-quota', 'set-privacy', 'create-spark-shadow']) @@ -135,7 +136,7 @@ const isShadow = computed(() => props.account?.parent_account_id != null) const isOpenAIOAuthParent = computed(() => isOpenAIOAuth.value && !isShadow.value) const supportsPrivacy = computed(() => (isAntigravityOAuth.value || isOpenAIOAuth.value) && !isShadow.value) const hasQuotaLimit = computed(() => { - return (props.account?.type === 'apikey' || props.account?.type === 'bedrock') && ( + return supportsAccountQuota(props.account?.type) && ( (props.account?.quota_limit ?? 0) > 0 || (props.account?.quota_daily_limit ?? 0) > 0 || (props.account?.quota_weekly_limit ?? 0) > 0 diff --git a/frontend/src/utils/accountQuota.ts b/frontend/src/utils/accountQuota.ts new file mode 100644 index 000000000000..22c54ccbe282 --- /dev/null +++ b/frontend/src/utils/accountQuota.ts @@ -0,0 +1,6 @@ +import type { AccountType } from '@/types' + +/** 支持账号配额（总/日/周上限、quota_used 累加、超限停调度）的账号类型，与后端 AccountTypeSupportsQuota 一致。 */ +export function supportsAccountQuota(type?: AccountType | null): boolean { + return type === 'apikey' || type === 'bedrock' || type === 'service_account' +} From 47cb84b8b047ff883d3ab6214151c96ff8b93584 Mon Sep 17 00:00:00 2001 From: feeeei Date: Sat, 17 Oct 2026 00:07:44 +0800 Subject: [PATCH 12/29] =?UTF-8?q?fix(gemini):=20=E5=88=86=E7=BB=84?= =?UTF-8?q?=E3=80=8C=E5=85=81=E8=AE=B8=E5=BD=93=E5=89=8D=E5=88=86=E7=BB=84?= =?UTF-8?q?=E7=94=9F=E5=9B=BE=E3=80=8D=E5=BC=80=E5=85=B3=E5=AF=B9=20Gemini?= =?UTF-8?q?=20=E7=B3=BB=E5=85=A5=E5=8F=A3=E7=94=9F=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 门禁作用于展示该开关的 Gemini 系分组(gemini/antigravity/composite);无分组 Key、 anthropic 等其他平台分组不受影响,OpenAI/Grok 仍走各自门禁 - 生图判定:复用图片接口的 gemini-*-image 判定(isGeminiCompatibleImageModel),或请求体 generationConfig.responseModalities 含 IMAGE(camelCase/snake_case、字符串/整数枚举、 重复键逐个检查);countTokens 不拦 - 两道检查:/v1/messages、/v1/chat/completions、/v1/responses、/v1beta/models 入口在 选号前按客户端请求名与渠道映射名预检;Gemini 兼容与 Antigravity 的 6 个转发函数在 账号模型映射后再按上游模型名检查,覆盖账号级别名 - 拒绝统一返回 400(Gemini 原生 INVALID_ARGUMENT,其余 invalid_request_error), 不触发换号与账号处罚,下游 sub2api 把本实例当上游时不会因此停用账号; 请求标记为本地功能门禁 - 升级兼容:迁移 246 把存量 gemini/antigravity 分组的生图开关置为开启;新建分组与简易 模式默认分组对 gemini/antigravity 默认开启(与 grok 同为 OR 语义,创建时传 false 不生效), 前端创建表单同步默认值;composite 不回填(同一开关还控制 OpenAI 生图),未开启的 composite 分组升级后 Gemini 生图返回 400 - 未覆盖:最终模型名不符合 gemini-*-image 形态(如上游中转自定义的 nano-banana 别名) 且请求体未声明 IMAGE 模态;门禁按 Key 所属分组判断,Claude Code 专用分组降级到 其他分组时仍按原分组 --- backend/internal/handler/gateway_handler.go | 4 + .../gateway_handler_chat_completions.go | 4 + .../handler/gateway_handler_responses.go | 4 + .../handler/gemini_image_generation_gate.go | 17 ++ .../gemini_image_generation_gate_test.go | 168 +++++++++++ .../internal/handler/gemini_v1beta_handler.go | 14 +- .../repository/simple_mode_default_groups.go | 2 +- ...le_mode_default_groups_integration_test.go | 17 +- backend/internal/service/admin_group.go | 17 +- .../service/admin_service_group_test.go | 68 ++--- .../service/antigravity_gateway_claude.go | 3 + .../service/antigravity_gateway_compat.go | 3 + .../service/antigravity_gateway_gemini.go | 3 + .../gemini_chat_completions_compat_service.go | 3 + .../service/gemini_image_generation_gate.go | 94 +++++++ .../gemini_image_generation_gate_test.go | 263 ++++++++++++++++++ .../service/gemini_messages_compat_service.go | 6 + ..._enable_gemini_image_generation_groups.sql | 6 + ..._image_generation_groups_migration_test.go | 24 ++ frontend/src/views/admin/GroupsView.vue | 2 + .../__tests__/groupsImagePricing.spec.ts | 9 + .../src/views/admin/groupsImagePricing.ts | 6 + 22 files changed, 691 insertions(+), 46 deletions(-) create mode 100644 backend/internal/handler/gemini_image_generation_gate.go create mode 100644 backend/internal/handler/gemini_image_generation_gate_test.go create mode 100644 backend/internal/service/gemini_image_generation_gate.go create mode 100644 backend/internal/service/gemini_image_generation_gate_test.go create mode 100644 backend/migrations/246_enable_gemini_image_generation_groups.sql create mode 100644 backend/migrations/gemini_image_generation_groups_migration_test.go diff --git a/backend/internal/handler/gateway_handler.go b/backend/internal/handler/gateway_handler.go index e990e3dd0a77..73eff575a6e0 100644 --- a/backend/internal/handler/gateway_handler.go +++ b/backend/internal/handler/gateway_handler.go @@ -228,6 +228,10 @@ func (h *GatewayHandler) Messages(c *gin.Context) { if rejectSystemOneOnlyPlatform(c, apiKey, h.errorResponse) { return } + if geminiImageGenerationRejected(c, apiKey, nil, reqModel, channelMapping.MappedModel) { + h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", service.ImageGenerationPermissionMessage()) + return + } if decision := h.checkSecurityAudit(c, reqLog, apiKey, subject, service.ContentModerationProtocolAnthropicMessages, reqModel, body); decision != nil && !decision.AllowNextStage { h.anthropicSecurityAuditError(c, decision) diff --git a/backend/internal/handler/gateway_handler_chat_completions.go b/backend/internal/handler/gateway_handler_chat_completions.go index 561ed1bca0a1..85fceffd7540 100644 --- a/backend/internal/handler/gateway_handler_chat_completions.go +++ b/backend/internal/handler/gateway_handler_chat_completions.go @@ -113,6 +113,10 @@ func (h *GatewayHandler) ChatCompletions(c *gin.Context) { "This group is restricted to Claude Code clients (/v1/messages only)") return } + if geminiImageGenerationRejected(c, apiKey, nil, reqModel, channelMapping.MappedModel) { + h.chatCompletionsErrorResponse(c, http.StatusBadRequest, "invalid_request_error", service.ImageGenerationPermissionMessage()) + return + } if decision := h.checkSecurityAudit(c, reqLog, apiKey, subject, service.ContentModerationProtocolOpenAIChat, reqModel, body); decision != nil && !decision.AllowNextStage { h.openAISecurityAuditError(c, decision) diff --git a/backend/internal/handler/gateway_handler_responses.go b/backend/internal/handler/gateway_handler_responses.go index 5ef6bace91cb..c25d3cfa9dcb 100644 --- a/backend/internal/handler/gateway_handler_responses.go +++ b/backend/internal/handler/gateway_handler_responses.go @@ -117,6 +117,10 @@ func (h *GatewayHandler) Responses(c *gin.Context) { "This group is restricted to Claude Code clients (/v1/messages only)") return } + if geminiImageGenerationRejected(c, apiKey, nil, reqModel, channelMapping.MappedModel) { + h.responsesErrorResponse(c, http.StatusBadRequest, "invalid_request_error", service.ImageGenerationPermissionMessage()) + return + } if decision := h.checkSecurityAudit(c, reqLog, apiKey, subject, service.ContentModerationProtocolOpenAIResponses, reqModel, body); decision != nil && !decision.AllowNextStage { h.responsesSecurityAuditError(c, decision) diff --git a/backend/internal/handler/gemini_image_generation_gate.go b/backend/internal/handler/gemini_image_generation_gate.go new file mode 100644 index 000000000000..f1667ee6d5cd --- /dev/null +++ b/backend/internal/handler/gemini_image_generation_gate.go @@ -0,0 +1,17 @@ +package handler + +import ( + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/gin-gonic/gin" +) + +// geminiImageGenerationRejected 在选号之前按客户端请求名、渠道映射名与请求体 +// 执行 Gemini 系分组的生图门禁。命中时把请求标记为本地功能门禁并返回 true， +// 调用方按入口协议写出 400；账号级模型映射之后的判定由转发层负责。 +func geminiImageGenerationRejected(c *gin.Context, apiKey *service.APIKey, body []byte, models ...string) bool { + if apiKey == nil || !service.GeminiImageGenerationDenied(apiKey.Group, body, models...) { + return false + } + service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate) + return true +} diff --git a/backend/internal/handler/gemini_image_generation_gate_test.go b/backend/internal/handler/gemini_image_generation_gate_test.go new file mode 100644 index 000000000000..048a7a7dd90c --- /dev/null +++ b/backend/internal/handler/gemini_image_generation_gate_test.go @@ -0,0 +1,168 @@ +//go:build unit + +package handler + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" + middleware "github.com/Wei-Shaw/sub2api/internal/server/middleware" + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type geminiImageGateEndpoint struct { + name string + path string + body string + params gin.Params + forcePlatform string + call func(*GatewayHandler, *gin.Context) +} + +func geminiImageGateEndpoints(model string) []geminiImageGateEndpoint { + return []geminiImageGateEndpoint{ + { + name: "messages", path: "/v1/messages", + body: `{"model":"` + model + `","max_tokens":16,"messages":[{"role":"user","content":"draw a cat"}]}`, + call: (*GatewayHandler).Messages, + }, + { + name: "chat completions", path: "/v1/chat/completions", + body: `{"model":"` + model + `","messages":[{"role":"user","content":"draw a cat"}]}`, + call: (*GatewayHandler).ChatCompletions, + }, + { + name: "responses", path: "/v1/responses", + body: `{"model":"` + model + `","input":"draw a cat"}`, + call: (*GatewayHandler).Responses, + }, + { + name: "gemini native", path: "/v1beta/models/" + model + ":generateContent", + body: `{"contents":[{"role":"user","parts":[{"text":"draw a cat"}]}]}`, + params: gin.Params{{Key: "modelAction", Value: "/" + model + ":generateContent"}}, + call: (*GatewayHandler).GeminiV1BetaModels, + }, + } +} + +func runGeminiImageGateEndpoint(t *testing.T, ep geminiImageGateEndpoint, group *service.Group, accountPlatform string) (*httptest.ResponseRecorder, *gin.Context, *groupScopedSchedulerCache) { + t.Helper() + const accountID = int64(9411) + schedulerCache := &groupScopedSchedulerCache{fakeSchedulerCache: &fakeSchedulerCache{accounts: []*service.Account{{ + ID: accountID, Platform: accountPlatform, Type: service.AccountTypeAPIKey, + Status: service.StatusActive, Schedulable: true, Concurrency: 1, + AccountGroups: []service.AccountGroup{{AccountID: accountID, GroupID: group.ID}}, + }}}} + gatewayService := service.NewGatewayService( + nil, &groupMapRepo{fakeGroupRepo: &fakeGroupRepo{}, groups: map[int64]*service.Group{group.ID: group}}, + nil, nil, nil, nil, nil, nil, nil, + service.NewSchedulerSnapshotService(schedulerCache, nil, nil, nil, nil), + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + ) + cfg := &config.Config{RunMode: config.RunModeSimple} + billingCacheService := service.NewBillingCacheService(nil, nil, nil, nil, nil, nil, cfg, nil) + t.Cleanup(billingCacheService.Stop) + h := &GatewayHandler{ + gatewayService: gatewayService, + billingCacheService: billingCacheService, + concurrencyHelper: NewConcurrencyHelper(service.NewConcurrencyService(&fakeConcurrencyCache{}), SSEPingFormatClaude, 0), + maxAccountSwitches: 1, + cfg: cfg, + } + + groupID := group.ID + apiKey := &service.APIKey{ + ID: 9420, UserID: 9430, GroupID: &groupID, Group: group, Status: service.StatusActive, + User: &service.User{ID: 9430, Concurrency: 10, Balance: 100}, + } + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + ctx := context.WithValue(context.Background(), ctxkey.Group, group) + req := httptest.NewRequest(http.MethodPost, ep.path, bytes.NewBufferString(ep.body)).WithContext(ctx) + req.Header.Set("Content-Type", "application/json") + c.Request = req + c.Params = ep.params + if ep.forcePlatform != "" { + c.Set(string(middleware.ContextKeyForcePlatform), ep.forcePlatform) + } + c.Set(string(middleware.ContextKeyAPIKey), apiKey) + c.Set(string(middleware.ContextKeyUser), middleware.AuthSubject{UserID: apiKey.UserID, Concurrency: 10}) + + ep.call(h, c) + return recorder, c, schedulerCache +} + +func TestGatewayHandlers_GeminiImageGenerationDisabledGroupRejectsBeforeSelection(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, platform := range []string{service.PlatformGemini, service.PlatformAntigravity} { + for _, ep := range geminiImageGateEndpoints("gemini-3-pro-image") { + if platform == service.PlatformAntigravity && ep.params != nil { + ep.path = "/antigravity" + ep.path + ep.forcePlatform = service.PlatformAntigravity + } + t.Run(platform+"/"+ep.name, func(t *testing.T) { + group := &service.Group{ + ID: 9400, Hydrated: true, Platform: platform, Status: service.StatusActive, + AllowImageGeneration: false, + } + recorder, c, schedulerCache := runGeminiImageGateEndpoint(t, ep, group, platform) + + require.Equal(t, http.StatusBadRequest, recorder.Code) + require.Contains(t, recorder.Body.String(), service.ImageGenerationPermissionMessage()) + _, reachedSelection := c.Get(opsAccountIDKey) + require.False(t, reachedSelection) + require.Empty(t, schedulerCache.queriedGroupIDs()) + }) + } + } +} + +func TestGatewayHandlers_GeminiImageGenerationGateSkipsGroupsWithoutTheSwitch(t *testing.T) { + gin.SetMode(gin.TestMode) + // Anthropic 分组在后台不展示生图开关，混合调度 Antigravity 账号时不受门禁约束。 + for _, ep := range geminiImageGateEndpoints("gemini-3-pro-image")[:3] { + t.Run(ep.name, func(t *testing.T) { + group := &service.Group{ + ID: 9401, Hydrated: true, Platform: service.PlatformAnthropic, Status: service.StatusActive, + AllowImageGeneration: false, + } + recorder, _, schedulerCache := runGeminiImageGateEndpoint(t, ep, group, service.PlatformAnthropic) + + require.NotContains(t, recorder.Body.String(), service.ImageGenerationPermissionMessage()) + require.NotEmpty(t, schedulerCache.queriedGroupIDs()) + }) + } +} + +func TestGatewayHandlers_GeminiImageGenerationGateAllowsTextModelsAndEnabledGroups(t *testing.T) { + gin.SetMode(gin.TestMode) + cases := []struct { + name string + model string + allow bool + }{ + {name: "text model in disabled group", model: "gemini-3.7-flash", allow: false}, + {name: "image model in enabled group", model: "gemini-3-pro-image", allow: true}, + } + for _, tc := range cases { + for _, ep := range geminiImageGateEndpoints(tc.model)[:3] { + t.Run(tc.name+"/"+ep.name, func(t *testing.T) { + group := &service.Group{ + ID: 9402, Hydrated: true, Platform: service.PlatformGemini, Status: service.StatusActive, + AllowImageGeneration: tc.allow, + } + recorder, _, schedulerCache := runGeminiImageGateEndpoint(t, ep, group, service.PlatformGemini) + + require.NotContains(t, recorder.Body.String(), service.ImageGenerationPermissionMessage()) + require.NotEmpty(t, schedulerCache.queriedGroupIDs()) + }) + } + } +} diff --git a/backend/internal/handler/gemini_v1beta_handler.go b/backend/internal/handler/gemini_v1beta_handler.go index 95c550302e6c..0e2187c9cbf9 100644 --- a/backend/internal/handler/gemini_v1beta_handler.go +++ b/backend/internal/handler/gemini_v1beta_handler.go @@ -328,6 +328,7 @@ func (h *GatewayHandler) GeminiV1BetaModels(c *gin.Context) { googleError(c, http.StatusBadRequest, "Invalid model in URL") return } + urlModel := modelName if resolvedModel, ok := service.ResolvedUpstreamModelFromContext(c.Request.Context()); ok && strings.TrimSpace(resolvedModel) != "" { modelName = strings.TrimSpace(resolvedModel) } @@ -358,14 +359,19 @@ func (h *GatewayHandler) GeminiV1BetaModels(c *gin.Context) { pricingCtx, pricingAt := service.WithGatewayTokenRequestPricing(c.Request.Context()) c.Request = c.Request.WithContext(pricingCtx) - if decision := h.checkSecurityAudit(c, reqLog, apiKey, authSubject, service.ContentModerationProtocolGemini, modelName, body); decision != nil && !decision.AllowNextStage { + // 解析渠道级模型映射 + channelMapping, _ := h.gatewayService.ResolveChannelMappingAndRestrict(c.Request.Context(), apiKey.GroupID, modelName) + reqModel := modelName // 保存映射前的原始模型名 + if action != "countTokens" && geminiImageGenerationRejected(c, apiKey, body, urlModel, reqModel, channelMapping.MappedModel) { + googleError(c, http.StatusBadRequest, service.ImageGenerationPermissionMessage()) + return + } + + if decision := h.checkSecurityAudit(c, reqLog, apiKey, authSubject, service.ContentModerationProtocolGemini, reqModel, body); decision != nil && !decision.AllowNextStage { googleSecurityAuditError(c, decision) return } - // 解析渠道级模型映射 - channelMapping, _ := h.gatewayService.ResolveChannelMappingAndRestrict(c.Request.Context(), apiKey.GroupID, modelName) - reqModel := modelName // 保存映射前的原始模型名 if channelMapping.Mapped { modelName = channelMapping.MappedModel } diff --git a/backend/internal/repository/simple_mode_default_groups.go b/backend/internal/repository/simple_mode_default_groups.go index 2b9daa4d6411..25f3d6df5f95 100644 --- a/backend/internal/repository/simple_mode_default_groups.go +++ b/backend/internal/repository/simple_mode_default_groups.go @@ -77,7 +77,7 @@ func createGroupIfNotExists(ctx context.Context, client *dbent.Client, name, pla SetSubscriptionType(service.SubscriptionTypeStandard). SetRateMultiplier(1.0). SetIsExclusive(false). - SetAllowImageGeneration(platform == service.PlatformGrok). + SetAllowImageGeneration(service.DefaultAllowImageGenerationForPlatform(platform)). Save(ctx) if err != nil { if dbent.IsConstraintError(err) { diff --git a/backend/internal/repository/simple_mode_default_groups_integration_test.go b/backend/internal/repository/simple_mode_default_groups_integration_test.go index d6a96bbfa248..e21d1b0b4d86 100644 --- a/backend/internal/repository/simple_mode_default_groups_integration_test.go +++ b/backend/internal/repository/simple_mode_default_groups_integration_test.go @@ -34,11 +34,18 @@ func TestEnsureSimpleModeDefaultGroups_CreatesMissingDefaults(t *testing.T) { assertGroupExists(service.PlatformAntigravity + "-default-1") assertGroupExists(service.PlatformAntigravity + "-default-2") - grokDefault, err := client.Group.Query(). - Where(group.NameEQ(service.PlatformGrok+"-default"), group.DeletedAtIsNil()). - Only(seedCtx) - require.NoError(t, err) - require.True(t, grokDefault.AllowImageGeneration) + for _, name := range []string{ + service.PlatformGrok + "-default", + service.PlatformGemini + "-default", + service.PlatformAntigravity + "-default-1", + service.PlatformAntigravity + "-default-2", + } { + imageDefault, err := client.Group.Query(). + Where(group.NameEQ(name), group.DeletedAtIsNil()). + Only(seedCtx) + require.NoError(t, err) + require.True(t, imageDefault.AllowImageGeneration, "expected group %s to allow image generation", name) + } } func TestEnsureSimpleModeDefaultGroups_BackfillsOnlyAutoCreatedGrokDefault(t *testing.T) { diff --git a/backend/internal/service/admin_group.go b/backend/internal/service/admin_group.go index 96c8092a8ec0..f2c06a9a839a 100644 --- a/backend/internal/service/admin_group.go +++ b/backend/internal/service/admin_group.go @@ -311,10 +311,17 @@ func defaultModelsListCandidateIDs(platform string) []string { } } -func defaultAllowImageGenerationForPlatform(platform string) bool { - // Grok image and video generation routes share the legacy image-generation gate. - // Older clients send the false zero value, so Grok groups must default enabled. - return platform == PlatformGrok +// DefaultAllowImageGenerationForPlatform reports whether new groups on the platform allow +// image generation. CreateGroup ORs the requested value with it, so a false in the create +// request does not turn it off. Composite defaults disabled because the same switch also +// gates OpenAI image generation there. +func DefaultAllowImageGenerationForPlatform(platform string) bool { + switch platform { + case PlatformGrok, PlatformGemini, PlatformAntigravity: + return true + default: + return false + } } func compositeDefaultModelsListCandidateIDs() []string { @@ -525,7 +532,7 @@ func (s *adminServiceImpl) CreateGroup(ctx context.Context, input *CreateGroupIn mcpXMLInject = *input.MCPXMLInject } - allowImageGeneration := input.AllowImageGeneration || defaultAllowImageGenerationForPlatform(platform) + allowImageGeneration := input.AllowImageGeneration || DefaultAllowImageGenerationForPlatform(platform) allowBatchImageGeneration := input.AllowBatchImageGeneration && allowImageGeneration && platform == PlatformGemini // 如果指定了复制账号的源分组，先获取账号 ID 列表 diff --git a/backend/internal/service/admin_service_group_test.go b/backend/internal/service/admin_service_group_test.go index 56ef6d7e3d71..af12cf0e4f72 100644 --- a/backend/internal/service/admin_service_group_test.go +++ b/backend/internal/service/admin_service_group_test.go @@ -593,41 +593,47 @@ func TestAdminService_CreateGroup_DefaultsGrokMediaGenerationEnabled(t *testing. require.True(t, group.AllowImageGeneration) } -func TestAdminService_CreateGroup_PreservesNonGrokImageGenerationDisabled(t *testing.T) { - repo := &groupRepoStubForAdmin{} - svc := &adminServiceImpl{groupRepo: repo} +func TestAdminService_CreateGroup_DefaultsGeminiFamilyImageGenerationEnabled(t *testing.T) { + for _, platform := range []string{PlatformGemini, PlatformAntigravity} { + t.Run(platform, func(t *testing.T) { + repo := &groupRepoStubForAdmin{} + svc := &adminServiceImpl{groupRepo: repo} - group, err := svc.CreateGroup(context.Background(), &CreateGroupInput{ - Name: "anthropic-text", - Description: "Anthropic text group", - Platform: PlatformAnthropic, - RateMultiplier: 1.0, - }) - require.NoError(t, err) - require.NotNil(t, group) - require.NotNil(t, repo.created) - require.False(t, repo.created.AllowImageGeneration) - require.False(t, group.AllowImageGeneration) + group, err := svc.CreateGroup(context.Background(), &CreateGroupInput{ + Name: platform + "-images", + Description: "Gemini family group", + Platform: platform, + RateMultiplier: 1.0, + }) + require.NoError(t, err) + require.NotNil(t, group) + require.NotNil(t, repo.created) + require.True(t, repo.created.AllowImageGeneration) + require.True(t, group.AllowImageGeneration) + require.False(t, repo.created.AllowBatchImageGeneration) + }) + } } -func TestAdminService_CreateGroup_DisablesBatchImageWhenImageGenerationDisabled(t *testing.T) { - repo := &groupRepoStubForAdmin{} - svc := &adminServiceImpl{groupRepo: repo} +func TestAdminService_CreateGroup_PreservesImageGenerationDisabledWithoutPlatformDefault(t *testing.T) { + for _, platform := range []string{PlatformAnthropic, PlatformComposite} { + t.Run(platform, func(t *testing.T) { + repo := &groupRepoStubForAdmin{} + svc := &adminServiceImpl{groupRepo: repo} - group, err := svc.CreateGroup(context.Background(), &CreateGroupInput{ - Name: "gemini-no-image", - Description: "Gemini group without image generation", - Platform: PlatformGemini, - RateMultiplier: 1.0, - AllowImageGeneration: false, - AllowBatchImageGeneration: true, - }) - require.NoError(t, err) - require.NotNil(t, group) - require.NotNil(t, repo.created) - require.False(t, repo.created.AllowImageGeneration) - require.False(t, repo.created.AllowBatchImageGeneration) - require.False(t, group.AllowBatchImageGeneration) + group, err := svc.CreateGroup(context.Background(), &CreateGroupInput{ + Name: platform + "-text", + Description: "Group without image default", + Platform: platform, + RateMultiplier: 1.0, + }) + require.NoError(t, err) + require.NotNil(t, group) + require.NotNil(t, repo.created) + require.False(t, repo.created.AllowImageGeneration) + require.False(t, group.AllowImageGeneration) + }) + } } func TestAdminService_CreateGroup_DisablesBatchImageForNonGeminiPlatform(t *testing.T) { diff --git a/backend/internal/service/antigravity_gateway_claude.go b/backend/internal/service/antigravity_gateway_claude.go index 264cbe998094..70230dd48888 100644 --- a/backend/internal/service/antigravity_gateway_claude.go +++ b/backend/internal/service/antigravity_gateway_claude.go @@ -61,6 +61,9 @@ func (s *AntigravityGatewayService) Forward(ctx context.Context, c *gin.Context, // 应用 thinking 模式自动后缀：如果 thinking 开启且目标是 claude-sonnet-4-5，自动改为 thinking 版本 thinkingEnabled := claudeReq.Thinking != nil && (claudeReq.Thinking.Type == "enabled" || claudeReq.Thinking.Type == "adaptive") mappedModel = applyThinkingModelSuffix(mappedModel, thinkingEnabled) + if geminiImageGenerationDenied(c, nil, originalModel, mappedModel) { + return nil, s.writeClaudeError(c, http.StatusBadRequest, "invalid_request_error", ImageGenerationPermissionMessage()) + } billingModel := mappedModel // 获取 access_token diff --git a/backend/internal/service/antigravity_gateway_compat.go b/backend/internal/service/antigravity_gateway_compat.go index 60ecd977ed5e..2869675449f7 100644 --- a/backend/internal/service/antigravity_gateway_compat.go +++ b/backend/internal/service/antigravity_gateway_compat.go @@ -225,6 +225,9 @@ func (s *AntigravityGatewayService) prepareAntigravityCompatCall( thinkingEnabled := claudeRequest.Thinking != nil && (claudeRequest.Thinking.Type == "enabled" || claudeRequest.Thinking.Type == "adaptive") mappedModel = applyThinkingModelSuffix(mappedModel, thinkingEnabled) + if geminiImageGenerationDenied(c, nil, request.originalModel, mappedModel) { + return nil, s.writeAntigravityCompatError(c, http.StatusBadRequest, "invalid_request_error", ImageGenerationPermissionMessage()) + } if s.tokenProvider == nil { return nil, s.writeAntigravityCompatError(c, http.StatusBadGateway, "api_error", "Antigravity token provider not configured") diff --git a/backend/internal/service/antigravity_gateway_gemini.go b/backend/internal/service/antigravity_gateway_gemini.go index b6396cb08a91..0d12fe77590e 100644 --- a/backend/internal/service/antigravity_gateway_gemini.go +++ b/backend/internal/service/antigravity_gateway_gemini.go @@ -96,6 +96,9 @@ func (s *AntigravityGatewayService) ForwardGemini(ctx context.Context, c *gin.Co MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate) return nil, s.writeGoogleError(c, http.StatusForbidden, fmt.Sprintf("model %s not in whitelist", originalModel)) } + if geminiImageGenerationDenied(c, body, originalModel, mappedModel) { + return nil, s.writeGoogleError(c, http.StatusBadRequest, ImageGenerationPermissionMessage()) + } forwardedModel := mappedModel // 获取 access_token diff --git a/backend/internal/service/gemini_chat_completions_compat_service.go b/backend/internal/service/gemini_chat_completions_compat_service.go index f85ac0577a2e..fbfcd4b17940 100644 --- a/backend/internal/service/gemini_chat_completions_compat_service.go +++ b/backend/internal/service/gemini_chat_completions_compat_service.go @@ -100,6 +100,9 @@ func (s *GeminiMessagesCompatService) forwardClaudeBodyAsChatCompletions( if err != nil { return nil, s.writeChatCompletionsError(c, http.StatusBadRequest, "invalid_request_error", err.Error()) } + if geminiImageGenerationDenied(c, geminiReq, originalModel, req.Model, mappedModel) { + return nil, s.writeChatCompletionsError(c, http.StatusBadRequest, "invalid_request_error", ImageGenerationPermissionMessage()) + } geminiReq = ensureGeminiFunctionCallThoughtSignatures(geminiReq) geminiReq = applyGeminiCompatThinkingConfig(geminiReq, claudeBody, mappedModel, clientStream) diff --git a/backend/internal/service/gemini_image_generation_gate.go b/backend/internal/service/gemini_image_generation_gate.go new file mode 100644 index 000000000000..89230836856f --- /dev/null +++ b/backend/internal/service/gemini_image_generation_gate.go @@ -0,0 +1,94 @@ +package service + +import ( + "slices" + "strings" + + "github.com/gin-gonic/gin" + "github.com/tidwall/gjson" +) + +// geminiImageModalityEnum 是 proto JSON 中 Modality.IMAGE 的整数枚举值。 +const geminiImageModalityEnum = 2 + +// GeminiImageGenerationDenied 判断 Gemini 系请求是否因分组关闭生图而必须拒绝。 +// 门禁只作用于后台展示该开关的 Gemini 系分组（gemini / antigravity / composite）； +// 无分组的 Key 与其他平台分组不受影响。models 中任一名字属于生图模型， +// 或请求体要求输出 IMAGE 模态，即视为生图请求。 +func GeminiImageGenerationDenied(group *Group, geminiBody []byte, models ...string) bool { + if group == nil || group.AllowImageGeneration { + return false + } + switch group.Platform { + case PlatformGemini, PlatformAntigravity, PlatformComposite: + default: + return false + } + return isGeminiImageGenerationIntent(geminiBody, models...) +} + +// geminiImageGenerationDenied 是转发层在账号模型映射之后的门禁：models 传入 +// 转发时可见的模型名（入参模型名与账号映射后的上游模型名）。命中时把请求标记为 +// 本地功能门禁，调用方负责按各自协议写出 400。 +func geminiImageGenerationDenied(c *gin.Context, geminiBody []byte, models ...string) bool { + if c == nil || !GeminiImageGenerationDenied(apiKeyGroup(getAPIKeyFromContext(c)), geminiBody, models...) { + return false + } + MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate) + return true +} + +func isGeminiImageGenerationIntent(geminiBody []byte, models ...string) bool { + return slices.ContainsFunc(models, isGeminiImageOutputModel) || geminiRequestAsksForImageOutput(geminiBody) +} + +// isGeminiImageOutputModel 匹配以图片为输出的 Gemini 模型名，与图片接口共用 +// isGeminiCompatibleImageModel 判定。 +func isGeminiImageOutputModel(model string) bool { + return isGeminiCompatibleImageModel(strings.TrimPrefix(strings.ToLower(strings.TrimSpace(model)), "models/")) +} + +// geminiRequestAsksForImageOutput 判断 generationConfig.responseModalities 是否包含 IMAGE。 +// Gemini REST 同时接受 camelCase 与 snake_case 字段名、字符串与整数枚举； +// 重复出现的键逐个检查。 +func geminiRequestAsksForImageOutput(body []byte) bool { + if len(body) == 0 || !gjson.ValidBytes(body) { + return false + } + found := false + gjson.ParseBytes(body).ForEach(func(key, config gjson.Result) bool { + if key.Str == "generationConfig" || key.Str == "generation_config" { + found = geminiGenerationConfigAsksForImage(config) + } + return !found + }) + return found +} + +func geminiGenerationConfigAsksForImage(config gjson.Result) bool { + if !config.IsObject() { + return false + } + found := false + config.ForEach(func(key, modalities gjson.Result) bool { + if key.Str == "responseModalities" || key.Str == "response_modalities" { + found = geminiModalitiesContainImage(modalities) + } + return !found + }) + return found +} + +func geminiModalitiesContainImage(modalities gjson.Result) bool { + found := false + modalities.ForEach(func(_, modality gjson.Result) bool { + switch modality.Type { + case gjson.String: + found = strings.EqualFold(strings.TrimSpace(modality.Str), "IMAGE") + case gjson.Number: + found = modality.Num == geminiImageModalityEnum + } + return !found + }) + return found +} diff --git a/backend/internal/service/gemini_image_generation_gate_test.go b/backend/internal/service/gemini_image_generation_gate_test.go new file mode 100644 index 000000000000..c384563127fa --- /dev/null +++ b/backend/internal/service/gemini_image_generation_gate_test.go @@ -0,0 +1,263 @@ +//go:build unit + +package service + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestIsGeminiImageGenerationIntent(t *testing.T) { + cases := []struct { + name string + body string + models []string + want bool + }{ + {name: "pro image", models: []string{"gemini-3-pro-image"}, want: true}, + {name: "flash image preview with prefix", models: []string{"models/gemini-3.1-flash-image-preview"}, want: true}, + {name: "image generation experimental", models: []string{"gemini-2.0-flash-exp-image-generation"}, want: true}, + {name: "imageless suffix", models: []string{"gemini-3-pro-imageless"}, want: false}, + {name: "image without gemini prefix", models: []string{"unknown-image", "my-image-chat"}, want: false}, + {name: "banana alias without resolved image model", models: []string{"nano-banana-2"}, want: false}, + {name: "any resolved name", models: []string{"my-alias", "gemini-2.5-flash-image"}, want: true}, + {name: "text model", models: []string{"gemini-3.7-flash", "gemini-3.1-pro-preview"}, want: false}, + {name: "empty", models: []string{"", " "}, want: false}, + {name: "camelCase modalities", body: `{"generationConfig":{"responseModalities":["TEXT","IMAGE"]}}`, models: []string{"gemini-3.7-flash"}, want: true}, + {name: "snake_case modalities", body: `{"generation_config":{"response_modalities":["image"]}}`, want: true}, + {name: "mixed case keys", body: `{"generationConfig":{"response_modalities":[" Image "]}}`, want: true}, + {name: "text only modalities", body: `{"generationConfig":{"responseModalities":["TEXT"]}}`, models: []string{"gemini-3.7-flash"}, want: false}, + {name: "modalities string not array", body: `{"generationConfig":{"responseModalities":"IMAGE"}}`, want: true}, + {name: "no generation config", body: `{"contents":[{"parts":[{"text":"IMAGE"}]}]}`, want: false}, + {name: "integer enum modalities", body: `{"generationConfig":{"responseModalities":[1,2]}}`, want: true}, + {name: "integer enum text only", body: `{"generationConfig":{"responseModalities":[1]}}`, want: false}, + {name: "duplicate generationConfig keys", body: `{"generationConfig":{"responseModalities":["TEXT"]},"generationConfig":{"responseModalities":["IMAGE"]}}`, want: true}, + {name: "duplicate modalities keys", body: `{"generationConfig":{"responseModalities":["TEXT"],"responseModalities":["IMAGE"]}}`, want: true}, + {name: "invalid json", body: `{"generationConfig":{"responseModalities":["IMAGE"]`, want: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, isGeminiImageGenerationIntent([]byte(tc.body), tc.models...)) + }) + } +} + +func TestGeminiImageGenerationDenied_GroupScope(t *testing.T) { + cases := []struct { + name string + group *Group + want bool + }{ + {name: "ungrouped key", group: nil, want: false}, + {name: "gemini disabled", group: &Group{Platform: PlatformGemini}, want: true}, + {name: "antigravity disabled", group: &Group{Platform: PlatformAntigravity}, want: true}, + {name: "composite disabled", group: &Group{Platform: PlatformComposite}, want: true}, + {name: "gemini enabled", group: &Group{Platform: PlatformGemini, AllowImageGeneration: true}, want: false}, + {name: "anthropic group without the switch", group: &Group{Platform: PlatformAnthropic}, want: false}, + {name: "openai group gated elsewhere", group: &Group{Platform: PlatformOpenAI}, want: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, GeminiImageGenerationDenied(tc.group, nil, "gemini-3-pro-image")) + }) + } +} + +func withGeminiImageGateAPIKey(c *gin.Context, group *Group) { + c.Set("api_key", &APIKey{ID: 1, Group: group}) +} + +func imageDisabledGroup() *Group { + return &Group{ID: 10, Platform: PlatformGemini, AllowImageGeneration: false} +} + +func requireGeminiImagePermissionError(t *testing.T, rec *httptest.ResponseRecorder, wantGoogleStatus string) { + t.Helper() + require.Equal(t, http.StatusBadRequest, rec.Code) + var payload map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &payload)) + errObj, ok := payload["error"].(map[string]any) + require.True(t, ok, "error object: %s", rec.Body.String()) + require.Equal(t, ImageGenerationPermissionMessage(), errObj["message"]) + if wantGoogleStatus != "" { + require.Equal(t, wantGoogleStatus, errObj["status"]) + } else { + require.Equal(t, "invalid_request_error", errObj["type"]) + } +} + +func TestGeminiForwardNative_ImageGenerationDisabledGroupRejectsBeforeUpstream(t *testing.T) { + gin.SetMode(gin.TestMode) + textBody := []byte(`{"contents":[{"role":"user","parts":[{"text":"draw a cat"}]}]}`) + imageModalityBody := []byte(`{"contents":[{"role":"user","parts":[{"text":"draw a cat"}]}],"generationConfig":{"responseModalities":["TEXT","IMAGE"]}}`) + + cases := []struct { + name string + model string + mapping map[string]any + body []byte + }{ + {name: "image model", model: "gemini-3-pro-image", body: textBody}, + {name: "account mapping alias to image model", model: "my-draw", mapping: map[string]any{"my-draw": "gemini-3.1-flash-image"}, body: textBody}, + {name: "text model asking for image modality", model: "gemini-3.7-flash", body: imageModalityBody}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + svc, httpStub := newGeminiVertexAPIKeyTestService() + c, rec := newGeminiNativeTestContext(t) + withGeminiImageGateAPIKey(c, imageDisabledGroup()) + account := geminiVertexAPIKeyTestAccount("") + if tc.mapping != nil { + account.Credentials["model_mapping"] = tc.mapping + } + + result, err := svc.ForwardNative(context.Background(), c, account, tc.model, "generateContent", false, tc.body) + require.Error(t, err) + require.Nil(t, result) + require.Zero(t, httpStub.calls) + requireGeminiImagePermissionError(t, rec, "INVALID_ARGUMENT") + }) + } +} + +func TestGeminiForwardNative_ImageGenerationGateAllowsPermittedRequests(t *testing.T) { + gin.SetMode(gin.TestMode) + body := []byte(`{"contents":[{"role":"user","parts":[{"text":"draw a cat"}]}]}`) + + cases := []struct { + name string + group *Group + model string + action string + }{ + {name: "group allows image generation", group: &Group{ID: 34, Platform: PlatformGemini, AllowImageGeneration: true}, model: "gemini-3-pro-image", action: "generateContent"}, + {name: "ungrouped key", group: nil, model: "gemini-3-pro-image", action: "generateContent"}, + {name: "text model in disabled group", group: imageDisabledGroup(), model: "gemini-3.7-flash", action: "generateContent"}, + {name: "countTokens on image model", group: imageDisabledGroup(), model: "gemini-3-pro-image", action: "countTokens"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + svc, httpStub := newGeminiVertexAPIKeyTestService() + c, rec := newGeminiNativeTestContext(t) + withGeminiImageGateAPIKey(c, tc.group) + + _, _ = svc.ForwardNative(context.Background(), c, geminiVertexAPIKeyTestAccount(""), tc.model, tc.action, false, body) + require.Equal(t, 1, httpStub.calls) + require.NotEqual(t, http.StatusBadRequest, rec.Code) + }) + } +} + +func TestGeminiCompatForward_ImageGenerationDisabledGroupRejectsBeforeUpstream(t *testing.T) { + gin.SetMode(gin.TestMode) + svc, httpStub := newGeminiVertexAPIKeyTestService() + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil) + withGeminiImageGateAPIKey(c, imageDisabledGroup()) + body := []byte(`{"model":"gemini-3-pro-image","max_tokens":16,"messages":[{"role":"user","content":"draw a cat"}]}`) + + result, err := svc.Forward(context.Background(), c, geminiVertexAPIKeyTestAccount(""), body) + require.Error(t, err) + require.Nil(t, result) + require.Zero(t, httpStub.calls) + requireGeminiImagePermissionError(t, rec, "") +} + +func TestGeminiForwardAsChatCompletions_ImageGenerationDisabledGroupRejectsBeforeUpstream(t *testing.T) { + gin.SetMode(gin.TestMode) + svc, httpStub := newGeminiVertexAPIKeyTestService() + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + body := []byte(`{"model":"gemini-3.1-flash-image","messages":[{"role":"user","content":"draw a cat"}]}`) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) + withGeminiImageGateAPIKey(c, imageDisabledGroup()) + + result, err := svc.ForwardAsChatCompletions(context.Background(), c, geminiVertexAPIKeyTestAccount(""), body) + require.Error(t, err) + require.Nil(t, result) + require.Zero(t, httpStub.calls) + requireGeminiImagePermissionError(t, rec, "") +} + +func newAntigravityImageGateAccount() *Account { + account := newAntigravityCompatAccount(AccountTypeOAuth) + account.Credentials["model_mapping"] = map[string]any{ + "gemini-3-pro-image": "gemini-3-pro-image", + "my-draw": "gemini-3.1-flash-image", + } + return account +} + +func TestAntigravityForwards_ImageGenerationGate(t *testing.T) { + gin.SetMode(gin.TestMode) + geminiBody := []byte(`{"contents":[{"role":"user","parts":[{"text":"draw a cat"}]}]}`) + claudeBody := []byte(`{"model":"my-draw","max_tokens":16,"stream":true,"messages":[{"role":"user","content":"draw a cat"}]}`) + chatBody := []byte(`{"model":"gemini-3-pro-image","stream":true,"messages":[{"role":"user","content":"draw a cat"}]}`) + + forwards := []struct { + name string + path string + body []byte + wantGoogleStatus string + forward func(*AntigravityGatewayService, *gin.Context, *Account) (*ForwardResult, error) + }{ + { + name: "ForwardGemini", + path: "/v1beta/models/gemini-3-pro-image:streamGenerateContent", + body: geminiBody, + wantGoogleStatus: "INVALID_ARGUMENT", + forward: func(svc *AntigravityGatewayService, c *gin.Context, account *Account) (*ForwardResult, error) { + return svc.ForwardGemini(context.Background(), c, account, "gemini-3-pro-image", "streamGenerateContent", true, geminiBody, false) + }, + }, + { + name: "Forward", + path: "/v1/messages", + body: claudeBody, + forward: func(svc *AntigravityGatewayService, c *gin.Context, account *Account) (*ForwardResult, error) { + return svc.Forward(context.Background(), c, account, claudeBody, false) + }, + }, + { + name: "ForwardAsChatCompletions", + path: "/v1/chat/completions", + body: chatBody, + forward: func(svc *AntigravityGatewayService, c *gin.Context, account *Account) (*ForwardResult, error) { + return svc.ForwardAsChatCompletions(context.Background(), c, account, chatBody, nil) + }, + }, + } + for _, fw := range forwards { + t.Run(fw.name+"/disabled group rejects", func(t *testing.T) { + upstream := &queuedHTTPUpstreamStub{} + svc := newAntigravityCompatService(config.GatewayConfig{MaxLineSize: defaultMaxLineSize}, upstream) + c, rec := newAntigravityCompatContext(http.MethodPost, fw.path, fw.body) + withGeminiImageGateAPIKey(c, &Group{ID: 20, Platform: PlatformAntigravity, AllowImageGeneration: false}) + + result, err := fw.forward(svc, c, newAntigravityImageGateAccount()) + require.Error(t, err) + require.Nil(t, result) + require.Zero(t, upstream.callCount) + requireGeminiImagePermissionError(t, rec, fw.wantGoogleStatus) + }) + t.Run(fw.name+"/enabled group reaches upstream", func(t *testing.T) { + upstream := &queuedHTTPUpstreamStub{responses: []*http.Response{antigravityCompatSuccessResponse()}} + svc := newAntigravityCompatService(config.GatewayConfig{MaxLineSize: defaultMaxLineSize}, upstream) + c, rec := newAntigravityCompatContext(http.MethodPost, fw.path, fw.body) + withGeminiImageGateAPIKey(c, &Group{ID: 21, Platform: PlatformAntigravity, AllowImageGeneration: true}) + + _, _ = fw.forward(svc, c, newAntigravityImageGateAccount()) + require.Equal(t, 1, upstream.callCount) + require.NotEqual(t, http.StatusBadRequest, rec.Code) + }) + } +} diff --git a/backend/internal/service/gemini_messages_compat_service.go b/backend/internal/service/gemini_messages_compat_service.go index 48b095fb3620..497bff022339 100644 --- a/backend/internal/service/gemini_messages_compat_service.go +++ b/backend/internal/service/gemini_messages_compat_service.go @@ -555,6 +555,9 @@ func (s *GeminiMessagesCompatService) Forward(ctx context.Context, c *gin.Contex if err != nil { return nil, s.writeClaudeError(c, http.StatusBadRequest, "invalid_request_error", err.Error()) } + if geminiImageGenerationDenied(c, geminiReq, originalModel, mappedModel) { + return nil, s.writeClaudeError(c, http.StatusBadRequest, "invalid_request_error", ImageGenerationPermissionMessage()) + } geminiReq = ensureGeminiFunctionCallThoughtSignatures(geminiReq) includeThoughts := req.Stream && claudeRequestEnablesThinking(body) geminiReq = applyGeminiCompatThinkingConfig(geminiReq, body, mappedModel, includeThoughts) @@ -1111,6 +1114,9 @@ func (s *GeminiMessagesCompatService) ForwardNative(ctx context.Context, c *gin. if account.Type == AccountTypeAPIKey || account.Type == AccountTypeServiceAccount { mappedModel = account.GetMappedModel(originalModel) } + if action != "countTokens" && geminiImageGenerationDenied(c, body, originalModel, mappedModel) { + return nil, s.writeGoogleError(c, http.StatusBadRequest, ImageGenerationPermissionMessage()) + } proxyURL := "" if account.ProxyID != nil && account.Proxy != nil { diff --git a/backend/migrations/246_enable_gemini_image_generation_groups.sql b/backend/migrations/246_enable_gemini_image_generation_groups.sql new file mode 100644 index 000000000000..ae37f206fce3 --- /dev/null +++ b/backend/migrations/246_enable_gemini_image_generation_groups.sql @@ -0,0 +1,6 @@ +-- gemini / antigravity 分组的生图开关同时约束 Gemini 系入口，存量分组统一置为开启。 +-- composite 分组的同一开关还约束 OpenAI 生图，不在本迁移范围内。 +UPDATE groups +SET allow_image_generation = true +WHERE platform IN ('gemini', 'antigravity') + AND allow_image_generation = false; diff --git a/backend/migrations/gemini_image_generation_groups_migration_test.go b/backend/migrations/gemini_image_generation_groups_migration_test.go new file mode 100644 index 000000000000..81630a151bcb --- /dev/null +++ b/backend/migrations/gemini_image_generation_groups_migration_test.go @@ -0,0 +1,24 @@ +package migrations + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEnableGeminiImageGenerationGroupsMigration(t *testing.T) { + content, err := FS.ReadFile("246_enable_gemini_image_generation_groups.sql") + require.NoError(t, err) + + var statements []string + for _, line := range strings.Split(string(content), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "--") { + statements = append(statements, line) + } + } + sql := strings.Join(strings.Fields(strings.Join(statements, " ")), " ") + require.Equal(t, + "UPDATE groups SET allow_image_generation = true WHERE platform IN ('gemini', 'antigravity') AND allow_image_generation = false;", + sql) +} diff --git a/frontend/src/views/admin/GroupsView.vue b/frontend/src/views/admin/GroupsView.vue index 3fb2c95cd52b..8572f7b0bf8a 100644 --- a/frontend/src/views/admin/GroupsView.vue +++ b/frontend/src/views/admin/GroupsView.vue @@ -4364,6 +4364,7 @@ import { type ReasoningEffortMappingRow, } from "./groupsReasoningEffort"; import { + defaultAllowImageGenerationForPlatform, getDefaultImagePreviewPrice, getDefaultVideoPreviewPrice, getImagePricePlaceholder, @@ -6726,6 +6727,7 @@ watch( createForm.require_oauth_only = false; createForm.require_privacy_set = false; } + createForm.allow_image_generation = defaultAllowImageGenerationForPlatform(newVal); resetDisabledBatchImagePricing(createForm); resetModelAllowlistState(createModelAllowlistState); loadModelAllowlistCandidates("create", 0, newVal); diff --git a/frontend/src/views/admin/__tests__/groupsImagePricing.spec.ts b/frontend/src/views/admin/__tests__/groupsImagePricing.spec.ts index 1a37b05778d5..57a3f7d7eee7 100644 --- a/frontend/src/views/admin/__tests__/groupsImagePricing.spec.ts +++ b/frontend/src/views/admin/__tests__/groupsImagePricing.spec.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { + defaultAllowImageGenerationForPlatform, getDefaultImagePreviewPrice, getDefaultVideoPreviewPrice, getImagePricePlaceholder, @@ -27,6 +28,14 @@ describe("groups image pricing platform support", () => { expect(supportsImagePricingPlatform("anthropic")).toBe(false); }); + it("defaults image generation on for Grok and Gemini-family groups only", () => { + expect(defaultAllowImageGenerationForPlatform("grok")).toBe(true); + expect(defaultAllowImageGenerationForPlatform("gemini")).toBe(true); + expect(defaultAllowImageGenerationForPlatform("antigravity")).toBe(true); + expect(defaultAllowImageGenerationForPlatform("composite")).toBe(false); + expect(defaultAllowImageGenerationForPlatform("openai")).toBe(false); + }); + it("includes Composite groups in image pricing controls", () => { expect(supportsImagePricingPlatform("composite")).toBe(true); expect(imagePricingPlatforms.has("composite")).toBe(true); diff --git a/frontend/src/views/admin/groupsImagePricing.ts b/frontend/src/views/admin/groupsImagePricing.ts index 95152e3efd78..e4f3de5e120a 100644 --- a/frontend/src/views/admin/groupsImagePricing.ts +++ b/frontend/src/views/admin/groupsImagePricing.ts @@ -9,6 +9,12 @@ export const imagePricingPlatforms = new Set([ export const supportsImagePricingPlatform = (platform: string): boolean => imagePricingPlatforms.has(platform); +// 与后端 DefaultAllowImageGenerationForPlatform 一致：这些平台新建分组默认允许生图。 +const imageGenerationDefaultPlatforms = new Set(["antigravity", "gemini", "grok"]); + +export const defaultAllowImageGenerationForPlatform = (platform: string): boolean => + imageGenerationDefaultPlatforms.has(platform); + export const supportsVideoPricingPlatform = (platform: string): boolean => platform === "grok"; From 4355236e00230de65226ad6ba710cffd1a512473 Mon Sep 17 00:00:00 2001 From: feeeei Date: Tue, 20 Oct 2026 01:25:19 +0800 Subject: [PATCH 13/29] =?UTF-8?q?fix(gateway):=20=E5=85=BC=E5=AE=B9?= =?UTF-8?q?=E6=B5=81=E4=B8=8A=E6=B8=B8=E6=9C=AA=E6=AD=A3=E5=B8=B8=E7=BB=93?= =?UTF-8?q?=E6=9D=9F=E6=97=B6=E6=8A=A5=E9=94=99=E5=B9=B6=E5=AF=B9=E9=BD=90?= =?UTF-8?q?=E5=8E=9F=E7=94=9F=E6=94=B6=E5=B0=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 客户端在线时，Anthropic 账号的 chat/responses 兼容流在上游未发出结束事件 （message_stop、带 stop_reason 的 message_delta 或 [DONE]）就中断，不再补发 finish_reason / [DONE] / response.completed 报成功： - 读间隔超时、读错误、message_start 之后直接 EOF：service 写出协议内终止 事件（CC 为 error 帧，Responses 为沿用流内 id 与 sequence_number 的 response.failed），返回错误并带上已观测到的 usage；读间隔超时调用 HandleStreamTimeout。 - 流内 event: error：立即结束读取，与 /v1/messages 共用 sseStreamErrorEventFailover 转为 UpstreamFailoverError，未输出前换号重试， 客户端在线时不计费。 - 两个 handler 的错误分支对带结果的错误照常入账。 客户端断开后的读取：stream_data_interval_timeout 未启用时按 180s 启用读间隔 上限；请求 ctx 取消也视为断开。复用 anthropicNativeStreamInterval。行泵在 计时器到期且已有缓冲行时优先交付数据，下游写入慢不再被判为上游空闲。 --- .../gateway_compat_stream_usage_test.go | 229 ++++++ .../gateway_handler_chat_completions.go | 75 +- .../handler/gateway_handler_responses.go | 75 +- backend/internal/service/gateway_forward.go | 104 +-- .../gateway_forward_as_chat_completions.go | 93 +-- .../service/gateway_forward_as_responses.go | 49 +- ...y_forward_compat_client_disconnect_test.go | 671 +++++++++++++++++- .../service/gateway_forward_compat_stream.go | 291 ++++++++ .../openai_gateway_anthropic_native_pump.go | 66 +- ...enai_gateway_anthropic_native_pump_test.go | 81 +++ ...teway_chat_completions_anthropic_native.go | 4 +- ...enai_gateway_responses_anthropic_native.go | 4 +- 12 files changed, 1498 insertions(+), 244 deletions(-) create mode 100644 backend/internal/handler/gateway_compat_stream_usage_test.go create mode 100644 backend/internal/service/gateway_forward_compat_stream.go diff --git a/backend/internal/handler/gateway_compat_stream_usage_test.go b/backend/internal/handler/gateway_compat_stream_usage_test.go new file mode 100644 index 000000000000..84f2e587d431 --- /dev/null +++ b/backend/internal/handler/gateway_compat_stream_usage_test.go @@ -0,0 +1,229 @@ +//go:build unit + +package handler + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" + middleware "github.com/Wei-Shaw/sub2api/internal/server/middleware" + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type compatStreamUsageUpstream struct { + service.HTTPUpstream + response *http.Response + calls int +} + +func (u *compatStreamUsageUpstream) DoWithTLS(*http.Request, string, int64, int, *tlsfingerprint.Profile) (*http.Response, error) { + u.calls++ + return u.response, nil +} + +type compatStreamUsageBillingRepo struct { + service.UsageBillingRepository + mu sync.Mutex + commands []*service.UsageBillingCommand +} + +func (r *compatStreamUsageBillingRepo) Apply(ctx context.Context, cmd *service.UsageBillingCommand) (*service.UsageBillingApplyResult, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.commands = append(r.commands, cmd) + return &service.UsageBillingApplyResult{Applied: true}, nil +} + +const compatStreamUsagePartialStream = "event: message_start\n" + + `data: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-sonnet-4.5","usage":{"input_tokens":12,"output_tokens":1}}}` + "\n\n" + + "event: content_block_start\n" + + `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}` + "\n\n" + + "event: content_block_delta\n" + + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"partial"}}` + "\n\n" + +const compatStreamUsageCompletion = "event: content_block_stop\n" + + `data: {"type":"content_block_stop","index":0}` + "\n\n" + + "event: message_delta\n" + + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":7}}` + "\n\n" + + "event: message_stop\n" + + `data: {"type":"message_stop"}` + "\n\n" + +type compatStreamUsageRun struct { + rec *httptest.ResponseRecorder + upstream *compatStreamUsageUpstream + usageRepo *openAIWSUsageHandlerUsageLogRepoStub + billingRepo *compatStreamUsageBillingRepo +} + +// runCompatStreamUsageHandler 经 GatewayHandler 转发一次流式请求；上游写完 upstreamStream 后， +// closeUpstream 为 false 时保持连接不再发送数据。 +func runCompatStreamUsageHandler(t *testing.T, endpoint, upstreamStream string, closeUpstream bool) compatStreamUsageRun { + t.Helper() + cfg := &config.Config{Gateway: config.GatewayConfig{StreamDataIntervalTimeout: 1}} + groupID := int64(9300) + group := &service.Group{ID: groupID, Hydrated: true, Platform: service.PlatformAnthropic, Status: service.StatusActive, RateMultiplier: 1} + account := &service.Account{ + ID: 9301, Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey, + Status: service.StatusActive, Schedulable: true, Concurrency: 1, + Credentials: map[string]any{"api_key": "test-key"}, + AccountGroups: []service.AccountGroup{{AccountID: 9301, GroupID: groupID}}, + } + snapshot := service.NewSchedulerSnapshotService(&fakeSchedulerCache{accounts: []*service.Account{account}}, nil, nil, nil, nil) + run := compatStreamUsageRun{ + rec: httptest.NewRecorder(), + usageRepo: &openAIWSUsageHandlerUsageLogRepoStub{created: make(chan *service.UsageLog, 2)}, + billingRepo: &compatStreamUsageBillingRepo{}, + } + billingCache := service.NewBillingCacheService(newHandlerInflightCache(100), nil, nil, nil, nil, nil, cfg, nil) + t.Cleanup(billingCache.Stop) + + pr, pw := io.Pipe() + t.Cleanup(func() { _ = pr.Close(); _ = pw.Close() }) + go func() { + _, _ = io.WriteString(pw, upstreamStream) + if closeUpstream { + _ = pw.Close() + } + }() + run.upstream = &compatStreamUsageUpstream{response: &http.Response{ + StatusCode: http.StatusOK, Body: pr, + Header: http.Header{"X-Request-Id": []string{"compat-stream-usage"}}, + }} + gw := service.NewGatewayService( + nil, &fakeGroupRepo{group: group}, run.usageRepo, run.billingRepo, nil, nil, nil, nil, cfg, + snapshot, nil, service.NewBillingService(cfg, nil), nil, nil, nil, run.upstream, + &service.DeferredService{}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + ) + h := &GatewayHandler{ + gatewayService: gw, billingCacheService: billingCache, cfg: cfg, maxAccountSwitches: 1, + concurrencyHelper: NewConcurrencyHelper(service.NewConcurrencyService(&fakeConcurrencyCache{}), SSEPingFormatClaude, 0), + } + + body := `{"model":"claude-sonnet-4.5","stream":true,"input":"hello"}` + if endpoint == "chat/completions" { + body = `{"model":"claude-sonnet-4.5","stream":true,"messages":[{"role":"user","content":"hello"}]}` + } + c, _ := gin.CreateTestContext(run.rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/"+endpoint, bytes.NewBufferString(body)) + c.Request.Header.Set("Content-Type", "application/json") + apiKey := &service.APIKey{ID: 9302, UserID: 9303, GroupID: &groupID, Group: group, User: &service.User{ID: 9303, Balance: 100}} + c.Set(string(middleware.ContextKeyAPIKey), apiKey) + c.Set(string(middleware.ContextKeyUser), middleware.AuthSubject{UserID: 9303}) + + if endpoint == "chat/completions" { + h.ChatCompletions(c) + } else { + h.Responses(c) + } + return run +} + +// compatStreamUsageMarkers 返回成功收尾与终止错误的标记。 +func compatStreamUsageMarkers(endpoint string) (success, failure string) { + if endpoint == "responses" { + return "event: response.completed", "event: response.failed" + } + return "[DONE]", `data: {"error":` +} + +func TestGatewayCompatibilityHandlers_StreamUsageRecordedOnceWithTerminalEvent(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, endpoint := range []string{"chat/completions", "responses"} { + for _, tc := range []struct { + name string + completion string + wantOutput int + }{ + {name: "complete", completion: compatStreamUsageCompletion, wantOutput: 7}, + {name: "idle_before_message_stop", wantOutput: 1}, + } { + t.Run(endpoint+"/"+tc.name, func(t *testing.T) { + successMarker, failureMarker := compatStreamUsageMarkers(endpoint) + wantMarker, denyMarker := successMarker, failureMarker + if tc.completion == "" { + wantMarker, denyMarker = failureMarker, successMarker + } + + run := runCompatStreamUsageHandler(t, endpoint, compatStreamUsagePartialStream+tc.completion, tc.completion != "") + + require.Equal(t, 1, run.upstream.calls) + out := run.rec.Body.String() + require.Contains(t, out, "partial") + require.Contains(t, out, wantMarker) + require.NotContains(t, out, denyMarker) + require.Equal(t, 1, strings.Count(out, wantMarker), "exactly one terminal event") + require.NotContains(t, out, `data: {"type":"error"`, "the handler must not append a fallback frame") + if tc.completion == "" { + require.Contains(t, out, "stream_timeout") + } + + run.billingRepo.mu.Lock() + defer run.billingRepo.mu.Unlock() + require.Len(t, run.billingRepo.commands, 1, "usage must be settled exactly once") + require.Equal(t, 12, run.billingRepo.commands[0].InputTokens) + require.Equal(t, tc.wantOutput, run.billingRepo.commands[0].OutputTokens) + require.Len(t, run.usageRepo.created, 1) + }) + } + } +} + +func TestGatewayCompatibilityHandlers_IdleBeforeAnyEventWritesStreamError(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, endpoint := range []string{"chat/completions", "responses"} { + t.Run(endpoint, func(t *testing.T) { + successMarker, failureMarker := compatStreamUsageMarkers(endpoint) + + run := runCompatStreamUsageHandler(t, endpoint, "", false) + + require.Equal(t, 1, run.upstream.calls) + require.Equal(t, http.StatusOK, run.rec.Code) + require.Equal(t, "text/event-stream", run.rec.Header().Get("Content-Type")) + out := run.rec.Body.String() + require.Equal(t, 1, strings.Count(out, failureMarker), "exactly one terminal event") + require.NotContains(t, out, `data: {"type":"error"`, "the handler must not append a fallback frame") + require.Contains(t, out, "stream_timeout") + require.NotContains(t, out, successMarker) + + run.billingRepo.mu.Lock() + defer run.billingRepo.mu.Unlock() + require.Empty(t, run.billingRepo.commands, "no usage was observed") + require.Empty(t, run.usageRepo.created) + }) + } +} + +func TestGatewayCompatibilityHandlers_UpstreamErrorEventAfterOutputIsNotBilled(t *testing.T) { + gin.SetMode(gin.TestMode) + upstreamError := "event: error\n" + `data: {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}` + "\n\n" + for _, endpoint := range []string{"chat/completions", "responses"} { + t.Run(endpoint, func(t *testing.T) { + successMarker, failureMarker := compatStreamUsageMarkers(endpoint) + + run := runCompatStreamUsageHandler(t, endpoint, compatStreamUsagePartialStream+upstreamError, true) + + require.Equal(t, 1, run.upstream.calls) + out := run.rec.Body.String() + require.Contains(t, out, "partial") + require.Equal(t, 1, strings.Count(out, failureMarker), "exactly one terminal event") + require.NotContains(t, out, `data: {"type":"error"`, "the handler must not append a fallback frame") + require.Contains(t, out, "Overloaded") + require.NotContains(t, out, successMarker) + + run.billingRepo.mu.Lock() + defer run.billingRepo.mu.Unlock() + require.Empty(t, run.billingRepo.commands, "an upstream error event is not billed while the client is online") + require.Empty(t, run.usageRepo.created) + }) + } +} diff --git a/backend/internal/handler/gateway_handler_chat_completions.go b/backend/internal/handler/gateway_handler_chat_completions.go index 561ed1bca0a1..ec0a43d88b38 100644 --- a/backend/internal/handler/gateway_handler_chat_completions.go +++ b/backend/internal/handler/gateway_handler_chat_completions.go @@ -317,6 +317,44 @@ func (h *GatewayHandler) ChatCompletions(c *gin.Context) { accountReleaseFunc() } + // 提交 usage 记录。成功路径与"流中断但 Forward 已观测到 usage 的部分结果" + // 错误路径共用；failover 错误恒定 result=nil，不会重复计费。 + submitForwardUsage := func(result *service.ForwardResult) { + userAgent := c.GetHeader("User-Agent") + clientIP := ip.GetClientIP(c) + requestPayloadHash := service.HashUsageRequestPayload(body) + inboundEndpoint := GetInboundEndpoint(c) + upstreamEndpoint := GetUpstreamEndpoint(c, account.Platform) + + quotaPlatform := service.QuotaPlatform(c.Request.Context(), apiKey) + sessionID := service.ExtractClientSessionID(c) + stampForwardRequestedReasoningEffort(result, service.RequestedReasoningEffortFromContext(c.Request.Context())) + h.submitUsageRecordTask(c.Request.Context(), func(ctx context.Context) { + if err := h.gatewayService.RecordUsage(ctx, &service.RecordUsageInput{ + Result: result, + QuotaPlatform: quotaPlatform, + APIKey: apiKey, + User: apiKey.User, + Account: account, + Subscription: subscription, + PricingAt: pricingAt, + InboundEndpoint: inboundEndpoint, + UpstreamEndpoint: upstreamEndpoint, + UserAgent: userAgent, + IPAddress: clientIP, + RequestPayloadHash: requestPayloadHash, + APIKeyService: h.apiKeyService, + SessionID: sessionID, + ChannelUsageFields: clientRequestedUsageFields(c, channelMapping, reqModel, result.UpstreamModel), + }); err != nil { + reqLog.Error("gateway.cc.record_usage_failed", + zap.Int64("account_id", account.ID), + zap.Error(err), + ) + } + }) + } + if err != nil { var failoverErr *service.UpstreamFailoverError if errors.As(err, &failoverErr) { @@ -347,43 +385,14 @@ func (h *GatewayHandler) ChatCompletions(c *gin.Context) { zap.Bool("upstream_error_response_already_written", upstreamErrorAlreadyCommunicated), zap.Error(err), ) + if result != nil { + submitForwardUsage(result) + } return } // 6. Record usage - userAgent := c.GetHeader("User-Agent") - clientIP := ip.GetClientIP(c) - requestPayloadHash := service.HashUsageRequestPayload(body) - inboundEndpoint := GetInboundEndpoint(c) - upstreamEndpoint := GetUpstreamEndpoint(c, account.Platform) - - quotaPlatform := service.QuotaPlatform(c.Request.Context(), apiKey) - sessionID := service.ExtractClientSessionID(c) - stampForwardRequestedReasoningEffort(result, service.RequestedReasoningEffortFromContext(c.Request.Context())) - h.submitUsageRecordTask(c.Request.Context(), func(ctx context.Context) { - if err := h.gatewayService.RecordUsage(ctx, &service.RecordUsageInput{ - Result: result, - QuotaPlatform: quotaPlatform, - APIKey: apiKey, - User: apiKey.User, - Account: account, - Subscription: subscription, - PricingAt: pricingAt, - InboundEndpoint: inboundEndpoint, - UpstreamEndpoint: upstreamEndpoint, - UserAgent: userAgent, - IPAddress: clientIP, - RequestPayloadHash: requestPayloadHash, - APIKeyService: h.apiKeyService, - SessionID: sessionID, - ChannelUsageFields: clientRequestedUsageFields(c, channelMapping, reqModel, result.UpstreamModel), - }); err != nil { - reqLog.Error("gateway.cc.record_usage_failed", - zap.Int64("account_id", account.ID), - zap.Error(err), - ) - } - }) + submitForwardUsage(result) return } } diff --git a/backend/internal/handler/gateway_handler_responses.go b/backend/internal/handler/gateway_handler_responses.go index 5ef6bace91cb..3d143c8ca24a 100644 --- a/backend/internal/handler/gateway_handler_responses.go +++ b/backend/internal/handler/gateway_handler_responses.go @@ -299,6 +299,44 @@ func (h *GatewayHandler) Responses(c *gin.Context) { accountReleaseFunc() } + // 提交 usage 记录。成功路径与"流中断但 Forward 已观测到 usage 的部分结果" + // 错误路径共用；failover 错误恒定 result=nil，不会重复计费。 + submitForwardUsage := func(result *service.ForwardResult) { + userAgent := c.GetHeader("User-Agent") + clientIP := ip.GetClientIP(c) + requestPayloadHash := service.HashUsageRequestPayload(body) + inboundEndpoint := GetInboundEndpoint(c) + upstreamEndpoint := GetUpstreamEndpoint(c, account.Platform) + + quotaPlatform := service.QuotaPlatform(c.Request.Context(), apiKey) + sessionID := service.ExtractClientSessionID(c) + stampForwardRequestedReasoningEffort(result, service.RequestedReasoningEffortFromContext(c.Request.Context())) + h.submitUsageRecordTask(c.Request.Context(), func(ctx context.Context) { + if err := h.gatewayService.RecordUsage(ctx, &service.RecordUsageInput{ + Result: result, + QuotaPlatform: quotaPlatform, + APIKey: apiKey, + User: apiKey.User, + Account: account, + Subscription: subscription, + PricingAt: pricingAt, + InboundEndpoint: inboundEndpoint, + UpstreamEndpoint: upstreamEndpoint, + UserAgent: userAgent, + IPAddress: clientIP, + RequestPayloadHash: requestPayloadHash, + APIKeyService: h.apiKeyService, + SessionID: sessionID, + ChannelUsageFields: clientRequestedUsageFields(c, channelMapping, reqModel, result.UpstreamModel), + }); err != nil { + reqLog.Error("gateway.responses.record_usage_failed", + zap.Int64("account_id", account.ID), + zap.Error(err), + ) + } + }) + } + if err != nil { var failoverErr *service.UpstreamFailoverError if errors.As(err, &failoverErr) { @@ -330,43 +368,14 @@ func (h *GatewayHandler) Responses(c *gin.Context) { zap.Bool("upstream_error_response_already_written", upstreamErrorAlreadyCommunicated), zap.Error(err), ) + if result != nil { + submitForwardUsage(result) + } return } // 6. Record usage - userAgent := c.GetHeader("User-Agent") - clientIP := ip.GetClientIP(c) - requestPayloadHash := service.HashUsageRequestPayload(body) - inboundEndpoint := GetInboundEndpoint(c) - upstreamEndpoint := GetUpstreamEndpoint(c, account.Platform) - - quotaPlatform := service.QuotaPlatform(c.Request.Context(), apiKey) - sessionID := service.ExtractClientSessionID(c) - stampForwardRequestedReasoningEffort(result, service.RequestedReasoningEffortFromContext(c.Request.Context())) - h.submitUsageRecordTask(c.Request.Context(), func(ctx context.Context) { - if err := h.gatewayService.RecordUsage(ctx, &service.RecordUsageInput{ - Result: result, - QuotaPlatform: quotaPlatform, - APIKey: apiKey, - User: apiKey.User, - Account: account, - Subscription: subscription, - PricingAt: pricingAt, - InboundEndpoint: inboundEndpoint, - UpstreamEndpoint: upstreamEndpoint, - UserAgent: userAgent, - IPAddress: clientIP, - RequestPayloadHash: requestPayloadHash, - APIKeyService: h.apiKeyService, - SessionID: sessionID, - ChannelUsageFields: clientRequestedUsageFields(c, channelMapping, reqModel, result.UpstreamModel), - }); err != nil { - reqLog.Error("gateway.responses.record_usage_failed", - zap.Int64("account_id", account.ID), - zap.Error(err), - ) - } - }) + submitForwardUsage(result) return } } diff --git a/backend/internal/service/gateway_forward.go b/backend/internal/service/gateway_forward.go index 388f2495a942..b0d59656add7 100644 --- a/backend/internal/service/gateway_forward.go +++ b/backend/internal/service/gateway_forward.go @@ -823,55 +823,7 @@ func (s *GatewayService) Forward(ctx context.Context, c *gin.Context, account *A if err != nil { var sseErr *sseStreamErrorEventError if errors.As(err, &sseErr) { - // 上游 HTTP 200 + SSE 流体内出现 event:error 帧。 - body := []byte(sseErr.RawData) - semanticStatus := http.StatusForbidden - if c.Writer.Size() == writerSizeBeforeStream && gjson.GetBytes(body, "error.type").String() == "overloaded_error" { - semanticStatus = 529 - syntheticResp := &http.Response{ - StatusCode: semanticStatus, - Header: resp.Header.Clone(), - Body: io.NopCloser(bytes.NewReader(body)), - } - s.handleFailoverSideEffects(ctx, syntheticResp, account, reqModel) - } - - upstreamMsg := sanitizeUpstreamErrorMessage( - strings.TrimSpace(extractUpstreamErrorMessage(body)), - ) - - upstreamDetail := "" - if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { - maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes - if maxBytes <= 0 { - maxBytes = 2048 - } - upstreamDetail = truncateString(sseErr.RawData, maxBytes) - } - - appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ - ProxyID: opsUpstreamProxyID(account), - ProxyName: opsUpstreamProxyName(account), - Platform: account.Platform, - AccountID: account.ID, - AccountName: account.Name, - UpstreamStatusCode: semanticStatus, - UpstreamRequestID: resp.Header.Get("x-request-id"), - Kind: "stream_error", - Message: upstreamMsg, - Detail: upstreamDetail, - }) - - logger.LegacyPrintf("service.gateway", - "[Forward] SSE error event in stream: Account=%d(%s) RequestID=%s Body=%s", - account.ID, account.Name, resp.Header.Get("x-request-id"), - truncateString(sseErr.RawData, 1000), - ) - - return nil, &UpstreamFailoverError{ - StatusCode: semanticStatus, - ResponseBody: body, - } + return nil, s.sseStreamErrorEventFailover(ctx, c, account, resp, sseErr.RawData, reqModel, c.Writer.Size() != writerSizeBeforeStream) } // 流中断（缺失 terminal 事件、读错误、数据间隔超时等）时保留已观测到的 // usage 与错误一起返回，handler 在错误处理完成后照常提交 usage 记录。 @@ -1064,3 +1016,57 @@ func (s *GatewayService) isStickyAccountUpstreamRestricted(ctx context.Context, } return s.isUpstreamModelRestrictedByChannel(ctx, *groupID, account, requestedModel) } + +// sseStreamErrorEventFailover 把上游 HTTP 200 + SSE 流体内的 event:error 帧转为 +// UpstreamFailoverError：尚未向客户端写出内容且错误类型为 overloaded_error 时按 529 +// 处理并触发账号侧副作用，其余按 403；同时登记 stream_error 运维记录。 +func (s *GatewayService) sseStreamErrorEventFailover(ctx context.Context, c *gin.Context, account *Account, resp *http.Response, rawData, model string, outputStarted bool) *UpstreamFailoverError { + body := []byte(rawData) + semanticStatus := http.StatusForbidden + if !outputStarted && gjson.GetBytes(body, "error.type").String() == "overloaded_error" { + semanticStatus = 529 + syntheticResp := &http.Response{ + StatusCode: semanticStatus, + Header: resp.Header.Clone(), + Body: io.NopCloser(bytes.NewReader(body)), + } + s.handleFailoverSideEffects(ctx, syntheticResp, account, model) + } + + upstreamMsg := sanitizeUpstreamErrorMessage( + strings.TrimSpace(extractUpstreamErrorMessage(body)), + ) + + upstreamDetail := "" + if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { + maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes + if maxBytes <= 0 { + maxBytes = 2048 + } + upstreamDetail = truncateString(rawData, maxBytes) + } + + appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ + ProxyID: opsUpstreamProxyID(account), + ProxyName: opsUpstreamProxyName(account), + Platform: account.Platform, + AccountID: account.ID, + AccountName: account.Name, + UpstreamStatusCode: semanticStatus, + UpstreamRequestID: resp.Header.Get("x-request-id"), + Kind: "stream_error", + Message: upstreamMsg, + Detail: upstreamDetail, + }) + + logger.LegacyPrintf("service.gateway", + "[Forward] SSE error event in stream: Account=%d(%s) RequestID=%s Body=%s", + account.ID, account.Name, resp.Header.Get("x-request-id"), + truncateString(rawData, 1000), + ) + + return &UpstreamFailoverError{ + StatusCode: semanticStatus, + ResponseBody: body, + } +} diff --git a/backend/internal/service/gateway_forward_as_chat_completions.go b/backend/internal/service/gateway_forward_as_chat_completions.go index bb96b28a6b28..24a281d61200 100644 --- a/backend/internal/service/gateway_forward_as_chat_completions.go +++ b/backend/internal/service/gateway_forward_as_chat_completions.go @@ -187,7 +187,15 @@ func (s *GatewayService) ForwardAsChatCompletions( var result *ForwardResult var handleErr error if clientStream { + writerSizeBeforeStream := c.Writer.Size() result, handleErr = s.handleCCStreamingFromAnthropic(resp, c, originalModel, mappedModel, reasoningEffort, startTime) + var sseErr *sseStreamErrorEventError + switch { + case errors.As(handleErr, &sseErr): + return nil, s.sseStreamErrorEventFailover(ctx, c, account, resp, sseErr.RawData, mappedModel, c.Writer.Size() != writerSizeBeforeStream) + case errors.Is(handleErr, errAnthropicNativeStreamIdle) && s.rateLimitService != nil: + s.rateLimitService.HandleStreamTimeout(ctx, account, originalModel) + } } else { result, handleErr = s.handleCCBufferedFromAnthropic(resp, c, originalModel, mappedModel, reasoningEffort, startTime) } @@ -381,21 +389,10 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( var usage ClaudeUsage var firstTokenMs *int firstChunk := true - // 客户端断开后停止向下游写入，继续读完上游以合并 message_delta 的最终 usage。 - clientDisconnected := false + writerSizeBeforeStream := c.Writer.Size() - scanner := bufio.NewScanner(resp.Body) - maxLineSize := defaultMaxLineSize - if s.cfg != nil && s.cfg.Gateway.MaxLineSize > 0 { - maxLineSize = s.cfg.Gateway.MaxLineSize - } - scanner.Buffer(make([]byte, 0, 64*1024), maxLineSize) - streamInterval := s.anthropicCompatStreamInterval() - pump := newAnthropicNativeLinePump(scanner, streamInterval) - defer pump.stop() - nextLine := func() (string, bool) { - return nextAnthropicCompatLine(pump, resp.Body, streamInterval, "forward_as_cc", requestID, clientDisconnected) - } + stream := newAnthropicCompatStream(s.cfg, c, resp, "forward_as_cc", requestID) + defer stream.stop() resultWithUsage := func() *ForwardResult { return &ForwardResult{ @@ -408,17 +405,10 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( Stream: true, Duration: time.Since(startTime), FirstTokenMs: firstTokenMs, - ClientDisconnect: clientDisconnected, + ClientDisconnect: stream.clientDisconnected, } } - markClientDisconnected := func() { - clientDisconnected = true - logger.L().Info("forward_as_cc stream: client disconnected, continuing to drain upstream for billing", - zap.String("request_id", requestID), - ) - } - writeChunk := func(chunk apicompat.ChatCompletionsChunk) bool { sse, err := apicompat.ChatChunkToSSE(chunk) if err != nil { @@ -438,11 +428,11 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( return } if event.Type == "ping" { - if clientDisconnected { + if stream.clientDisconnected { return } if _, err := fmt.Fprint(c.Writer, ": ping\n\n"); err != nil { - markClientDisconnected() + stream.markClientDisconnected() return } c.Writer.Flush() @@ -462,7 +452,7 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( if event.Type == "message_start" && event.Message != nil { mergeAnthropicUsage(&usage, event.Message.Usage) } - if clientDisconnected { + if stream.clientDisconnected { return } @@ -477,7 +467,7 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( ccChunks := apicompat.ResponsesEventToChatChunks(&resEvt, ccState) for _, chunk := range ccChunks { if disconnected := writeChunk(chunk); disconnected { - markClientDisconnected() + stream.markClientDisconnected() return } } @@ -486,7 +476,7 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( } for { - line, ok := nextLine() + line, ok := stream.nextLine() if !ok { break } @@ -495,7 +485,7 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( continue } - dataLine, ok := nextLine() + dataLine, ok := stream.nextLine() if !ok { break } @@ -513,10 +503,19 @@ func (s *GatewayService) handleCCStreamingFromAnthropic( // The intermediate Responses converter synthesizes usage even when absent. ccState.IncludeUsage = ccState.IncludeUsage || anthropicChatStreamHasUsage(&event, payload) + stream.observe(&event, payload) + if stream.upstreamErrType != "" { + break + } processAnthropicEvent(&event) } - if clientDisconnected { + if failure := stream.failure(); failure != nil { + return finishAnthropicCompatStreamFailure(c, writerSizeBeforeStream, failure, resultWithUsage(), func() { + writeAnthropicCompatCCStreamFailure(c, failure) + }) + } + if stream.clientDisconnected { return resultWithUsage(), nil } @@ -552,42 +551,6 @@ func writeGatewayCCError(c *gin.Context, statusCode int, errType, message string }) } -// anthropicCompatStreamInterval 返回 Anthropic 上游协议转换流的逐行读间隔上限， -// gateway.stream_data_interval_timeout <= 0 时不限。 -func (s *GatewayService) anthropicCompatStreamInterval() time.Duration { - if s.cfg != nil && s.cfg.Gateway.StreamDataIntervalTimeout > 0 { - return time.Duration(s.cfg.Gateway.StreamDataIntervalTimeout) * time.Second - } - return 0 -} - -// nextAnthropicCompatLine 从行泵取下一行；ok=false 表示上游流已结束（EOF、读错误 -// 或读间隔超时），调用方按已累计的 usage 收尾。读间隔超时时关闭 body，解除泵 -// goroutine 阻塞中的读并释放上游连接。 -func nextAnthropicCompatLine(pump *anthropicNativeLinePump, body io.Closer, interval time.Duration, logPrefix, requestID string, clientDisconnected bool) (string, bool) { - line, err := pump.next() - if err == nil { - return line, true - } - switch { - case errors.Is(err, errAnthropicNativeStreamIdle): - _ = body.Close() - logger.L().Warn(logPrefix+" stream: data interval timeout", - zap.String("request_id", requestID), - zap.Duration("interval", interval), - zap.Bool("client_disconnected", clientDisconnected), - ) - case errors.Is(err, io.EOF), errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): - // 正常收流或取消：不记日志。 - default: - logger.L().Warn(logPrefix+" stream: read error", - zap.Error(err), - zap.String("request_id", requestID), - ) - } - return "", false -} - // anthropicChatStreamHasUsage distinguishes an explicit zero-valued usage object // from omitted/null usage before the Anthropic→Responses conversion loses that distinction. func anthropicChatStreamHasUsage(event *apicompat.AnthropicStreamEvent, payload string) bool { diff --git a/backend/internal/service/gateway_forward_as_responses.go b/backend/internal/service/gateway_forward_as_responses.go index fa2d601a0536..f7f4897d5aa7 100644 --- a/backend/internal/service/gateway_forward_as_responses.go +++ b/backend/internal/service/gateway_forward_as_responses.go @@ -198,7 +198,15 @@ func (s *GatewayService) ForwardAsResponses( var result *ForwardResult var handleErr error if clientStream { + writerSizeBeforeStream := c.Writer.Size() result, handleErr = s.handleResponsesStreamingResponse(resp, c, originalModel, mappedModel, reasoningEffort, startTime, clientToolMapping) + var sseErr *sseStreamErrorEventError + switch { + case errors.As(handleErr, &sseErr): + return nil, s.sseStreamErrorEventFailover(ctx, c, account, resp, sseErr.RawData, mappedModel, c.Writer.Size() != writerSizeBeforeStream) + case errors.Is(handleErr, errAnthropicNativeStreamIdle) && s.rateLimitService != nil: + s.rateLimitService.HandleStreamTimeout(ctx, account, originalModel) + } } else { result, handleErr = s.handleResponsesBufferedStreamingResponse(resp, c, originalModel, mappedModel, reasoningEffort, startTime, clientToolMapping) } @@ -545,21 +553,10 @@ func (s *GatewayService) handleResponsesStreamingResponse( var usage ClaudeUsage var firstTokenMs *int firstChunk := true - // 客户端断开后停止向下游写入，继续读完上游以合并 message_delta 的最终 usage。 - clientDisconnected := false + writerSizeBeforeStream := c.Writer.Size() - scanner := bufio.NewScanner(resp.Body) - maxLineSize := defaultMaxLineSize - if s.cfg != nil && s.cfg.Gateway.MaxLineSize > 0 { - maxLineSize = s.cfg.Gateway.MaxLineSize - } - scanner.Buffer(make([]byte, 0, 64*1024), maxLineSize) - streamInterval := s.anthropicCompatStreamInterval() - pump := newAnthropicNativeLinePump(scanner, streamInterval) - defer pump.stop() - nextLine := func() (string, bool) { - return nextAnthropicCompatLine(pump, resp.Body, streamInterval, "forward_as_responses", requestID, clientDisconnected) - } + stream := newAnthropicCompatStream(s.cfg, c, resp, "forward_as_responses", requestID) + defer stream.stop() resultWithUsage := func() *ForwardResult { return &ForwardResult{ @@ -572,7 +569,7 @@ func (s *GatewayService) handleResponsesStreamingResponse( Stream: true, Duration: time.Since(startTime), FirstTokenMs: firstTokenMs, - ClientDisconnect: clientDisconnected, + ClientDisconnect: stream.clientDisconnected, } } @@ -592,7 +589,7 @@ func (s *GatewayService) handleResponsesStreamingResponse( if event.Type == "message_start" && event.Message != nil { mergeAnthropicUsage(&usage, event.Message.Usage) } - if clientDisconnected { + if stream.clientDisconnected { return } @@ -625,10 +622,7 @@ func (s *GatewayService) handleResponsesStreamingResponse( for _, restored := range payloads { eventType := gjson.GetBytes(restored, "type").String() if _, err := fmt.Fprintf(c.Writer, "event: %s\ndata: %s\n\n", eventType, restored); err != nil { - clientDisconnected = true - logger.L().Info("forward_as_responses stream: client disconnected, continuing to drain upstream for billing", - zap.String("request_id", requestID), - ) + stream.markClientDisconnected() return } } @@ -639,7 +633,12 @@ func (s *GatewayService) handleResponsesStreamingResponse( } finalizeStream := func() (*ForwardResult, error) { - if clientDisconnected { + if failure := stream.failure(); failure != nil { + return finishAnthropicCompatStreamFailure(c, writerSizeBeforeStream, failure, resultWithUsage(), func() { + writeAnthropicCompatResponsesStreamFailure(c, state, clientToolRestorer, failure) + }) + } + if stream.clientDisconnected { return resultWithUsage(), nil } if finalEvents := apicompat.FinalizeAnthropicResponsesStream(state); len(finalEvents) > 0 { @@ -658,7 +657,7 @@ func (s *GatewayService) handleResponsesStreamingResponse( // Read Anthropic SSE events for { - line, ok := nextLine() + line, ok := stream.nextLine() if !ok { break } @@ -668,7 +667,7 @@ func (s *GatewayService) handleResponsesStreamingResponse( } // Read data line - dataLine, ok := nextLine() + dataLine, ok := stream.nextLine() if !ok { break } @@ -687,6 +686,10 @@ func (s *GatewayService) handleResponsesStreamingResponse( continue } + stream.observe(&event, payload) + if stream.upstreamErrType != "" { + break + } processEvent(&event) } diff --git a/backend/internal/service/gateway_forward_compat_client_disconnect_test.go b/backend/internal/service/gateway_forward_compat_client_disconnect_test.go index cfa55a0f0ab1..d41e42a4ada4 100644 --- a/backend/internal/service/gateway_forward_compat_client_disconnect_test.go +++ b/backend/internal/service/gateway_forward_compat_client_disconnect_test.go @@ -3,11 +3,14 @@ package service import ( + "bufio" + "context" "errors" "fmt" "io" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -16,6 +19,7 @@ import ( "github.com/Wei-Shaw/sub2api/internal/pkg/apicompat" "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" ) // disconnectOnMarkerResponseWriter 在首次写入包含 marker 的数据时返回错误，模拟客户端中途断开。 @@ -97,8 +101,10 @@ func newCompatStreamIdleTestService(intervalSec int) *GatewayService { return &GatewayService{cfg: &config.Config{Gateway: config.GatewayConfig{StreamDataIntervalTimeout: intervalSec}}} } -func TestAnthropicCompatStreaming_IdleUpstreamEndsWithinInterval(t *testing.T) { - handlers := map[string]func(*GatewayService, *http.Response, *gin.Context) (*ForwardResult, error){ +type compatStreamHandler func(*GatewayService, *http.Response, *gin.Context) (*ForwardResult, error) + +func compatStreamHandlers() map[string]compatStreamHandler { + return map[string]compatStreamHandler{ "chat_completions": func(svc *GatewayService, resp *http.Response, c *gin.Context) (*ForwardResult, error) { return svc.handleCCStreamingFromAnthropic(resp, c, "claude-sonnet-4-6", "claude-sonnet-4-6", nil, time.Now()) }, @@ -106,8 +112,54 @@ func TestAnthropicCompatStreaming_IdleUpstreamEndsWithinInterval(t *testing.T) { return svc.handleResponsesStreamingResponse(resp, c, "claude-sonnet-4-6", "claude-sonnet-4-6", nil, time.Now(), apicompat.ResponsesClientToolMapping{}) }, } - stream := strings.TrimSuffix(anthropicStreamWithMidStreamText(), "event: message_stop\ndata: {\"type\":\"message_stop\"}\n") - for name, handle := range handlers { +} + +// compatStreamTerminalMarkers 是两条兼容流按成功收尾时写出的完成标记。 +var compatStreamTerminalMarkers = map[string]string{ + "chat_completions": "[DONE]", + "responses": "response.completed", +} + +// compatStreamFailureMarkers 是两条兼容流写出终止错误时的标记。 +var compatStreamFailureMarkers = map[string]string{ + "chat_completions": `data: {"error":`, + "responses": "event: response.failed", +} + +// anthropicStreamWithoutMessageStop 以带 stop_reason 的 message_delta 结束，不含 message_stop。 +func anthropicStreamWithoutMessageStop() string { + return strings.TrimSuffix(anthropicStreamWithMidStreamText(), "event: message_stop\ndata: {\"type\":\"message_stop\"}\n") +} + +// anthropicStreamTruncatedBeforeMessageDelta 在 message_delta 之前中断，usage 只有 message_start 的部分。 +func anthropicStreamTruncatedBeforeMessageDelta() string { + full := anthropicStreamWithMidStreamText() + return full[:strings.Index(full, "event: message_delta")] +} + +type compatStreamOutcome struct { + result *ForwardResult + err error +} + +func runCompatStreamWithDeadline(t *testing.T, handle compatStreamHandler, svc *GatewayService, resp *http.Response, c *gin.Context) compatStreamOutcome { + t.Helper() + done := make(chan compatStreamOutcome, 1) + go func() { + result, err := handle(svc, resp, c) + done <- compatStreamOutcome{result: result, err: err} + }() + select { + case got := <-done: + return got + case <-time.After(5 * time.Second): + t.Fatal("stream did not end after upstream idle interval") + return compatStreamOutcome{} + } +} + +func TestAnthropicCompatStreaming_IdleUpstreamEndsWithinInterval(t *testing.T) { + for name, handle := range compatStreamHandlers() { for _, disconnect := range []bool{false, true} { t.Run(fmt.Sprintf("%s/client_disconnected=%t", name, disconnect), func(t *testing.T) { gin.SetMode(gin.TestMode) @@ -117,28 +169,25 @@ func TestAnthropicCompatStreaming_IdleUpstreamEndsWithinInterval(t *testing.T) { } writer := &disconnectOnMarkerResponseWriter{ResponseRecorder: httptest.NewRecorder(), marker: marker} c, _ := gin.CreateTestContext(writer) - resp, pw := stallingAnthropicUpstream(t, stream) + resp, pw := stallingAnthropicUpstream(t, anthropicStreamTruncatedBeforeMessageDelta()) - type outcome struct { - result *ForwardResult - err error - } - done := make(chan outcome, 1) - go func() { - result, err := handle(newCompatStreamIdleTestService(1), resp, c) - done <- outcome{result: result, err: err} - }() - var got outcome - select { - case got = <-done: - case <-time.After(5 * time.Second): - t.Fatal("stream did not end after upstream idle interval") - } + got := runCompatStreamWithDeadline(t, handle, newCompatStreamIdleTestService(1), resp, c) - require.NoError(t, got.err) + if disconnect { + require.NoError(t, got.err) + } else { + require.ErrorIs(t, got.err, errAnthropicNativeStreamIdle) + body := writer.Body.String() + require.NotContains(t, body, compatStreamTerminalMarkers[name], "a stream cut by idle timeout must not be reported as completed") + require.Contains(t, body, compatStreamFailureMarkers[name]) + require.Contains(t, body, "stream_timeout") + require.True(t, IsResponseCommitted(c)) + } + require.NotNil(t, got.result) require.Equal(t, disconnect, got.result.ClientDisconnect) require.Equal(t, disconnect, writer.failed) - require.Equal(t, 42, got.result.Usage.OutputTokens) + require.Equal(t, 20, got.result.Usage.InputTokens) + require.Equal(t, 1, got.result.Usage.OutputTokens) require.Zero(t, writer.writesAfterError) _, werr := io.WriteString(pw, "event: ping\n") require.Error(t, werr, "upstream body must be closed after idle timeout") @@ -147,6 +196,584 @@ func TestAnthropicCompatStreaming_IdleUpstreamEndsWithinInterval(t *testing.T) { } } +func TestAnthropicCompatStreaming_IdleAfterTerminalEventCompletes(t *testing.T) { + for name, handle := range compatStreamHandlers() { + for streamName, stream := range map[string]string{ + "message_stop": anthropicStreamWithMidStreamText(), + "stop_reason": anthropicStreamWithoutMessageStop(), + } { + t.Run(name+"/"+streamName, func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + resp, _ := stallingAnthropicUpstream(t, stream) + + got := runCompatStreamWithDeadline(t, handle, newCompatStreamIdleTestService(1), resp, c) + + require.NoError(t, got.err) + require.NotNil(t, got.result) + require.Equal(t, 42, got.result.Usage.OutputTokens) + require.Contains(t, rec.Body.String(), compatStreamTerminalMarkers[name]) + require.NotContains(t, rec.Body.String(), compatStreamFailureMarkers[name]) + }) + } + } +} + +// errAfterReader 读完 data 后返回 err，模拟上游连接中途断开。 +type errAfterReader struct { + data *strings.Reader + err error +} + +func (r *errAfterReader) Read(p []byte) (int, error) { + if r.data.Len() > 0 { + return r.data.Read(p) + } + return 0, r.err +} + +func TestAnthropicCompatStreaming_ReadErrorBeforeTerminalEventFails(t *testing.T) { + messageStartOnly := strings.Join([]string{ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_disc","type":"message","role":"assistant","content":[],"model":"claude-sonnet-4-6","usage":{"input_tokens":20,"output_tokens":1}}}`, + ``, + }, "\n") + "\n" + for name, handle := range compatStreamHandlers() { + for _, tc := range []struct { + name string + stream string + wantErr bool + wantResult bool + }{ + {name: "after_text", stream: anthropicStreamTruncatedBeforeMessageDelta(), wantErr: true, wantResult: true}, + {name: "after_message_start", stream: messageStartOnly, wantErr: true, wantResult: true}, + {name: "before_any_event", stream: "", wantErr: true}, + {name: "after_stop_reason", stream: anthropicStreamWithoutMessageStop(), wantResult: true}, + {name: "after_message_stop_without_stop_reason", stream: strings.Replace(anthropicStreamWithMidStreamText(), `"delta":{"stop_reason":"end_turn"}`, `"delta":{}`, 1), wantResult: true}, + } { + t.Run(name+"/"+tc.name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + body := io.NopCloser(&errAfterReader{data: strings.NewReader(tc.stream), err: io.ErrUnexpectedEOF}) + + result, err := handle(&GatewayService{}, &http.Response{Header: http.Header{}, Body: body}, c) + + if tc.wantErr { + require.ErrorIs(t, err, io.ErrUnexpectedEOF) + require.NotErrorIs(t, err, errAnthropicNativeStreamIdle) + require.NotContains(t, rec.Body.String(), compatStreamTerminalMarkers[name]) + require.Contains(t, rec.Body.String(), compatStreamFailureMarkers[name]) + require.Contains(t, rec.Body.String(), "stream_read_error") + } else { + require.NoError(t, err) + require.Contains(t, rec.Body.String(), compatStreamTerminalMarkers[name]) + } + if !tc.wantResult { + require.Nil(t, result) + return + } + require.NotNil(t, result) + require.False(t, result.ClientDisconnect) + require.Equal(t, 20, result.Usage.InputTokens) + }) + } + } +} + +// compatResponsesSSEEvents 解析 Responses SSE 输出中各 data 帧的 type 与 sequence_number。 +func compatResponsesSSEEvents(t *testing.T, body string) []gjson.Result { + t.Helper() + var events []gjson.Result + for _, line := range strings.Split(body, "\n") { + if payload, ok := strings.CutPrefix(line, "data: "); ok { + require.True(t, gjson.Valid(payload), payload) + events = append(events, gjson.Parse(payload)) + } + } + return events +} + +func TestAnthropicCompatStreaming_EOFWithoutTerminalEventFails(t *testing.T) { + truncatedToolUse := strings.Join([]string{ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_disc","type":"message","role":"assistant","content":[],"model":"claude-sonnet-4-6","usage":{"input_tokens":20,"output_tokens":1}}}`, + ``, + `event: content_block_start`, + `data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"write_file","input":{}}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"path\":\"/tmp/a\",\"content\":\"hal"}}`, + ``, + }, "\n") + "\n" + for name, handle := range compatStreamHandlers() { + for _, tc := range []struct { + name string + stream string + wantErrText string + wantType string + }{ + {name: "truncated", stream: anthropicStreamTruncatedBeforeMessageDelta(), wantErrText: "missing terminal event", wantType: "stream_incomplete"}, + {name: "truncated_tool_use", stream: truncatedToolUse, wantErrText: "missing terminal event", wantType: "stream_incomplete"}, + } { + t.Run(name+"/"+tc.name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + + result, err := handle(&GatewayService{}, &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(tc.stream))}, c) + + require.ErrorContains(t, err, tc.wantErrText) + require.NotNil(t, result) + require.Equal(t, 20, result.Usage.InputTokens) + require.True(t, IsResponseCommitted(c)) + body := rec.Body.String() + require.NotContains(t, body, compatStreamTerminalMarkers[name]) + require.NotContains(t, body, `"finish_reason":"`) + require.Equal(t, 1, strings.Count(body, compatStreamFailureMarkers[name])) + if name != "responses" { + require.Contains(t, body, tc.wantType) + return + } + events := compatResponsesSSEEvents(t, body) + require.GreaterOrEqual(t, len(events), 2) + failed := events[len(events)-1] + require.Equal(t, "response.failed", failed.Get("type").String()) + require.Equal(t, "msg_disc", failed.Get("response.id").String()) + require.Equal(t, "failed", failed.Get("response.status").String()) + require.Equal(t, tc.wantType, failed.Get("response.error.code").String()) + require.Equal(t, events[len(events)-2].Get("sequence_number").Int()+1, failed.Get("sequence_number").Int()) + }) + } + } +} + +func TestAnthropicCompatStreaming_EOFBeforeMessageStartKeepsExistingEnding(t *testing.T) { + for name, handle := range compatStreamHandlers() { + t.Run(name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + + result, err := handle(&GatewayService{}, &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader("event: ping\ndata: {\"type\":\"ping\"}\n\n"))}, c) + + require.NoError(t, err) + require.NotNil(t, result) + require.NotContains(t, rec.Body.String(), compatStreamFailureMarkers[name]) + }) + } +} + +func TestAnthropicCompatStreaming_DoneLineCountsAsTerminalEvent(t *testing.T) { + for name, handle := range compatStreamHandlers() { + t.Run(name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + stream := anthropicStreamTruncatedBeforeMessageDelta() + "data: [DONE]\n\n" + + result, err := handle(&GatewayService{}, &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(stream))}, c) + + require.NoError(t, err) + require.NotNil(t, result) + require.NotContains(t, rec.Body.String(), compatStreamFailureMarkers[name]) + }) + } +} + +func anthropicStreamWithInBandError(messageBeforeError, after string) string { + return strings.Join([]string{ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_disc","type":"message","role":"assistant","content":[],"model":"claude-sonnet-4-6","usage":{"input_tokens":20,"output_tokens":1}}}`, + ``, + `event: content_block_start`, + `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"second"}}`, + ``, + `event: error`, + `data: {"type":"error","error":{"type":"overloaded_error","message":` + strconv.Quote(messageBeforeError) + `}}`, + ``, + }, "\n") + "\n" + after +} + +func TestAnthropicCompatStreaming_UpstreamErrorEventEndsStreamWithoutWaiting(t *testing.T) { + for name, handle := range compatStreamHandlers() { + for _, disconnect := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/client_disconnected=%t", name, disconnect), func(t *testing.T) { + gin.SetMode(gin.TestMode) + marker := "\x00never-written" + if disconnect { + marker = "second" + } + writer := &disconnectOnMarkerResponseWriter{ResponseRecorder: httptest.NewRecorder(), marker: marker} + c, _ := gin.CreateTestContext(writer) + resp, pw := stallingAnthropicUpstream(t, anthropicStreamWithInBandError("Overloaded", "")) + + got := runCompatStreamWithDeadline(t, handle, newCompatStreamIdleTestService(0), resp, c) + + if disconnect { + require.NoError(t, got.err) + require.NotNil(t, got.result) + require.True(t, got.result.ClientDisconnect) + require.Equal(t, 20, got.result.Usage.InputTokens) + require.Zero(t, writer.writesAfterError) + } else { + var sseErr *sseStreamErrorEventError + require.ErrorAs(t, got.err, &sseErr) + require.Nil(t, got.result, "an upstream error event is not billed while the client is online") + require.Equal(t, 1, strings.Count(writer.Body.String(), compatStreamFailureMarkers[name])) + require.Contains(t, writer.Body.String(), "Overloaded") + } + _, werr := io.WriteString(pw, "event: ping\n") + require.Error(t, werr, "upstream body must be closed after an upstream error event") + }) + } + } +} + +func TestAnthropicCompatStreaming_UpstreamErrorEventStopsReadingBufferedEvents(t *testing.T) { + after := strings.Join([]string{ + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"LEAKED"}}`, + ``, + `event: message_delta`, + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":9}}`, + ``, + `event: message_stop`, + `data: {"type":"message_stop"}`, + ``, + }, "\n") + "\n" + for name, handle := range compatStreamHandlers() { + t.Run(name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + + result, err := handle(&GatewayService{}, &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(anthropicStreamWithInBandError("Overloaded", after)))}, c) + + var sseErr *sseStreamErrorEventError + require.ErrorAs(t, err, &sseErr) + require.Nil(t, result) + body := rec.Body.String() + require.NotContains(t, body, "LEAKED") + require.NotContains(t, body, compatStreamTerminalMarkers[name]) + require.Equal(t, 1, strings.Count(body, compatStreamFailureMarkers[name])) + }) + } +} + +func TestAnthropicCompatStreaming_UpstreamErrorEventMessageIsSanitized(t *testing.T) { + for name, handle := range compatStreamHandlers() { + t.Run(name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + stream := anthropicStreamWithInBandError("upstream https://example.test/v1?key=SECRET failed", "") + + _, err := handle(&GatewayService{}, &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(stream))}, c) + + require.Error(t, err) + require.Contains(t, rec.Body.String(), "key=***") + require.NotContains(t, rec.Body.String(), "SECRET") + }) + } +} + +func TestAnthropicCompatStreaming_UpstreamErrorEventBeforeOutputWritesNothing(t *testing.T) { + stream := "event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded\"}}\n\n" + for name, handle := range compatStreamHandlers() { + t.Run(name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + + result, err := handle(&GatewayService{}, &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(stream))}, c) + + var sseErr *sseStreamErrorEventError + require.ErrorAs(t, err, &sseErr) + require.Nil(t, result) + require.Zero(t, rec.Body.Len(), "the handler fails over to another account") + require.False(t, IsResponseCommitted(c)) + }) + } +} + +func TestForwardAsCompatStreaming_UpstreamErrorEventFailsOver(t *testing.T) { + beforeOutput := "event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded\"}}\n\n" + for _, endpoint := range []string{"chat_completions", "responses"} { + for _, tc := range []struct { + name string + stream string + wantStatus int + }{ + {name: "before_output", stream: beforeOutput, wantStatus: 529}, + {name: "after_output", stream: anthropicStreamWithInBandError("Overloaded", ""), wantStatus: http.StatusForbidden}, + } { + t.Run(endpoint+"/"+tc.name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + body := `{"model":"claude-sonnet-4-6","messages":[{"role":"user","content":"hi"}],"stream":true}` + path := "/v1/chat/completions" + if endpoint == "responses" { + body = `{"model":"claude-sonnet-4-6","input":"hi","stream":true}` + path = "/v1/responses" + } + c.Request = httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)) + resp := &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(tc.stream))} + svc := &GatewayService{ + cfg: rawChatCompletionsTestConfig(), + httpUpstream: &anthropicHTTPUpstreamRecorder{resp: resp}, + rateLimitService: &RateLimitService{accountRepo: &errorPolicyRepoStub{}, cfg: &config.Config{}}, + } + account := &Account{ID: 1, Platform: PlatformAnthropic, Type: AccountTypeAPIKey, Credentials: map[string]any{"api_key": "sk-test"}} + + var result *ForwardResult + var err error + if endpoint == "responses" { + result, err = svc.ForwardAsResponses(context.Background(), c, account, []byte(body), nil) + } else { + result, err = svc.ForwardAsChatCompletions(context.Background(), c, account, []byte(body), nil) + } + + var failoverErr *UpstreamFailoverError + require.ErrorAs(t, err, &failoverErr) + require.Equal(t, tc.wantStatus, failoverErr.StatusCode) + require.Nil(t, result) + }) + } + } +} + +func TestAnthropicCompatStreaming_LineTooLongFails(t *testing.T) { + for name, handle := range compatStreamHandlers() { + t.Run(name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + svc := &GatewayService{cfg: &config.Config{Gateway: config.GatewayConfig{MaxLineSize: 128 * 1024}}} + stream := anthropicStreamTruncatedBeforeMessageDelta() + "event: content_block_delta\ndata: " + strings.Repeat("x", 256*1024) + "\n\n" + + result, err := handle(svc, &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(stream))}, c) + + require.ErrorIs(t, err, bufio.ErrTooLong) + require.NotNil(t, result) + require.Contains(t, rec.Body.String(), compatStreamFailureMarkers[name]) + require.Contains(t, rec.Body.String(), "response_too_large") + }) + } +} + +func TestHandleResponsesStreamingResponse_FailureSequenceFollowsRestoredEvents(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + stream := strings.Join([]string{ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_tool","type":"message","role":"assistant","content":[],"model":"claude-sonnet-4-6","usage":{"input_tokens":20,"output_tokens":1}}}`, + ``, + `event: content_block_start`, + `data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"apply_patch","input":{}}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"input\":"}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"*** Begin"}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":" Patch"}}`, + ``, + }, "\n") + "\n" + mapping := apicompat.ResponsesClientToolMapping{CustomTools: map[string]bool{"apply_patch": true}} + + _, err := (&GatewayService{}).handleResponsesStreamingResponse( + &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(stream))}, + c, "claude-sonnet-4-6", "claude-sonnet-4-6", nil, time.Now(), mapping, + ) + + require.ErrorContains(t, err, "missing terminal event") + events := compatResponsesSSEEvents(t, rec.Body.String()) + require.NotEmpty(t, events) + require.Equal(t, "response.failed", events[len(events)-1].Get("type").String()) + for i, event := range events { + require.Equal(t, int64(i), event.Get("sequence_number").Int(), "sequence_number must be contiguous: %s", event.Raw) + } +} + +func TestAnthropicCompatStream_ClientGoneAtStreamEndMarksDisconnected(t *testing.T) { + gin.SetMode(gin.TestMode) + // 泵 goroutine 结束与请求 ctx 取消同时就绪时 select 随机选择，多轮重复以覆盖两种选择。 + for round := 0; round < 64; round++ { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + c.Request = httptest.NewRequest(http.MethodPost, "/", nil).WithContext(ctx) + stream := newAnthropicCompatStream(nil, c, &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(""))}, "test", "req") + time.Sleep(time.Millisecond) + + _, ok := stream.nextLine() + + require.False(t, ok) + require.True(t, stream.clientDisconnected, "round %d", round) + require.Nil(t, stream.failure()) + stream.stop() + } +} + +func TestAnthropicCompatStreaming_RequestContextCanceledDrainsWithoutWriting(t *testing.T) { + for name, handle := range compatStreamHandlers() { + for _, delayed := range []bool{true, false} { + t.Run(fmt.Sprintf("%s/upstream_delayed=%t", name, delayed), func(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + c.Request = httptest.NewRequest(http.MethodPost, "/", nil).WithContext(ctx) + var body io.ReadCloser = io.NopCloser(strings.NewReader(anthropicStreamWithMidStreamText())) + if delayed { + pr, pw := io.Pipe() + t.Cleanup(func() { _ = pr.Close() }) + go func() { + time.Sleep(20 * time.Millisecond) + _, _ = io.WriteString(pw, anthropicStreamWithMidStreamText()) + _ = pw.Close() + }() + body = pr + } + + got := runCompatStreamWithDeadline(t, handle, newCompatStreamIdleTestService(0), &http.Response{Header: http.Header{}, Body: body}, c) + + require.NoError(t, got.err) + require.NotNil(t, got.result) + require.True(t, got.result.ClientDisconnect) + require.Equal(t, 20, got.result.Usage.InputTokens) + require.Equal(t, 42, got.result.Usage.OutputTokens) + require.NotContains(t, rec.Body.String(), compatStreamTerminalMarkers[name]) + }) + } + } +} + +func TestAnthropicCompatStream_ClientDisconnectBoundsDrainWhenIntervalDisabled(t *testing.T) { + for _, tc := range []struct { + name string + intervalSec int + wantInterval time.Duration + }{ + {name: "interval_disabled", intervalSec: 0, wantInterval: anthropicCompatDisconnectDrainInterval}, + {name: "interval_configured", intervalSec: 30, wantInterval: 30 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + pr, pw := io.Pipe() + t.Cleanup(func() { _ = pw.Close(); _ = pr.Close() }) + cfg := &config.Config{Gateway: config.GatewayConfig{StreamDataIntervalTimeout: tc.intervalSec}} + stream := newAnthropicCompatStream(cfg, c, &http.Response{Header: http.Header{}, Body: pr}, "test", "req") + defer stream.stop() + require.Equal(t, tc.intervalSec > 0, stream.pump.timer != nil) + + stream.markClientDisconnected() + + require.True(t, stream.clientDisconnected) + require.NotNil(t, stream.pump.timer) + require.Equal(t, tc.wantInterval, stream.pump.interval) + }) + } +} + +func TestAnthropicCompatStreaming_DisconnectDrainEndsWhenIntervalDisabled(t *testing.T) { + original := anthropicCompatDisconnectDrainInterval + anthropicCompatDisconnectDrainInterval = 200 * time.Millisecond + t.Cleanup(func() { anthropicCompatDisconnectDrainInterval = original }) + + for name, handle := range compatStreamHandlers() { + t.Run(name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + writer := &disconnectOnMarkerResponseWriter{ResponseRecorder: httptest.NewRecorder(), marker: "second"} + c, _ := gin.CreateTestContext(writer) + resp, pw := stallingAnthropicUpstream(t, anthropicStreamTruncatedBeforeMessageDelta()) + + got := runCompatStreamWithDeadline(t, handle, newCompatStreamIdleTestService(0), resp, c) + + require.NoError(t, got.err) + require.NotNil(t, got.result) + require.True(t, got.result.ClientDisconnect) + require.Equal(t, 20, got.result.Usage.InputTokens) + require.Zero(t, writer.writesAfterError) + _, werr := io.WriteString(pw, "event: ping\n") + require.Error(t, werr, "upstream body must be closed after the disconnect drain bound") + }) + } +} + +func TestForwardAsCompatStreaming_IdleTimeoutTriggersStreamTimeoutHandling(t *testing.T) { + for _, endpoint := range []string{"chat_completions", "responses"} { + for _, tc := range []struct { + name string + stream string + marker string + wantErr bool + wantOutput int + wantPenalty int + }{ + {name: "client_online", stream: anthropicStreamTruncatedBeforeMessageDelta(), marker: "\x00never-written", wantErr: true, wantOutput: 1, wantPenalty: 1}, + {name: "client_disconnected", stream: anthropicStreamTruncatedBeforeMessageDelta(), marker: "second", wantOutput: 1}, + {name: "after_message_stop", stream: anthropicStreamWithMidStreamText(), marker: "\x00never-written", wantOutput: 42}, + {name: "after_stop_reason", stream: anthropicStreamWithoutMessageStop(), marker: "\x00never-written", wantOutput: 42}, + } { + t.Run(endpoint+"/"+tc.name, func(t *testing.T) { + gin.SetMode(gin.TestMode) + writer := &disconnectOnMarkerResponseWriter{ResponseRecorder: httptest.NewRecorder(), marker: tc.marker} + c, _ := gin.CreateTestContext(writer) + body := `{"model":"claude-sonnet-4-6","messages":[{"role":"user","content":"hi"}],"stream":true}` + path := "/v1/chat/completions" + if endpoint == "responses" { + body = `{"model":"claude-sonnet-4-6","input":"hi","stream":true}` + path = "/v1/responses" + } + c.Request = httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)) + resp, _ := stallingAnthropicUpstream(t, tc.stream) + resp.StatusCode = http.StatusOK + repo := &errorPolicyRepoStub{} + settings := NewSettingService(&settingRepoStub{values: map[string]string{ + SettingKeyStreamTimeoutSettings: `{"enabled":true,"action":"temp_unsched","temp_unsched_minutes":1,"threshold_count":1,"threshold_window_minutes":1}`, + }}, nil) + cfg := rawChatCompletionsTestConfig() + cfg.Gateway.StreamDataIntervalTimeout = 1 + svc := &GatewayService{ + cfg: cfg, + httpUpstream: &anthropicHTTPUpstreamRecorder{resp: resp}, + rateLimitService: &RateLimitService{accountRepo: repo, settingService: settings}, + } + account := &Account{ID: 1, Platform: PlatformAnthropic, Type: AccountTypeAPIKey, Credentials: map[string]any{"api_key": "sk-test"}} + + var result *ForwardResult + var err error + if endpoint == "responses" { + result, err = svc.ForwardAsResponses(context.Background(), c, account, []byte(body), nil) + } else { + result, err = svc.ForwardAsChatCompletions(context.Background(), c, account, []byte(body), nil) + } + + if tc.wantErr { + require.ErrorIs(t, err, errAnthropicNativeStreamIdle) + } else { + require.NoError(t, err) + } + require.NotNil(t, result) + require.Equal(t, tc.wantOutput, result.Usage.OutputTokens) + require.Equal(t, tc.wantPenalty, repo.tempCalls) + }) + } + } +} + func TestHandleCCStreamingFromAnthropic_ClientDisconnectDrainsUpstreamUsage(t *testing.T) { gin.SetMode(gin.TestMode) writer := &disconnectOnMarkerResponseWriter{ResponseRecorder: httptest.NewRecorder(), marker: "second"} diff --git a/backend/internal/service/gateway_forward_compat_stream.go b/backend/internal/service/gateway_forward_compat_stream.go new file mode 100644 index 000000000000..7be3e85c35f2 --- /dev/null +++ b/backend/internal/service/gateway_forward_compat_stream.go @@ -0,0 +1,291 @@ +package service + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/apicompat" + "github.com/Wei-Shaw/sub2api/internal/pkg/logger" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/tidwall/gjson" + "go.uber.org/zap" +) + +// anthropicCompatDisconnectDrainInterval 是 gateway.stream_data_interval_timeout +// 未启用时，客户端断开后继续读取上游的读间隔上限，与该配置项的默认值一致。 +var anthropicCompatDisconnectDrainInterval = 180 * time.Second + +// anthropicCompatStream 是 Anthropic 上游经 chat/responses 兼容入口流式转发时的 +// 上游读取状态。上游请求 ctx 与客户端解绑，客户端断开后调用方停止向下游写入、 +// 继续读取上游以合并最终 usage。逐行读间隔受 gateway.stream_data_interval_timeout +// 限制；该配置未启用时，客户端断开后以 anthropicCompatDisconnectDrainInterval 限制。 +type anthropicCompatStream struct { + pump *anthropicNativeLinePump + body io.Closer + maxLineSize int + logPrefix string + requestID string + // clientGone 是请求 ctx 的 Done，客户端断开后为 nil。 + clientGone <-chan struct{} + + clientDisconnected bool + sawMessageStart bool + // sawTerminal 表示上游已发出 message_stop、带 stop_reason 的 message_delta 或 [DONE]。 + sawTerminal bool + // upstreamErr* 记录上游流内的 error 事件；收到后调用方结束读取。 + upstreamErrType string + upstreamErrMessage string + upstreamErrPayload string + // readErr 是上游流结束的原因（io.EOF、读错误或 errAnthropicNativeStreamIdle）。 + readErr error +} + +// anthropicCompatStreamFailure 描述客户端在线时上游流未正常结束的原因： +// err 返回给调用方，errType / message 写入下游的终止错误事件。 +type anthropicCompatStreamFailure struct { + err error + errType string + message string +} + +// newAnthropicCompatStream 启动上游行泵；调用方 defer stop()。 +func newAnthropicCompatStream(cfg *config.Config, c *gin.Context, resp *http.Response, logPrefix, requestID string) *anthropicCompatStream { + scanner := bufio.NewScanner(resp.Body) + maxLineSize := defaultMaxLineSize + if cfg != nil && cfg.Gateway.MaxLineSize > 0 { + maxLineSize = cfg.Gateway.MaxLineSize + } + scanner.Buffer(make([]byte, 0, 64*1024), maxLineSize) + stream := &anthropicCompatStream{ + pump: newAnthropicNativeLinePump(scanner, anthropicNativeStreamInterval(cfg)), + body: resp.Body, + maxLineSize: maxLineSize, + logPrefix: logPrefix, + requestID: requestID, + } + if c.Request != nil { + stream.clientGone = c.Request.Context().Done() + } + return stream +} + +func (st *anthropicCompatStream) stop() { + st.pump.stop() +} + +// markClientDisconnected 标记客户端已断开，此后调用方不再向下游写入；读间隔上限 +// 未启用时以 anthropicCompatDisconnectDrainInterval 启用。 +func (st *anthropicCompatStream) markClientDisconnected() { + if st.clientDisconnected { + return + } + st.clientDisconnected = true + st.clientGone = nil + st.pump.enableInterval(anthropicCompatDisconnectDrainInterval) + logger.L().Info(st.logPrefix+" stream: client disconnected, continuing to drain upstream for billing", + zap.String("request_id", st.requestID), + ) +} + +// nextLine 返回上游下一行；ok=false 表示上游流已结束，原因记入 readErr。 +// 读间隔超时时关闭 body，解除泵 goroutine 阻塞中的读并释放上游连接。 +// 上游流结束时请求 ctx 已取消的，按客户端已断开收尾。 +func (st *anthropicCompatStream) nextLine() (string, bool) { + for { + line, err := st.pump.nextOrClientGone(st.clientGone) + if err == nil { + if payload, ok := extractOpenAISSEDataLine(line); ok && strings.TrimSpace(payload) == "[DONE]" { + st.sawTerminal = true + } + return line, true + } + if errors.Is(err, errAnthropicNativeClientGone) { + st.markClientDisconnected() + continue + } + if st.clientGone != nil { + select { + case <-st.clientGone: + st.clientDisconnected = true + st.clientGone = nil + default: + } + } + st.readErr = err + switch { + case errors.Is(err, errAnthropicNativeStreamIdle): + _ = st.body.Close() + logger.L().Warn(st.logPrefix+" stream: data interval timeout", + zap.String("request_id", st.requestID), + zap.Duration("interval", st.pump.interval), + zap.Bool("client_disconnected", st.clientDisconnected), + ) + case errors.Is(err, io.EOF), errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + // 正常收流或取消：不记日志。 + default: + logger.L().Warn(st.logPrefix+" stream: read error", + zap.Error(err), + zap.String("request_id", st.requestID), + ) + } + return "", false + } +} + +// observe 记录决定收尾方式的上游事件。收到上游 error 事件时关闭上游 body， +// 调用方不再处理该事件并结束读取。 +func (st *anthropicCompatStream) observe(event *apicompat.AnthropicStreamEvent, payload string) { + switch event.Type { + case "message_start": + st.sawMessageStart = true + case "message_stop": + st.sawTerminal = true + case "message_delta": + if event.Delta != nil && event.Delta.StopReason != "" { + st.sawTerminal = true + } + case "error": + st.upstreamErrPayload = payload + st.upstreamErrType = firstNonEmpty(gjson.Get(payload, "error.type").String(), "upstream_error") + st.upstreamErrMessage = firstNonEmpty(sanitizeUpstreamErrorMessage(gjson.Get(payload, "error.message").String()), "upstream stream error") + _ = st.body.Close() + logger.L().Warn(st.logPrefix+" stream: upstream error event", + zap.String("request_id", st.requestID), + zap.String("error_type", st.upstreamErrType), + zap.String("error_message", st.upstreamErrMessage), + zap.Bool("client_disconnected", st.clientDisconnected), + ) + } +} + +// failure 返回客户端在线、上游未发出结束事件即中断时的失败原因：上游流内 +// error 事件、读间隔超时、读错误，或 message_start 之后正常关闭。此时调用方不补发 +// 完成事件，而是写出协议内的终止错误。上游已发出结束事件、客户端已断开，或上游 +// 未发出 message_start 即正常关闭时返回 nil，调用方按已累计的 usage 正常收尾。 +// 读间隔超时时 err 为 errAnthropicNativeStreamIdle。 +func (st *anthropicCompatStream) failure() *anthropicCompatStreamFailure { + if st.clientDisconnected || st.sawTerminal { + return nil + } + switch { + case st.upstreamErrType != "": + return &anthropicCompatStreamFailure{ + err: &sseStreamErrorEventError{RawData: st.upstreamErrPayload}, + errType: st.upstreamErrType, + message: st.upstreamErrMessage, + } + case errors.Is(st.readErr, errAnthropicNativeStreamIdle): + return &anthropicCompatStreamFailure{ + err: errAnthropicNativeStreamIdle, + errType: "stream_timeout", + message: fmt.Sprintf("upstream stream idle for %s", st.pump.interval), + } + case errors.Is(st.readErr, bufio.ErrTooLong): + return &anthropicCompatStreamFailure{ + err: fmt.Errorf("stream usage incomplete: %w", st.readErr), + errType: "response_too_large", + message: fmt.Sprintf("upstream SSE line exceeded %d bytes", st.maxLineSize), + } + case st.readErr != nil && !errors.Is(st.readErr, io.EOF): + return &anthropicCompatStreamFailure{ + err: fmt.Errorf("stream usage incomplete: %w", st.readErr), + errType: "stream_read_error", + message: "upstream stream disconnected: " + sanitizeStreamError(st.readErr), + } + case st.sawMessageStart: + return &anthropicCompatStreamFailure{ + err: errors.New("stream usage incomplete: missing terminal event"), + errType: "stream_incomplete", + message: "upstream stream ended without a terminal event", + } + default: + return nil + } +} + +// finishAnthropicCompatStreamFailure 按失败原因收尾，返回交给调用方的结果与错误。 +// 上游流内 error 事件与 /v1/messages 一致：不计费，返回 sseStreamErrorEventError +// 由调用方转为 UpstreamFailoverError；尚未向客户端写出内容时不写终止错误，由 +// handler 换号重试。其余失败写出终止错误，并带上已观测到的 usage；未观测到任何 +// usage 时结果为 nil。 +func finishAnthropicCompatStreamFailure(c *gin.Context, writerSizeBeforeStream int, failure *anthropicCompatStreamFailure, result *ForwardResult, writeFailure func()) (*ForwardResult, error) { + var sseErr *sseStreamErrorEventError + if errors.As(failure.err, &sseErr) { + if c.Writer.Size() != writerSizeBeforeStream { + writeFailure() + } + return nil, failure.err + } + writeFailure() + if !result.Usage.hasObservedTokens() { + return nil, failure.err + } + return result, failure.err +} + +// writeAnthropicCompatCCStreamFailure 向 Chat Completions 流写出终止错误帧， +// 并标记响应已提交，handler 不再追加兜底错误。 +func writeAnthropicCompatCCStreamFailure(c *gin.Context, failure *anthropicCompatStreamFailure) { + payload, _ := json.Marshal(map[string]any{ + "error": map[string]string{ + "type": failure.errType, + "message": failure.message, + }, + }) + commitAnthropicCompatStreamFailure(c, failure, "data: "+string(payload)+"\n\n") +} + +// writeAnthropicCompatResponsesStreamFailure 向 Responses 流写出 response.failed +// 终止事件，沿用流内的 response id；sequence_number 经 restorer 与已写出的事件 +// 保持连续。写出后标记响应已提交。 +func writeAnthropicCompatResponsesStreamFailure(c *gin.Context, state *apicompat.AnthropicEventToResponsesState, restorer *apicompat.ResponsesClientToolStreamRestorer, failure *anthropicCompatStreamFailure) { + responseID := state.ResponseID + if responseID == "" { + responseID = "resp_" + strings.ReplaceAll(uuid.NewString(), "-", "") + } + evt := apicompat.ResponsesStreamEvent{ + Type: "response.failed", + SequenceNumber: state.SequenceNumber, + Response: &apicompat.ResponsesResponse{ + ID: responseID, + Object: "response", + CreatedAt: state.Created, + Model: state.Model, + Status: "failed", + Output: []apicompat.ResponsesOutput{}, + Error: &apicompat.ResponsesError{Code: failure.errType, Message: failure.message}, + }, + } + state.SequenceNumber++ + payload, err := json.Marshal(evt) + if err != nil { + return + } + payloads, _, err := restorer.RestoreEvent(payload) + if err != nil { + return + } + var frames strings.Builder + for _, restored := range payloads { + fmt.Fprintf(&frames, "event: %s\ndata: %s\n\n", gjson.GetBytes(restored, "type").String(), restored) + } + commitAnthropicCompatStreamFailure(c, failure, frames.String()) +} + +func commitAnthropicCompatStreamFailure(c *gin.Context, failure *anthropicCompatStreamFailure, frame string) { + MarkOpsStreamError(c, failure.errType, failure.message, http.StatusBadGateway) + MarkResponseCommitted(c) + if _, err := io.WriteString(c.Writer, frame); err != nil { + return + } + c.Writer.Flush() +} diff --git a/backend/internal/service/openai_gateway_anthropic_native_pump.go b/backend/internal/service/openai_gateway_anthropic_native_pump.go index 8a1967770568..3b679bc4e8cd 100644 --- a/backend/internal/service/openai_gateway_anthropic_native_pump.go +++ b/backend/internal/service/openai_gateway_anthropic_native_pump.go @@ -1,8 +1,10 @@ package service -// 国产供应商 Anthropic 协议转换路径的上游 SSE 行泵。 +// Anthropic 协议转换路径的上游 SSE 行泵，用于 OpenAI 侧国产供应商的 +// CC×anthropic / Responses×anthropic 转换链，以及 Anthropic 账号的 +// chat/responses 兼容入口（anthropicCompatStream）。 // -// 这两条转换链（CC×anthropic / Responses×anthropic）的上游 ctx 是 +// 这些转换链的上游 ctx 是 // WithoutCancel（detachStreamUpstreamContext）、http.Client 无整体 Timeout， // 客户端断开后的排水阶段若上游挂住 SSE（不发数据也不断连）， // scanner.Scan() 将永久阻塞：goroutine 钉死、resp.Body 不归还、连接池位 @@ -18,11 +20,17 @@ import ( "errors" "io" "time" + + "github.com/Wei-Shaw/sub2api/internal/config" ) // errAnthropicNativeStreamIdle 表示上游流读间隔超时（见上方文件注释）。 var errAnthropicNativeStreamIdle = errors.New("stream data interval timeout") +// errAnthropicNativeClientGone 表示等待上游数据期间客户端已离开；泵状态不变， +// 调用方可继续读取。 +var errAnthropicNativeClientGone = errors.New("client gone") + // anthropicNativeLineEvent 是行泵交付的单次读取结果：line 为一行 SSE 文本， // err 为 scanner 读错误（流自然结束时 next 返回 io.EOF，不经过本字段）。 type anthropicNativeLineEvent struct { @@ -31,7 +39,7 @@ type anthropicNativeLineEvent struct { } // anthropicNativeLinePump 以独立 goroutine 泵送 scanner 的行，并对逐行到达 -// 间隔施加 interval 上限（<=0 表示禁用，保持无界读的旧行为）。 +// 间隔施加 interval 上限（<=0 表示禁用，可经 enableInterval 启用）。 type anthropicNativeLinePump struct { events chan anthropicNativeLineEvent done chan struct{} @@ -69,23 +77,51 @@ func newAnthropicNativeLinePump(scanner *bufio.Scanner, interval time.Duration) } // next 阻塞返回下一行。返回 io.EOF 表示上游正常收流；errAnthropicNativeStreamIdle -// 表示 interval 内无任何数据到达（计时从收到上一行时起算，事件处理耗时不算入， -// 与 readOpenAICompatBufferedTerminal 的 resetTimeout 语义一致）。 +// 表示自交付上一行起已过 interval 且没有可交付的行；计时器到期时若已有缓冲的行， +// 优先交付该行，调用方处理事件期间上游送达的数据不会被判为上游空闲。 func (p *anthropicNativeLinePump) next() (string, error) { + return p.nextOrClientGone(nil) +} + +// nextOrClientGone 与 next 相同，另在等待期间 clientGone 关闭时返回 +// errAnthropicNativeClientGone。clientGone 为 nil 时不监听。 +func (p *anthropicNativeLinePump) nextOrClientGone(clientGone <-chan struct{}) (string, error) { var timeoutCh <-chan time.Time if p.timer != nil { timeoutCh = p.timer.C } select { case ev, ok := <-p.events: - if !ok { - return "", io.EOF - } - p.resetTimer() - return ev.line, ev.err + return p.deliver(ev, ok) case <-timeoutCh: - return "", errAnthropicNativeStreamIdle + select { + case ev, ok := <-p.events: + return p.deliver(ev, ok) + default: + return "", errAnthropicNativeStreamIdle + } + case <-clientGone: + return "", errAnthropicNativeClientGone + } +} + +// deliver 交付一次读取结果；ok=false 表示泵 goroutine 已结束。 +func (p *anthropicNativeLinePump) deliver(ev anthropicNativeLineEvent, ok bool) (string, error) { + if !ok { + return "", io.EOF + } + p.resetTimer() + return ev.line, ev.err +} + +// enableInterval 在读间隔上限未启用时以 interval 启用，计时从调用时起算； +// 已启用时不变。只能在调用 next 的 goroutine 中调用。 +func (p *anthropicNativeLinePump) enableInterval(interval time.Duration) { + if p.timer != nil || interval <= 0 { + return } + p.interval = interval + p.timer = time.NewTimer(interval) } // resetTimer 在收到一行后重启间隔计时器。 @@ -116,11 +152,11 @@ func (p *anthropicNativeLinePump) stop() { } } -// anthropicNativeStreamInterval 返回本组转换路径适用的读间隔上限； +// anthropicNativeStreamInterval 返回 Anthropic 上游协议转换路径适用的读间隔上限； // gateway.stream_data_interval_timeout <= 0 时视为禁用。 -func (s *OpenAIGatewayService) anthropicNativeStreamInterval() time.Duration { - if s.cfg != nil && s.cfg.Gateway.StreamDataIntervalTimeout > 0 { - return time.Duration(s.cfg.Gateway.StreamDataIntervalTimeout) * time.Second +func anthropicNativeStreamInterval(cfg *config.Config) time.Duration { + if cfg != nil && cfg.Gateway.StreamDataIntervalTimeout > 0 { + return time.Duration(cfg.Gateway.StreamDataIntervalTimeout) * time.Second } return 0 } diff --git a/backend/internal/service/openai_gateway_anthropic_native_pump_test.go b/backend/internal/service/openai_gateway_anthropic_native_pump_test.go index e5b9d0dd84bb..328f5c99b643 100644 --- a/backend/internal/service/openai_gateway_anthropic_native_pump_test.go +++ b/backend/internal/service/openai_gateway_anthropic_native_pump_test.go @@ -130,6 +130,87 @@ func TestAnthropicNativeLinePump_DataResetsTimer(t *testing.T) { } } +func TestAnthropicNativeLinePump_SlowConsumerGetsBufferedLineInsteadOfIdle(t *testing.T) { + // 计时器与缓冲行同时就绪时 select 随机选择，多轮重复以覆盖两种选择。 + for round := 0; round < 64; round++ { + pump := &anthropicNativeLinePump{ + events: make(chan anthropicNativeLineEvent, 1), + done: make(chan struct{}), + timer: time.NewTimer(0), + interval: time.Hour, + } + pump.events <- anthropicNativeLineEvent{line: "data: {}"} + time.Sleep(time.Millisecond) + + line, err := pump.next() + pump.stop() + if err != nil || line != "data: {}" { + t.Fatalf("round %d: expected buffered line, got %q err=%v", round, line, err) + } + } +} + +func TestAnthropicNativeLinePump_ClientGoneKeepsPumpReadable(t *testing.T) { + pr, pw := io.Pipe() + scanner := bufio.NewScanner(pr) + pump := newAnthropicNativeLinePump(scanner, 0) + defer pump.stop() + defer func() { _ = pr.Close() }() + + clientGone := make(chan struct{}) + close(clientGone) + if _, err := pump.nextOrClientGone(clientGone); err != errAnthropicNativeClientGone { + t.Fatalf("expected client gone, got %v", err) + } + + go func() { + _, _ = pw.Write([]byte("event: ping\n")) + _ = pw.Close() + }() + line, err := pump.nextOrClientGone(nil) + if err != nil || line != "event: ping" { + t.Fatalf("expected the pending line after client gone, got %q err=%v", line, err) + } + if _, err := pump.next(); err != io.EOF { + t.Fatalf("expected EOF, got %v", err) + } +} + +func TestAnthropicNativeLinePump_EnableIntervalBoundsDisabledPump(t *testing.T) { + pr, _ := io.Pipe() + scanner := bufio.NewScanner(pr) + defer func() { _ = pr.Close() }() + + pump := newAnthropicNativeLinePump(scanner, 0) + defer pump.stop() + pump.enableInterval(50 * time.Millisecond) + + start := time.Now() + if _, err := pump.next(); err != errAnthropicNativeStreamIdle { + t.Fatalf("expected interval timeout after enableInterval, got %v", err) + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Fatalf("timeout not respected: %v", elapsed) + } +} + +func TestAnthropicNativeLinePump_EnableIntervalKeepsConfiguredInterval(t *testing.T) { + pr, _ := io.Pipe() + scanner := bufio.NewScanner(pr) + defer func() { _ = pr.Close() }() + + pump := newAnthropicNativeLinePump(scanner, 50*time.Millisecond) + defer pump.stop() + pump.enableInterval(time.Hour) + + if pump.interval != 50*time.Millisecond { + t.Fatalf("configured interval must not change, got %v", pump.interval) + } + if _, err := pump.next(); err != errAnthropicNativeStreamIdle { + t.Fatalf("expected interval timeout, got %v", err) + } +} + func TestCCStreamingFromNativeAnthropic_HangTimesOut(t *testing.T) { gin.SetMode(gin.TestMode) svc := newNativeAnthropicHangTestService(1) diff --git a/backend/internal/service/openai_gateway_chat_completions_anthropic_native.go b/backend/internal/service/openai_gateway_chat_completions_anthropic_native.go index f76a45737077..f6333661f5db 100644 --- a/backend/internal/service/openai_gateway_chat_completions_anthropic_native.go +++ b/backend/internal/service/openai_gateway_chat_completions_anthropic_native.go @@ -169,7 +169,7 @@ func (s *OpenAIGatewayService) handleCCBufferedFromNativeAnthropic( var usage ClaudeUsage // 读间隔上限：上游挂住 SSE 时中止组装（缓冲路径尚未提交响应头，可回 502）。 - streamInterval := s.anthropicNativeStreamInterval() + streamInterval := anthropicNativeStreamInterval(s.cfg) pump := newAnthropicNativeLinePump(scanner, streamInterval) defer pump.stop() @@ -356,7 +356,7 @@ func (s *OpenAIGatewayService) handleCCStreamingFromNativeAnthropic( // 读间隔上限：上游挂住 SSE（不发数据也不断连）时结束排水。上游 ctx 为 // WithoutCancel 且 http.Client 无整体 Timeout，无此界限则客户端断开后 // scanner.Scan() 永久阻塞（见 anthropic native pump 文件注释）。 - streamInterval := s.anthropicNativeStreamInterval() + streamInterval := anthropicNativeStreamInterval(s.cfg) pump := newAnthropicNativeLinePump(scanner, streamInterval) defer pump.stop() diff --git a/backend/internal/service/openai_gateway_responses_anthropic_native.go b/backend/internal/service/openai_gateway_responses_anthropic_native.go index 09278d29149f..ec4fada7906f 100644 --- a/backend/internal/service/openai_gateway_responses_anthropic_native.go +++ b/backend/internal/service/openai_gateway_responses_anthropic_native.go @@ -171,7 +171,7 @@ func (s *OpenAIGatewayService) handleResponsesBufferedFromNativeAnthropic( var usage ClaudeUsage // 读间隔上限：上游挂住 SSE 时中止组装（缓冲路径尚未提交响应头，可回 502）。 - streamInterval := s.anthropicNativeStreamInterval() + streamInterval := anthropicNativeStreamInterval(s.cfg) pump := newAnthropicNativeLinePump(scanner, streamInterval) defer pump.stop() @@ -367,7 +367,7 @@ func (s *OpenAIGatewayService) handleResponsesStreamingFromNativeAnthropic( // 读间隔上限：上游挂住 SSE（不发数据也不断连）时结束转换循环。上游 ctx 为 // WithoutCancel 且 http.Client 无整体 Timeout，无此界限则 scanner.Scan() // 永久阻塞（见 anthropic native pump 文件注释）。 - streamInterval := s.anthropicNativeStreamInterval() + streamInterval := anthropicNativeStreamInterval(s.cfg) pump := newAnthropicNativeLinePump(scanner, streamInterval) defer pump.stop() From 89654873ff4b8d586e333a1dd21d194aa82facf0 Mon Sep 17 00:00:00 2001 From: feeeei Date: Tue, 20 Oct 2026 20:33:13 +0800 Subject: [PATCH 14/29] =?UTF-8?q?perf(account):=20=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=98=A0=E5=B0=84=E7=AD=BE=E5=90=8D=E6=94=B9=E4=B8=BA=E4=B8=8E?= =?UTF-8?q?=E9=81=8D=E5=8E=86=E9=A1=BA=E5=BA=8F=E6=97=A0=E5=85=B3=E3=80=81?= =?UTF-8?q?=E9=9B=B6=E5=88=86=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GetModelMapping 命中缓存前要计算 model_mapping 的签名：收集全部 key、 排序、再逐个写入 fnv64a。签名只存放在 Account 的未导出字段里，不持久化、 不序列化、不跨进程比较，排序的唯一作用是消除 Go map 的随机遍历顺序。 改为逐条目计算 FNV-1a（key、分隔字节、value 或非字符串标记）并经 fmix64 收尾后求和：内容相同签名相同，单条改值、改键、两个 key 互换 value 都能检出， 且不分配内存。30 条映射单次计算约 2.3µs/1 次分配降到约 0.8µs/0 次分配； 选号时每个候选账号每次调用 3 次。 --- backend/internal/service/account.go | 59 ++++++++---- .../account_model_mapping_signature_test.go | 96 +++++++++++++++++++ 2 files changed, 139 insertions(+), 16 deletions(-) create mode 100644 backend/internal/service/account_model_mapping_signature_test.go diff --git a/backend/internal/service/account.go b/backend/internal/service/account.go index 06b94b754e5d..ae2b25012268 100644 --- a/backend/internal/service/account.go +++ b/backend/internal/service/account.go @@ -4,7 +4,6 @@ package service import ( "encoding/json" "errors" - "hash/fnv" "log/slog" "net/url" "reflect" @@ -717,28 +716,56 @@ func mapPtr(m map[string]any) uintptr { return reflect.ValueOf(m).Pointer() } +const ( + modelMappingSignatureFNVOffset = 14695981039346656037 + modelMappingSignatureFNVPrime = 1099511628211 +) + +// modelMappingSignature 计算原始映射内容的进程内校验值，与 map 遍历顺序无关且不分配内存。 +// 每个条目把 key、分隔字节 0 与 value（string 写其字节，其它类型写标记字节 1） +// 放进同一个 FNV-1a 哈希并经 fmix64 收尾，再对所有条目求和； +// key 与 value 同在一个条目哈希里，交换两个 key 的 value 会改变结果。 func modelMappingSignature(rawMapping map[string]any) uint64 { if len(rawMapping) == 0 { return 0 } - keys := make([]string, 0, len(rawMapping)) - for k := range rawMapping { - keys = append(keys, k) - } - sort.Strings(keys) - - h := fnv.New64a() - for _, k := range keys { - _, _ = h.Write([]byte(k)) - _, _ = h.Write([]byte{0}) - if v, ok := rawMapping[k].(string); ok { - _, _ = h.Write([]byte(v)) + var sum uint64 + for k, v := range rawMapping { + h := uint64(modelMappingSignatureFNVOffset) + h = fnv1aString(h, k) + h = fnv1aByte(h, 0) + if s, ok := v.(string); ok { + h = fnv1aString(h, s) } else { - _, _ = h.Write([]byte{1}) + h = fnv1aByte(h, 1) } - _, _ = h.Write([]byte{0xff}) + sum += fmix64(h) } - return h.Sum64() + return sum +} + +func fnv1aString(h uint64, s string) uint64 { + for i := 0; i < len(s); i++ { + h ^= uint64(s[i]) + h *= modelMappingSignatureFNVPrime + } + return h +} + +func fnv1aByte(h uint64, b byte) uint64 { + h ^= uint64(b) + h *= modelMappingSignatureFNVPrime + return h +} + +// fmix64 是 MurmurHash3 的 64 位收尾函数，使条目哈希各位充分混合后再求和。 +func fmix64(h uint64) uint64 { + h ^= h >> 33 + h *= 0xff51afd7ed558ccd + h ^= h >> 33 + h *= 0xc4ceb9fe1a85ec53 + h ^= h >> 33 + return h } // antigravityDefaultPassthroughModels 是 Antigravity 显式 model_mapping 运行时补齐的默认透传条目： diff --git a/backend/internal/service/account_model_mapping_signature_test.go b/backend/internal/service/account_model_mapping_signature_test.go new file mode 100644 index 000000000000..a83c05dc81ab --- /dev/null +++ b/backend/internal/service/account_model_mapping_signature_test.go @@ -0,0 +1,96 @@ +//go:build unit + +package service + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestModelMappingSignature_IndependentOfInsertionOrder(t *testing.T) { + const size = 64 + forward := make(map[string]any, size) + backward := make(map[string]any, size) + for i := 0; i < size; i++ { + forward[fmt.Sprintf("model-%02d", i)] = fmt.Sprintf("upstream-%02d", i) + } + for i := size - 1; i >= 0; i-- { + backward[fmt.Sprintf("model-%02d", i)] = fmt.Sprintf("upstream-%02d", i) + } + forward["non-string"] = 42 + backward["non-string"] = 42 + + want := modelMappingSignature(forward) + require.NotZero(t, want) + for i := 0; i < 20; i++ { + require.Equal(t, want, modelMappingSignature(forward)) + require.Equal(t, want, modelMappingSignature(backward)) + } +} + +func TestModelMappingSignature_DetectsContentChanges(t *testing.T) { + base := map[string]any{ + "claude-sonnet": "sonnet-a", + "claude-opus": "opus-a", + "gpt-5": "gpt-5-upstream", + } + baseSig := modelMappingSignature(base) + + cases := map[string]map[string]any{ + "value changed": { + "claude-sonnet": "sonnet-b", + "claude-opus": "opus-a", + "gpt-5": "gpt-5-upstream", + }, + "key renamed": { + "claude-sonnet-x": "sonnet-a", + "claude-opus": "opus-a", + "gpt-5": "gpt-5-upstream", + }, + "values swapped between keys": { + "claude-sonnet": "opus-a", + "claude-opus": "sonnet-a", + "gpt-5": "gpt-5-upstream", + }, + "string value replaced by non-string": { + "claude-sonnet": 1, + "claude-opus": "opus-a", + "gpt-5": "gpt-5-upstream", + }, + "entry removed": { + "claude-sonnet": "sonnet-a", + "claude-opus": "opus-a", + }, + } + for name, changed := range cases { + require.NotEqual(t, baseSig, modelMappingSignature(changed), name) + } + + // 非 string 的 value 统一按类型标记参与签名，具体取值不影响结果。 + require.Equal(t, + modelMappingSignature(map[string]any{"m": 1}), + modelMappingSignature(map[string]any{"m": true}), + ) +} + +func TestModelMappingSignature_EmptyIsZero(t *testing.T) { + require.Zero(t, modelMappingSignature(nil)) + require.Zero(t, modelMappingSignature(map[string]any{})) +} + +func TestModelMappingSignature_DoesNotAllocate(t *testing.T) { + mapping := map[string]any{ + "claude-sonnet": "sonnet-a", + "claude-opus": "opus-a", + "gpt-5": "gpt-5-upstream", + "non-string": 3.14, + } + var sink uint64 + allocs := testing.AllocsPerRun(1000, func() { + sink += modelMappingSignature(mapping) + }) + require.Zero(t, allocs) + require.NotZero(t, sink) +} From 38b72d734e0aa92ce7547a5ea8618abb7290cf98 Mon Sep 17 00:00:00 2001 From: feeeei Date: Tue, 20 Oct 2026 20:33:13 +0800 Subject: [PATCH 15/29] =?UTF-8?q?fix(ratelimit):=20=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E9=99=90=E6=B5=81=E5=BF=AB=E7=85=A7=E5=86=99=E5=85=A5=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E5=86=99=E6=97=B6=E5=A4=8D=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setAccountModelRateLimitSnapshot 原地写 account.Extra 以及其中的 model_rate_limits map。选号阶段的 Anthropic Fable 阈值过滤 （filterAccountsBySchedulingThreshold → ApplyAccountSchedulingThreshold） 会对候选账号调用它；候选账号的 Extra 一旦与其它请求共享，原地写就会造成 并发 map 读写并污染共享对象。 改为复制旧的 limits 与 Extra 顶层、写入新条目后替换 account.Extra， 不修改任何已有 map，本请求后续读取到的结果不变。其余调用方 （ratelimit_cline、cline_usage、openai_images_balance）都通过同一个 *Account 读取结果，不依赖原地修改可见。 --- backend/internal/service/model_rate_limit.go | 21 +- .../model_rate_limit_snapshot_cow_test.go | 207 ++++++++++++++++++ 2 files changed, 220 insertions(+), 8 deletions(-) create mode 100644 backend/internal/service/model_rate_limit_snapshot_cow_test.go diff --git a/backend/internal/service/model_rate_limit.go b/backend/internal/service/model_rate_limit.go index a75fa6209477..46e3419e03e8 100644 --- a/backend/internal/service/model_rate_limit.go +++ b/backend/internal/service/model_rate_limit.go @@ -2,6 +2,7 @@ package service import ( "context" + "maps" "strings" "time" @@ -209,18 +210,13 @@ func (a *Account) modelRateLimitReason(scope string) string { return reason } +// setAccountModelRateLimitSnapshot 把模型级限流写进本账号对象的 Extra 快照。 +// 写时复制：Extra 与其中的 model_rate_limits 可能与调度快照母本及其它副本共享， +// 这里只读取旧 map，写入新复制的 map 后替换 account.Extra，不修改任何已有 map。 func setAccountModelRateLimitSnapshot(account *Account, scope string, resetAt time.Time, reason string, now time.Time) { if account == nil || strings.TrimSpace(scope) == "" { return } - if account.Extra == nil { - account.Extra = make(map[string]any) - } - limits, ok := account.Extra[modelRateLimitsKey].(map[string]any) - if !ok { - limits = make(map[string]any) - account.Extra[modelRateLimitsKey] = limits - } payload := map[string]any{ "rate_limited_at": now.UTC().Format(time.RFC3339), "rate_limit_reset_at": resetAt.UTC().Format(time.RFC3339), @@ -228,5 +224,14 @@ func setAccountModelRateLimitSnapshot(account *Account, scope string, resetAt ti if reason = strings.TrimSpace(reason); reason != "" { payload["reason"] = reason } + + oldLimits, _ := account.Extra[modelRateLimitsKey].(map[string]any) + limits := make(map[string]any, len(oldLimits)+1) + maps.Copy(limits, oldLimits) limits[scope] = payload + + extra := make(map[string]any, len(account.Extra)+1) + maps.Copy(extra, account.Extra) + extra[modelRateLimitsKey] = limits + account.Extra = extra } diff --git a/backend/internal/service/model_rate_limit_snapshot_cow_test.go b/backend/internal/service/model_rate_limit_snapshot_cow_test.go new file mode 100644 index 000000000000..97e35c296ef0 --- /dev/null +++ b/backend/internal/service/model_rate_limit_snapshot_cow_test.go @@ -0,0 +1,207 @@ +//go:build unit + +package service + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/stretchr/testify/require" +) + +func TestSetAccountModelRateLimitSnapshot_CopyOnWrite(t *testing.T) { + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Second) + existingLimit := map[string]any{ + "rate_limited_at": now.Format(time.RFC3339), + "rate_limit_reset_at": now.Add(time.Hour).Format(time.RFC3339), + } + limits := map[string]any{"claude-opus-5": existingLimit} + extra := map[string]any{ + modelRateLimitsKey: limits, + "session_window_utilization": 0.5, + } + account := &Account{ID: 1, Platform: PlatformAnthropic, Extra: extra} + before := *account + + setAccountModelRateLimitSnapshot(account, anthropicFableRateLimitKey, now.Add(2*time.Hour), " threshold ", now) + + // 调用前的 Extra 与嵌套 limits 原样保留。 + require.Len(t, extra, 2) + require.Equal(t, 0.5, extra["session_window_utilization"]) + require.Len(t, limits, 1) + require.Equal(t, existingLimit, limits["claude-opus-5"]) + require.NotContains(t, limits, anthropicFableRateLimitKey) + require.False(t, before.isModelRateLimitedWithContext(ctx, "claude-fable-5")) + + // 新 Extra 同时带有旧条目与新限流，本请求后续读取结果与原地写入一致。 + require.Equal(t, 0.5, account.Extra["session_window_utilization"]) + newLimits, ok := account.Extra[modelRateLimitsKey].(map[string]any) + require.True(t, ok) + require.Len(t, newLimits, 2) + require.Equal(t, existingLimit, newLimits["claude-opus-5"]) + require.Equal(t, map[string]any{ + "rate_limited_at": now.Format(time.RFC3339), + "rate_limit_reset_at": now.Add(2 * time.Hour).Format(time.RFC3339), + "reason": "threshold", + }, newLimits[anthropicFableRateLimitKey]) + require.Equal(t, "threshold", account.modelRateLimitReason(anthropicFableRateLimitKey)) + require.True(t, account.isModelRateLimitedWithContext(ctx, "claude-fable-5")) + require.True(t, account.isModelRateLimitedWithContext(ctx, "claude-opus-5")) + require.False(t, account.isModelRateLimitedWithContext(ctx, "claude-sonnet-5")) +} + +func TestSetAccountModelRateLimitSnapshot_InitializesMissingOrInvalidLimits(t *testing.T) { + ctx := context.Background() + now := time.Now().UTC() + + empty := &Account{Platform: PlatformAnthropic} + setAccountModelRateLimitSnapshot(empty, "claude-opus-5", now.Add(time.Hour), "", now) + require.True(t, empty.isModelRateLimitedWithContext(ctx, "claude-opus-5")) + require.Empty(t, empty.modelRateLimitReason("claude-opus-5")) + + invalidExtra := map[string]any{modelRateLimitsKey: "corrupted", "keep": true} + invalid := &Account{Platform: PlatformAnthropic, Extra: invalidExtra} + setAccountModelRateLimitSnapshot(invalid, "claude-opus-5", now.Add(time.Hour), "r", now) + require.True(t, invalid.isModelRateLimitedWithContext(ctx, "claude-opus-5")) + require.Equal(t, true, invalid.Extra["keep"]) + require.Equal(t, "corrupted", invalidExtra[modelRateLimitsKey]) + + unchanged := &Account{Platform: PlatformAnthropic, Extra: map[string]any{"k": 1}} + setAccountModelRateLimitSnapshot(unchanged, " ", now.Add(time.Hour), "r", now) + require.Equal(t, map[string]any{"k": 1}, unchanged.Extra) +} + +// 同一母本的浅拷贝并发写各自的限流快照：-race 下不得出现竞争，母本与其它副本都看不到这些写入。 +func TestSetAccountModelRateLimitSnapshot_ConcurrentCopiesOfSharedMaster(t *testing.T) { + ctx := context.Background() + now := time.Now().UTC() + master := &Account{ + ID: 7, + Platform: PlatformAnthropic, + Extra: map[string]any{ + modelRateLimitsKey: map[string]any{ + "claude-opus-5": map[string]any{"rate_limit_reset_at": now.Add(time.Hour).Format(time.RFC3339)}, + }, + "session_window_utilization": 0.2, + }, + } + + const workers = 16 + var wg sync.WaitGroup + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func(worker int) { + defer wg.Done() + own := fmt.Sprintf("model-%d", worker) + other := fmt.Sprintf("model-%d", (worker+1)%workers) + for i := 0; i < 50; i++ { + replica := *master + setAccountModelRateLimitSnapshot(&replica, own, now.Add(time.Hour), "r", now) + if !replica.isModelRateLimitedWithContext(ctx, own) { + t.Errorf("worker %d: own limit missing", worker) + return + } + if !replica.isModelRateLimitedWithContext(ctx, "claude-opus-5") { + t.Errorf("worker %d: inherited limit missing", worker) + return + } + if replica.isModelRateLimitedWithContext(ctx, other) { + t.Errorf("worker %d: saw another replica's limit", worker) + return + } + probe := *master + if probe.isModelRateLimitedWithContext(ctx, own) { + t.Errorf("worker %d: master was mutated", worker) + return + } + } + }(worker) + } + wg.Wait() + + limits, ok := master.Extra[modelRateLimitsKey].(map[string]any) + require.True(t, ok) + require.Len(t, limits, 1) + require.Len(t, master.Extra, 2) +} + +type sharedSnapshotFableRepoStub struct { + AccountRepository + modelCalls atomic.Int64 +} + +func (r *sharedSnapshotFableRepoStub) SetModelRateLimit(_ context.Context, _ int64, _ string, _ time.Time, _ ...string) error { + r.modelCalls.Add(1) + return nil +} + +// Fable 阈值在选号阶段写入候选对象：同一母本的其它副本（包括并发读取的副本）与母本本身都不得出现该限流。 +func TestApplyAccountSchedulingThreshold_FableLimitStaysOnOwnSnapshotCopy(t *testing.T) { + accountSchedulingThresholdsSF.Forget(SettingKeyAccountSchedulingThresholds) + accountSchedulingThresholdsCache.Store(&cachedAccountSchedulingThresholds{}) + t.Cleanup(func() { + accountSchedulingThresholdsSF.Forget(SettingKeyAccountSchedulingThresholds) + accountSchedulingThresholdsCache.Store(&cachedAccountSchedulingThresholds{}) + }) + + settingsRepo := newMockSettingRepo() + settingsRepo.data[SettingKeyAccountSchedulingThresholds] = `{"anthropic":100}` + repo := &sharedSnapshotFableRepoStub{} + limiter := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + limiter.SetSettingService(NewSettingService(settingsRepo, &config.Config{})) + + now := time.Now().UTC() + master := &Account{ + ID: 1006, + Platform: PlatformAnthropic, + Status: StatusActive, + Schedulable: true, + Credentials: map[string]any{"account_scheduling_threshold": 60}, + Extra: map[string]any{ + "passive_usage_7d_utilization": 0.40, + "passive_usage_7d_reset": float64(now.Add(3 * 24 * time.Hour).Unix()), + "passive_usage_7d_oi_utilization": 0.61, + "passive_usage_7d_oi_reset": float64(now.Add(4 * 24 * time.Hour).Unix()), + }, + } + + const pairs = 8 + var wg sync.WaitGroup + for i := 0; i < pairs; i++ { + wg.Add(2) + go func() { + defer wg.Done() + candidate := *master + if limiter.ApplyAccountSchedulingThreshold(context.Background(), &candidate) { + t.Error("the Fable-only window must not pause the whole account") + } + if candidate.IsSchedulableForModel("claude-fable-5") { + t.Error("the candidate that crossed the threshold must stop scheduling Fable") + } + if !candidate.IsSchedulableForModel("claude-opus-5") { + t.Error("non-Fable models must stay schedulable") + } + }() + go func() { + defer wg.Done() + for j := 0; j < 20; j++ { + reader := *master + if !reader.IsSchedulableForModel("claude-fable-5") { + t.Error("a sibling copy must not see the Fable limit") + return + } + } + }() + } + wg.Wait() + + require.Equal(t, int64(pairs), repo.modelCalls.Load()) + require.NotContains(t, master.Extra, modelRateLimitsKey) + require.True(t, master.IsSchedulableForModel("claude-fable-5")) +} From d5dbb5c36be3e2b76e7f0197e6ef7f3f214df6fc Mon Sep 17 00:00:00 2001 From: feeeei Date: Tue, 20 Oct 2026 17:23:26 +0800 Subject: [PATCH 16/29] =?UTF-8?q?perf(gateway):=20=E5=8D=95=20antigravity?= =?UTF-8?q?=20=E8=B4=A6=E5=8F=B7=E5=88=86=E7=BB=84=E5=88=A4=E5=AE=9A?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E6=8C=89=E9=9C=80=E6=B1=82=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 三个入口（Gemini 原生、/v1/messages 的 Gemini 分支与 Claude 路径） 每个请求都会查询分组的 antigravity:forced 快照桶。纯 Gemini/Anthropic 分组的该桶恒为空，空快照按未命中处理，于是每个请求都回源 PG 并重新 发布快照，而判定结果只被 antigravity 账号的 503 原地重试与模型限流 预检查读取。 入口改为挂载延迟判定：antigravity 账号首次需要时才查询快照，同一请求 内只查一次；显式设置的 SingleAccountRetry 标记仍优先。其他平台的请求 不再触发该查询。 --- backend/internal/handler/gateway_handler.go | 18 +- ...eway_handler_warmup_intercept_unit_test.go | 6 +- .../handler/gemini_client_cancel_test.go | 16 +- .../internal/handler/gemini_v1beta_handler.go | 9 +- .../single_account_retry_probe_unit_test.go | 120 +++++++++++ .../service/antigravity_gateway_retry.go | 8 +- .../internal/service/gateway_scheduling.go | 20 +- backend/internal/service/request_metadata.go | 33 +++ .../single_account_retry_probe_test.go | 191 ++++++++++++++++++ 9 files changed, 388 insertions(+), 33 deletions(-) create mode 100644 backend/internal/handler/single_account_retry_probe_unit_test.go create mode 100644 backend/internal/service/single_account_retry_probe_test.go diff --git a/backend/internal/handler/gateway_handler.go b/backend/internal/handler/gateway_handler.go index e990e3dd0a77..c332e38ea04b 100644 --- a/backend/internal/handler/gateway_handler.go +++ b/backend/internal/handler/gateway_handler.go @@ -339,12 +339,9 @@ func (h *GatewayHandler) Messages(c *gin.Context) { if platform == service.PlatformGemini { fs := NewFailoverState(h.maxAccountSwitchesGemini, hasBoundSession) - // 单账号分组提前设置 SingleAccountRetry 标记，让 Service 层首次 503 就不设模型限流标记。 - // 避免单账号分组收到 503 (MODEL_CAPACITY_EXHAUSTED) 时设 29s 限流，导致后续请求连续快速失败。 - if h.gatewayService.IsSingleAntigravityAccountGroup(c.Request.Context(), apiKey.GroupID) { - ctx := service.WithSingleAccountRetry(c.Request.Context(), true, h.metadataBridgeEnabled()) - c.Request = c.Request.WithContext(ctx) - } + // 单账号分组首次 503 就不设模型限流标记，避免设 29s 限流导致后续请求连续快速失败； + // 是否单账号分组在 antigravity 账号遇到 503 或模型限流预检查时才判定。 + c.Request = c.Request.WithContext(h.gatewayService.WithSingleAntigravityAccountGroupProbe(c.Request.Context(), apiKey.GroupID)) for { selection, err := h.gatewayService.SelectAccountWithLoadAwareness(c.Request.Context(), apiKey.GroupID, sessionKey, reqModel, fs.FailedAccountIDs, "", int64(0)) // Gemini 不使用会话限制 @@ -639,12 +636,9 @@ func (h *GatewayHandler) Messages(c *gin.Context) { } fallbackUsed := false - // 单账号分组提前设置 SingleAccountRetry 标记，让 Service 层首次 503 就不设模型限流标记。 - // 避免单账号分组收到 503 (MODEL_CAPACITY_EXHAUSTED) 时设 29s 限流，导致后续请求连续快速失败。 - if h.gatewayService.IsSingleAntigravityAccountGroup(c.Request.Context(), currentAPIKey.GroupID) { - ctx := service.WithSingleAccountRetry(c.Request.Context(), true, h.metadataBridgeEnabled()) - c.Request = c.Request.WithContext(ctx) - } + // 单账号分组首次 503 就不设模型限流标记，避免设 29s 限流导致后续请求连续快速失败； + // 是否单账号分组在 antigravity 账号遇到 503 或模型限流预检查时才判定。 + c.Request = c.Request.WithContext(h.gatewayService.WithSingleAntigravityAccountGroupProbe(c.Request.Context(), currentAPIKey.GroupID)) // 会话槽失败释放：failover 链上每个选中的 Anthropic OAuth 账号都注册过同一会话 // （checkAndRegisterSession）。若请求最终失败（转发失败/客户端中断/换号耗尽）， diff --git a/backend/internal/handler/gateway_handler_warmup_intercept_unit_test.go b/backend/internal/handler/gateway_handler_warmup_intercept_unit_test.go index aad6fcaad1f7..a4e37119f21a 100644 --- a/backend/internal/handler/gateway_handler_warmup_intercept_unit_test.go +++ b/backend/internal/handler/gateway_handler_warmup_intercept_unit_test.go @@ -158,8 +158,12 @@ func (f *fakeConcurrencyCache) CleanupStaleProcessSlots(context.Context, string) func newTestGatewayHandler(t *testing.T, group *service.Group, accounts []*service.Account) (*GatewayHandler, func()) { t.Helper() + return newTestGatewayHandlerWithSchedulerCache(t, group, &fakeSchedulerCache{accounts: accounts}) +} + +func newTestGatewayHandlerWithSchedulerCache(t *testing.T, group *service.Group, schedulerCache service.SchedulerCache) (*GatewayHandler, func()) { + t.Helper() - schedulerCache := &fakeSchedulerCache{accounts: accounts} schedulerSnapshot := service.NewSchedulerSnapshotService(schedulerCache, nil, nil, nil, nil) gwSvc := service.NewGatewayService( diff --git a/backend/internal/handler/gemini_client_cancel_test.go b/backend/internal/handler/gemini_client_cancel_test.go index aad811f025b2..6fda17db5a9a 100644 --- a/backend/internal/handler/gemini_client_cancel_test.go +++ b/backend/internal/handler/gemini_client_cancel_test.go @@ -34,11 +34,12 @@ func (u *clientCancelUpstream) Do(req *http.Request, _ string, _ int64, _ int) ( } type geminiClientCancelFixture struct { - handler *GatewayHandler - group *service.Group - apiKey *service.APIKey - upstream *clientCancelUpstream - ctx context.Context + handler *GatewayHandler + group *service.Group + apiKey *service.APIKey + upstream *clientCancelUpstream + schedulerCache *bucketRecordingSchedulerCache + ctx context.Context } func newGeminiClientCancelFixture(t *testing.T) *geminiClientCancelFixture { @@ -57,7 +58,8 @@ func newGeminiClientCancelFixture(t *testing.T) *geminiClientCancelFixture { Priority: 1, AccountGroups: []service.AccountGroup{{AccountID: accountID, GroupID: groupID}}, } - h, cleanup := newTestGatewayHandler(t, group, []*service.Account{account}) + schedulerCache := &bucketRecordingSchedulerCache{fakeSchedulerCache: &fakeSchedulerCache{accounts: []*service.Account{account}}} + h, cleanup := newTestGatewayHandlerWithSchedulerCache(t, group, schedulerCache) t.Cleanup(cleanup) ctx, cancel := context.WithCancel(context.Background()) @@ -69,7 +71,7 @@ func newGeminiClientCancelFixture(t *testing.T) *geminiClientCancelFixture { ID: 9202, UserID: 9203, GroupID: &groupID, Group: group, Status: service.StatusActive, User: &service.User{ID: 9203, Concurrency: 10, Balance: 100}, } - return &geminiClientCancelFixture{handler: h, group: group, apiKey: apiKey, upstream: upstream, ctx: ctx} + return &geminiClientCancelFixture{handler: h, group: group, apiKey: apiKey, upstream: upstream, schedulerCache: schedulerCache, ctx: ctx} } // serve 经 ops 错误日志中间件执行请求，返回响应与 ops 队列中的条目数。 diff --git a/backend/internal/handler/gemini_v1beta_handler.go b/backend/internal/handler/gemini_v1beta_handler.go index 95c550302e6c..ee2ee55583f6 100644 --- a/backend/internal/handler/gemini_v1beta_handler.go +++ b/backend/internal/handler/gemini_v1beta_handler.go @@ -546,12 +546,9 @@ func (h *GatewayHandler) GeminiV1BetaModels(c *gin.Context) { } fs := NewFailoverState(maxAccountSwitches, hasBoundSession) - // 单账号分组提前设置 SingleAccountRetry 标记，让 Service 层首次 503 就不设模型限流标记。 - // 避免单账号分组收到 503 (MODEL_CAPACITY_EXHAUSTED) 时设 29s 限流，导致后续请求连续快速失败。 - if h.gatewayService.IsSingleAntigravityAccountGroup(c.Request.Context(), apiKey.GroupID) { - ctx := service.WithSingleAccountRetry(c.Request.Context(), true, h.metadataBridgeEnabled()) - c.Request = c.Request.WithContext(ctx) - } + // 单账号分组首次 503 就不设模型限流标记，避免设 29s 限流导致后续请求连续快速失败； + // 是否单账号分组在 antigravity 账号遇到 503 或模型限流预检查时才判定。 + c.Request = c.Request.WithContext(h.gatewayService.WithSingleAntigravityAccountGroupProbe(c.Request.Context(), apiKey.GroupID)) for { excludedAccountIDs := fs.FailedAccountIDs diff --git a/backend/internal/handler/single_account_retry_probe_unit_test.go b/backend/internal/handler/single_account_retry_probe_unit_test.go new file mode 100644 index 000000000000..8b73067e254d --- /dev/null +++ b/backend/internal/handler/single_account_retry_probe_unit_test.go @@ -0,0 +1,120 @@ +//go:build unit + +package handler + +import ( + "bytes" + "context" + "net/http/httptest" + "sync" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" + middleware "github.com/Wei-Shaw/sub2api/internal/server/middleware" + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +// bucketRecordingSchedulerCache 记录读取过的快照桶平台。 +type bucketRecordingSchedulerCache struct { + *fakeSchedulerCache + mu sync.Mutex + platforms []string +} + +func (c *bucketRecordingSchedulerCache) GetSnapshot(ctx context.Context, bucket service.SchedulerBucket) ([]*service.Account, bool, error) { + c.mu.Lock() + c.platforms = append(c.platforms, bucket.Platform) + c.mu.Unlock() + return c.fakeSchedulerCache.GetSnapshot(ctx, bucket) +} + +func (c *bucketRecordingSchedulerCache) readPlatforms() []string { + c.mu.Lock() + defer c.mu.Unlock() + return append([]string(nil), c.platforms...) +} + +// 非 antigravity 账号的请求不读取单 antigravity 账号分组判定，选号之外不查询 antigravity 快照桶。 +func TestGeminiGroupRequestsDoNotReadAntigravityBucket(t *testing.T) { + gin.SetMode(gin.TestMode) + tests := []struct { + name string + route string + path string + body string + call func(h *GatewayHandler) func(*gin.Context) + }{ + { + name: "gemini native", + route: "/v1beta/models/*modelAction", + path: "/v1beta/models/gemini-2.5-flash:generateContent", + body: `{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`, + call: func(h *GatewayHandler) func(*gin.Context) { return h.GeminiV1BetaModels }, + }, + { + name: "messages on gemini group", + route: "/v1/messages", + path: "/v1/messages", + body: `{"model":"gemini-2.5-flash","max_tokens":16,"messages":[{"role":"user","content":"hi"}]}`, + call: func(h *GatewayHandler) func(*gin.Context) { return h.Messages }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newGeminiClientCancelFixture(t) + + f.serve(t, tt.route, tt.path, tt.body, tt.call(f.handler)) + + require.Equal(t, int32(1), f.upstream.calls.Load(), "请求应完成选号并到达上游") + platforms := f.schedulerCache.readPlatforms() + require.NotEmpty(t, platforms) + require.NotContains(t, platforms, service.PlatformAntigravity) + }) + } +} + +func TestMessagesAnthropicGroupDoesNotReadAntigravityBucket(t *testing.T) { + gin.SetMode(gin.TestMode) + groupID := int64(9400) + accountID := int64(9401) + group := &service.Group{ID: groupID, Hydrated: true, Platform: service.PlatformAnthropic, Status: service.StatusActive} + account := &service.Account{ + ID: accountID, + Platform: service.PlatformAnthropic, + Type: service.AccountTypeAPIKey, + Credentials: map[string]any{"api_key": "test-key", "intercept_warmup_requests": true}, + Status: service.StatusActive, + Schedulable: true, + Concurrency: 1, + Priority: 1, + AccountGroups: []service.AccountGroup{{AccountID: accountID, GroupID: groupID}}, + } + schedulerCache := &bucketRecordingSchedulerCache{fakeSchedulerCache: &fakeSchedulerCache{accounts: []*service.Account{account}}} + h, cleanup := newTestGatewayHandlerWithSchedulerCache(t, group, schedulerCache) + t.Cleanup(cleanup) + + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + body := `{"model":"claude-sonnet-4-5","max_tokens":256,"messages":[{"role":"user","content":[{"type":"text","text":"Warmup"}]}]}` + req := httptest.NewRequest("POST", "/v1/messages", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + c.Request = req.WithContext(context.WithValue(req.Context(), ctxkey.Group, group)) + apiKey := &service.APIKey{ + ID: 9402, UserID: 9403, GroupID: &groupID, Group: group, Status: service.StatusActive, + User: &service.User{ID: 9403, Concurrency: 10, Balance: 100}, + } + c.Set(string(middleware.ContextKeyAPIKey), apiKey) + c.Set(string(middleware.ContextKeyUser), middleware.AuthSubject{UserID: apiKey.UserID, Concurrency: 10}) + + h.Messages(c) + + require.Equal(t, 200, rec.Code) + selected, ok := c.Get(opsAccountIDKey) + require.True(t, ok) + require.Equal(t, accountID, selected) + platforms := schedulerCache.readPlatforms() + require.NotEmpty(t, platforms) + require.NotContains(t, platforms, service.PlatformAntigravity) +} diff --git a/backend/internal/service/antigravity_gateway_retry.go b/backend/internal/service/antigravity_gateway_retry.go index ee7a9834813d..9a317c1a061d 100644 --- a/backend/internal/service/antigravity_gateway_retry.go +++ b/backend/internal/service/antigravity_gateway_retry.go @@ -850,10 +850,12 @@ func sleepAntigravityBackoffWithContext(ctx context.Context, attempt int) bool { } } -// isSingleAccountRetry 检查 context 中是否设置了单账号退避重试标记 +// isSingleAccountRetry 检查是否处于单账号退避重试模式：显式标记优先，否则使用请求挂载的延迟判定。 func isSingleAccountRetry(ctx context.Context) bool { - v, _ := SingleAccountRetryFromContext(ctx) - return v + if v, ok := SingleAccountRetryFromContext(ctx); ok { + return v + } + return singleAccountRetryProbeResult(ctx) } // setModelRateLimitByModelName 使用官方模型 ID 设置模型级限流 diff --git a/backend/internal/service/gateway_scheduling.go b/backend/internal/service/gateway_scheduling.go index 059d7e2b2cb5..c2c9f419c696 100644 --- a/backend/internal/service/gateway_scheduling.go +++ b/backend/internal/service/gateway_scheduling.go @@ -1122,10 +1122,22 @@ func (s *GatewayService) listSchedulableAccounts(ctx context.Context, groupID *i return accounts, useMixed, nil } -// IsSingleAntigravityAccountGroup 检查指定分组是否只有一个 antigravity 平台的可调度账号。 -// 用于 Handler 层在首次请求时提前设置 SingleAccountRetry context， -// 避免单账号分组收到 503 时错误地设置模型限流标记导致后续请求连续快速失败。 -func (s *GatewayService) IsSingleAntigravityAccountGroup(ctx context.Context, groupID *int64) bool { +// WithSingleAntigravityAccountGroupProbe 为请求挂载「分组是否只有一个 antigravity 可调度账号」的延迟判定。 +// 只有 antigravity 账号遇到 503 或模型限流预检查时才会查询；其他平台的请求不读取该判定。 +func (s *GatewayService) WithSingleAntigravityAccountGroupProbe(ctx context.Context, groupID *int64) context.Context { + var probeGroupID *int64 + if groupID != nil { + id := *groupID + probeGroupID = &id + } + return WithSingleAccountRetryProbe(ctx, func(probeCtx context.Context) bool { + return s.isSingleAntigravityAccountGroup(probeCtx, probeGroupID) + }) +} + +// isSingleAntigravityAccountGroup 检查指定分组是否只有一个 antigravity 平台的可调度账号。 +// 单账号分组收到 503 时原地重试而不设置模型限流，避免后续请求连续快速失败。 +func (s *GatewayService) isSingleAntigravityAccountGroup(ctx context.Context, groupID *int64) bool { accounts, _, err := s.listSchedulableAccounts(ctx, groupID, PlatformAntigravity, true) if err != nil { return false diff --git a/backend/internal/service/request_metadata.go b/backend/internal/service/request_metadata.go index 5c81bbf1220c..4a521c20f610 100644 --- a/backend/internal/service/request_metadata.go +++ b/backend/internal/service/request_metadata.go @@ -2,6 +2,7 @@ package service import ( "context" + "sync" "sync/atomic" "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" @@ -11,6 +12,15 @@ type requestMetadataContextKey struct{} var requestMetadataKey = requestMetadataContextKey{} +type singleAccountRetryProbeContextKey struct{} + +// singleAccountRetryProbe 在首次需要时判定是否进入单账号退避重试模式，同一请求内至多判定一次。 +type singleAccountRetryProbe struct { + once sync.Once + check func(ctx context.Context) bool + result bool +} + type RequestMetadata struct { IsMaxTokensOneHaikuRequest *bool ThinkingEnabled *bool @@ -107,6 +117,29 @@ func WithSingleAccountRetry(ctx context.Context, value bool, bridgeOldKeys bool) }) } +// WithSingleAccountRetryProbe 挂载单账号退避重试的延迟判定；显式设置的 SingleAccountRetry 标记优先于它。 +func WithSingleAccountRetryProbe(ctx context.Context, check func(ctx context.Context) bool) context.Context { + if ctx == nil || check == nil { + return ctx + } + return context.WithValue(ctx, singleAccountRetryProbeContextKey{}, &singleAccountRetryProbe{check: check}) +} + +func singleAccountRetryProbeResult(ctx context.Context) bool { + if ctx == nil { + return false + } + probe, _ := ctx.Value(singleAccountRetryProbeContextKey{}).(*singleAccountRetryProbe) + if probe == nil { + return false + } + // 判定结果在整个请求内复用，不能因调用时 ctx 已取消而被记成 false。 + probe.once.Do(func() { + probe.result = probe.check(context.WithoutCancel(ctx)) + }) + return probe.result +} + func WithAccountSwitchCount(ctx context.Context, value int, bridgeOldKeys bool) context.Context { return updateRequestMetadata(ctx, bridgeOldKeys, func(md *RequestMetadata) { v := value diff --git a/backend/internal/service/single_account_retry_probe_test.go b/backend/internal/service/single_account_retry_probe_test.go new file mode 100644 index 000000000000..660762b908b8 --- /dev/null +++ b/backend/internal/service/single_account_retry_probe_test.go @@ -0,0 +1,191 @@ +//go:build unit + +package service + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" + "github.com/stretchr/testify/require" +) + +func TestIsSingleAccountRetry_ProbeEvaluatedLazilyOnce(t *testing.T) { + calls := 0 + ctx := WithSingleAccountRetryProbe(context.Background(), func(context.Context) bool { + calls++ + return true + }) + require.Zero(t, calls, "挂载时不应判定") + + require.True(t, isSingleAccountRetry(ctx)) + require.True(t, isSingleAccountRetry(ctx)) + require.Equal(t, 1, calls, "同一请求内只判定一次") +} + +func TestIsSingleAccountRetry_ProbeFalse(t *testing.T) { + ctx := WithSingleAccountRetryProbe(context.Background(), func(context.Context) bool { return false }) + require.False(t, isSingleAccountRetry(ctx)) +} + +func TestIsSingleAccountRetry_ExplicitFlagOverridesProbe(t *testing.T) { + calls := 0 + probe := func(context.Context) bool { + calls++ + return false + } + + ctx := WithSingleAccountRetryProbe(context.Background(), probe) + ctx = WithSingleAccountRetry(ctx, true, false) + require.True(t, isSingleAccountRetry(ctx)) + + legacy := context.WithValue(WithSingleAccountRetryProbe(context.Background(), probe), ctxkey.SingleAccountRetry, true) + require.True(t, isSingleAccountRetry(legacy)) + + explicitFalse := WithSingleAccountRetry(WithSingleAccountRetryProbe(context.Background(), func(context.Context) bool { + calls++ + return true + }), false, false) + require.False(t, isSingleAccountRetry(explicitFalse)) + + require.Zero(t, calls, "显式标记存在时不应触发判定") +} + +func TestIsSingleAccountRetry_ProbeIgnoresCanceledContext(t *testing.T) { + ctx, cancel := context.WithCancel(WithSingleAccountRetryProbe(context.Background(), func(probeCtx context.Context) bool { + return probeCtx.Err() == nil + })) + cancel() + require.True(t, isSingleAccountRetry(ctx)) +} + +func TestIsSingleAccountRetry_ProbeSurvivesMetadataUpdates(t *testing.T) { + ctx := WithSingleAccountRetryProbe(context.Background(), func(context.Context) bool { return true }) + ctx = WithAccountSwitchCount(ctx, 1, false) + ctx = WithThinkingEnabled(ctx, true, false) + require.True(t, isSingleAccountRetry(ctx)) +} + +// groupRecordingAccountRepo 记录单平台分组查询的分组与平台。 +type groupRecordingAccountRepo struct { + *mockAccountRepoForPlatform + groupIDs []int64 + platforms []string +} + +func (r *groupRecordingAccountRepo) ListSchedulableByGroupIDAndPlatform(ctx context.Context, groupID int64, platform string) ([]Account, error) { + r.groupIDs = append(r.groupIDs, groupID) + r.platforms = append(r.platforms, platform) + return r.mockAccountRepoForPlatform.ListSchedulableByGroupIDAndPlatform(ctx, groupID, platform) +} + +func newSingleAntigravityProbeTestService(accounts ...Account) (*GatewayService, *groupRecordingAccountRepo) { + repo := &groupRecordingAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{accounts: accounts}} + return &GatewayService{accountRepo: repo}, repo +} + +func probeTestAccount(id int64, platform string) Account { + return Account{ID: id, Platform: platform, Status: StatusActive, Schedulable: true} +} + +func TestWithSingleAntigravityAccountGroupProbe_QueriesOnlyWhenRead(t *testing.T) { + svc, repo := newSingleAntigravityProbeTestService(probeTestAccount(1, PlatformAntigravity)) + groupID := int64(42) + + ctx := svc.WithSingleAntigravityAccountGroupProbe(context.Background(), &groupID) + require.Empty(t, repo.groupIDs, "挂载时不应查询账号") + + require.True(t, isSingleAccountRetry(ctx)) + require.True(t, isSingleAccountRetry(ctx)) + require.Equal(t, []int64{42}, repo.groupIDs) + require.Equal(t, []string{PlatformAntigravity}, repo.platforms) +} + +func TestWithSingleAntigravityAccountGroupProbe_UsesGroupAtMountTime(t *testing.T) { + svc, repo := newSingleAntigravityProbeTestService(probeTestAccount(1, PlatformAntigravity)) + groupID := int64(42) + + ctx := svc.WithSingleAntigravityAccountGroupProbe(context.Background(), &groupID) + groupID = 99 + + require.True(t, isSingleAccountRetry(ctx)) + require.Equal(t, []int64{42}, repo.groupIDs) +} + +func TestWithSingleAntigravityAccountGroupProbe_CountsOnlyAntigravityAccounts(t *testing.T) { + groupID := int64(42) + tests := []struct { + name string + accounts []Account + want bool + }{ + {name: "no antigravity accounts", accounts: []Account{probeTestAccount(1, PlatformGemini)}, want: false}, + {name: "single antigravity account", accounts: []Account{probeTestAccount(1, PlatformAntigravity), probeTestAccount(2, PlatformGemini)}, want: true}, + {name: "two antigravity accounts", accounts: []Account{probeTestAccount(1, PlatformAntigravity), probeTestAccount(2, PlatformAntigravity)}, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc, _ := newSingleAntigravityProbeTestService(tt.accounts...) + ctx := svc.WithSingleAntigravityAccountGroupProbe(context.Background(), &groupID) + require.Equal(t, tt.want, isSingleAccountRetry(ctx)) + }) + } +} + +func runRateLimitedPreCheckWithProbe(t *testing.T, single bool) (*antigravityRetryLoopResult, *recordingOKUpstream, error) { + t.Helper() + upstream := &recordingOKUpstream{} + account := &Account{ + ID: 30, + Type: AccountTypeOAuth, + Platform: PlatformAntigravity, + Schedulable: true, + Status: StatusActive, + Concurrency: 1, + Extra: map[string]any{ + modelRateLimitsKey: map[string]any{ + "claude-sonnet-4-5": map[string]any{ + "rate_limit_reset_at": time.Now().Add(30 * time.Second).Format(time.RFC3339), + }, + }, + }, + } + ctx := WithSingleAccountRetryProbe(context.Background(), func(context.Context) bool { return single }) + svc := &AntigravityGatewayService{} + result, err := svc.antigravityRetryLoop(antigravityRetryLoopParams{ + ctx: ctx, + prefix: "[test]", + account: account, + accessToken: "token", + action: "generateContent", + body: []byte(`{"input":"test"}`), + httpUpstream: upstream, + requestedModel: "claude-sonnet-4-5", + handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { + return nil + }, + }) + return result, upstream, err +} + +func TestAntigravityRetryLoop_PreCheck_ProbeSingleAccount_SkipsRateLimit(t *testing.T) { + result, upstream, err := runRateLimitedPreCheckWithProbe(t, true) + + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, result.resp) + require.Equal(t, http.StatusOK, result.resp.StatusCode) + require.Equal(t, 1, upstream.calls) +} + +func TestAntigravityRetryLoop_PreCheck_ProbeMultiAccount_SwitchesOnRateLimit(t *testing.T) { + result, upstream, err := runRateLimitedPreCheckWithProbe(t, false) + + require.Nil(t, result) + var switchErr *AntigravityAccountSwitchError + require.ErrorAs(t, err, &switchErr) + require.Equal(t, "claude-sonnet-4-5", switchErr.RateLimitedModel) + require.Zero(t, upstream.calls) +} From 53002f677c3d8e182ba11858e00b301079107f6e Mon Sep 17 00:00:00 2001 From: feeeei Date: Tue, 20 Oct 2026 20:33:13 +0800 Subject: [PATCH 17/29] =?UTF-8?q?perf(scheduler):=20=E8=B0=83=E5=BA=A6?= =?UTF-8?q?=E5=BF=AB=E7=85=A7=E8=A7=A3=E7=A0=81=E7=BB=93=E6=9E=9C=E5=B8=B8?= =?UTF-8?q?=E9=A9=BB=E8=BF=9B=E7=A8=8B=E5=86=85=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GetSnapshot 每次选号都 ZRANGE 全组账号 ID、分块 MGET 全组 sched:meta 并逐条 JSON 解码，Redis 出口、CPU 与 GC 都随分组号数线性增长。 - 每个 bucket 的解码结果常驻进程内存。每次读取用一次 MGET 取 ready、 active、全局修订号与 bucket 平台修订号（mixed 另加 antigravity），全部与 条目一致且条目未超过 10 秒才复用，否则重新加载。last_used 仍每次读取， 套用在本次调用独占的副本上，LRU 语义不变。 - 修订号是随机串：精简记录改写后写入所涉平台的修订号，与最后一批写入同一 pipeline 发出；删除记录、账号平台变化（SET ... GET 取回旧值比对）、 写入出错时写入全局修订号，出错路径在后台合并写入。 - 母本发布前预解析模型映射；副本在指针、长度与运行时版本都匹配时直接复用， 不计算签名。 - 未命中、空快照、记录缺失、解码失败、Redis 出错的返回与原实现一致；选中后 的整账号读取仍每次重新解码，转发路径对账号的原地修改不触及常驻对象。 真实 Redis 端到端（SelectAccountWithLoadAwareness，本机）： 100 号：3.83ms → 0.73ms，Redis 出口 251KB → 7.0KB，分配 2.03MB → 0.23MB 1000 号：27.8ms → 1.92ms，Redis 出口 2.47MB → 25KB，分配 20.2MB → 1.97MB 每次都未命中：100 号 3.83ms → 3.10ms，1000 号 27.6ms → 21.9ms 常驻内存约为每号 7KB × 活跃 bucket 数；闲置超过 100 秒的条目在写入新 条目时清除。 --- .../internal/repository/scheduler_cache.go | 368 ++++++- ...scheduler_cache_snapshot_memo_unit_test.go | 980 ++++++++++++++++++ backend/internal/service/account.go | 21 + .../service/account_shared_snapshot_test.go | 82 ++ .../model_rate_limit_snapshot_cow_test.go | 2 + backend/internal/service/scheduler_cache.go | 2 + 6 files changed, 1417 insertions(+), 38 deletions(-) create mode 100644 backend/internal/repository/scheduler_cache_snapshot_memo_unit_test.go create mode 100644 backend/internal/service/account_shared_snapshot_test.go diff --git a/backend/internal/repository/scheduler_cache.go b/backend/internal/repository/scheduler_cache.go index 827a750577b1..2aa73c61ea3b 100644 --- a/backend/internal/repository/scheduler_cache.go +++ b/backend/internal/repository/scheduler_cache.go @@ -5,13 +5,18 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "errors" "fmt" "log/slog" + "slices" "strconv" + "sync" + "sync/atomic" "time" "github.com/Wei-Shaw/sub2api/internal/service" "github.com/redis/go-redis/v9" + "github.com/tidwall/gjson" ) const ( @@ -27,6 +32,12 @@ const ( schedulerRetiredPrefix = "sched:retired:" schedulerSnapshotPrefix = "sched:" schedulerLockPrefix = "sched:lock:" + // 账号精简记录（sched:meta:）的修订号：全局修订号与按平台划分的修订号（前缀后接平台名）。 + // 改写精简记录后写入所涉平台的修订号；删除记录、改写平台为空或平台发生变化的记录、写入出错时写入全局修订号。 + // 取值都是新的随机串，读侧只比较是否相等，键丢失后重建也不会与旧取值相同； + // Redis 回滚恢复出缓存记录过的旧取值时，由最长复用时长兜底。 + schedulerAccountMetaRevisionKey = "sched:meta:rev" + schedulerAccountMetaPlatformRevisionPrefix = "sched:meta:rev:" defaultSchedulerSnapshotMGetChunkSize = 128 defaultSchedulerSnapshotWriteChunkSize = 256 @@ -35,6 +46,16 @@ const ( // snapshotGraceTTLSeconds 旧快照过期的宽限期（秒）。 // 替代立即 DEL，让正在读取旧版本的 reader 有足够时间完成 ZRANGE。 snapshotGraceTTLSeconds = 60 + + // schedulerSnapshotCacheMaxAge 是进程内快照缓存条目的最长复用时长， + // 限定了未推进修订号的精简记录改写在本进程的最长可见延迟，与 outbox_lag_rebuild_seconds 默认 10 秒的陈旧容忍度相同。 + schedulerSnapshotCacheMaxAge = 10 * time.Second + // schedulerSnapshotCacheEvictAge：写入条目时，加载时间早于该时长的其它条目一并清除， + // 已退休分组的条目不会长期驻留。 + schedulerSnapshotCacheEvictAge = 10 * schedulerSnapshotCacheMaxAge + + schedulerAccountMetaRevisionBytes = 16 + schedulerAccountMetaRevisionWriteTimeout = 3 * time.Second ) const ( @@ -223,6 +244,28 @@ type schedulerCache struct { rdb *redis.Client mgetChunkSize int writeChunkSize int + + // now 是进程内快照缓存判定条目年龄的时钟。 + now func() time.Time + + snapshotMu sync.RWMutex + snapshotEntries map[string]*schedulerSnapshotEntry + + // globalRevisionPending 与 globalRevisionWriting 合并写入出错后的后台全局修订号写入，见 scheduleGlobalAccountMetaRevision。 + globalRevisionPending atomic.Bool + globalRevisionWriting atomic.Bool +} + +// schedulerSnapshotEntry 是一个 bucket 的已解码快照。 +// revisions 依次是全局修订号与该 bucket 成员可能涉及的各平台修订号（见 schedulerSnapshotTagKeys）。 +// accounts 是按快照顺序排列的母本，发布后只读，指针不外泄； +// 每次读取都按值复制出本次请求独占的副本再套用 last_used。 +type schedulerSnapshotEntry struct { + active string + revisions []string + loadedAt time.Time + accounts []*service.Account + lastUsedKeys []string } func NewSchedulerCache(rdb *redis.Client) service.SchedulerCache { @@ -237,42 +280,67 @@ func newSchedulerCacheWithChunkSizes(rdb *redis.Client, mgetChunkSize, writeChun writeChunkSize = defaultSchedulerSnapshotWriteChunkSize } return &schedulerCache{ - rdb: rdb, - mgetChunkSize: mgetChunkSize, - writeChunkSize: writeChunkSize, + rdb: rdb, + mgetChunkSize: mgetChunkSize, + writeChunkSize: writeChunkSize, + now: time.Now, + snapshotEntries: make(map[string]*schedulerSnapshotEntry), } } +// GetSnapshot 读取 bucket 的激活快照。解码后的精简记录按 bucket 常驻进程内存， +// 只有 ready、active 版本、全局修订号与该 bucket 所涉平台的修订号都与 Redis 一致且未超过最长复用时长时才复用； +// 否则重新从 Redis 加载。last_used 每次都从 Redis 读取。 +// 返回的账号对象由本次调用独占，可以修改其顶层字段；map、切片底层数组与指针指向的值与其它调用共享，不得原地修改。 func (c *schedulerCache) GetSnapshot(ctx context.Context, bucket service.SchedulerBucket) ([]*service.Account, bool, error) { - readyKey := schedulerBucketKey(schedulerReadyPrefix, bucket) - readyVal, err := c.rdb.Get(ctx, readyKey).Result() - if err == redis.Nil { - return nil, false, nil - } + bucketKey := bucket.String() + tagKeys := schedulerSnapshotTagKeys(bucket) + tags, err := c.rdb.MGet(ctx, tagKeys...).Result() if err != nil { return nil, false, err } - if readyVal != "1" { - return nil, false, nil + if len(tags) != len(tagKeys) { + return nil, false, fmt.Errorf("unexpected scheduler snapshot tag count: got %d, want %d", len(tags), len(tagKeys)) } - - activeKey := schedulerBucketKey(schedulerActivePrefix, bucket) - activeVal, err := c.rdb.Get(ctx, activeKey).Result() - if err == redis.Nil { - return nil, false, nil + readyVal, err := schedulerCacheOptionalString(tags[0]) + if err != nil { + return nil, false, err } + activeVal, err := schedulerCacheOptionalString(tags[1]) if err != nil { return nil, false, err } + // 修订号缺失（nil）按空字符串比较：它是一个独立取值，不会与任何写入过的随机修订号相等。 + revisions := make([]string, 0, len(tags)-2) + for _, tag := range tags[2:] { + revision, err := schedulerCacheOptionalString(tag) + if err != nil { + return nil, false, err + } + revisions = append(revisions, revision) + } + if readyVal != "1" || activeVal == "" { + c.dropSnapshotEntry(bucketKey) + return nil, false, nil + } + + now := c.now() + if entry := c.lookupSnapshotEntry(bucketKey); entry != nil && entry.matches(activeVal, revisions, now) { + accounts, err := c.snapshotAccountsFromEntry(ctx, entry) + if err != nil { + return nil, false, err + } + return accounts, true, nil + } - snapshotKey := schedulerSnapshotKey(bucket, activeVal) - ids, err := c.rdb.ZRange(ctx, snapshotKey, 0, -1).Result() + ids, err := c.rdb.ZRange(ctx, schedulerSnapshotKey(bucket, activeVal), 0, -1).Result() if err != nil { return nil, false, err } if len(ids) == 0 { // 空快照视为缓存未命中，触发数据库回退查询 // 这解决了新分组创建后立即绑定账号时的竞态条件问题 + c.dropSnapshotEntry(bucketKey) return nil, false, nil } @@ -286,27 +354,141 @@ func (c *schedulerCache) GetSnapshot(ctx context.Context, bucket service.Schedul if err != nil { return nil, false, err } - lastUsedValues, err := c.mgetChunked(ctx, lastUsedKeys) - if err != nil { - return nil, false, err - } - accounts := make([]*service.Account, 0, len(values)) - for i, val := range values { + masters := make([]*service.Account, 0, len(values)) + for _, val := range values { if val == nil { + c.dropSnapshotEntry(bucketKey) return nil, false, nil } account, err := decodeCachedAccount(val) if err != nil { return nil, false, err } + masters = append(masters, account) + } + // 母本发布进缓存前由本 goroutine 完成预处理，发布后只读。 + for _, master := range masters { + master.PrepareSharedSnapshot() + } + + // 条目记录的是加载前读到的 (active, 修订号) 与当时的时钟：精简记录读取晚于这组取值， + // 之后若再有改写，修订号或 active 必然不同，或在最长复用时长后过期。 + entry := &schedulerSnapshotEntry{ + active: activeVal, + revisions: revisions, + loadedAt: now, + accounts: masters, + lastUsedKeys: lastUsedKeys, + } + c.storeSnapshotEntry(bucketKey, entry) + + accounts, err := c.snapshotAccountsFromEntry(ctx, entry) + if err != nil { + return nil, false, err + } + return accounts, true, nil +} + +func (e *schedulerSnapshotEntry) matches(active string, revisions []string, now time.Time) bool { + if e.active != active || !slices.Equal(e.revisions, revisions) { + return false + } + age := now.Sub(e.loadedAt) + return age >= 0 && age < schedulerSnapshotCacheMaxAge +} + +// snapshotAccountsFromEntry 读取最新 last_used，并为本次调用一次性分配一组母本副本。 +func (c *schedulerCache) snapshotAccountsFromEntry(ctx context.Context, entry *schedulerSnapshotEntry) ([]*service.Account, error) { + lastUsedValues, err := c.mgetChunked(ctx, entry.lastUsedKeys) + if err != nil { + return nil, err + } + if len(lastUsedValues) != len(entry.accounts) { + return nil, fmt.Errorf("unexpected scheduler last_used value count: got %d, want %d", len(lastUsedValues), len(entry.accounts)) + } + slab := make([]service.Account, len(entry.accounts)) + accounts := make([]*service.Account, len(entry.accounts)) + for i, master := range entry.accounts { + slab[i] = *master + account := &slab[i] if err := applySchedulerLastUsed(account, lastUsedValues[i]); err != nil { - return nil, false, err + return nil, err } - accounts = append(accounts, account) + accounts[i] = account } + return accounts, nil +} - return accounts, true, nil +func (c *schedulerCache) lookupSnapshotEntry(bucketKey string) *schedulerSnapshotEntry { + c.snapshotMu.RLock() + defer c.snapshotMu.RUnlock() + return c.snapshotEntries[bucketKey] +} + +// storeSnapshotEntry 写入条目，并清除加载时间早于 schedulerSnapshotCacheEvictAge 的其它条目。 +// 已有条目的加载时间晚于新条目时保留已有条目：它基于更晚读到的 (active, 修订号)。 +func (c *schedulerCache) storeSnapshotEntry(bucketKey string, entry *schedulerSnapshotEntry) { + c.snapshotMu.Lock() + defer c.snapshotMu.Unlock() + if current := c.snapshotEntries[bucketKey]; current == nil || !current.loadedAt.After(entry.loadedAt) { + c.snapshotEntries[bucketKey] = entry + } + for key, other := range c.snapshotEntries { + if key != bucketKey && entry.loadedAt.Sub(other.loadedAt) > schedulerSnapshotCacheEvictAge { + delete(c.snapshotEntries, key) + } + } +} + +func (c *schedulerCache) dropSnapshotEntry(bucketKey string) { + c.snapshotMu.RLock() + _, ok := c.snapshotEntries[bucketKey] + c.snapshotMu.RUnlock() + if !ok { + return + } + c.snapshotMu.Lock() + delete(c.snapshotEntries, bucketKey) + c.snapshotMu.Unlock() +} + +// schedulerSnapshotTagKeys 返回 GetSnapshot 一次读取的键：ready、active、全局修订号、bucket 平台的修订号， +// mixed 模式再加 antigravity 平台的修订号。按平台判定失效依赖 bucket 成员只取自 bucket 平台 +// （mixed 模式另含开启混合调度的 antigravity 账号）；账号平台发生变化时，写入方另行写入全局修订号。 +func schedulerSnapshotTagKeys(bucket service.SchedulerBucket) []string { + keys := make([]string, 0, 5) + keys = append(keys, + schedulerBucketKey(schedulerReadyPrefix, bucket), + schedulerBucketKey(schedulerActivePrefix, bucket), + schedulerAccountMetaRevisionKey, + schedulerAccountMetaRevisionKeyForPlatform(bucket.Platform), + ) + if bucket.Mode == service.SchedulerModeMixed && bucket.Platform != service.PlatformAntigravity { + keys = append(keys, schedulerAccountMetaRevisionKeyForPlatform(service.PlatformAntigravity)) + } + return keys +} + +// schedulerAccountMetaRevisionKeyForPlatform 返回平台修订号的键；平台为空时返回全局修订号的键。 +func schedulerAccountMetaRevisionKeyForPlatform(platform string) string { + if platform == "" { + return schedulerAccountMetaRevisionKey + } + return schedulerAccountMetaPlatformRevisionPrefix + platform +} + +func schedulerCacheOptionalString(value any) (string, error) { + switch typed := value.(type) { + case nil: + return "", nil + case string: + return typed, nil + case []byte: + return string(typed), nil + default: + return "", fmt.Errorf("unexpected scheduler cache value type: %T", value) + } } func (c *schedulerCache) CaptureBucketWriteToken(ctx context.Context, bucket service.SchedulerBucket) (service.SchedulerBucketWriteToken, error) { @@ -604,7 +786,13 @@ func (c *schedulerCache) DeleteAccount(ctx context.Context, accountID int64) err return nil } id := strconv.FormatInt(accountID, 10) - return c.rdb.Del(ctx, schedulerAccountKey(id), schedulerAccountMetaKey(id), schedulerLastUsedKey(id)).Err() + // 删除精简记录与推进全局修订号在同一个 MULTI 中原子执行：读到删除结果的读者也读到新修订号， + // 进程内快照缓存不会给出已删除的账号。 + pipe := c.rdb.TxPipeline() + pipe.Del(ctx, schedulerAccountKey(id), schedulerAccountMetaKey(id), schedulerLastUsedKey(id)) + pipe.Set(ctx, schedulerAccountMetaRevisionKey, newSchedulerAccountMetaRevision(), 0) + _, err := pipe.Exec(ctx) + return err } func (c *schedulerCache) UpdateLastUsed(ctx context.Context, updates map[int64]time.Time) error { @@ -614,6 +802,7 @@ func (c *schedulerCache) UpdateLastUsed(ctx context.Context, updates map[int64]t pipe := c.rdb.Pipeline() queued := 0 + metaDeleted := false keys := make([]string, 0, schedulerLastUsedUpdateChunkSize*2) args := make([]any, 0, schedulerLastUsedUpdateChunkSize) queueBatch := func() { @@ -638,6 +827,7 @@ func (c *schedulerCache) UpdateLastUsed(ctx context.Context, updates map[int64]t idText := strconv.FormatInt(id, 10) pipe.Del(ctx, schedulerAccountKey(idText), schedulerAccountMetaKey(idText), schedulerLastUsedKey(idText)) queued++ + metaDeleted = true continue } idText := strconv.FormatInt(id, 10) @@ -651,6 +841,10 @@ func (c *schedulerCache) UpdateLastUsed(ctx context.Context, updates map[int64]t if queued == 0 { return nil } + if metaDeleted { + // 被删账号的平台未知，全局修订号排在同一 pipeline 的 DEL 之后。 + pipe.Set(ctx, schedulerAccountMetaRevisionKey, newSchedulerAccountMetaRevision(), 0) + } _, err := pipe.Exec(ctx) return err } @@ -776,21 +970,53 @@ func decodeCachedAccount(val any) (*service.Account, error) { return &account, nil } +// writeAccountIDs 分块写入整账号与精简记录，返回成功编码并写入的有序账号 ID。 +// 每块在追加下一个账号前才执行，最后一块总是非空；所涉平台的修订号追加在最后一块末尾， +// 与最后一批精简记录同一次往返发出，且排在全部精简记录写入之后。 +// 精简记录用 SET ... GET 写入并取回旧值：旧值的平台与新平台不同时，旧平台 bucket 中可能仍缓存着该账号， +// 全部写入完成后另行写入全局修订号。该检测尽力而为（客户端在 pipeline 内部重试时取回的可能已是本次写入的值）， +// 漏检由进程内快照缓存的最长复用时长兜底。 +// 任一块执行出错时，此前已执行的块与出错的块都可能已有精简记录生效，返回时启动后台写入全局修订号。 func (c *schedulerCache) writeAccountIDs(ctx context.Context, accounts []service.Account) ([]int64, error) { if len(accounts) == 0 { return nil, nil } + type metaWrite struct { + platform string + previous *redis.StatusCmd + } pipe := c.rdb.Pipeline() accountIDs := make([]int64, 0, len(accounts)) + platforms := make([]string, 0, 1) + writes := make([]metaWrite, 0, min(len(accounts), c.writeChunkSize)) pending := 0 - flush := func() error { - if pending == 0 { - return nil + execFailed := false + platformChanged := false + defer func() { + if execFailed { + c.scheduleGlobalAccountMetaRevision(ctx) } - if _, err := pipe.Exec(ctx); err != nil { + }() + exec := func() error { + cmds, err := pipe.Exec(ctx) + if err != nil && !errors.Is(err, redis.Nil) { + execFailed = true return err } + // SET ... GET 在键原本不存在时返回 redis.Nil，不算失败；其余命令的错误逐条检查。 + for _, cmd := range cmds { + if cmdErr := cmd.Err(); cmdErr != nil && !errors.Is(cmdErr, redis.Nil) { + execFailed = true + return cmdErr + } + } + for _, write := range writes { + if schedulerMetaPlatformChanged(write.previous, write.platform) { + platformChanged = true + } + } + writes = writes[:0] pipe = c.rdb.Pipeline() pending = 0 return nil @@ -805,27 +1031,93 @@ func (c *schedulerCache) writeAccountIDs(ctx context.Context, accounts []service ) continue } + if pending >= c.writeChunkSize { + if err := exec(); err != nil { + return nil, err + } + } id := strconv.FormatInt(account.ID, 10) pipe.Set(ctx, schedulerAccountKey(id), fullPayload, 0) - pipe.Set(ctx, schedulerAccountMetaKey(id), metaPayload, 0) + writes = append(writes, metaWrite{ + platform: account.Platform, + previous: pipe.SetArgs(ctx, schedulerAccountMetaKey(id), metaPayload, redis.SetArgs{Get: true}), + }) // Keep the hot LastUsedAt side key untouched: a lagging snapshot rebuild // must not overwrite a newer scheduler update. accountIDs = append(accountIDs, account.ID) - pending++ - if pending >= c.writeChunkSize { - if err := flush(); err != nil { - return nil, err - } + if !slices.Contains(platforms, account.Platform) { + platforms = append(platforms, account.Platform) } + pending++ } - if err := flush(); err != nil { + if pending == 0 { + return accountIDs, nil + } + for _, platform := range platforms { + pipe.Set(ctx, schedulerAccountMetaRevisionKeyForPlatform(platform), newSchedulerAccountMetaRevision(), 0) + } + if err := exec(); err != nil { return nil, err } + if platformChanged { + c.advanceGlobalAccountMetaRevision(ctx) + } return accountIDs, nil } +// schedulerMetaPlatformChanged 判断 SET ... GET 取回的旧精简记录是否属于另一个平台； +// 旧值不是合法 JSON 或缺少字符串类型的顶层 Platform 时按已变化处理。 +func schedulerMetaPlatformChanged(previous *redis.StatusCmd, platform string) bool { + raw, err := previous.Result() + if err != nil { + return false + } + if !gjson.Valid(raw) { + return true + } + old := gjson.Get(raw, "Platform") + return old.Type != gjson.String || old.Str != platform +} + +// scheduleGlobalAccountMetaRevision 在后台写入全局修订号，不阻塞调用方。 +// 同一时刻至多一个后台写入：写入前先清除待写标记，写完后标记再次被置位就再写一轮， +// 因此每次调用之后都至少还有一次完整的写入发生在它之后。 +func (c *schedulerCache) scheduleGlobalAccountMetaRevision(ctx context.Context) { + c.globalRevisionPending.Store(true) + if !c.globalRevisionWriting.CompareAndSwap(false, true) { + return + } + go func() { + for { + c.globalRevisionPending.Store(false) + c.advanceGlobalAccountMetaRevision(ctx) + c.globalRevisionWriting.Store(false) + if !c.globalRevisionPending.Load() || !c.globalRevisionWriting.CompareAndSwap(false, true) { + return + } + } + }() +} + +// advanceGlobalAccountMetaRevision 单独写入新的全局修订号，使所有 bucket 的进程内快照失效。 +// 不受调用方取消影响；失败只记录日志，由进程内快照缓存的最长复用时长兜底。 +func (c *schedulerCache) advanceGlobalAccountMetaRevision(ctx context.Context) { + writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), schedulerAccountMetaRevisionWriteTimeout) + defer cancel() + if err := c.rdb.Set(writeCtx, schedulerAccountMetaRevisionKey, newSchedulerAccountMetaRevision(), 0).Err(); err != nil { + slog.Warn("scheduler cache failed to advance account metadata revision", "error", err) + } +} + +func newSchedulerAccountMetaRevision() string { + raw := make([]byte, schedulerAccountMetaRevisionBytes) + // crypto/rand.Read 读取失败时直接终止进程，不会返回错误。 + _, _ = rand.Read(raw) + return hex.EncodeToString(raw) +} + func marshalSchedulerCacheAccount(account service.Account) ([]byte, []byte, error) { fullPayload, err := json.Marshal(account) if err != nil { diff --git a/backend/internal/repository/scheduler_cache_snapshot_memo_unit_test.go b/backend/internal/repository/scheduler_cache_snapshot_memo_unit_test.go new file mode 100644 index 000000000000..9b3d92ea2635 --- /dev/null +++ b/backend/internal/repository/scheduler_cache_snapshot_memo_unit_test.go @@ -0,0 +1,980 @@ +//go:build unit + +package repository + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +// schedulerCacheCommandRecorder 记录发往 Redis 的命令（名称 + 首个参数）与每次 pipeline 执行的命令批次， +// 并可让第 N 次 pipeline 执行失败。新连接的握手命令不计入。 +type schedulerCacheCommandRecorder struct { + mu sync.Mutex + commands []string + batches [][]string + pipelines int + failPipelineAt int +} + +func (r *schedulerCacheCommandRecorder) DialHook(next redis.DialHook) redis.DialHook { return next } + +func (r *schedulerCacheCommandRecorder) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + if !isSchedulerCacheHandshakeCommand(cmd) { + r.record(cmd) + } + return next(ctx, cmd) + } +} + +func (r *schedulerCacheCommandRecorder) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return func(ctx context.Context, cmds []redis.Cmder) error { + handshake := len(cmds) > 0 + for _, cmd := range cmds { + handshake = handshake && isSchedulerCacheHandshakeCommand(cmd) + } + if handshake { + return next(ctx, cmds) + } + r.mu.Lock() + r.pipelines++ + fail := r.failPipelineAt > 0 && r.pipelines == r.failPipelineAt + r.mu.Unlock() + batch := make([]string, 0, len(cmds)) + for _, cmd := range cmds { + batch = append(batch, r.record(cmd)) + } + r.mu.Lock() + r.batches = append(r.batches, batch) + r.mu.Unlock() + if fail { + err := errors.New("injected pipeline failure") + for _, cmd := range cmds { + cmd.SetErr(err) + } + return err + } + return next(ctx, cmds) + } +} + +func isSchedulerCacheHandshakeCommand(cmd redis.Cmder) bool { + switch strings.ToLower(cmd.Name()) { + case "hello", "auth", "client", "select": + return true + default: + return false + } +} + +func (r *schedulerCacheCommandRecorder) record(cmd redis.Cmder) string { + entry := strings.ToLower(cmd.Name()) + if args := cmd.Args(); len(args) > 1 { + entry += " " + fmt.Sprint(args[1]) + } + r.mu.Lock() + r.commands = append(r.commands, entry) + r.mu.Unlock() + return entry +} + +func (r *schedulerCacheCommandRecorder) reset() { + r.mu.Lock() + r.commands = nil + r.batches = nil + r.pipelines = 0 + r.mu.Unlock() +} + +func (r *schedulerCacheCommandRecorder) pipelineBatches() [][]string { + r.mu.Lock() + defer r.mu.Unlock() + return append([][]string(nil), r.batches...) +} + +func (r *schedulerCacheCommandRecorder) setFailPipelineAt(n int) { + r.mu.Lock() + r.failPipelineAt = n + r.mu.Unlock() +} + +func (r *schedulerCacheCommandRecorder) recorded() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.commands...) +} + +func (r *schedulerCacheCommandRecorder) countPrefix(prefix string) int { + count := 0 + for _, command := range r.recorded() { + if strings.HasPrefix(command, prefix) { + count++ + } + } + return count +} + +func newSchedulerCacheMemoUnit(t *testing.T, writeChunkSize int) (*schedulerCache, *miniredis.Miniredis, *schedulerCacheCommandRecorder) { + t.Helper() + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + recorder := &schedulerCacheCommandRecorder{} + rdb.AddHook(recorder) + cache, ok := newSchedulerCacheWithChunkSizes(rdb, defaultSchedulerSnapshotMGetChunkSize, writeChunkSize).(*schedulerCache) + require.True(t, ok) + return cache, mr, recorder +} + +func memoTestBucket(groupID int64) service.SchedulerBucket { + return service.SchedulerBucket{GroupID: groupID, Platform: service.PlatformAnthropic, Mode: service.SchedulerModeSingle} +} + +func memoTestAccount(id int64, name string) service.Account { + return memoTestAccountOn(service.PlatformAnthropic, id, name) +} + +func memoTestAccountOn(platform string, id int64, name string) service.Account { + return service.Account{ + ID: id, + Name: name, + Platform: platform, + Type: service.AccountTypeOAuth, + Status: service.StatusActive, + Schedulable: true, + Concurrency: 2, + Credentials: map[string]any{ + "model_mapping": map[string]any{"claude-sonnet": "sonnet-" + name}, + "access_token": "dropped-by-projection", + }, + Extra: map[string]any{"session_window_utilization": 0.25}, + } +} + +func memoPublish(t *testing.T, cache *schedulerCache, bucket service.SchedulerBucket, accounts ...service.Account) { + t.Helper() + ctx := context.Background() + token, err := cache.CaptureBucketWriteToken(ctx, bucket) + require.NoError(t, err) + require.NoError(t, cache.SetSnapshot(ctx, bucket, token, accounts)) +} + +func memoRead(t *testing.T, cache *schedulerCache, bucket service.SchedulerBucket) []*service.Account { + t.Helper() + accounts, hit, err := cache.GetSnapshot(context.Background(), bucket) + require.NoError(t, err) + require.True(t, hit) + return accounts +} + +func memoRequireMiss(t *testing.T, cache *schedulerCache, bucket service.SchedulerBucket) { + t.Helper() + accounts, hit, err := cache.GetSnapshot(context.Background(), bucket) + require.NoError(t, err) + require.False(t, hit) + require.Nil(t, accounts) +} + +func memoIDs(accounts []*service.Account) []int64 { + ids := make([]int64, 0, len(accounts)) + for _, account := range accounts { + ids = append(ids, account.ID) + } + return ids +} + +func memoRevision(t *testing.T, cache *schedulerCache, key string) string { + t.Helper() + revision, err := cache.rdb.Get(context.Background(), key).Result() + if errors.Is(err, redis.Nil) { + return "" + } + require.NoError(t, err) + return revision +} + +// memoRevisionValue 读取修订号，不做断言，供 require.Eventually 的条件函数使用。 +func memoRevisionValue(cache *schedulerCache, key string) string { + revision, _ := cache.rdb.Get(context.Background(), key).Result() + return revision +} + +var memoAnthropicRevisionKey = schedulerAccountMetaRevisionKeyForPlatform(service.PlatformAnthropic) + +func memoEntryCount(cache *schedulerCache) int { + cache.snapshotMu.RLock() + defer cache.snapshotMu.RUnlock() + return len(cache.snapshotEntries) +} + +func TestSchedulerCacheSnapshotMemoSecondReadOnlyFetchesTagsAndLastUsed(t *testing.T) { + cache, _, recorder := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(101) + memoPublish(t, cache, bucket, memoTestAccount(1, "a"), memoTestAccount(2, "b"), memoTestAccount(3, "c")) + + recorder.reset() + first := memoRead(t, cache, bucket) + require.Equal(t, 1, recorder.countPrefix("zrange ")) + require.Equal(t, 1, recorder.countPrefix("mget "+schedulerAccountMetaPrefix)) + + recorder.reset() + second := memoRead(t, cache, bucket) + require.Equal(t, []string{ + "mget " + schedulerBucketKey(schedulerReadyPrefix, bucket), + "mget " + schedulerLastUsedKey("1"), + }, recorder.recorded()) + + require.Equal(t, []int64{1, 2, 3}, memoIDs(second)) + require.Equal(t, first, second) + for i := range first { + require.NotSame(t, first[i], second[i]) + } + require.Equal(t, "sonnet-b", second[1].GetMappedModel("claude-sonnet")) + require.NotContains(t, second[1].Credentials, "access_token") +} + +func TestSchedulerCacheSnapshotMemoLastUsedIsFreshAndCopiesAreIndependent(t *testing.T) { + ctx := context.Background() + cache, _, recorder := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(102) + embedded := time.Now().UTC().Truncate(time.Millisecond).Add(-time.Hour) + account := memoTestAccount(11, "a") + account.LastUsedAt = &embedded + memoPublish(t, cache, bucket, account) + + first := memoRead(t, cache, bucket) + require.Equal(t, embedded, *first[0].LastUsedAt) + + latest := embedded.Add(42 * time.Second) + require.NoError(t, cache.UpdateLastUsed(ctx, map[int64]time.Time{account.ID: latest})) + recorder.reset() + second := memoRead(t, cache, bucket) + require.Zero(t, recorder.countPrefix("zrange "), "last_used updates must not invalidate the decoded snapshot") + require.Equal(t, latest, *second[0].LastUsedAt) + require.Equal(t, embedded, *first[0].LastUsedAt, "an earlier result must not observe later reads") + + // 修改返回对象的顶层字段只影响本次结果。 + second[0].Schedulable = false + second[0].Name = "mutated" + second[0].LastUsedAt = nil + second[0].Credentials = map[string]any{"model_mapping": map[string]any{"claude-sonnet": "replaced"}} + require.Equal(t, "replaced", second[0].GetMappedModel("claude-sonnet")) + + // 删除 last_used 侧键（不改精简记录）：若母本曾被套用 last_used，这里会读到 latest。 + require.NoError(t, cache.rdb.Del(ctx, schedulerLastUsedKey("11")).Err()) + recorder.reset() + third := memoRead(t, cache, bucket) + require.Zero(t, recorder.countPrefix("zrange ")) + require.Equal(t, embedded, *third[0].LastUsedAt) + require.True(t, third[0].Schedulable) + require.Equal(t, "a", third[0].Name) + require.Equal(t, "sonnet-a", third[0].GetMappedModel("claude-sonnet")) +} + +func TestSchedulerCacheSnapshotMemoSetAccountIsVisibleImmediately(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(103) + account := memoTestAccount(21, "a") + memoPublish(t, cache, bucket, account, memoTestAccount(22, "b")) + require.True(t, memoRead(t, cache, bucket)[0].Schedulable) + + globalBefore := memoRevision(t, cache, schedulerAccountMetaRevisionKey) + platformBefore := memoRevision(t, cache, memoAnthropicRevisionKey) + account.Schedulable = false + account.Extra = map[string]any{"session_window_utilization": 0.9} + require.NoError(t, cache.SetAccount(ctx, &account)) + require.NotEqual(t, platformBefore, memoRevision(t, cache, memoAnthropicRevisionKey)) + require.Equal(t, globalBefore, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) + + got := memoRead(t, cache, bucket) + require.Equal(t, []int64{21, 22}, memoIDs(got)) + require.False(t, got[0].Schedulable) + require.Equal(t, 0.9, got[0].Extra["session_window_utilization"]) + require.True(t, got[1].Schedulable) +} + +func TestSchedulerCacheSnapshotMemoDeleteAccountMisses(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(104) + memoPublish(t, cache, bucket, memoTestAccount(31, "a"), memoTestAccount(32, "b")) + memoRead(t, cache, bucket) + + revisionBefore := memoRevision(t, cache, schedulerAccountMetaRevisionKey) + require.NoError(t, cache.DeleteAccount(ctx, 32)) + require.NotEqual(t, revisionBefore, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) + memoRequireMiss(t, cache, bucket) + require.Zero(t, memoEntryCount(cache)) +} + +func TestSchedulerCacheSnapshotMemoCrossBucketMetadataRewrite(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucketA := memoTestBucket(105) + bucketB := memoTestBucket(106) + shared := memoTestAccount(41, "shared") + memoPublish(t, cache, bucketA, shared) + memoPublish(t, cache, bucketB, shared, memoTestAccount(42, "b-only")) + require.Equal(t, "shared", memoRead(t, cache, bucketB)[0].Name) + activeB := cache.rdb.Get(ctx, schedulerBucketKey(schedulerActivePrefix, bucketB)).Val() + + shared.Name = "shared-renamed" + shared.Priority = 9 + memoPublish(t, cache, bucketA, shared) + + require.Equal(t, activeB, cache.rdb.Get(ctx, schedulerBucketKey(schedulerActivePrefix, bucketB)).Val()) + got := memoRead(t, cache, bucketB) + require.Equal(t, []int64{41, 42}, memoIDs(got)) + require.Equal(t, "shared-renamed", got[0].Name) + require.Equal(t, 9, got[0].Priority) +} + +func TestSchedulerCacheSnapshotMemoUnencodableLastUsedDeletionMisses(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(107) + memoPublish(t, cache, bucket, memoTestAccount(51, "a"), memoTestAccount(52, "b")) + memoRead(t, cache, bucket) + + revisionBefore := memoRevision(t, cache, schedulerAccountMetaRevisionKey) + invalid := time.Date(10000, time.January, 1, 0, 0, 0, 0, time.UTC) + require.NoError(t, cache.UpdateLastUsed(ctx, map[int64]time.Time{52: invalid})) + require.NotEqual(t, revisionBefore, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) + memoRequireMiss(t, cache, bucket) + + // 正常的 last_used 写入不推进任何修订号。 + globalAfterDelete := memoRevision(t, cache, schedulerAccountMetaRevisionKey) + platformAfterDelete := memoRevision(t, cache, memoAnthropicRevisionKey) + require.NoError(t, cache.UpdateLastUsed(ctx, map[int64]time.Time{51: time.Now()})) + require.Equal(t, globalAfterDelete, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) + require.Equal(t, platformAfterDelete, memoRevision(t, cache, memoAnthropicRevisionKey)) +} + +func TestSchedulerCacheSnapshotMemoMembershipChangeByAccountIDs(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(108) + token, err := cache.CaptureBucketWriteToken(ctx, bucket) + require.NoError(t, err) + ids, err := cache.SetSnapshotAndReturnAccountIDs(ctx, bucket, token, []service.Account{memoTestAccount(61, "a"), memoTestAccount(62, "b")}) + require.NoError(t, err) + require.Equal(t, []int64{61, 62}, ids) + require.Equal(t, []int64{61, 62}, memoIDs(memoRead(t, cache, bucket))) + + globalBefore := memoRevision(t, cache, schedulerAccountMetaRevisionKey) + platformBefore := memoRevision(t, cache, memoAnthropicRevisionKey) + activeBefore := cache.rdb.Get(ctx, schedulerBucketKey(schedulerActivePrefix, bucket)).Val() + token, err = cache.CaptureBucketWriteToken(ctx, bucket) + require.NoError(t, err) + require.NoError(t, cache.SetSnapshotByAccountIDs(ctx, bucket, token, []int64{62})) + + require.Equal(t, globalBefore, memoRevision(t, cache, schedulerAccountMetaRevisionKey), "membership-only publishing does not touch metadata") + require.Equal(t, platformBefore, memoRevision(t, cache, memoAnthropicRevisionKey), "membership-only publishing does not touch metadata") + require.NotEqual(t, activeBefore, cache.rdb.Get(ctx, schedulerBucketKey(schedulerActivePrefix, bucket)).Val()) + require.Equal(t, []int64{62}, memoIDs(memoRead(t, cache, bucket))) +} + +func TestSchedulerCacheSnapshotMemoRetireReopenAndRepublish(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(109) + memoPublish(t, cache, bucket, memoTestAccount(71, "before")) + memoRead(t, cache, bucket) + + require.NoError(t, cache.RetireBucket(ctx, bucket)) + memoRequireMiss(t, cache, bucket) + require.Zero(t, memoEntryCount(cache)) + + token, err := cache.ReopenBucket(ctx, bucket) + require.NoError(t, err) + memoRequireMiss(t, cache, bucket) + + require.NoError(t, cache.SetSnapshot(ctx, bucket, token, []service.Account{memoTestAccount(72, "after")})) + got := memoRead(t, cache, bucket) + require.Equal(t, []int64{72}, memoIDs(got)) + require.Equal(t, "after", got[0].Name) +} + +func TestSchedulerCacheSnapshotMemoEmptyAndNilMetadataAreNotCached(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + + t.Run("empty zset", func(t *testing.T) { + bucket := memoTestBucket(110) + _, err := cache.writeAccountIDs(ctx, []service.Account{memoTestAccount(81, "a")}) + require.NoError(t, err) + memoPublish(t, cache, bucket) + memoRequireMiss(t, cache, bucket) + + active := cache.rdb.Get(ctx, schedulerBucketKey(schedulerActivePrefix, bucket)).Val() + require.NotEmpty(t, active) + require.NoError(t, cache.rdb.ZAdd(ctx, schedulerSnapshotKey(bucket, active), redis.Z{Score: 0, Member: "81"}).Err()) + require.Equal(t, []int64{81}, memoIDs(memoRead(t, cache, bucket))) + }) + + t.Run("nil metadata", func(t *testing.T) { + bucket := memoTestBucket(111) + memoPublish(t, cache, bucket, memoTestAccount(91, "a"), memoTestAccount(92, "b")) + metaKey := schedulerAccountMetaKey("92") + payload, err := cache.rdb.Get(ctx, metaKey).Bytes() + require.NoError(t, err) + require.NoError(t, cache.rdb.Del(ctx, metaKey).Err()) + memoRequireMiss(t, cache, bucket) + + require.NoError(t, cache.rdb.Set(ctx, metaKey, payload, 0).Err()) + require.Equal(t, []int64{91, 92}, memoIDs(memoRead(t, cache, bucket))) + }) +} + +func TestSchedulerCacheSnapshotMemoDecodeFailureIsNotCached(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(112) + memoPublish(t, cache, bucket, memoTestAccount(101, "a")) + metaKey := schedulerAccountMetaKey("101") + payload, err := cache.rdb.Get(ctx, metaKey).Bytes() + require.NoError(t, err) + require.NoError(t, cache.rdb.Set(ctx, metaKey, "{not-json", 0).Err()) + + accounts, hit, err := cache.GetSnapshot(ctx, bucket) + require.Error(t, err) + require.False(t, hit) + require.Nil(t, accounts) + require.Zero(t, memoEntryCount(cache)) + + require.NoError(t, cache.rdb.Set(ctx, metaKey, payload, 0).Err()) + require.Equal(t, "a", memoRead(t, cache, bucket)[0].Name) +} + +func TestSchedulerCacheSnapshotMemoRedisErrorIsReturned(t *testing.T) { + cache, mr, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(113) + memoPublish(t, cache, bucket, memoTestAccount(111, "a")) + memoRead(t, cache, bucket) + + mr.SetError("injected failure") + accounts, hit, err := cache.GetSnapshot(context.Background(), bucket) + require.Error(t, err) + require.False(t, hit) + require.Nil(t, accounts) + + mr.SetError("") + require.Equal(t, []int64{111}, memoIDs(memoRead(t, cache, bucket))) +} + +func TestSchedulerCacheSnapshotMemoConcurrentReadersAndWriters(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(114) + accounts := []service.Account{memoTestAccount(121, "a"), memoTestAccount(122, "b"), memoTestAccount(123, "c")} + memoPublish(t, cache, bucket, accounts...) + + var wg sync.WaitGroup + for reader := 0; reader < 8; reader++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 40; i++ { + got, hit, err := cache.GetSnapshot(ctx, bucket) + if err != nil || !hit { + t.Errorf("GetSnapshot: hit=%v err=%v", hit, err) + return + } + for _, account := range got { + if account.GetModelMapping()["claude-sonnet"] == "" { + t.Errorf("account %d lost its model mapping", account.ID) + return + } + _ = account.IsSchedulableForModel("claude-sonnet") + _ = account.GetHeaderOverrides() + account.Schedulable = !account.Schedulable + account.LastUsedAt = nil + } + } + }() + } + for writer := 0; writer < 2; writer++ { + wg.Add(1) + go func(writer int) { + defer wg.Done() + for i := 0; i < 20; i++ { + account := memoTestAccount(int64(121+writer), fmt.Sprintf("w%d-%d", writer, i)) + account.Extra = map[string]any{"session_window_utilization": float64(i) / 100} + if err := cache.SetAccount(ctx, &account); err != nil { + t.Errorf("SetAccount: %v", err) + return + } + if err := cache.UpdateLastUsed(ctx, map[int64]time.Time{account.ID: time.Now()}); err != nil { + t.Errorf("UpdateLastUsed: %v", err) + return + } + } + }(writer) + } + wg.Wait() + + got := memoRead(t, cache, bucket) + require.Equal(t, "w0-19", got[0].Name) + require.Equal(t, "w1-19", got[1].Name) + require.Equal(t, "c", got[2].Name) +} + +func TestSchedulerCacheSnapshotMemoWriteAccountIDsAdvancesRevision(t *testing.T) { + ctx := context.Background() + geminiRevisionKey := schedulerAccountMetaRevisionKeyForPlatform(service.PlatformGemini) + + t.Run("revisions share the last chunk", func(t *testing.T) { + cache, _, recorder := newSchedulerCacheMemoUnit(t, 2) + accounts := []service.Account{ + memoTestAccount(131, "a"), + memoTestAccountOn(service.PlatformGemini, 132, "b"), + memoTestAccount(133, "c"), + } + recorder.reset() + ids, err := cache.writeAccountIDs(ctx, accounts) + require.NoError(t, err) + require.Equal(t, []int64{131, 132, 133}, ids) + + batches := recorder.pipelineBatches() + require.Len(t, batches, 2, "one execution per chunk") + require.Equal(t, []string{ + "set " + schedulerAccountKey("133"), + "set " + schedulerAccountMetaKey("133"), + "set " + memoAnthropicRevisionKey, + "set " + geminiRevisionKey, + }, batches[1]) + require.NotContains(t, strings.Join(batches[0], "\n"), schedulerAccountMetaRevisionKey) + require.Equal(t, 2, recorder.countPrefix("set "+schedulerAccountMetaRevisionKey), "no revision write outside the last chunk") + require.Empty(t, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) + first := memoRevision(t, cache, memoAnthropicRevisionKey) + require.Len(t, first, schedulerAccountMetaRevisionBytes*2) + require.Len(t, memoRevision(t, cache, geminiRevisionKey), schedulerAccountMetaRevisionBytes*2) + + _, err = cache.writeAccountIDs(ctx, accounts[:1]) + require.NoError(t, err) + require.NotEqual(t, first, memoRevision(t, cache, memoAnthropicRevisionKey)) + }) + + t.Run("account count is a multiple of the chunk size", func(t *testing.T) { + cache, _, recorder := newSchedulerCacheMemoUnit(t, 2) + accounts := []service.Account{ + memoTestAccount(141, "a"), memoTestAccount(142, "b"), + memoTestAccount(143, "c"), memoTestAccount(144, "d"), + } + recorder.reset() + _, err := cache.writeAccountIDs(ctx, accounts) + require.NoError(t, err) + + batches := recorder.pipelineBatches() + require.Len(t, batches, 2, "no extra execution for the revision") + require.Equal(t, []string{ + "set " + schedulerAccountKey("143"), + "set " + schedulerAccountMetaKey("143"), + "set " + schedulerAccountKey("144"), + "set " + schedulerAccountMetaKey("144"), + "set " + memoAnthropicRevisionKey, + }, batches[1]) + require.Len(t, recorder.recorded(), 9, "every command travels inside the two pipelines") + }) + + t.Run("failure in a later chunk advances the global revision", func(t *testing.T) { + cache, _, recorder := newSchedulerCacheMemoUnit(t, 1) + accounts := []service.Account{memoTestAccount(151, "a"), memoTestAccount(152, "b"), memoTestAccount(153, "c")} + bucket := memoTestBucket(115) + memoPublish(t, cache, bucket, accounts...) + memoRead(t, cache, bucket) + globalBefore := memoRevision(t, cache, schedulerAccountMetaRevisionKey) + platformBefore := memoRevision(t, cache, memoAnthropicRevisionKey) + + renamed := []service.Account{memoTestAccount(151, "a2"), memoTestAccount(152, "b2"), memoTestAccount(153, "c2")} + recorder.reset() + recorder.setFailPipelineAt(2) + _, err := cache.writeAccountIDs(ctx, renamed) + require.Error(t, err) + recorder.setFailPipelineAt(0) + require.Eventually(t, func() bool { + return memoRevisionValue(cache, schedulerAccountMetaRevisionKey) != globalBefore + }, 2*time.Second, 5*time.Millisecond, "the global revision is advanced in the background") + require.Equal(t, platformBefore, memoRevision(t, cache, memoAnthropicRevisionKey)) + + recorder.reset() + got := memoRead(t, cache, bucket) + require.Equal(t, 1, recorder.countPrefix("zrange "), "the cached snapshot must be reloaded") + require.Equal(t, []string{"a2", "b", "c"}, []string{got[0].Name, got[1].Name, got[2].Name}) + }) + + t.Run("failure in the last chunk advances the global revision", func(t *testing.T) { + cache, _, recorder := newSchedulerCacheMemoUnit(t, 2) + recorder.reset() + recorder.setFailPipelineAt(1) + _, err := cache.writeAccountIDs(ctx, []service.Account{memoTestAccount(161, "a")}) + require.Error(t, err) + recorder.setFailPipelineAt(0) + require.Eventually(t, func() bool { + return memoRevisionValue(cache, schedulerAccountMetaRevisionKey) != "" + }, 2*time.Second, 5*time.Millisecond, "the global revision is advanced in the background") + require.Empty(t, memoRevision(t, cache, memoAnthropicRevisionKey)) + }) + + t.Run("nothing encodable", func(t *testing.T) { + cache, _, recorder := newSchedulerCacheMemoUnit(t, 1) + invalid := time.Date(10000, time.January, 1, 0, 0, 0, 0, time.UTC) + recorder.reset() + ids, err := cache.writeAccountIDs(ctx, []service.Account{{ID: 171, Platform: service.PlatformAnthropic, ExpiresAt: &invalid}}) + require.NoError(t, err) + require.Empty(t, ids) + require.Empty(t, recorder.recorded()) + require.Empty(t, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) + require.Empty(t, memoRevision(t, cache, memoAnthropicRevisionKey)) + }) + + t.Run("empty platform uses the global revision", func(t *testing.T) { + cache, _, _ := newSchedulerCacheMemoUnit(t, 2) + _, err := cache.writeAccountIDs(ctx, []service.Account{{ID: 181, Name: "no-platform"}}) + require.NoError(t, err) + require.NotEmpty(t, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) + }) +} + +// memoReadExpectHit 读取一次并报告是否重新加载（发出过 ZRANGE）。 +func memoReadExpectHit(t *testing.T, cache *schedulerCache, recorder *schedulerCacheCommandRecorder, bucket service.SchedulerBucket) ([]*service.Account, bool) { + t.Helper() + recorder.reset() + accounts := memoRead(t, cache, bucket) + return accounts, recorder.countPrefix("zrange ") > 0 +} + +func TestSchedulerCacheSnapshotMemoPlatformRevisionScopesInvalidation(t *testing.T) { + ctx := context.Background() + cache, _, recorder := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + geminiSingle := service.SchedulerBucket{GroupID: 201, Platform: service.PlatformGemini, Mode: service.SchedulerModeSingle} + gemini := memoTestAccountOn(service.PlatformGemini, 211, "gemini") + memoPublish(t, cache, geminiSingle, gemini) + _, reloaded := memoReadExpectHit(t, cache, recorder, geminiSingle) + require.True(t, reloaded) + + anthropic := memoTestAccount(212, "anthropic") + require.NoError(t, cache.SetAccount(ctx, &anthropic)) + recorder.reset() + memoRead(t, cache, geminiSingle) + require.Equal(t, []string{ + "mget " + schedulerBucketKey(schedulerReadyPrefix, geminiSingle), + "mget " + schedulerLastUsedKey("211"), + }, recorder.recorded(), "an Anthropic account write must not invalidate a Gemini bucket") + + gemini.Name = "gemini-renamed" + require.NoError(t, cache.SetAccount(ctx, &gemini)) + got, reloaded := memoReadExpectHit(t, cache, recorder, geminiSingle) + require.True(t, reloaded) + require.Equal(t, "gemini-renamed", got[0].Name) +} + +func TestSchedulerCacheSnapshotMemoPlatformChangeInvalidatesOldPlatformBuckets(t *testing.T) { + ctx := context.Background() + cache, _, recorder := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + geminiSingle := service.SchedulerBucket{GroupID: 202, Platform: service.PlatformGemini, Mode: service.SchedulerModeSingle} + moved := memoTestAccountOn(service.PlatformGemini, 221, "moved") + memoPublish(t, cache, geminiSingle, moved) + globalBefore := memoRevision(t, cache, schedulerAccountMetaRevisionKey) + + moved.Name = "same-platform" + require.NoError(t, cache.SetAccount(ctx, &moved)) + require.Equal(t, globalBefore, memoRevision(t, cache, schedulerAccountMetaRevisionKey), "a same-platform rewrite only advances its platform revision") + memoRead(t, cache, geminiSingle) + + moved.Platform = service.PlatformAnthropic + moved.Name = "now-anthropic" + require.NoError(t, cache.SetAccount(ctx, &moved)) + require.NotEqual(t, globalBefore, memoRevision(t, cache, schedulerAccountMetaRevisionKey), "a platform change advances the global revision") + + got, reloaded := memoReadExpectHit(t, cache, recorder, geminiSingle) + require.True(t, reloaded, "the old platform bucket must drop the cached account") + require.Equal(t, service.PlatformAnthropic, got[0].Platform) + require.Equal(t, "now-anthropic", got[0].Name) +} + +func TestSchedulerCacheSnapshotMemoNewAccountOnlyAdvancesPlatformRevision(t *testing.T) { + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + fresh := memoTestAccountOn(service.PlatformGemini, 231, "fresh") + require.NoError(t, cache.SetAccount(context.Background(), &fresh)) + require.Empty(t, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) + require.NotEmpty(t, memoRevision(t, cache, schedulerAccountMetaRevisionKeyForPlatform(service.PlatformGemini))) +} + +func TestSchedulerCacheSnapshotMemoAntigravityRevisionInvalidatesMixedBuckets(t *testing.T) { + ctx := context.Background() + cache, _, recorder := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + antigravity := memoTestAccountOn(service.PlatformAntigravity, 221, "ag") + antigravity.Extra = map[string]any{"mixed_scheduling": true} + gemini := memoTestAccountOn(service.PlatformGemini, 222, "gemini") + anthropic := memoTestAccount(223, "anthropic") + + geminiMixed := service.SchedulerBucket{GroupID: 202, Platform: service.PlatformGemini, Mode: service.SchedulerModeMixed} + anthropicMixed := service.SchedulerBucket{GroupID: 202, Platform: service.PlatformAnthropic, Mode: service.SchedulerModeMixed} + geminiSingle := service.SchedulerBucket{GroupID: 202, Platform: service.PlatformGemini, Mode: service.SchedulerModeSingle} + geminiForced := service.SchedulerBucket{GroupID: 202, Platform: service.PlatformGemini, Mode: service.SchedulerModeForced} + memoPublish(t, cache, geminiMixed, gemini, antigravity) + memoPublish(t, cache, anthropicMixed, anthropic, antigravity) + memoPublish(t, cache, geminiSingle, gemini) + memoPublish(t, cache, geminiForced, gemini) + for _, bucket := range []service.SchedulerBucket{geminiMixed, anthropicMixed, geminiSingle, geminiForced} { + memoRead(t, cache, bucket) + } + + antigravity.Name = "ag-renamed" + require.NoError(t, cache.SetAccount(ctx, &antigravity)) + + got, reloaded := memoReadExpectHit(t, cache, recorder, geminiMixed) + require.True(t, reloaded) + require.Equal(t, "ag-renamed", got[1].Name) + got, reloaded = memoReadExpectHit(t, cache, recorder, anthropicMixed) + require.True(t, reloaded) + require.Equal(t, "ag-renamed", got[1].Name) + _, reloaded = memoReadExpectHit(t, cache, recorder, geminiSingle) + require.False(t, reloaded, "single buckets never contain Antigravity accounts") + _, reloaded = memoReadExpectHit(t, cache, recorder, geminiForced) + require.False(t, reloaded, "forced buckets never contain Antigravity accounts") +} + +func TestSchedulerCacheSnapshotMemoGlobalRevisionInvalidatesEveryPlatform(t *testing.T) { + ctx := context.Background() + buckets := []service.SchedulerBucket{ + {GroupID: 203, Platform: service.PlatformGemini, Mode: service.SchedulerModeSingle}, + {GroupID: 203, Platform: service.PlatformAnthropic, Mode: service.SchedulerModeSingle}, + {GroupID: 203, Platform: service.PlatformOpenAI, Mode: service.SchedulerModeSingle}, + } + setup := func(t *testing.T) (*schedulerCache, *schedulerCacheCommandRecorder) { + cache, _, recorder := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + for i, bucket := range buckets { + memoPublish(t, cache, bucket, memoTestAccountOn(bucket.Platform, int64(231+i), bucket.Platform)) + memoRead(t, cache, bucket) + } + // 不属于任何上述 bucket 的账号。 + unrelated := memoTestAccountOn(service.PlatformGrok, 239, "grok") + require.NoError(t, cache.SetAccount(ctx, &unrelated)) + for _, bucket := range buckets { + _, reloaded := memoReadExpectHit(t, cache, recorder, bucket) + require.False(t, reloaded) + } + return cache, recorder + } + + t.Run("DeleteAccount", func(t *testing.T) { + cache, recorder := setup(t) + require.NoError(t, cache.DeleteAccount(ctx, 239)) + for _, bucket := range buckets { + _, reloaded := memoReadExpectHit(t, cache, recorder, bucket) + require.True(t, reloaded, bucket.String()) + } + }) + + t.Run("UpdateLastUsed unencodable deletion", func(t *testing.T) { + cache, recorder := setup(t) + invalid := time.Date(10000, time.January, 1, 0, 0, 0, 0, time.UTC) + require.NoError(t, cache.UpdateLastUsed(ctx, map[int64]time.Time{239: invalid})) + for _, bucket := range buckets { + _, reloaded := memoReadExpectHit(t, cache, recorder, bucket) + require.True(t, reloaded, bucket.String()) + } + }) +} + +func TestSchedulerCacheSnapshotMemoFlushAllAndRepublishSameVersion(t *testing.T) { + ctx := context.Background() + cache, mr, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(116) + memoPublish(t, cache, bucket, memoTestAccount(141, "before")) + require.Equal(t, "before", memoRead(t, cache, bucket)[0].Name) + activeBefore := cache.rdb.Get(ctx, schedulerBucketKey(schedulerActivePrefix, bucket)).Val() + + mr.FlushAll() + memoPublish(t, cache, bucket, memoTestAccount(141, "after")) + require.Equal(t, activeBefore, cache.rdb.Get(ctx, schedulerBucketKey(schedulerActivePrefix, bucket)).Val()) + require.Equal(t, "after", memoRead(t, cache, bucket)[0].Name) +} + +type memoTestClock struct { + mu sync.Mutex + now time.Time +} + +func (c *memoTestClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *memoTestClock) Advance(d time.Duration) { + c.mu.Lock() + c.now = c.now.Add(d) + c.mu.Unlock() +} + +func TestSchedulerCacheSnapshotMemoMaxAgeBoundsUnversionedWrites(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + clock := &memoTestClock{now: time.Date(2026, 10, 20, 8, 0, 0, 0, time.UTC)} + cache.now = clock.Now + bucket := memoTestBucket(117) + memoPublish(t, cache, bucket, memoTestAccount(151, "old")) + require.Equal(t, "old", memoRead(t, cache, bucket)[0].Name) + + // 模拟不推进修订号的旧实例：直接改写精简记录。 + _, metaPayload, err := marshalSchedulerCacheAccount(memoTestAccount(151, "new")) + require.NoError(t, err) + globalBefore := memoRevision(t, cache, schedulerAccountMetaRevisionKey) + platformBefore := memoRevision(t, cache, memoAnthropicRevisionKey) + require.NoError(t, cache.rdb.Set(ctx, schedulerAccountMetaKey("151"), metaPayload, 0).Err()) + require.Equal(t, globalBefore, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) + require.Equal(t, platformBefore, memoRevision(t, cache, memoAnthropicRevisionKey)) + + clock.Advance(schedulerSnapshotCacheMaxAge - time.Millisecond) + require.Equal(t, "old", memoRead(t, cache, bucket)[0].Name) + + clock.Advance(time.Millisecond) + require.Equal(t, "new", memoRead(t, cache, bucket)[0].Name) +} + +func TestSchedulerCacheSnapshotMemoEvictsLongIdleEntries(t *testing.T) { + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + clock := &memoTestClock{now: time.Date(2026, 10, 20, 8, 0, 0, 0, time.UTC)} + cache.now = clock.Now + idle := memoTestBucket(118) + recent := memoTestBucket(119) + active := memoTestBucket(120) + memoPublish(t, cache, idle, memoTestAccount(161, "idle")) + memoPublish(t, cache, recent, memoTestAccount(162, "recent")) + memoPublish(t, cache, active, memoTestAccount(163, "active")) + + memoRead(t, cache, idle) + clock.Advance(schedulerSnapshotCacheEvictAge / 2) + memoRead(t, cache, recent) + clock.Advance(schedulerSnapshotCacheEvictAge/2 + time.Second) + memoRead(t, cache, active) + + cache.snapshotMu.RLock() + defer cache.snapshotMu.RUnlock() + require.NotContains(t, cache.snapshotEntries, idle.String()) + require.Contains(t, cache.snapshotEntries, recent.String()) + require.Contains(t, cache.snapshotEntries, active.String()) +} + +func TestSchedulerCacheSnapshotMemoKeepsNewerEntryOnLateStore(t *testing.T) { + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + base := time.Date(2026, 10, 20, 8, 0, 0, 0, time.UTC) + newer := &schedulerSnapshotEntry{active: "2", revisions: []string{"r2"}, loadedAt: base.Add(time.Second)} + older := &schedulerSnapshotEntry{active: "1", revisions: []string{"r1"}, loadedAt: base} + cache.storeSnapshotEntry("b", newer) + cache.storeSnapshotEntry("b", older) + require.Same(t, newer, cache.lookupSnapshotEntry("b")) +} + +// 母本在发布前预解析了模型映射：副本命中时不计算签名。此处故意原地改写共享的原始映射， +// 只用来观察副本走的是免签名路径（生产代码不会原地修改该 map）。 +func TestSchedulerCacheSnapshotMemoPreparesSharedModelMapping(t *testing.T) { + cache, _, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + bucket := memoTestBucket(121) + memoPublish(t, cache, bucket, memoTestAccount(171, "a")) + memoRead(t, cache, bucket) + + hit := memoRead(t, cache, bucket)[0] + require.Equal(t, "sonnet-a", hit.GetMappedModel("claude-sonnet")) + + replaced := memoRead(t, cache, bucket)[0] + replaced.Credentials = map[string]any{"model_mapping": map[string]any{"claude-sonnet": "sonnet-replaced"}} + require.Equal(t, "sonnet-replaced", replaced.GetMappedModel("claude-sonnet")) + + probe := memoRead(t, cache, bucket)[0] + rawMapping, ok := probe.Credentials["model_mapping"].(map[string]any) + require.True(t, ok) + rawMapping["claude-sonnet"] = "sonnet-mutated-in-place" + require.Equal(t, "sonnet-a", probe.GetMappedModel("claude-sonnet")) +} + +func TestSchedulerCacheSnapshotMemoRevisionValuesAreUnique(t *testing.T) { + seen := make(map[string]struct{}, 64) + for i := 0; i < 64; i++ { + revision := newSchedulerAccountMetaRevision() + require.Len(t, revision, schedulerAccountMetaRevisionBytes*2) + _, err := strconv.ParseUint(revision[:8], 16, 32) + require.NoError(t, err) + _, dup := seen[revision] + require.False(t, dup) + seen[revision] = struct{}{} + } +} + +func TestSchedulerCacheSnapshotMemoPlatformChangeInEarlierChunk(t *testing.T) { + ctx := context.Background() + cache, _, _ := newSchedulerCacheMemoUnit(t, 1) + require.NoError(t, cache.SetAccount(ctx, ptrAccount(memoTestAccountOn(service.PlatformGemini, 241, "moved")))) + require.NoError(t, cache.SetAccount(ctx, ptrAccount(memoTestAccount(242, "stays")))) + globalBefore := memoRevision(t, cache, schedulerAccountMetaRevisionKey) + + _, err := cache.writeAccountIDs(ctx, []service.Account{ + memoTestAccount(241, "moved"), + memoTestAccount(242, "stays"), + }) + require.NoError(t, err) + require.NotEqual(t, globalBefore, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) +} + +func TestSchedulerCacheSnapshotMemoUnparsableOldMetadataAdvancesGlobalRevision(t *testing.T) { + ctx := context.Background() + cache, mr, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + require.NoError(t, mr.Set(schedulerAccountMetaKey("251"), "not-json")) + + require.NoError(t, cache.SetAccount(ctx, ptrAccount(memoTestAccount(251, "replaced")))) + require.NotEmpty(t, memoRevision(t, cache, schedulerAccountMetaRevisionKey)) +} + +func TestSchedulerCacheSnapshotMemoNilBeforeRealPipelineError(t *testing.T) { + ctx := context.Background() + cache, mr, _ := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + mr.HSet(schedulerAccountMetaKey("262"), "field", "value") + + _, err := cache.writeAccountIDs(ctx, []service.Account{ + memoTestAccount(261, "fresh"), + memoTestAccount(262, "wrong-type"), + }) + require.Error(t, err, "a WRONGTYPE reply after a nil SET ... GET reply must fail the write") + require.Eventually(t, func() bool { + return memoRevisionValue(cache, schedulerAccountMetaRevisionKey) != "" + }, 2*time.Second, 5*time.Millisecond, "the global revision is advanced in the background") +} + +func TestSchedulerCacheSnapshotMemoCoalescesBackgroundGlobalRevisionWrites(t *testing.T) { + cache, _, recorder := newSchedulerCacheMemoUnit(t, defaultSchedulerSnapshotWriteChunkSize) + recorder.reset() + for range 50 { + cache.scheduleGlobalAccountMetaRevision(context.Background()) + } + require.Eventually(t, func() bool { + return !cache.globalRevisionWriting.Load() && memoRevisionValue(cache, schedulerAccountMetaRevisionKey) != "" + }, 2*time.Second, 5*time.Millisecond) + writes := 0 + for _, command := range recorder.recorded() { + if command == "set "+schedulerAccountMetaRevisionKey { + writes++ + } + } + require.GreaterOrEqual(t, writes, 1) + require.LessOrEqual(t, writes, 3, "concurrent schedules collapse into at most a couple of background writes") +} + +func ptrAccount(account service.Account) *service.Account { + return &account +} diff --git a/backend/internal/service/account.go b/backend/internal/service/account.go index 06b94b754e5d..27a9bbccc8a5 100644 --- a/backend/internal/service/account.go +++ b/backend/internal/service/account.go @@ -76,6 +76,9 @@ type Account struct { modelMappingCacheRawLen int modelMappingCacheRawSig uint64 modelMappingCacheRuntimeVersion uint64 + // modelMappingCacheShared 为 true 时缓存来自只读共享母本（见 PrepareSharedSnapshot）， + // 指针、长度与运行时版本匹配即直接复用预解析结果，不计算签名。 + modelMappingCacheShared bool // header_overrides 热路径缓存（非持久化字段，同 model_mapping 缓存先例） headerOverrideCache map[string]string @@ -635,6 +638,9 @@ func (a *Account) GetModelMapping() map[string]string { a.modelMappingCacheRawPtr == rawPtr && a.modelMappingCacheRawLen == rawLen && a.modelMappingCacheRuntimeVersion == runtimeVersion { + if a.modelMappingCacheShared { + return a.modelMappingCache + } rawSig = modelMappingSignature(rawMapping) rawSigReady = true if a.modelMappingCacheRawSig == rawSig { @@ -654,9 +660,24 @@ func (a *Account) GetModelMapping() map[string]string { a.modelMappingCacheRawLen = rawLen a.modelMappingCacheRawSig = rawSig a.modelMappingCacheRuntimeVersion = runtimeVersion + a.modelMappingCacheShared = false return mapping } +// PrepareSharedSnapshot 预解析模型映射，并把账号标记为只读共享母本。 +// 母本按值复制出的副本与母本共用 Credentials 及其中 model_mapping 的同一 map： +// 这些内层 map 在运行期不被原地修改，母本留在缓存期间持有这些 map，其地址不会被复用， +// 因此副本在指针、长度与运行时版本都匹配时直接复用预解析结果，不计算签名； +// 副本替换 Credentials 等导致任一项不匹配时回到普通的重建路径。 +// 只能由构造该账号的 goroutine 在其发布给其它 goroutine 之前调用，发布后母本不得再被修改。 +func (a *Account) PrepareSharedSnapshot() { + if a == nil { + return + } + a.GetModelMapping() + a.modelMappingCacheShared = true +} + func (a *Account) resolveModelMapping(rawMapping map[string]any) map[string]string { if a.Credentials == nil { // Antigravity 平台使用默认映射 diff --git a/backend/internal/service/account_shared_snapshot_test.go b/backend/internal/service/account_shared_snapshot_test.go new file mode 100644 index 000000000000..2d3421193d6d --- /dev/null +++ b/backend/internal/service/account_shared_snapshot_test.go @@ -0,0 +1,82 @@ +//go:build unit + +package service + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// 共享母本的副本命中预解析缓存时不计算签名：此处故意原地改写共享的原始映射， +// 只用来观察副本走的是免签名路径（生产代码不会原地修改该 map）。 +func TestAccountPrepareSharedSnapshot_CopiesReusePreparedMappingWithoutSignature(t *testing.T) { + rawMapping := map[string]any{"claude-sonnet": "sonnet-a"} + master := &Account{ + Platform: PlatformAnthropic, + Credentials: map[string]any{"model_mapping": rawMapping}, + } + master.PrepareSharedSnapshot() + + replica := *master + require.Equal(t, map[string]string{"claude-sonnet": "sonnet-a"}, replica.GetModelMapping()) + + rawMapping["claude-sonnet"] = "sonnet-b" + require.Equal(t, "sonnet-a", replica.GetModelMapping()["claude-sonnet"], + "the shared fast path must not recompute the signature") + require.Equal(t, "sonnet-a", replica.GetMappedModel("claude-sonnet")) + + // 未标记为共享的账号仍按签名检出原地改写。 + plainRaw := map[string]any{"claude-sonnet": "sonnet-a"} + plain := &Account{ + Platform: PlatformAnthropic, + Credentials: map[string]any{"model_mapping": plainRaw}, + } + require.Equal(t, "sonnet-a", plain.GetModelMapping()["claude-sonnet"]) + plainRaw["claude-sonnet"] = "sonnet-b" + require.Equal(t, "sonnet-b", plain.GetModelMapping()["claude-sonnet"]) +} + +func TestAccountPrepareSharedSnapshot_ReplacedCredentialsFallBackToSignaturePath(t *testing.T) { + master := &Account{ + Platform: PlatformAnthropic, + Credentials: map[string]any{"model_mapping": map[string]any{"claude-sonnet": "sonnet-a"}}, + } + master.PrepareSharedSnapshot() + + replica := *master + replacedRaw := map[string]any{"claude-sonnet": "sonnet-b"} + replica.Credentials = map[string]any{"model_mapping": replacedRaw} + require.Equal(t, "sonnet-b", replica.GetModelMapping()["claude-sonnet"]) + + // 重建后副本回到普通路径：同一 map 的原地改写由签名检出。 + replacedRaw["claude-sonnet"] = "sonnet-c" + require.Equal(t, "sonnet-c", replica.GetModelMapping()["claude-sonnet"]) + + // 母本不受副本重建影响。 + require.Equal(t, "sonnet-a", master.GetModelMapping()["claude-sonnet"]) +} + +func TestAccountPrepareSharedSnapshot_MappingLengthChangeRebuilds(t *testing.T) { + rawMapping := map[string]any{"claude-sonnet": "sonnet-a"} + master := &Account{ + Platform: PlatformAnthropic, + Credentials: map[string]any{"model_mapping": rawMapping}, + } + master.PrepareSharedSnapshot() + + replica := *master + rawMapping["claude-opus"] = "opus-a" + require.Equal(t, "opus-a", replica.GetModelMapping()["claude-opus"]) +} + +func TestAccountPrepareSharedSnapshot_NilAccountAndNoMapping(t *testing.T) { + var nilAccount *Account + require.NotPanics(t, nilAccount.PrepareSharedSnapshot) + + master := &Account{Platform: PlatformAnthropic} + master.PrepareSharedSnapshot() + replica := *master + require.Nil(t, replica.GetModelMapping()) + require.Equal(t, "claude-sonnet", replica.GetMappedModel("claude-sonnet")) +} diff --git a/backend/internal/service/model_rate_limit_snapshot_cow_test.go b/backend/internal/service/model_rate_limit_snapshot_cow_test.go index 97e35c296ef0..a73fcff5fe9a 100644 --- a/backend/internal/service/model_rate_limit_snapshot_cow_test.go +++ b/backend/internal/service/model_rate_limit_snapshot_cow_test.go @@ -91,6 +91,7 @@ func TestSetAccountModelRateLimitSnapshot_ConcurrentCopiesOfSharedMaster(t *test "session_window_utilization": 0.2, }, } + master.PrepareSharedSnapshot() const workers = 16 var wg sync.WaitGroup @@ -170,6 +171,7 @@ func TestApplyAccountSchedulingThreshold_FableLimitStaysOnOwnSnapshotCopy(t *tes "passive_usage_7d_oi_reset": float64(now.Add(4 * 24 * time.Hour).Unix()), }, } + master.PrepareSharedSnapshot() const pairs = 8 var wg sync.WaitGroup diff --git a/backend/internal/service/scheduler_cache.go b/backend/internal/service/scheduler_cache.go index d507d6ce2ad2..5e5dbed1643b 100644 --- a/backend/internal/service/scheduler_cache.go +++ b/backend/internal/service/scheduler_cache.go @@ -76,6 +76,8 @@ func ParseSchedulerBucket(raw string) (SchedulerBucket, bool) { // SchedulerCache 负责调度快照与账号快照的缓存读写。 type SchedulerCache interface { // GetSnapshot 读取快照并返回命中与否（ready + active + 数据完整）。 + // 返回的账号对象由本次调用独占，可以修改其顶层字段；map、切片底层数组与指针指向的值 + // 与其它调用共享，不得原地修改。 GetSnapshot(ctx context.Context, bucket SchedulerBucket) ([]*Account, bool, error) // CaptureBucketWriteToken captures the current open epoch without changing // retirement state. A tombstoned bucket returns ErrSchedulerBucketRetired. From e0fa4b94f9b962949da039c09e63968358324e26 Mon Sep 17 00:00:00 2001 From: wucm667 Date: Wed, 21 Oct 2026 02:34:30 +0800 Subject: [PATCH 18/29] fix(ratelimit): avoid penalizing Cloudflare 1010 blocks --- backend/internal/service/ratelimit_service.go | 78 +++++++++++++++---- .../ratelimit_service_403_html_test.go | 65 +++++++++++++++- 2 files changed, 128 insertions(+), 15 deletions(-) diff --git a/backend/internal/service/ratelimit_service.go b/backend/internal/service/ratelimit_service.go index 7aa64632c404..2f2f70564008 100644 --- a/backend/internal/service/ratelimit_service.go +++ b/backend/internal/service/ratelimit_service.go @@ -1003,10 +1003,19 @@ func buildForbiddenErrorMessage(prefix string, upstreamMsg string, responseBody return prefix + fallback } -// handle403 处理 403 Forbidden 错误 -// Antigravity 平台区分 validation/violation/generic 三种类型，均 SetError 永久禁用； -// 其他平台保持原有 SetError 行为。 +// handle403 处理 403 Forbidden 错误。Cloudflare 1010 是请求/边缘级拦截， +// 不应将其视为账号级故障；其他真实 403 继续按平台策略处理。 func (s *RateLimitService) handle403(ctx context.Context, account *Account, upstreamMsg string, responseBody []byte) (shouldDisable bool) { + if isCloudflareBotBlockResponse(responseBody) { + slog.Warn( + "403_cloudflare_bot_block_skips_account_penalty", + "account_id", account.ID, + "platform", account.Platform, + "upstream_message", upstreamMsg, + ) + return false + } + if account.Platform == PlatformAntigravity { return s.handleAntigravity403(ctx, account, upstreamMsg, responseBody) } @@ -1064,16 +1073,6 @@ func (s *RateLimitService) handleOpenAI403(ctx context.Context, account *Account ) return false } - if isCloudflareBotBlockResponse(responseBody) { - slog.Warn( - "openai_403_cloudflare_bot_block_skips_account_penalty", - "account_id", account.ID, - "platform", account.Platform, - "upstream_message", upstreamMsg, - ) - return false - } - msg := buildForbiddenErrorMessage( "Access forbidden (403):", upstreamMsg, @@ -1123,9 +1122,62 @@ func (s *RateLimitService) handleOpenAI403(ctx context.Context, account *Account // request/edge-level failure and must not consume the account 403 strike budget. func isCloudflareBotBlockResponse(body []byte) bool { normalized := strings.ToLower(strings.TrimSpace(string(body))) + var payload any + if json.Unmarshal(body, &payload) == nil { + return containsCloudflareBotBlockCode(payload) + } return strings.Contains(normalized, "error code: 1010") } +func containsCloudflareBotBlockCode(value any) bool { + switch node := value.(type) { + case map[string]any: + cloudflareError := false + for key, child := range node { + if strings.EqualFold(key, "cloudflare_error") { + if is, ok := child.(bool); ok && is { + cloudflareError = true + } + } + } + for key, child := range node { + if strings.EqualFold(key, "error_name") { + if name, ok := child.(string); ok && strings.EqualFold(strings.TrimSpace(name), "browser_signature_banned") { + return true + } + } + if cloudflareError && (strings.EqualFold(key, "code") || strings.EqualFold(key, "error_code")) { + if isCloudflareBotBlockCode(child) { + return true + } + } + if containsCloudflareBotBlockCode(child) { + return true + } + } + case []any: + for _, child := range node { + if containsCloudflareBotBlockCode(child) { + return true + } + } + } + return false +} + +func isCloudflareBotBlockCode(value any) bool { + switch code := value.(type) { + case float64: + return code == 1010 + case json.Number: + return code == "1010" + case string: + return strings.TrimSpace(code) == "1010" + default: + return false + } +} + // handleAntigravity403 处理 Antigravity 平台的 403 错误 // validation（需要验证）→ 永久 SetError（需人工去 Google 验证后恢复） // violation（违规封号）→ 永久 SetError（需人工处理） diff --git a/backend/internal/service/ratelimit_service_403_html_test.go b/backend/internal/service/ratelimit_service_403_html_test.go index 8768b41fea61..8048dd3cabe8 100644 --- a/backend/internal/service/ratelimit_service_403_html_test.go +++ b/backend/internal/service/ratelimit_service_403_html_test.go @@ -104,8 +104,8 @@ func TestHandleUpstreamError_OpenAIHTML403RepeatedNeverEscalates(t *testing.T) { h.requireNoAccountPenalty(t) } -func TestHandleUpstreamError_OpenAICloudflare1010DoesNotPenalizeAccount(t *testing.T) { - for _, platform := range []string{PlatformOpenAI, PlatformOpenCodeGo} { +func TestHandleUpstreamError_Cloudflare1010DoesNotPenalizeAccount(t *testing.T) { + for _, platform := range []string{PlatformOpenAI, PlatformOpenCodeGo, PlatformAnthropic, PlatformGemini, PlatformAntigravity} { t.Run(platform, func(t *testing.T) { repo := &rateLimitAccountRepoStub{} counter := &countingOpenAI403CounterCache{ @@ -130,6 +130,67 @@ func TestHandleUpstreamError_OpenAICloudflare1010DoesNotPenalizeAccount(t *testi } } +func TestHandleUpstreamError_Cloudflare1010JSONDoesNotPenalizeAccount(t *testing.T) { + for _, body := range []string{ + `{"cloudflare_error":true,"error_code":1010,"error_name":"browser_signature_banned","detail":"blocked"}`, + `{"error_name":"browser_signature_banned","detail":"blocked by Cloudflare"}`, + } { + t.Run(body, func(t *testing.T) { + repo := &rateLimitAccountRepoStub{} + counter := &countingOpenAI403CounterCache{ + openAI403CounterCacheStub: openAI403CounterCacheStub{counts: []int64{openAI403DisableThreshold}}, + } + svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + svc.SetOpenAI403CounterCache(counter) + account := &Account{ID: 602, Platform: PlatformAnthropic, Type: AccountTypeAPIKey} + + require.False(t, svc.HandleUpstreamError( + context.Background(), account, http.StatusForbidden, http.Header{}, []byte(body), + )) + require.Equal(t, 0, counter.increments) + require.Equal(t, 0, repo.setErrorCalls) + require.Equal(t, 0, repo.tempCalls) + }) + } +} + +func TestIsCloudflareBotBlockResponseOnlyMatches1010(t *testing.T) { + cases := []struct { + name string + body string + want bool + }{ + {"plain text", "error code: 1010", true}, + {"root numeric JSON code", `{"cloudflare_error":true,"code":1010}`, true}, + {"nested string JSON code", `{"errors":[{"cloudflare_error":true,"code":"1010"}]}`, true}, + {"cloudflare error code", `{"cloudflare_error":true,"error_code":1010}`, true}, + {"browser signature name", `{"error_name":"browser_signature_banned"}`, true}, + {"different JSON code", `{"code":403,"message":"forbidden"}`, false}, + {"unrelated code 1010", `{"error":{"code":1010,"message":"account banned"}}`, false}, + {"message mentions 1010", `{"message":"error code: 1010"}`, false}, + {"malformed JSON", `{"code":1010`, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, isCloudflareBotBlockResponse([]byte(tc.body))) + }) + } +} + +func TestHandleUpstreamError_AnthropicGenuine403StillPenalizes(t *testing.T) { + repo := &rateLimitAccountRepoStub{} + svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + account := &Account{ID: 603, Platform: PlatformAnthropic, Type: AccountTypeAPIKey} + + require.True(t, svc.HandleUpstreamError( + context.Background(), account, http.StatusForbidden, http.Header{}, + []byte(`{"error":{"type":"permission_error","message":"invalid API key"}}`), + )) + require.Equal(t, 1, repo.setErrorCalls) + require.Contains(t, repo.lastErrorMsg, "invalid API key") +} + // 对照不变式：真正的结构化 JSON 403 是账号级证据，处罚链路必须原样保留。 // 缺了这组断言，上面的跳过逻辑一旦写宽就会把真实的封号 403 也放过去。 func TestHandleUpstreamError_OpenAIStructured403StillPenalizes(t *testing.T) { From cacc80b8620e11e9655ab307471f71eb9ee31a0e Mon Sep 17 00:00:00 2001 From: wucm667 Date: Wed, 21 Oct 2026 15:27:29 +0800 Subject: [PATCH 19/29] fix(security): clear dependency security scan failures --- .github/workflows/security-scan.yml | 2 +- backend/go.mod | 2 +- frontend/package.json | 3 ++- frontend/pnpm-lock.yaml | 9 +++++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 8caec3ff4221..88da184e5dc5 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -23,7 +23,7 @@ jobs: cache-dependency-path: backend/go.sum - name: Verify Go version run: | - go version | grep -q 'go1.27.2' + go version | grep -q 'go1.27.3' - name: Run govulncheck working-directory: backend run: | diff --git a/backend/go.mod b/backend/go.mod index 2e68ebc9ac2b..50865afb0c7b 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,6 +1,6 @@ module github.com/Wei-Shaw/sub2api -go 1.27.2 +go 1.27.3 require ( entgo.io/ent v0.14.5 diff --git a/frontend/package.json b/frontend/package.json index e714a702f834..177259daa3c6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -66,7 +66,8 @@ "form-data@<4.0.6": ">=4.0.6", "postcss@<8.5.18": ">=8.5.18", "dompurify@<3.4.14": ">=3.4.14", - "source-map-js@<1.2.2": ">=1.2.2" + "source-map-js@<1.2.2": ">=1.2.2", + "follow-redirects@<1.16.1": ">=1.16.1" } } } diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 1e36493ef0e6..5f02f3f2a8dc 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -10,6 +10,7 @@ overrides: postcss@<8.5.18: '>=8.5.18' dompurify@<3.4.14: '>=3.4.14' source-map-js@<1.2.2: '>=1.2.2' + follow-redirects@<1.16.1: '>=1.16.1' importers: @@ -2560,8 +2561,8 @@ packages: flatted@3.3.3: resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} - follow-redirects@1.16.0: - resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} + follow-redirects@1.16.1: + resolution: {integrity: sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==} engines: {node: '>=4.0'} peerDependencies: debug: '*' @@ -6552,7 +6553,7 @@ snapshots: axios@1.20.0: dependencies: - follow-redirects: 1.16.0 + follow-redirects: 1.16.1 form-data: 4.0.6 https-proxy-agent: 5.0.1 proxy-from-env: 2.1.0 @@ -7294,7 +7295,7 @@ snapshots: flatted@3.3.3: {} - follow-redirects@1.16.0: {} + follow-redirects@1.16.1: {} for-in@1.0.2: {} From 4c940c3cb7069d23cd304fc378f29ffae1ceb80f Mon Sep 17 00:00:00 2001 From: wucm667 Date: Wed, 21 Oct 2026 15:35:03 +0800 Subject: [PATCH 20/29] fix(ci): synchronize Go patch version checks --- .github/workflows/backend-ci.yml | 4 ++-- .github/workflows/release.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index 7367f680cf90..c9a140f88633 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -35,7 +35,7 @@ jobs: cache-dependency-path: backend/go.sum - name: Verify Go version run: | - go version | grep -q 'go1.27.2' + go version | grep -q 'go1.27.3' - name: Unit tests working-directory: backend run: make test-unit @@ -75,7 +75,7 @@ jobs: cache-dependency-path: backend/go.sum - name: Verify Go version run: | - go version | grep -q 'go1.27.2' + go version | grep -q 'go1.27.3' - name: golangci-lint uses: golangci/golangci-lint-action@v9 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e70f1680e007..9cef765b6237 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -137,7 +137,7 @@ jobs: check-latest: false cache: false - name: Verify Go version - run: go version | grep -q 'go1.27.2' + run: go version | grep -q 'go1.27.3' - name: Restore target-specific Go cache id: go-cache uses: actions/cache/restore@v6 From 67dca9687eddcd4a21d44b354c2afe2d921942a6 Mon Sep 17 00:00:00 2001 From: feeeei Date: Wed, 21 Oct 2026 15:36:36 +0800 Subject: [PATCH 21/29] =?UTF-8?q?perf(billing):=20=E8=B4=A6=E5=8F=B7?= =?UTF-8?q?=E9=85=8D=E9=A2=9D=E7=B4=AF=E5=8A=A0=E6=8C=89=E5=B7=B2=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E7=9A=84=E5=91=A8=E6=9C=9F=E7=BB=B4=E5=BA=A6=E9=80=89?= =?UTF-8?q?=E6=8B=A9=E8=AF=AD=E5=8F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 计费事务中账号配额累加位于余额（或订阅用量）扣减之后、提交之前，与同一用户 共享的余额行锁处在同一持有区间。累加语句约 9.4KB，无论账号设置了哪些上限， 都要解析、规划并求值总额度、日、周三个维度的全部分支；lib/pq 每次执行都重新 规划，同一用户的并发计费在这段上串行。 - 预先拼好 4 条语句：只累加总额度、加日维度、加周维度、日周都累加（与原语句 去空白后逐字相同）。按账号设置的日/周上限选择，只设总额度的账号使用约 1KB 的语句。 - 选择依据来自构造计费命令时的账号配置：UsageBillingCommand 新增 AccountQuotaHasDailyLimit / AccountQuotaHasWeeklyLimit，不参与幂等指纹。 省略维度的语句在 WHERE 中要求 extra 为对象、被省略维度未设上限（键不存在、 JSON null 或不大于 0 的数字）；库中配置不符时命中 0 行，再执行日周都累加的 语句，结果与原语句一致。条件只用 ->、jsonb_typeof 与 jsonb 比较，对任何行 求值都不报错。 - 事务内语句顺序、加锁顺序、越线入队 scheduler_outbox 与返回的额度状态不变。 集成测试对 15 种 extra 配置 × 4 种选择组合，与原语句比对 extra、updated_at、 返回值与报错码，并断言各语句是否命中。 单条语句耗时（PostgreSQL 18，BEGIN/ROLLBACK 内，扣除空事务）： 只设总额度 0.77ms → 0.24ms 设日上限 0.79ms → 0.50ms 设周上限 0.79ms → 0.57ms 日周都设 不变 压测（PostgreSQL 18、NVMe、lib/pq；1 个用户经 41 个只设总额度的账号计费， 同时运行 usage_logs 批量写入；TPS）： 并发 32：488 → 1209 并发 64：424 → 874 并发 128：357 → 575 --- .../internal/repository/usage_billing_repo.go | 173 +++++++++++++----- .../usage_billing_repo_integration_test.go | 136 ++++++++++++++ .../usage_billing_repo_unit_test.go | 102 ++++++++++- .../internal/service/gateway_usage_billing.go | 2 + ...ateway_usage_billing_account_quota_test.go | 92 ++++++++++ backend/internal/service/usage_billing.go | 5 + 6 files changed, 461 insertions(+), 49 deletions(-) create mode 100644 backend/internal/service/gateway_usage_billing_account_quota_test.go diff --git a/backend/internal/repository/usage_billing_repo.go b/backend/internal/repository/usage_billing_repo.go index 01f5f857f5b2..6b509215b301 100644 --- a/backend/internal/repository/usage_billing_repo.go +++ b/backend/internal/repository/usage_billing_repo.go @@ -204,7 +204,7 @@ func (r *usageBillingRepository) applyUsageBillingEffects(ctx context.Context, t // 账号已不存在时跳过其配额计数，其余结算项不受影响。 if cmd.AccountQuotaCost > 0 && service.AccountTypeSupportsQuota(cmd.AccountType) { - quotaState, err := incrementUsageBillingAccountQuota(ctx, tx, cmd.AccountID, cmd.AccountQuotaCost) + quotaState, err := incrementUsageBillingAccountQuota(ctx, tx, cmd.AccountID, cmd.AccountQuotaCost, cmd.AccountQuotaHasDailyLimit, cmd.AccountQuotaHasWeeklyLimit) if err != nil && !errors.Is(err, service.ErrAccountNotFound) { return err } @@ -466,57 +466,157 @@ func incrementUsageBillingAPIKeyRateLimit(ctx context.Context, tx *sql.Tx, apiKe return nil } -func incrementUsageBillingAccountQuota(ctx context.Context, tx *sql.Tx, accountID int64, amount float64) (*service.AccountQuotaState, error) { - rows, err := tx.QueryContext(ctx, - `UPDATE accounts SET extra = ( - COALESCE(extra, '{}'::jsonb) - || jsonb_build_object('quota_used', COALESCE((extra->>'quota_used')::numeric, 0) + $1) +// accountQuotaTotalIncrementExpr 累加总额度已用量。 +const accountQuotaTotalIncrementExpr = `COALESCE(extra, '{}'::jsonb) + || jsonb_build_object('quota_used', COALESCE((extra->>'quota_used')::numeric, 0) + $1)` + +// accountQuotaDailyIncrementExpr 在库中设置了日上限时累加日用量，周期过期时从本次金额重新计数。 +const accountQuotaDailyIncrementExpr = ` || CASE WHEN COALESCE((extra->>'quota_daily_limit')::numeric, 0) > 0 THEN jsonb_build_object( 'quota_daily_used', - CASE WHEN `+dailyExpiredExpr+` + CASE WHEN ` + dailyExpiredExpr + ` THEN $1 ELSE COALESCE((extra->>'quota_daily_used')::numeric, 0) + $1 END, 'quota_daily_start', - CASE WHEN `+dailyExpiredExpr+` - THEN `+nowUTC+` - ELSE COALESCE(extra->>'quota_daily_start', `+nowUTC+`) END + CASE WHEN ` + dailyExpiredExpr + ` + THEN ` + nowUTC + ` + ELSE COALESCE(extra->>'quota_daily_start', ` + nowUTC + `) END ) - || CASE WHEN `+dailyExpiredExpr+` AND `+nextDailyResetAtExpr+` IS NOT NULL - THEN jsonb_build_object('quota_daily_reset_at', `+nextDailyResetAtExpr+`) + || CASE WHEN ` + dailyExpiredExpr + ` AND ` + nextDailyResetAtExpr + ` IS NOT NULL + THEN jsonb_build_object('quota_daily_reset_at', ` + nextDailyResetAtExpr + `) ELSE '{}'::jsonb END - ELSE '{}'::jsonb END + ELSE '{}'::jsonb END` + +// accountQuotaWeeklyIncrementExpr 在库中设置了周上限时累加周用量，周期过期时从本次金额重新计数。 +const accountQuotaWeeklyIncrementExpr = ` || CASE WHEN COALESCE((extra->>'quota_weekly_limit')::numeric, 0) > 0 THEN jsonb_build_object( 'quota_weekly_used', - CASE WHEN `+weeklyExpiredExpr+` + CASE WHEN ` + weeklyExpiredExpr + ` THEN $1 ELSE COALESCE((extra->>'quota_weekly_used')::numeric, 0) + $1 END, 'quota_weekly_start', - CASE WHEN `+weeklyExpiredExpr+` - THEN `+nowUTC+` - ELSE COALESCE(extra->>'quota_weekly_start', `+nowUTC+`) END + CASE WHEN ` + weeklyExpiredExpr + ` + THEN ` + nowUTC + ` + ELSE COALESCE(extra->>'quota_weekly_start', ` + nowUTC + `) END ) - || CASE WHEN `+weeklyExpiredExpr+` AND `+nextWeeklyResetAtExpr+` IS NOT NULL - THEN jsonb_build_object('quota_weekly_reset_at', `+nextWeeklyResetAtExpr+`) + || CASE WHEN ` + weeklyExpiredExpr + ` AND ` + nextWeeklyResetAtExpr + ` IS NOT NULL + THEN jsonb_build_object('quota_weekly_reset_at', ` + nextWeeklyResetAtExpr + `) ELSE '{}'::jsonb END - ELSE '{}'::jsonb END - ), updated_at = NOW() - WHERE id = $2 AND deleted_at IS NULL + ELSE '{}'::jsonb END` + +// accountQuotaExtraIsObjectCond 要求 extra 为 JSON 对象：省略某个维度的语句只在对象上与完整语句等价。 +// +// accountQuotaDailyLimitUnsetCond / accountQuotaWeeklyLimitUnsetCond 在库中未设置日/周上限时为真： +// 键不存在、值为 JSON null 或不大于 0 的数字。其他取值一律为假，交给包含该维度的语句处理。 +// 三个条件都只使用 ->、jsonb_typeof 与 jsonb 比较，对任何行求值都不会报错。 +const ( + accountQuotaExtraIsObjectCond = `jsonb_typeof(extra) = 'object'` + accountQuotaDailyLimitUnsetCond = `(extra->'quota_daily_limit' IS NULL + OR jsonb_typeof(extra->'quota_daily_limit') = 'null' + OR (jsonb_typeof(extra->'quota_daily_limit') = 'number' AND extra->'quota_daily_limit' <= '0'::jsonb))` + accountQuotaWeeklyLimitUnsetCond = `(extra->'quota_weekly_limit' IS NULL + OR jsonb_typeof(extra->'quota_weekly_limit') = 'null' + OR (jsonb_typeof(extra->'quota_weekly_limit') = 'number' AND extra->'quota_weekly_limit' <= '0'::jsonb))` +) + +const accountQuotaIncrementReturning = ` RETURNING COALESCE((extra->>'quota_used')::numeric, 0), COALESCE((extra->>'quota_limit')::numeric, 0), COALESCE((extra->>'quota_daily_used')::numeric, 0), COALESCE((extra->>'quota_daily_limit')::numeric, 0), COALESCE((extra->>'quota_weekly_used')::numeric, 0), - COALESCE((extra->>'quota_weekly_limit')::numeric, 0)`, - amount, accountID) + COALESCE((extra->>'quota_weekly_limit')::numeric, 0)` + +// accountQuotaIncrementSQLs 按 accountQuotaIncrementSQLIndex 存放账号配额累加语句。省略日或周维度的语句 +// 在 WHERE 中附加 accountQuotaExtraIsObjectCond 与被省略维度的未设置条件，条件不成立时命中 0 行； +// 同时包含日、周维度的语句不附加这些条件，对任何账号配置都给出完整的累加结果。 +var accountQuotaIncrementSQLs = buildAccountQuotaIncrementSQLs() + +func accountQuotaIncrementSQLIndex(daily, weekly bool) int { + index := 0 + if daily { + index |= 1 + } + if weekly { + index |= 2 + } + return index +} + +func buildAccountQuotaIncrementSQLs() [4]string { + var sqls [4]string + for _, daily := range []bool{false, true} { + for _, weekly := range []bool{false, true} { + set := accountQuotaTotalIncrementExpr + where := "id = $2 AND deleted_at IS NULL" + if !daily || !weekly { + where += "\n\t\t\tAND " + accountQuotaExtraIsObjectCond + } + if daily { + set += accountQuotaDailyIncrementExpr + } else { + where += "\n\t\t\tAND " + accountQuotaDailyLimitUnsetCond + } + if weekly { + set += accountQuotaWeeklyIncrementExpr + } else { + where += "\n\t\t\tAND " + accountQuotaWeeklyLimitUnsetCond + } + sqls[accountQuotaIncrementSQLIndex(daily, weekly)] = "UPDATE accounts SET extra = (\n\t\t\t" + set + + "\n\t\t), updated_at = NOW()\n\t\tWHERE " + where + accountQuotaIncrementReturning + } + } + return sqls +} + +// incrementUsageBillingAccountQuota 累加账号配额并返回累加后的状态。hasDaily/hasWeekly 是调用方所见的 +// 账号日/周上限设置，只决定先执行哪条语句；该语句的附加条件不成立而命中 0 行时，再执行同时包含日、周维度的语句。 +func incrementUsageBillingAccountQuota(ctx context.Context, tx *sql.Tx, accountID int64, amount float64, hasDaily, hasWeekly bool) (*service.AccountQuotaState, error) { + index := accountQuotaIncrementSQLIndex(hasDaily, hasWeekly) + fullIndex := accountQuotaIncrementSQLIndex(true, true) + state, err := execAccountQuotaIncrement(ctx, tx, accountQuotaIncrementSQLs[index], accountID, amount) if err != nil { return nil, err } + if state == nil && index != fullIndex { + state, err = execAccountQuotaIncrement(ctx, tx, accountQuotaIncrementSQLs[fullIndex], accountID, amount) + if err != nil { + return nil, err + } + } + if state == nil { + return nil, service.ErrAccountNotFound + } + // 任意维度额度在本次递增中从"未超"跨越到"已超"时，必须刷新调度快照， + // 否则 Redis 中缓存的 Account 仍显示旧的 used 值，后续请求会继续选中本账号， + // 最终观察到 daily_used / weekly_used 大幅超过配置的 limit。 + // 对于日/周额度，即使本次触发了周期重置（pre=0、post=amount）， + // 判定式 (post-amount) < limit 同样成立，逻辑与总额度保持一致。 + crossedTotal := state.TotalLimit > 0 && state.TotalUsed >= state.TotalLimit && (state.TotalUsed-amount) < state.TotalLimit + crossedDaily := state.DailyLimit > 0 && state.DailyUsed >= state.DailyLimit && (state.DailyUsed-amount) < state.DailyLimit + crossedWeekly := state.WeeklyLimit > 0 && state.WeeklyUsed >= state.WeeklyLimit && (state.WeeklyUsed-amount) < state.WeeklyLimit + if crossedTotal || crossedDaily || crossedWeekly { + if err := enqueueSchedulerOutbox(ctx, tx, service.SchedulerOutboxEventAccountChanged, &accountID, nil, nil); err != nil { + logger.LegacyPrintf("repository.usage_billing", "[SchedulerOutbox] enqueue quota exceeded failed: account=%d err=%v", accountID, err) + return nil, err + } + } + return state, nil +} - var state service.AccountQuotaState +// execAccountQuotaIncrement 执行一条账号配额累加语句，未命中任何行时返回 nil 状态。 +func execAccountQuotaIncrement(ctx context.Context, tx *sql.Tx, query string, accountID int64, amount float64) (*service.AccountQuotaState, error) { + rows, err := tx.QueryContext(ctx, query, amount, accountID) + if err != nil { + return nil, err + } + + var state *service.AccountQuotaState if rows.Next() { + state = &service.AccountQuotaState{} if err := rows.Scan( &state.TotalUsed, &state.TotalLimit, &state.DailyUsed, &state.DailyLimit, @@ -525,13 +625,6 @@ func incrementUsageBillingAccountQuota(ctx context.Context, tx *sql.Tx, accountI _ = rows.Close() return nil, err } - } else { - if err := rows.Err(); err != nil { - _ = rows.Close() - return nil, err - } - _ = rows.Close() - return nil, service.ErrAccountNotFound } if err := rows.Err(); err != nil { _ = rows.Close() @@ -543,19 +636,5 @@ func incrementUsageBillingAccountQuota(ctx context.Context, tx *sql.Tx, accountI if err := rows.Close(); err != nil { return nil, err } - // 任意维度额度在本次递增中从"未超"跨越到"已超"时，必须刷新调度快照， - // 否则 Redis 中缓存的 Account 仍显示旧的 used 值，后续请求会继续选中本账号， - // 最终观察到 daily_used / weekly_used 大幅超过配置的 limit。 - // 对于日/周额度，即使本次触发了周期重置（pre=0、post=amount）， - // 判定式 (post-amount) < limit 同样成立，逻辑与总额度保持一致。 - crossedTotal := state.TotalLimit > 0 && state.TotalUsed >= state.TotalLimit && (state.TotalUsed-amount) < state.TotalLimit - crossedDaily := state.DailyLimit > 0 && state.DailyUsed >= state.DailyLimit && (state.DailyUsed-amount) < state.DailyLimit - crossedWeekly := state.WeeklyLimit > 0 && state.WeeklyUsed >= state.WeeklyLimit && (state.WeeklyUsed-amount) < state.WeeklyLimit - if crossedTotal || crossedDaily || crossedWeekly { - if err := enqueueSchedulerOutbox(ctx, tx, service.SchedulerOutboxEventAccountChanged, &accountID, nil, nil); err != nil { - logger.LegacyPrintf("repository.usage_billing", "[SchedulerOutbox] enqueue quota exceeded failed: account=%d err=%v", accountID, err) - return nil, err - } - } - return &state, nil + return state, nil } diff --git a/backend/internal/repository/usage_billing_repo_integration_test.go b/backend/internal/repository/usage_billing_repo_integration_test.go index c7f2e6c52859..07971d08d987 100644 --- a/backend/internal/repository/usage_billing_repo_integration_test.go +++ b/backend/internal/repository/usage_billing_repo_integration_test.go @@ -5,11 +5,13 @@ package repository import ( "context" "fmt" + "maps" "strings" "testing" "time" "github.com/google/uuid" + "github.com/lib/pq" "github.com/stretchr/testify/require" "github.com/Wei-Shaw/sub2api/internal/service" @@ -442,6 +444,140 @@ func TestUsageBillingRepositoryApply_EnqueuesSchedulerOutboxOnQuotaCrossing(t *t require.NoError(t, err) require.Equal(t, 1, outboxCountFor(t, accountID), "single-shot crossing weekly limit should enqueue once") }) + + t.Run("total_first_crossing_enqueues", func(t *testing.T) { + apiKeyID, accountID := newFixture(t, map[string]any{ + "quota_limit": 10.0, + }) + _, err := repo.Apply(ctx, &service.UsageBillingCommand{ + RequestID: uuid.NewString(), + APIKeyID: apiKeyID, + AccountID: accountID, + AccountType: service.AccountTypeAPIKey, + AccountQuotaCost: 4, + }) + require.NoError(t, err) + require.Equal(t, 0, outboxCountFor(t, accountID), "below limit should not enqueue") + + _, err = repo.Apply(ctx, &service.UsageBillingCommand{ + RequestID: uuid.NewString(), + APIKeyID: apiKeyID, + AccountID: accountID, + AccountType: service.AccountTypeAPIKey, + AccountQuotaCost: 8, + }) + require.NoError(t, err) + require.Equal(t, 1, outboxCountFor(t, accountID), "crossing total limit should enqueue once") + }) +} + +func TestIncrementUsageBillingAccountQuota_StatementVariantsMatchFullStatement(t *testing.T) { + ctx := context.Background() + client := testEntClient(t) + now := time.Now().UTC() + at := func(offset time.Duration) string { return now.Add(offset).Format(time.RFC3339) } + + // dailyUnset/weeklyUnset 为库中该维度视为未设置上限：省略该维度的语句在这些配置上命中，其余配置命中 0 行。 + configs := []struct { + name string + extra map[string]any + rawExtra string + dailyUnset bool + weeklyUnset bool + wantErr bool + }{ + {name: "no_limits", extra: map[string]any{}, dailyUnset: true, weeklyUnset: true}, + {name: "total_only", extra: map[string]any{"quota_limit": 100.0, "quota_used": 3.0}, dailyUnset: true, weeklyUnset: true}, + {name: "zero_and_negative_period_limits", extra: map[string]any{"quota_limit": 5.0, "quota_daily_limit": 0.0, "quota_weekly_limit": -1.0}, dailyUnset: true, weeklyUnset: true}, + {name: "null_period_limits", extra: map[string]any{"quota_daily_limit": nil, "quota_weekly_limit": nil}, dailyUnset: true, weeklyUnset: true}, + {name: "string_period_limits", extra: map[string]any{"quota_daily_limit": "10", "quota_weekly_limit": "0"}}, + {name: "daily_rolling_active", extra: map[string]any{"quota_daily_limit": 10.0, "quota_daily_used": 4.0, "quota_daily_start": at(-time.Hour)}, weeklyUnset: true}, + {name: "daily_rolling_expired", extra: map[string]any{"quota_daily_limit": 10.0, "quota_daily_used": 9.0, "quota_daily_start": at(-30 * time.Hour)}, weeklyUnset: true}, + {name: "daily_fixed_expired", extra: map[string]any{ + "quota_daily_limit": 10.0, "quota_daily_used": 7.0, "quota_daily_reset_mode": "fixed", + "quota_daily_reset_hour": 3, "quota_reset_timezone": "Asia/Shanghai", "quota_daily_reset_at": at(-2 * time.Hour), + }, weeklyUnset: true}, + {name: "weekly_rolling_active", extra: map[string]any{"quota_weekly_limit": 50.0, "quota_weekly_used": 20.0, "quota_weekly_start": at(-48 * time.Hour)}, dailyUnset: true}, + {name: "weekly_fixed_expired", extra: map[string]any{ + "quota_weekly_limit": 50.0, "quota_weekly_used": 49.0, "quota_weekly_reset_mode": "fixed", + "quota_weekly_reset_day": 1, "quota_weekly_reset_hour": 0, "quota_reset_timezone": "UTC", "quota_weekly_reset_at": at(-time.Hour), + }, dailyUnset: true}, + {name: "daily_and_weekly", extra: map[string]any{ + "quota_limit": 100.0, "quota_daily_limit": 10.0, "quota_weekly_limit": 50.0, + "quota_daily_used": 1.0, "quota_weekly_used": 2.0, "quota_daily_start": at(-time.Hour), "quota_weekly_start": at(-200 * time.Hour), + }}, + {name: "array_extra", rawExtra: `[]`}, + {name: "scalar_extra", rawExtra: `"value"`}, + {name: "non_numeric_daily_limit", extra: map[string]any{"quota_daily_limit": "abc"}, weeklyUnset: true, wantErr: true}, + {name: "boolean_weekly_limit", extra: map[string]any{"quota_weekly_limit": true}, dailyUnset: true, wantErr: true}, + } + + for _, cfg := range configs { + for _, hints := range [][2]bool{{false, false}, {true, false}, {false, true}, {true, true}} { + t.Run(fmt.Sprintf("%s/daily=%v/weekly=%v", cfg.name, hints[0], hints[1]), func(t *testing.T) { + variant := mustCreateAccount(t, client, &service.Account{ + Name: "usage-billing-quota-variant-" + uuid.NewString(), + Type: service.AccountTypeAPIKey, + Extra: maps.Clone(cfg.extra), + }) + reference := mustCreateAccount(t, client, &service.Account{ + Name: "usage-billing-quota-reference-" + uuid.NewString(), + Type: service.AccountTypeAPIKey, + Extra: maps.Clone(cfg.extra), + }) + if cfg.rawExtra != "" { + _, err := integrationDB.ExecContext(ctx, "UPDATE accounts SET extra = $1::jsonb WHERE id IN ($2, $3)", cfg.rawExtra, variant.ID, reference.ID) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = integrationDB.ExecContext(ctx, "UPDATE accounts SET extra = '{}'::jsonb WHERE id IN ($1, $2)", variant.ID, reference.ID) + }) + } + + if cfg.wantErr { + sqlState := func(accountID int64, daily, weekly bool) pq.ErrorCode { + tx, err := integrationDB.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + _, err = incrementUsageBillingAccountQuota(ctx, tx, accountID, 2.5, daily, weekly) + var pqErr *pq.Error + require.ErrorAs(t, err, &pqErr) + return pqErr.Code + } + require.Equal(t, sqlState(reference.ID, true, true), sqlState(variant.ID, hints[0], hints[1])) + return + } + + tx, err := integrationDB.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + + isObject := cfg.rawExtra == "" + wantHit := (hints[0] && hints[1]) || (isObject && (hints[0] || cfg.dailyUnset) && (hints[1] || cfg.weeklyUnset)) + _, err = tx.ExecContext(ctx, "SAVEPOINT quota_variant_probe") + require.NoError(t, err) + probe, err := execAccountQuotaIncrement(ctx, tx, accountQuotaIncrementSQLs[accountQuotaIncrementSQLIndex(hints[0], hints[1])], variant.ID, 2.5) + require.NoError(t, err) + require.Equal(t, wantHit, probe != nil, "variant statement hit") + _, err = tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT quota_variant_probe") + require.NoError(t, err) + + got, err := incrementUsageBillingAccountQuota(ctx, tx, variant.ID, 2.5, hints[0], hints[1]) + require.NoError(t, err) + want, err := incrementUsageBillingAccountQuota(ctx, tx, reference.ID, 2.5, true, true) + require.NoError(t, err) + require.Equal(t, *want, *got) + + var gotExtra, wantExtra string + var sameExtra, sameUpdatedAt bool + require.NoError(t, tx.QueryRowContext(ctx, + "SELECT v.extra::text, r.extra::text, v.extra = r.extra, v.updated_at = r.updated_at FROM accounts v, accounts r WHERE v.id = $1 AND r.id = $2", + variant.ID, reference.ID, + ).Scan(&gotExtra, &wantExtra, &sameExtra, &sameUpdatedAt)) + require.True(t, sameExtra, "variant extra %s != reference extra %s", gotExtra, wantExtra) + require.True(t, sameUpdatedAt) + }) + } + } } func TestDashboardAggregationRepositoryCleanupUsageBillingDedup_BatchDeletesOldRows(t *testing.T) { diff --git a/backend/internal/repository/usage_billing_repo_unit_test.go b/backend/internal/repository/usage_billing_repo_unit_test.go index a745eb8d7301..73f29d2e4cf0 100644 --- a/backend/internal/repository/usage_billing_repo_unit_test.go +++ b/backend/internal/repository/usage_billing_repo_unit_test.go @@ -5,6 +5,7 @@ package repository import ( "context" "database/sql" + "strings" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -22,7 +23,7 @@ const ( userExistsForBillingSQL = `(?s)SELECT 1\s+FROM users\s+WHERE id = \$1 AND deleted_at IS NULL` apiKeyQuotaIncrementSQL = `(?s)UPDATE api_keys\s+SET quota_used = quota_used \+ \$1,.*WHERE id = \$2 AND deleted_at IS NULL\s+RETURNING` apiKeyRateLimitIncrementSQL = `(?s)UPDATE api_keys SET\s+usage_5h = .*WHERE id = \$2 AND deleted_at IS NULL` - accountQuotaIncrementSQL = `(?s)UPDATE accounts SET extra = .*WHERE id = \$2 AND deleted_at IS NULL\s+RETURNING` + accountQuotaIncrementSQL = `(?s)UPDATE accounts SET extra = .*WHERE id = \$2 AND deleted_at IS NULL.*RETURNING` ) func TestDeductUsageBillingBalance_UsesSufficientBalanceGuard(t *testing.T) { @@ -152,7 +153,10 @@ func TestApplyUsageBillingEffects_DeletedAccountStillBillsBalance(t *testing.T) WillReturnRows(sqlmock.NewRows([]string{"balance"}).AddRow(90.0)) mock.ExpectQuery(accountQuotaIncrementSQL). WithArgs(4.0, int64(9)). - WillReturnRows(sqlmock.NewRows([]string{"quota_used", "quota_limit", "quota_daily_used", "quota_daily_limit", "quota_weekly_used", "quota_weekly_limit"})) + WillReturnRows(sqlmock.NewRows(accountQuotaStateColumns)) + mock.ExpectQuery(accountQuotaIncrementSQL). + WithArgs(4.0, int64(9)). + WillReturnRows(sqlmock.NewRows(accountQuotaStateColumns)) mock.ExpectCommit() result := &service.UsageBillingApplyResult{Applied: true} @@ -218,6 +222,100 @@ func TestDeductUsageBillingBalance_ReturnsUserNotFoundWhenNoUserUpdated(t *testi require.NoError(t, mock.ExpectationsWereMet()) } +var accountQuotaStateColumns = []string{"quota_used", "quota_limit", "quota_daily_used", "quota_daily_limit", "quota_weekly_used", "quota_weekly_limit"} + +func TestAccountQuotaIncrementSQLs_GuardOmittedDimensions(t *testing.T) { + for _, daily := range []bool{false, true} { + for _, weekly := range []bool{false, true} { + query := accountQuotaIncrementSQLs[accountQuotaIncrementSQLIndex(daily, weekly)] + set, where, ok := strings.Cut(query, "WHERE ") + require.True(t, ok) + require.Equal(t, !daily || !weekly, strings.Contains(where, accountQuotaExtraIsObjectCond), "daily=%v weekly=%v", daily, weekly) + require.Equal(t, daily, strings.Contains(set, "'quota_daily_used'"), "daily=%v weekly=%v", daily, weekly) + require.Equal(t, weekly, strings.Contains(set, "'quota_weekly_used'"), "daily=%v weekly=%v", daily, weekly) + require.Equal(t, !daily, strings.Contains(where, accountQuotaDailyLimitUnsetCond), "daily=%v weekly=%v", daily, weekly) + require.Equal(t, !weekly, strings.Contains(where, accountQuotaWeeklyLimitUnsetCond), "daily=%v weekly=%v", daily, weekly) + } + } +} + +func TestIncrementUsageBillingAccountQuota_UsesStatementMatchingHints(t *testing.T) { + ctx := context.Background() + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + require.NoError(t, err) + defer func() { _ = db.Close() }() + + mock.ExpectBegin() + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + mock.ExpectQuery(accountQuotaIncrementSQLs[accountQuotaIncrementSQLIndex(true, false)]). + WithArgs(2.5, int64(9)). + WillReturnRows(sqlmock.NewRows(accountQuotaStateColumns).AddRow(12.5, 100.0, 2.5, 50.0, 0.0, 0.0)) + mock.ExpectCommit() + + state, err := incrementUsageBillingAccountQuota(ctx, tx, 9, 2.5, true, false) + require.NoError(t, err) + require.NotNil(t, state) + require.InDelta(t, 12.5, state.TotalUsed, 0.000001) + require.InDelta(t, 2.5, state.DailyUsed, 0.000001) + require.NoError(t, tx.Commit()) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestIncrementUsageBillingAccountQuota_FallsBackToFullStatementWhenGuardMisses(t *testing.T) { + ctx := context.Background() + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + require.NoError(t, err) + defer func() { _ = db.Close() }() + + mock.ExpectBegin() + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + mock.ExpectQuery(accountQuotaIncrementSQLs[accountQuotaIncrementSQLIndex(false, false)]). + WithArgs(2.5, int64(9)). + WillReturnRows(sqlmock.NewRows(accountQuotaStateColumns)) + mock.ExpectQuery(accountQuotaIncrementSQLs[accountQuotaIncrementSQLIndex(true, true)]). + WithArgs(2.5, int64(9)). + WillReturnRows(sqlmock.NewRows(accountQuotaStateColumns).AddRow(2.5, 0.0, 0.0, 0.0, 2.5, 20.0)) + mock.ExpectCommit() + + state, err := incrementUsageBillingAccountQuota(ctx, tx, 9, 2.5, false, false) + require.NoError(t, err) + require.NotNil(t, state) + require.InDelta(t, 2.5, state.WeeklyUsed, 0.000001) + require.InDelta(t, 20.0, state.WeeklyLimit, 0.000001) + require.NoError(t, tx.Commit()) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestIncrementUsageBillingAccountQuota_ReturnsNotFoundWhenFullStatementMisses(t *testing.T) { + for _, hints := range [][2]bool{{false, false}, {true, false}, {false, true}, {true, true}} { + ctx := context.Background() + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + require.NoError(t, err) + + mock.ExpectBegin() + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + mock.ExpectQuery(accountQuotaIncrementSQLs[accountQuotaIncrementSQLIndex(hints[0], hints[1])]). + WithArgs(2.5, int64(9)). + WillReturnRows(sqlmock.NewRows(accountQuotaStateColumns)) + if !(hints[0] && hints[1]) { + mock.ExpectQuery(accountQuotaIncrementSQLs[accountQuotaIncrementSQLIndex(true, true)]). + WithArgs(2.5, int64(9)). + WillReturnRows(sqlmock.NewRows(accountQuotaStateColumns)) + } + mock.ExpectRollback() + + state, err := incrementUsageBillingAccountQuota(ctx, tx, 9, 2.5, hints[0], hints[1]) + require.ErrorIs(t, err, service.ErrAccountNotFound, "hints=%v", hints) + require.Nil(t, state) + require.NoError(t, tx.Rollback()) + require.NoError(t, mock.ExpectationsWereMet(), "hints=%v", hints) + _ = db.Close() + } +} + func TestReserveUsageBillingBatchImageBalance_MovesAvailableToFrozen(t *testing.T) { ctx := context.Background() db, mock, err := sqlmock.New() diff --git a/backend/internal/service/gateway_usage_billing.go b/backend/internal/service/gateway_usage_billing.go index 426209b2966f..580ca81dc1ac 100644 --- a/backend/internal/service/gateway_usage_billing.go +++ b/backend/internal/service/gateway_usage_billing.go @@ -345,6 +345,8 @@ func buildUsageBillingCommand(requestID string, usageLog *UsageLog, p *postUsage } if p.shouldUpdateAccountQuota() { cmd.AccountQuotaCost = p.Cost.TotalCost * p.AccountRateMultiplier + cmd.AccountQuotaHasDailyLimit = p.Account.GetQuotaDailyLimit() > 0 + cmd.AccountQuotaHasWeeklyLimit = p.Account.GetQuotaWeeklyLimit() > 0 } cmd.Normalize() diff --git a/backend/internal/service/gateway_usage_billing_account_quota_test.go b/backend/internal/service/gateway_usage_billing_account_quota_test.go new file mode 100644 index 000000000000..8d45bcc54c5e --- /dev/null +++ b/backend/internal/service/gateway_usage_billing_account_quota_test.go @@ -0,0 +1,92 @@ +package service + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuildUsageBillingCommand_AccountQuotaPeriodHints(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + accountType string + extra map[string]any + wantQuota bool + wantDaily bool + wantWeekly bool + }{ + {name: "no_limits", extra: map[string]any{}}, + {name: "total_only", extra: map[string]any{"quota_limit": 10.0}, wantQuota: true}, + {name: "daily_only", extra: map[string]any{"quota_daily_limit": 5.0}, wantQuota: true, wantDaily: true}, + {name: "weekly_only", extra: map[string]any{"quota_weekly_limit": 5.0}, wantQuota: true, wantWeekly: true}, + { + name: "all_dimensions", + extra: map[string]any{"quota_limit": 10.0, "quota_daily_limit": 5.0, "quota_weekly_limit": 7.0}, + wantQuota: true, wantDaily: true, wantWeekly: true, + }, + { + name: "non_positive_period_limits", + extra: map[string]any{"quota_limit": 10.0, "quota_daily_limit": 0.0, "quota_weekly_limit": -1.0}, + wantQuota: true, + }, + { + name: "service_account_total_only", + accountType: AccountTypeServiceAccount, + extra: map[string]any{"quota_limit": 10.0}, + wantQuota: true, + }, + { + name: "service_account_daily", + accountType: AccountTypeServiceAccount, + extra: map[string]any{"quota_limit": 10.0, "quota_daily_limit": 5.0}, + wantQuota: true, wantDaily: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + accountType := tt.accountType + if accountType == "" { + accountType = AccountTypeAPIKey + } + p := &postUsageBillingParams{ + Cost: &CostBreakdown{TotalCost: 1, ActualCost: 1}, + User: &User{ID: 1}, + APIKey: &APIKey{ID: 2}, + Account: &Account{ID: 3, Type: accountType, Extra: tt.extra}, + AccountRateMultiplier: 1, + } + + cmd := buildUsageBillingCommand("req-1", nil, p) + require.NotNil(t, cmd) + require.Equal(t, tt.wantQuota, cmd.AccountQuotaCost > 0) + require.Equal(t, tt.wantDaily, cmd.AccountQuotaHasDailyLimit) + require.Equal(t, tt.wantWeekly, cmd.AccountQuotaHasWeeklyLimit) + }) + } +} + +func TestUsageBillingCommandFingerprintIgnoresAccountQuotaPeriodHints(t *testing.T) { + t.Parallel() + + base := UsageBillingCommand{ + RequestID: "req-1", + UserID: 1, + APIKeyID: 2, + AccountID: 3, + AccountType: AccountTypeAPIKey, + BalanceCost: 1, + AccountQuotaCost: 1, + } + hinted := base + hinted.AccountQuotaHasDailyLimit = true + hinted.AccountQuotaHasWeeklyLimit = true + + base.Normalize() + hinted.Normalize() + require.NotEmpty(t, base.RequestFingerprint) + require.Equal(t, base.RequestFingerprint, hinted.RequestFingerprint) +} diff --git a/backend/internal/service/usage_billing.go b/backend/internal/service/usage_billing.go index e058ecb1b029..447260ecb451 100644 --- a/backend/internal/service/usage_billing.go +++ b/backend/internal/service/usage_billing.go @@ -42,6 +42,11 @@ type UsageBillingCommand struct { APIKeyQuotaCost float64 APIKeyRateLimitCost float64 AccountQuotaCost float64 + + // AccountQuotaHasDailyLimit / AccountQuotaHasWeeklyLimit 记录构造命令时账号是否设置了日/周上限， + // 只用于选择账号配额累加语句，不参与幂等指纹；实际累加哪些维度以数据库中的账号配置为准。 + AccountQuotaHasDailyLimit bool + AccountQuotaHasWeeklyLimit bool } func (c *UsageBillingCommand) Normalize() { From 7da09cab4e875463baa79fc150adde700c5438df Mon Sep 17 00:00:00 2001 From: shaw Date: Wed, 21 Oct 2026 20:19:37 +0800 Subject: [PATCH 22/29] chore: update sponsors --- README.md | 5 +++++ README_CN.md | 5 +++++ README_JA.md | 5 +++++ assets/partners/logos/chenju_ai.png | Bin 0 -> 115826 bytes 4 files changed, 15 insertions(+) create mode 100644 assets/partners/logos/chenju_ai.png diff --git a/README.md b/README.md index 985201c14b3b..11955a37b6e9 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,11 @@ Please read the following carefully before using this project: Modelflare: Full strength, stable, nothing watered down. Global SOTA models at a lower cost. Operated by US-based Havenbyte LLC, Modelflare supports GPT, Claude, Gemini, Grok, and other leading models through common compatible protocols, plus 40+ image and video models. Cache compensation is a highlight: selected OpenAI groups (price-first, stable, and premium) guarantee daily cache hit rates of 65%, 75%, or 85%, and if eligible requests fall short, Modelflare refunds the difference, visible on the usage dashboard after settlement. Special offers: a single top-up of US$20 unlocks the GPT discount group at a 0.015x price multiplier, and a single top-up of US$50 unlocks the Claude discount group at a 0.03x multiplier. Register via this exclusive link to get started. + +Chenju +Thanks to Chenju for sponsoring this project! This is probably the only ad slot in this section that doesn't ask you to sign up for anything. Good open-source projects deserve to be paid for, so Chenju is chipping in a gift and raising a glass to the author. Find Chenju on X to chat about API relays, upstream providers, or just shoot the breeze. + + ## Overview diff --git a/README_CN.md b/README_CN.md index 9d126cb144ff..9064b68eacea 100644 --- a/README_CN.md +++ b/README_CN.md @@ -154,6 +154,11 @@ Modelflare：满血稳定不掺水，低成本享受全球 SOTA 模型。站点由美国 LLC 主体 Havenbyte LLC 运营，支持 GPT、Claude、Gemini、Grok 等主流模型和常用兼容协议，还支持 40 多种图片和视频模型。缓存补偿是一大亮点：部分 OpenAI 价格优先、稳定和高级分组提供每日 65%、75%、85% 的缓存命中率保障，符合条件的请求没达到目标的，平台补偿差额，结算后可在用量看板查看。特惠活动：单笔充值 US$20 即可解锁 0.015 倍价格倍率的 GPT 特惠分组，单笔充值 US$50 即可解锁 0.03 倍价格倍率的 Claude 特惠分组。通过专属链接注册即可参与 + +Chenju +感谢 陈局 赞助了本项目！这大概是本栏唯一一个不让你注册的广告位。好的开源项目值得被付费，陈局来随个份子，敬作者一杯。X 上找 陈局，聊中转、聊上游、吹吹水。 + + ## 项目概述 diff --git a/README_JA.md b/README_JA.md index bf5f13b930e3..367cce6373db 100644 --- a/README_JA.md +++ b/README_JA.md @@ -153,6 +153,11 @@ Modelflare：フルスペック、安定、水増しなし。世界の SOTA モデルを、より低いコストで。米国拠点の Havenbyte LLC が運営する Modelflare は、一般的な互換プロトコルを通じて GPT、Claude、Gemini、Grok などの主要モデルに対応し、さらに 40 以上の画像・動画モデルも利用できます。注目はキャッシュ補償です。対象の OpenAI グループ（価格優先・安定・プレミアム）は、1 日のキャッシュヒット率をそれぞれ 65%、75%、85% で保証します。対象リクエストがこの水準に達しない場合、Modelflare が差額を返金し、精算後に利用状況ダッシュボードで確認できます。特別オファー：US$20 を 1 回チャージすると、価格倍率 0.015 倍の GPT 割引グループが解放されます。US$50 を 1 回チャージすると、価格倍率 0.03 倍の Claude 割引グループが解放されます。こちらの専用リンクから登録して始めましょう。 + +Chenju +陈局（Chenju）様、本プロジェクトへのご支援ありがとうございます！これはおそらく、この欄で唯一「登録しなくていい」広告枠です。良いオープンソースには対価が払われるべき——というわけで、陈局がご祝儀を包んで作者に一杯おごります。X で 陈局 を見つけて、中継サービスや上流の話、雑談でもどうぞ。 + + ## 概要 diff --git a/assets/partners/logos/chenju_ai.png b/assets/partners/logos/chenju_ai.png new file mode 100644 index 0000000000000000000000000000000000000000..2e4affee5d60580d2a89f20a3805c7253e8347ae GIT binary patch literal 115826 zcmd43bySpJ+cr!|Nh94QC<4+YouX3G-5t`+kVBUeg1`_eNJvS?NFz0bba%th1I#e* z_^#)ZsF&&sG;^JS?nfiXp^r?8>Up=mj`ASjjK*h-Cal*n=SpeS8jg= zX&jaG44+k|9iq($nGT#DI7|1tx)~`CSZ;?%BUzdr+k@mssTGtd zTpEq6H;kGYI~LvQ-ygWF-5MlgasU4DfBh_O5rK@HEGNc4e(lfqJINB!GcF*JFPXIn z{_>ZZv9WAmco-6-f7#cc&0-8f$+5gy|Gq0DPyT}U_W{rG{==(4oV#)-SD_*a9<65w+EXM-_2{OS!;yWQzzPvye? z>*5#!a4hsZ+`pPD79HL8KN}2-r>6xq$pXL;`cce>PZn0S6W4 z(>)N*2iYR(zj_OI;6M8c<>&POY%r1^n^v+Mh;s`v)km0r4I}jbhp&3V!u}ryi^wz~ zDE$mze2o!j^;d5(Gyi8_g;A4e{%3>9j=fL#|8sC#bmZdJ9yc=uKsv(C!1sh-i5t3U z$dYqrv4mdLj%gbt@_u4yPkOT}9*X>1#^G*1N9Byh&ylB^6L*Y?r%YzyM+teQ8tO`~wD<$} z5BQ~DB+90o7P1^0%tnW|Z0gKPuXHkSOWLi6ski+j5tt2Z_?f#=8EDC(=8s4k2v~Qd zUI`$sKaF#k*?+!%)F;otX2y5TI2**@)=oH=g3Jv{2~QUMo0I?KQ)lxF(pdKnh8qZy z_KKnPoe(&c`21te*B9UX@5!W{~Kt*lW}3f4U^zg4SMae z=kp764gjf>e%LC>{Gd=Z{6KJk(f_0Rj|&>Mf7IdO5^7B#Anw&^x5^qWQDkG8#Mpg6 zC$S#h+AIZrZ+8k-u-VIhWDly?^oJ23$)aSh5HU>^LdKxvU%zDUHkHH!_xIQEcRZ+c zFr-%g=}nR!?AEeC0G{xYHN$0VMT2b6+85!+oa44No_WY#(Iv`vME^`x)M7oT#aQ)6 z--1^gay}88UUK$vj-BkO*n4p%HxU(&&&KAwcM_{(sJ6*2q;k-rHIG=wRN|7>jD_!- zs5ht0X~uAg*ML$G`AacNSyDi$Oevfo-mBafI{ev3vva` z|2$N}3zV2J1>gyqd!nwQnuH5@V_0`<2W2g08i=P0CKXB9z zsb_1dt*!d|lG9M3xZTWHu>js{CvI-W7vlSyS-Ad~h4jh!_Y;qUK9#m*asJ9-u4~_m z)q+Zz?f?J^Y&y?+cx z;!M#0dIL5#CN5lJ&1Svpbi7%+pC%hiYI!!OVrIJVz=Enzr61zx!l|q> z{_I!8eoa5-DEU7_AB!iRI(5zb=OE&y&fy7{L`mKiPAkrsi(>@8X-e)ppPxzo1?dcB z5s$y*w$6@EPKS)Hk8Gbgm`Qu)2R~j7{$~)&jzuJ+)7(Gc%6uUGStpOYSVmf8w9?gn z|7J4)D z2|(3?cBu8t!HWEx&7H5N~QE_cGG@O(z@OJ1`6)Afroq)(jP`?{3&!= zbu^D{8s`RxMwNUG7(Kf@)2NEJ6TRH3F`P!B+^Gz?Bkh^2>FwW7tmKY)DN1BNA>57b zPBiDn=YI2-Cn0ASMe!%od>{z)owR=1U0XJLeIif4ZB}=EM$ZVZCYLq=MPM7>CYJ^9e(0HM2MOEdhewA2?tHAv z@n|PWn)Lpn*l^+v=DYNiLk%Uac@`z#*hhn_&o+<=tCjQ5N{Kn2i;9R19QCAjY)yxqzx*e7kUN*awt?+@`%{5$iJ9^# zK8T6)$2GKwd=z*+K=kXXd-qNa=9N9SlF^+VDlB--g3r~xH9#=b(~)wu;m!xgOj$yR zMSO3g$ujp8)~jkJ*~x1pqk@Ae4C-7McuoCizPG|876Xe`F#Dh4 z9RGl^;ROM7sETv`dLJYmM-=He-7xMfB6^oSg7n*p-ei$fOsZfoOpUTx{dt86TPgLyhbu$y>%LEY5k17A;gO*JjU+5+uT(>(OITI zT{^y=T3hF8-y7`3i%R3uJ5Dp@#ge#G;FAIZUDO)Mxg-zH6`09OHZ)y@v0off@A(vi z7n0Hnekgf)?~5Q=l$J!7LtPl3~5Mb$axqu z7>9oH(+~nMe~Us7m#tyvum6?^tf)l5q=sL7YVgdS`xO^gClDAZ75Q#UPnMxCXV(^# zVPGxoR{6qtm@1dpR#ikm;qxp~>)A1J3l&w>Y!o*i7~Ashs<@?{K{wCsBT-{f(+$_A zun&p8uKZLW?z)1{fQ!3g;48u&o~Z$+Z_#C6&;xkhjQQ4gO)o`nhr4+vURXL<5wNj! zJBWKzgF=5W=$m?QWbzp&^dARdd;>jXv1YhaI4@Z2o$ju?7F+T8N)5W@-%kyuW*52e zz69kRfN~WTa?sX|k8xqYqW@B#2B3UOxcFu;LhNLEDQ@@v!A|an*uPmZxbtZ^IL( zAAfF~Vry@wEV`0dfxIPa6s6BPbl{0MqSKmMip%V9^S=w^7kWWZDHqcU2Jq?SfoZQz zeE5o1=7{kYa`@71d#5y`HizBcxN^Ww2BnXzf3t|Eo(w2;1qnRDJKFUCnc5@oyH!$= zi&Yq-;(s~B9z4TNUjuD~J*)z!E$Ag3l^>s25U!NHa!MmlBldAPK8$xB-dXU}peYMu zHV8Z5YbG7aa_>n%-bo0&_KnTUQ1xg@UG-$`=!(PAfAF5$DOE=+dUJlzT?pRqHaUOr z)4eLPsgh`}^0L~uzR1;|WJ#f0Y+^p13KYQjDbxKYkNy&{iBCI?Trc*JAwUzjJ?b89 z3ooK)hC9g=WylRa6lp}<1p^pFnm|v8&JDA4kS;G7$mO~e1BYQS1 zhRf83X|@wh29tdCrn{K&@$qo;7YO#uJOFwJ_|kHq`z*D!Tnq}lr&p!OCA59V3uEH0 zMw&jka{FMtA|n1?dIb1t(qC`krum47D-`DNQ$yr0g+UPr;Fq`X52EYChPw}MOCQPw z)E?*3AU+Vv7zu5bFbWMC^#&tVs z?)_sDxc8_SzQt76Q|>VgLjhBVOri+a%Ty*u;Grsck82XiJ1Vq&7RSoSbyWtJBUcBe zLxneQh27=?1*drz!ZhyN$mc8{po{M8$j`k!g}?;paF!uG9oN}w_S>MBcz9I=l}T>k z8=8ZYnER1N%zN>_!CUbJlmGzn2%BtNj90d7Sy6Uv0$_()zwsL`+m{fD@H#2l}yVnmOTd=+^-Xce|PGU{d>+#&|kjF|6(vQ-_{f}+{7DqP+9!O#)6{l|$ zHCq{~V{>U1KkjO8M%8-sbv)J+!Np{F!%%&(RD+$f$|)pvH)vir<;SQ*?1FPM@4aEA zFiE4n!kA&wBMlxK9Bz-#RX9t_X7H=%6$`|mus@P-p@3k-&>INyf$9TG3vh>&lvSmP zf>WcVP6n!x48m9K#kcW$8ngKcPZ2CXAx{AsN*s-C>|>lve>qTDyTK$Z7I%fjem*z0 z1K2G%g;bvkR|#N-T!@Rmb9JL`b@QCo;A^v6gLmB7684GD#U7K*oKz=3sdHzIdunkb{ zv{G1qb)EaW(vMmaygmtb8fVgSKi1_%#|b-lq8`Jx0{c%G^UEHrD^lV#afK`rC~ z`_UU!kkKbV%QNI&4=+iqahcY;&y2@umh`R?iS^~vtK07G(YxobN1;RVwU~A=d&G^x z`PXe8oZ7J$HV>#|p@xh^&x;`fh=t8r@SC{yh4k6FH{naDNBf&_Or|HRT<%cr1qgLL zp!tJ!rh4r~Oy<9-p}0oge~?l_=c5Np!flWnLNug?aa3$V-vPu9r(NU_1(5c0GwSgRO{N3rqoHBKZkV5*nVQ z0SD&`*I(CffZ$>xnj@b9Y3-P8Grqu}I}Zqh-$q%IzQTnqWqR68YRrzO@#U{>d@&w} zZYB|*(};Eu)MTK`i#cf^{DlDVL^Ug}^pNu6-LoOc=e`bC@mycIBdrg4b|3L*VVl#Uqv5py2O*$|GQWGQ=>+P?nBnp!%7cgjc?0l4J@)rGX= z1l}gxnPpELt5+AZ#g(#c-zYt9pVE&tlh~KZzDIoQ^kYe*0bHGX+z24>4&`NSITVu! z2P6VjBN0}2!UPCX%|o3)6bWi3mH zq|;%5ep36}r;o&+O9-5Ea7g)Frpmw+BswRU<_zt_JbG`Hy>oaLzjL6JF3qbQWb|1Y zNFF7k_0aRH{I3qcPvM=%D8LC32y2GCh$$S+7Tr0xCMXbl8FW0{0seBl7CnIbtLsgc zYo}6{X%Zq}ORS(=KDXh$AiSmpzlYW&wo!+9ypsU%U_EbmQN!l|8Gl+OA5r|imHB5? zW%%Yha`ws*l15R_q??HnOIQYvPwB~d(U)TKA&7ahWQP|gyX8|=HqQ{NKC!m2$u1Fi zhi7QRtB4mBxV6=+s@>C2p3WPRV7;*4DPCz8g|w$9 z+@#3-h!5-)xXV`^G+yGr$Kw`D053UBZi^+$38?RHjvce<=%{65I%D?{HDmV(By-TJ z<$4P2f9q?jv;uJSm@pr4CZ^n*mc0g`x=sJ1Jmb2PlC`WD3RZ5NWY7wdo!6j)@kE(X z_WgI-xE9);y>bs(^Ki^SDkAE<*v6W^kg2yzx7klM1P~P;0K$z8GUH}({y0Ndt*O(R zgj%oIvbLz|YuylpB#k1>)@tk4Is5ZfV%m{q`Ra$sNxeA~<)_2l;IL=UsLr4m1KsB= z4~JyjV1|zuOR73Jw<4ayo8JP#Bxhq}16^v&gxA@j(_<;+XcjVFROn~SNyCweJkAzg zzK0>G(}hO*OSQa>3RQ7^gLPkUG9gjXZX=i&wv+I~-)by9O`Hw;j)?IOPR~Iia3+Wp{8jnT*)ec2<*22D#c{Hf; zCV+klL(9cpR#bp&uQkZ7Pw?5rxKmAZlRn=%eHPLQH`jk4SE=FXGo@N-PeppTsxQg% zt+s1^BJo5DhQ@1B&uHFJLtYlE&BdlxxSgRuYBn5KS)TRAw#RT9L<6AI*HtCeVd-6r zPHav%nR`iVCdRQ6ok$LrF+7x4x&M)0Ra=p7?f%%>$~&FAndky9{)d>=^RWqhK)-`I zlam#BaGyV8Th^T3_$xq?x+4t=$An_NId7bK&MzWT%vZ@4Sv(e3*-tl|r)Dnfy`vrdc5VZZ#U7QBu)}7X zMPgjl!7!w67CR$DtYFnlh?6+Ht8Wc1#J;0!C9l{yH{yjqRR; zvSfS=MO+*LU0#K_e34IeaBL0ce)lHIkFIjdClnGq36B{12zoyDO3NsYuvnp3aJfO_ z`qmg-=4IL_FSg64vHcJnj0PQr(=oP>Nu31iRyN_flesT|^OBlb=@DJ#2)FSoQHy8! z`|b{}Ecw%A?qB{)N%#x=J7`G$FC#tEnRA8da0 z>(}6YUVp~!JLb8q_4L(J*%zEt>kR5Wl6hj9YqBoUn8DNJ1zpLi83Z$C0L8nEb%G=LzdxP4c(_v6bh6^T^ape9A})i!&zl# z8}+;A1jDhWeJyK8uIMsaJidA53}kA^J3jMljm(!k@R3E{ndcqnjHA8;txE%;@Evqe zi5H6qQ%T5*scEUk7TXCG1P3l=|H|dzn4oa8Eft0nQ4fnX_Yp1GZZqqD$w|0IkN!Z5 zNQ-Ql+0p$o`wOM1$$8An<%zIQ?R#|h+BVUYUZ(4U#kT%%91ZKaGs0QV>+5O_t;8NO zob@4Y580uH-E!HY8VnB!Ix9l;9@@NE!5Y`Qr)dtA^0n>QPly_l!Rt-hxvx=tQ-^iR zFu_Kj;k{{n2R6VYI_zN_g*v;wuB?sF+F1cllLbPe?P@LPorsA1_t8GEG@N7wd~Rm) zBUp-80>JjY87lGOs-2S*4snR>{+)Vu#)7Y>KxTyYQkBJq)Ykk-n)#si(Ua)AQa@z^ z!R}>S2T)>ko%Zxdez&o1$h<4{d47nc`tpLMt_Yj~)rKmD z^Hs?QPr9OmioG~8e}P7!dU_t9?nGlS9n@NyPF$g@t+#0E^j@JiRdLHShQRf|X948N3~@bno$Cg4c|wv%e1Zg8HyhGBj0Q|gszG66xz+1G+?iQI!7DDHKgly zv$_3G8wKMZE=(1K^C_bGF}ScAh)#SAQ?|#+C-o0BWh$jL<^?;+|COlB4xN2}j2EGE zKLZ|VN(7I3znfPEgwlkXmjv0Wl5#rQkgiA8L7__;)+1;2P0;s4r`N3G33fPgih&(Q;EG=^x7Shg??J6TGt*ue8b8vYS!%wa|B+t2mXy|s-cbdj#ErSWr; zY}TW^WEUbr>dt1(Wc#x)uiA>$gM&$Xft;TAVUQt{($(k{7z0~l_b;Ai%WVKlY31vU znY&zSnzg5xl+GHM{{qai?}+~NerTO7?B@%rj20e2)lVi5egp88U@c2tZ%(=l$icxj zij=HR43c*k0~om><}q8&B5V0fQppS+!Lf#v;Nf@#HkDQLSt!-|psDJF4y&$l(xVU_ zvP7R)!QZTU`bqw#!eri4v6DKiW^a=oSxdbardZ9IJDnzCbZgncX|gy{YJv`OLrVbd z4|^@Sb<8*)G1A#EJRWGbi2m>ZxA7;eiHQD>7z_wzN;JHz@fF;;KUAMrZrW(p4< zCS1=_{}JX9mN{GG{MH?*TiprH_Hx`r z${3O5){Mf1zKWtKh0W3(A`C+FFC5NnAd4z>0O0B)#4)iTuZ%aP=)3H*pLBtM#whLx zp(0$vwCGgDZ8`c(%m5vJ_O%;TJPPXrzvJEqJC%KB5A`g1&xB_Kk=_1+LbPdTCZB0t zRHu#73hG!hD-UKQNt{dNbDJqozZ#(^N@_ustVuH=o)I047U#-tdqV>W$MOqBYQcBn z>kaBX*xWi!T5&T=Sf|axdgcXeyyuzsgyfc7W7>vrvuCifXK=sthb^iLsu?jaW)T zf6`j9I!LDA^EW=%GD}y{3988v^ECBeEsU>fw9jQhZImBI?egwih#N1aa~#X{qM(JIqBpFy>;kCMQv+}nT$ z?kzMsRYoyL_ufDgpQSfVJB9tw18{q|C~FZ^-BUnXYmijeAR>xY>Z&Y(k3-5K8{irqj(kkGUs=fkgEtXW3_^SOX1J%&BCi*cWVX)Mhoku z;*;90AE`j+G#azm-xj2dEnDan((agUZFkc210UoZ24V_iqaj7CryKV9eZ4cm54oo+ zy8*oBk>^=ey@R@RH6Gn|E-f%M6YtZ;8jr~>gGNj8Y1by)A^nQJ_>6;Um!qd9!AD#E z`&;I$2GoQ;b?CLebnK=gs;%@S^&j6AG~n`Q+{AiPFc zC|bbyHiOaJ9}g$uq?d{rlbCpTvWgaXd&{Dw_6|vW+@C|??6EgEbUGZ#$9jhFk7f02 zipRFQvR(U<8l83HmST?e#7sb-oZH^!Hbz2%C1`am4AesznIj;u^JT}Gn@8m~$rur+ zK|!h&+HPn5_T13NZl>d^?{rD$mY88Z)Am7ND1^OXD<2$rWNcm^v=T9a!qj%wAD59_ z|FkrKhuP(1sTX;mUBRrd%}%T1yWl5Lf*+$@Z;`o<;>F5KYN1Imi5VRr35m zN&20hopG9Z(mJB6Jf_QfRp3G(1ykD&CZ)%<db6dv`uaF4i5&qS5*@_Ne`X3x?5&oyJ*I1YYbteuvarUb; z)enpg^j6Q_8OOur{{au3n&3cY`F+?q25EeIy}!5 z!ZSCv+P1*Ye*0;GztQrGNnR34aJ3V0Nf2r_Cuz30YBm|_hjQx-L;Z3J4xcN`yKT~5 zAC+?}f3Bm~O2&YG64LZDOkJ{#7<2_rxB|PZfy2(g^bB}w(tyXOv;|8=^%8WYJiXaA z9|wl3pORz5`bvcP{_BnvW_7d&GbRz70GRP1QUJ3k)ON z{z|FJS#tDd5s6UG&oMiz?Z|uFtV%EjmD_S6r+FZ{Rmc-8WuBhLJXp$vio>u^NveL- zX=bjnPL)J8yyRVjXEo)_bmLnONc6Hh{W2oWE0wlMxqma#&FHRh($&t>ug>9<7{A4i zUO=y(k+7Yw)NDG^tOiL@j`q}z!VgzGQ)seH^?HPNNj9EhK=NaHunB7=Tr`_qQ(FmB zJr8cP=dNvUbSrwmuZRsr@1&g1>?8V&BU{2nnpALBk53f57tLInFDWaG8i@4W@w-&2 zLX$7JIC|FROvn{zst5$~feEH|Hv9r?51UrJUhz=Ym~aR#)I#pJM#Dg3lw~NT#O{&v z(3M8VsrV-jL#Xp>^&)Y>gaxK$5P?8YBKY9PEAyKVEBrQ8a$hA4J}92PX?zg8oNZv( zA=7PaNCCA5z3Q$)aknrg$^V5F!~W3XsqyUWtG$%CCwYO1gz5!#&2)x$hF`&06Yr8B zVRiLubxjQ`$b7UfsfwBEH1}kxMe)r7p?ZV&(@ZPZXrp4Dhv?**hpuekN+dcc2#_UH zy=5OR?F&!@%0RvTd!U!($7ieRO+uLi0qMij;Z^5pcIP2L+rx`*eJ~icGIsyi@@n|g z!bgT_N^cof=OFi$l_1j$=16^lm+V7lS!^HlG3N3y+7SKM)s|Cs6${>YW}yhCf!|rI z-4z?PcYAsdqbRxUuWqiiam&?M%EnnL2cxJ>pL9IaVSD`{rspY}wi5x#ZwHpLDdzT< zkc?gK_UUiJx5KfG=%V8xi zuyhJNyAfY|6n0Pub!wimft35!4Ps1hLa=NbDfW5kJQrNXe&`k~hu~IXn(or>hqqLt z8M}k2=tRw;xkXpPj;cw-H^m{7!;bA?`36!g=&fgPoHc#grdV~boql2fl^mytChSV0@-3A{4~;`Z5~XPf-qDP6kkpa+Dc9Ghh(}o%1+yM=kK#BHlL+VIeuzb(-)Xrv5l%tkODDi zr#yTaLl9f`O!h||!8e?_6Z%x21D5klg*n&NKy({FnnK^Y84;Et>xg0J@Gj@bqV9-A zw#Vs>`UNdu3YD0kvBmlL#$|u1^32NKTj5Wqi{yMP;sWIF9j1SE&`&(a{G65(2H7xq zNoz{v=X{c=mbBxna`D9qrkesM5M)5H&KC#QodVl}<(WCOA3kW_@A8AqCgGjFxg*qc z)X;K`PM0`gvkQA6do|3_N*UH}CfL7P>bOSQ{00+1z++>}H?Ln8fdN=-Y~&wp@IrZi z0i3#JlC=^odm2Xp{$$%-_%?lyH6mpbOgc8s(>KMOKl205;eAfY)zl|^mV)3OYCo#%R5i@=g zb@>?A;0Yj)GGbmpJ)X#JnddEVNr*$x#?)>UdY&sj;2@~60&N{J5e)^+-i9`Wm~)U3 z?~`slLx-3pq7IQ-R1+9`;Ds_*JLU2Hvu?g?>Sp14!==YI0VBD=&i6b62i~3Qj!{7s zOm2QSuy5a*4hPGteJ_=o&in3a?#^vcZ-q)(X})a_-Y>Hy3Lrm+TRX+~6i~>uwg|js zW+lht>z(()-H2k$lsM|}e98?l_p+lfDq%cijlTQNq~$B$SqFpGD0X%zJ#Xn`@x{34-UyMUz&4XX%2#0GVD4~+c<4`LvZ2gu$+Pt!IL z_KHlG$iw@|L~-5ejCngS1KCVT$J;^A1dL((U|doq7ylBoqdW0YIl7}B4{TcC^XwpV znFvdzTFP%)%&tLQSdk`>`Yr!VuEx*l!x*1BkmUVes!PWAGWdU0Rxi=m0$#cNQx7k7*2291H3yruw-`-?7_pDEg` zk@}CI0lBZ@*oF#X8!a3xyyro5PPG(b-R7TCg^G8I*XSA`9l7UVrG3zgHGNq5H3Kee zMe8q>nVk^4Ua0bXdPU}lPj9)O559=IhF-cdl`rAyOTtjC~#WT^#h&31h`gDPA{4 z$5?#ArI$o6L+r!j*Wy7zHGkw>8yih zz*$-b_B_ggAcBE?yEk{i90->?nJa^Oly<9N8^Qdu5N$AC^o|D0-kNtrx)dXTgS6Omhy))OXy^r~;2u?v%tb)@6 zJoedc-+rveq+r9(J01}NO6CMm2KP8zl!}Az0HjL1ssg>YhxlgSMW18*wLo|RLBjs~ zwR0tiFJ%B5dWI0T-Cer-vcQ;e(HG&T)`WN2*C5!SOz;X5XCFVm5RtPbkH8$`bz7~t zpyQra@7t$x%CE&IiU*k!xx$%WP`S6kiKeTT6Lmj{tQ9X6l6)Fsu~|4a9O`V$^x}YH zd?3ZBLwBU)fvNLMlnJR;3R^ZdKWbH*Z9%;F+=Rj;A6ZJLWDSZxy{B}n|1#iWPjg|t z1xJ4G+4fXK5z!lF?)FCcwJ9&tOI_1sK^s`eS7Z-3iRN>!S6ElNl;JyEvafxf zOw2Qw>`AxI793VHmkbe69H6h-zJXtu)Q6pY#*eUmMe>^gRix)UyqgFki$gRAY@7DB zD`5a`ee0o2NzyCFXHjc6n#$CLxZ`pZTJeb_Q#v%kb5AH7Htye?xz6|>8Vsc}>f)1D zVRgj{93q-<`I#u?KS?5vZWn!V6jz#`tb*>o*ua{BA(4~Dq2q7$Wr}A^@fGf_k)I}; zuS-@i=oTz{+k!cy?ToMY80xke3E}fRkl_W2N1pO7mgV1ASFmq4~<-7 z>h4*^B@&lpiF=x7-EH1O)8L(tjnSa+laWUeH9>R@mg(a-;O*b$?nu(OQkJutl(m?W z|5-URDmvTq`?w(#3>~@ce&sYa_LP6gOL2U`Pr>?2q^VQ%6tioyf9-S&%hQu-`j+6LGB6R zTrDKoT3e~_jQR5CcljQ518V2&Gnk81`?SPiYg@6m)oA<+37vPNlzR5x>GvIh$B(>gOpMi#9gER}0RIxz1Ed@Z{`Gm@OQM<>mu#@m6% z2w^ztVh~QQ@n1^LX95}Ssa!Xlz0?Xi+rPEZ*qnZTy+>=Wrbyp zi4ZdxC(^Q6rq5|41Fh|3u(5T;CaOyHXgxqFcX_jI+hi4n0G5U$I0maLDa?)#=8L(0 zxwX15tXlVx`8jCd_@Sx9_jH-&i3sQ!MpWjuMambfCsWwaavzfq(xFTGgy4cg`U&+7 z)I3G?$UtTDim@&M3Ji8%yXKYb*WTH}w%pk4VTPE}4O2|U-ApqD=n%uO5aB@Fx{-U zvSMXrp@rzqVOJl2r#dhRfsC=QK*rKehTNlHwQwW$<7W>}A~6(0459|EA6D;7S~W?2 zZG2|uX@&8C?_OjTghauVl1We$kK=pPWXAD*H29VQb-l<(uOFcKG)AUaM4ErKhWH1Z zXU{%~1~cUFFL>}5s%RU76T;b4N-mY+d4Bm3=N2U*+3US*@7dm66Il6w6c1g&oEt<` z*}<~cm}KiaNV{-K47=Tdp7aF3>*PQJPW`o0!N=ONIO_` z(pIW(_NW7_H_wi@8(dQRNV_qTyXKlEw0s3OiM02I% z$n0Y6d~>PURIlliopvOR=zZWTCws&%H|H8%w2*vABTh6VwU*OMLtD;lwdzLlgNp9o{HBPAB zH9kqhPE3A%f?n2(8RQUR%c1N>@dW)b+*kffC7Q`|!2IHOV z(8|{kTfqHF7*p$C;GqBz%q#9QZ~i$jrhnAoD#adlr!qDr8Fk+|UUjhp>Y4TEmf`0I z)^w7l|KcI3t6zFCLjUriM>>&JK2R{Hbvdp#<5Po0_g7SNV3PRf%p+L`u9&eRl``Fw zN1qsfiP%96?LfA$OA-Si6JLIx+>HmaHI4W&*e$C^2SC+y;6YUz{-;dWu5!K(ywFk< z|Iz;bNZPA$)c)4+L(h|uCHNgDzXpy+I`-c97ZQ&N7M`wwj5C!`gT#T9jxu?}qaQN* z=_7e`-7BWXetrE7nd$QHjtJ{~-74eWgvmsje#E_4iN0Y5G;SF?Me>N^U6}mOHJgYB z)R7#PuyfSCLVdm9tAi_zLFjc#LA^V*DMDfQnCXlI)(n>xgdbx3)WJcXzP#`wl%s{z zme|=S@1@avk>tgLlvE*)b!0$$e#raWK-@0xf!GNP2&ZGy{h6^x=U|fmDIXwQ))=E^ z>1TFGVQ&-)m4DOY(RQ^pX+VCQ)A%$G91g_Iinx200 z@3-j}C2y<=!9C7h`DGZ2k_l-Ro$B5MA^ksFeP)jSX8TJFEAiJ^3O;n{~-OVL5@&2$I)E zQqjMwU)c0GsDDH_5ORu8?Sw8vj>QFw5=m5?S(2YbKL7fth+68seBx?|0l~7$msM9j zrA-x%#iJU~!BdHn%Q-7hCwwQA0(#zQB?7MyLcZQ<8DBOVI0m&>NCZjgb#7|-n%}Rc zU@%_a6zpA@80>e$j_=Sx-2cC50o~J40`Vy|A`Vm> zg3MyHm0joMU~JjoH1Oz@fNy3&>avN|@ojGnMAM zzr)NOd(7&vC10J9nB+%pe)GAD*cU)?BIkP5%1MtmcUblPsCNE@IdE7{&kDdAIhuGK z2lTHvT#i6YDaCp|Qdr!M>T+F^*e+kZrwR4#T>UtAopDvC9I9L@R_vyz7xN%4Nm)@s zt-8?&Q%iDT9pYK^bk!Ncd6|_ND5}me(sst@hLl$hn+vr=FqNDLbd23bl~}r1j5-?k*>)Mi-zA;HH9~ZYd*Fq zPeL)&0BiZ`{aYubI?rv8%GXPpgG%>R=?~%3e9!c{&2I-m+ulo!*7jmE+NA$I3xEr! zsd&mf#t-5fL|uIaTQ&U#284?|VgDb^9zD6UNEA-dw(AHe+-0=+0_P-9S*% z*QWGP>VkbcD!kxp$p&H2M2J((_0P-)oAs@$`_6I)nlgzZoNld0ZWRV~M1D*6&#fZ2 zQlSI`i9&bV!oZ9nXGCgfrVnt#A_I4PYw1p!}7+ zf#Ofd!-OfQe&3<^k#+L+7G$5lWaS5iLY^tA(@lY%?H9tg`hR_LMgtRU%g}|2nLbp% z*NLjOyqU58_k@BnC4{)Lb2H?P?lRFS9Q-WMh?K4;BtWq?w4;#7T#4HlvHEM`qIN&D z<7IUJAnfrO(L}rksTJzy{9@txSLj0e4J@MFxShxHvCZ|1s2EGCb6fscctH~iN*h|V zj!MdBQza?Bmjckr{oO^cLr~+X%lMdnCVF8 z1)$D^{Uh*f0wh1)fp=>C$3m;^Y+>m6$0na!lk1>{#~qVVo7g<8^u4O$4%;o~<*0e( zm4;yZ_f^kUdUPpOrHD{Uo8o6~Vt!qvfqk1P9LmxbL*Xu+jIb;o!smI@&^om!m#Qk9 zu$Czz1r^Nsm_V(W#Y9QBR&s6>X=7Tv#F{AbV&2S$B(YKU-xOqn-#~D;MeY@4u&K$-Jy*+Z-|F%8uSppq4G7la zGM;x!|M(iCO3)6M(R|Fi3kR8Gk)Uy%C7?vVrI1)k!Ms29)>Fwxc(k|yy(BemJ1^|6 z2uV)8NffIqN4+QCqVh$e!OVPEklDHsFW5|6u~pa~0rs^RK@Ar{zVqujXG94s4w3I&zwfCbfIG#{gb`5V^AhJ!M(Q{{$P;SvO*NVY82_Y%h|K6_w zfYXz)Weg?^v$S)sNNsAg?qw5s2zjZqx?DpZut5 z4`0tR+n%nkd-&+#)bCNnW(}lgpNCB=E7on(&OEekiwz}9|9?*<2SV`n)XZ0eQc+hh zR?ebCdTCi&maR(oa2$OcMSvj@7wqwXHo?H_2^KcFhaPR%dmgCL%cF1`u8n&|$;R^) z*Ab-sz)7M}@VR){Al58ZAp~{G6H9qk%|FHGm;c$5E)^POE2O&CxNR%XcXOzK2h+d=j-dlvx8J!?{CwlbWTM*HEZ_$Dn z(HXq+d*1Io=Xt(!{-0~EnZ5U2_FC&+`yZ&aWby8R3sf~qU?ndu5+?rgB}X^Ewo#Pk zo&g*z_9&v}`AoVyHPYlKrB&{tg&d`W(6Jq^sqfX>xH%Wa=h${FN3d)8A~>?zUazdN zLHV$5-hY4M^U%gv@@A7?&DqSz3;IvKT5+1bXUCMDZQRq`3rmsbG)=DKp!SFFRN#kS zR~jTF?MvYe$RR}Oir-YBcONMiiGd9c(IW+9WE$(R_f=UFWb?hYdDk<_=#rwlAhU~c|CrzZZ z!tuqd*gV@JzY=uaXC|GGC_kh*OwZf=N}9F7$f1*wMe~lO1$;=I^in*g z=^&l1%q-68{3JnvY35oDPBkgOTxvSHl#aT`YkTXk`J9w~6Gm}JD0UBW7)JK1UWZoL2p{1xEu zzhhRRUxTZSX%Gczp!BUHVk*Z?6TpYat-ek3Lm@FA+<~ctD-LB1w59eRIu*n}XNq7x zV2DoMf*OvX>Tw8HDD38$Kmkve4EYKX%Nz-w*hLj9QNU;%Ub zM#i&hswpjea|f-VrOJ4(h1~sW?Lh^csn1dh0Sb7awMxWXQ!c!;sMw{g|wiL7D1|W0#XQN|s2T8}_%&u0DM> z?ym_R58SuVgI_L1`{krCx_;tjYD{7NY;jXuO6zh_FA;2Ze?QcEb%zFa;&ksD=h?0J zdG~8_c|z#$17oC93q~J3v8716=Ijc0Hd3GS-SIf(?|sN-73|r+f(`t5m^N5f%f2yq zT!j+24T3^R!;{!iTwY%fCzUPP=MB8Qyd!0Q*k*;ERb$+S(nb^66sE3k9a9C68 zWLmzqVy)<)qfD@TMCtYK*~^Fd7UtpFN*?ABdfYW#yDF-;yk39WPTB>Rt}^SC)id1s z;rZ7Ez3vq71xf&|E{`)$KZq59j$iFSNSvbYO-`ZjO)@7J4q7Xtmrr$qCK%yF3=aul zV^WZNds2B9VcVt_qX<6 zKK3_npP0mdc34ok#Idj;m>q9pAnF}gy9<5=Fk~L|EL*nE4@z8l1R3-5&BkU{IGBmwp^gjUHb2y{fM%DeTi^K9wRRaKE}qS9sd~w{_%D(gX1) z`>F-9hX1DSo9Lfx92`eNA%+d#fHSl)oPJ(sLYP(v_&zW^&F2*2iM)0(e;bLTe){4` z(NP?py!(A-fP`K2M?ez|BjC&CSO$Ga(>Sfz?UY;mZ;kkcpFlkiGBQ{lEFMwbQ^!GK zGi_`pB_i;vC_pbTz!C5rE&eWN4~Ca`@WSz81yV*t=dV=4gWIn1_69_+ z3+=NpM-WWS_>dTnVzam&rp*XwBt6Nz_7U@WdO-ZJdVEFfwWWyrSbA~SHZaAOW41Nw zFX#VBRKLN`_PAubVSc7nOzZElzsp@qx~+;fy0$}=^uguF9FC+v08iTU)9s^XUWSh= z$8KBQm1uhynZ(W#-8TW=*p)rwkqC94TwW~$*KVv@aY6t%nKb&^o`gA^0vf~#be!M1 zPtookKSAg0fKk;R=|MTM`>A4Hlfqtk3%qJ9KdS+O>7;JO0AWjrWII-aJ}_wW%{(*I z=!w5BMenPpr^V4hBqZ1B&P=^0)bDAmQcLVWd8EOvOPE|tttumc$1tQlYDqzbJRv5v zRdg)K*yzB?xVQLk)3B#jvEg2apjY$K%Q|6m*+jIq!U6L1FoMqPD8$kFdSCzqxsP;g zl2lxzo(@3pGSIM_;5M2LDl6FQmiB&O;CoF5la8>+QJ**Wb1hpwVc<pCtdYt zIA+}nhspP9GWRqby~w8WXm9*&n=*JOObXj+V`y$Cib-2^%(-gmwq8YfauIMDv7AD^ z#^W%Wu>bG6J=w$9qN>IQ)ATC0J)Dp4K<3uHp0?%VgYbAnaknYCTDZQ?$VfU|OIHF6 zQlg`t83o@wLsGBfI7+tbqU8-d2^Gd{s_p8HY^-Ygocu9!i+!-{!eu9K4z zdQ44qZ2`Frhj#@Zi2@!M%)Y0H?haRZihcb;qtaSMMCWHPK@%h7KUr1d-BWaJBTI=X z#oqXlnW(m9HYcoGASvi45Ie1V-9&anncd#7wEt@|wf&9MA&>OE(*B_b)8Xy{x4HOj z;qnPHZvrAmudC{2uUK88E2-_z01EGFw)JTHJV;yD(B8Xq3X2WNSyFGcZ4mWp@mt%I z{uw`XuQS)-#4x|)MxFVBXodQp(}!Bck+1jENP7Sn_fg142; zrlBwU(WT`mFBq(-;s& z*iRvTZApgYb*<)pK||t67ou2di0hJ^DckdW2OG{Yj=ovN$M_W*_e53zy@7=kiXOOq zh$WPN{LSR3=?}Vi9L<{*zWZgO30i$RFQ!pZJO4VEvYvJeJ_p=u3EoZmVlAw3cFuUHwxa{6lB^a zd5W(!het&k(4Xx=uRU`7HgA{0!rJ`StjI!X(Dn7=)`D^J9`{S}KJcJdi2)y3dFhZSqx*sCZ;S)Es)0d_ul{r#>U!V&9JFKb zi^j~t_AS8n&7emw_eAeicDQy2>heb;qJH{>l+P^0XNL#0a{@+C!=d`Jzt35Gio>eY z^G|=9Kmqk6TzGPwv8W!h{v#J2rlxJbCOe64S-+1gYhwu><4ay?J@HgSBPnTb$k0V! zu7LmPjs{ItczrM0Fyq3>5BK;lhm35~QaTDuK6(fR{*7mc%IH%qpwqmJWc2|f zJD;SQy;QVrc;!tCVth!tL;=ku0n4Y;-U=}gFy)KXIb*xE3K2UP-V_#RCMneW9& zuyOCWGCz*90mF1CF+miFpku(YdC#tq{8QIa0I(0grNY$kyt7ODbx|CF;IA051{*AKDL1X3I@=9hTl%5KuJWF+nBLONGuqsb~$YSTaRdq20ePuMeV zlSX?RL5{1qx%R}SG(k$=G@wKW44}i5rA9>Y0jCNKusTQ4YIJc+4=`CUdG8IeBBWw4 z>xe`vu`el|rkK_$WW|yt7YH{#c*H+V>*2Lq97ivw7mFgHnc}Smy&`mBWxiHinW%A` znKcIZmQ(xEI5b*!*V!;~&hWtT8seX#YACmmioK+53l_fv&cx(=#6k(>&r`kVBJGth zghN_-vuof0|Ne{s>~RW{qpW_f3GA7aK( zzcj|a$w2CIEbb1Mom};c9SpJ=@Ha5T8T;ON`Fl0<^CKU^?%ylJDD_Y8-V=L=(r>wr zsYA`=GKD{|;K6txg4fdeCFH$a9EBikv8xGD#A#OCe2RZ0L}6I43}`hgq_#s-!GEiqApZqDK^`9C~!Y#P*G!iFHQ~IpI1}N`eCkhuNaYpO-{w=srg*ZV8 zKS2qj#$ZHp(x=oQJj)YiZ$ekAOG35eFi-2G*&uE*Z{>js$64YGiwYQ)N4 zo^6QX)UnbA!S&`Y0_NhxYYEW_M;L+6GU9bD2LiK++%L{ILamH?^mCqjKhzNS5Kjel z^%1E;xkErnlP0w1Xc_7-S3R~N7T;s@UME?56_Un^Ig=4v40PCp68e&fdGV3wZ@tx5 zPuuQ0d8Pqilf8?k+=g1)pH~wdQ_0qXvO^y@*f+~mg{6?Q^KHopjLw4cIRA9##*6hj!Bk4uUaLOE>OUrrD?5Op zy2ZGgO)Ugc&LrNT=WzTtrm7E0k2{-!`2OMRMsECDc(s(mySjWEi*%J2l&TEjN5qLR zISoR<=N&T{J@%kq3UEbB=ubI#vvA$dkn|5sl{mcHWiA~|LveP9 z7RI~8xVn-WHbEL1-ZP+`fQ&J->w5>R3j#=c>1^%<^p_z{XXsja=tQn`(A!||EIiU8 z0%=(Z)4z4q*$lx|0(aZNkG@fYDgLod4;{4Q{s^xFB){smk;_OvGjVhtr zwqyO(Rj+B|*4Fcsji8nkv%V{7glD{#OkOY|oEkf#%MFrsyQV%Ij*-Z6#|#C(SwJ_0 zlAzBY)O>89M=N(nJW53=fkm=KaqhVU?yb8wWyIe{bUeSmdEK;bger_L3fJxg&*_AD zcJK6lQWRs2ahA^+^X%I5-P@DEm@eo%3h-s=**U4Tp%Zk~-wZ`dJ$CJVnM3_Ai1-Sk;#_{TpXV!lYb(H?L1bfgfya-Lrm4@If_ux-%OnW^Be{`eR zIDM9b`up^;3ptTBa0%IdUF?8RwZ$03cwzm8MnC06e5z7>DnWcIAuyxVBZrwRfU&yt zMPN+>=C!3bbN?g-d83+WcRg*tH6bvKZW8HDlVC>3f=!rz=2k7s-YX|XEAgR=dPK0I zhxv_o+%m&0&A0XgxoghOCw1+98^_x?Z=Ar_w?6}$iiv&2{WkabaXPct5ZU}R5_RGm zhnvIM;}>7MKAFZ=F;I*d1sJ`#;cpjf-{2m>sDpXXkBN?B)QS5!XNL<~PTuW}_1PPN zGxv@Ot7N0N9zVZx9>KWUUbD*Xojz#~brQHSb1)Y|PbBNnrjRUAmo}e~w4Pzi8^TP` zBu;o6?p=UAjLlz0d?Y2WEI3!87usTo?|hY$9l)p&-b}{|kp2k_Ne2OHls1&rqc-h> zfV|gv6bP9pAf5H#E)U6k{RWwBfPYzt5Z&|6JaUD6$|icRn%1-mS$hi4@*n z^tkLLJ~YWstgjJiA3MKgW6?G|*6KBCcA0KAQ;7)PI=)(gjlJMdk=LsK$ic_(xIRuT zen)J~4h1(Hd0YwqceVUyk7Mk(%y7XCS=4Vn^P3xF-2+2D^x!vSiiRw7lKA zVD$oNu6qO!(orOx&vE8ocVM~gjmx;pF_o_t&-qkP_{Z;`;S?avNE{U8P;j;#u&&(wBJ)1*Umu)*IINLj3W0nHhs2;q zP*cqmg)S`n0C0#I`ZeGAhqp*Xovy#>>%r%aLn)ilM6TxQh$NVimn0$U1h7=PplUVD zn~Z9gWD2yWtBxk6=?CcRWQq zv0V!uPCaT6%#zrxle$*_(>gEQ)3iOPdnbK51#Z4yNwHtw{3~MqaF5$GerI1$SA(l= zqQSHI$87UI!u?-Iq3Zy=E_!8AWhe2%{D39;-F1{cc23ewDGXc>T=%2L1fO6v-{sxK z=W5q!xR!}C#GJ>;Uwf(jXdj4{Ohc!sg+>?-Gd-9pNLiV$gWkWOW<@5YS7GF+9ffm2 z)(%=mpX7~lZ8>R-zo$Y^V%r#^WP~dD^k-ZEt)Iew{Om8Qrt|h1*z?{W*mF5Osp>=G zzwQZ=eaAI2jZbg5mMZ8Z@rAP!`0Q0D5G7{kUGu%*_dPv$1c_PW5ZIFch&-|`S&iu! zjoG&rkB)6bDM+p5{r!o3?%E~=Ct}tD>Cfqpr5r%a=FCo(B85!~lzVC^UlzYo`&n3)M^ zU0PLnX#rTNp)>Nkf4=5@=CR^6XwZCFX3=c?fbRXtyLY+|gFZVuwaFQc!oU~w;C!g$ z{cB(ku>=N7zkGw8+|ahkP>q&dckg`G5XU3{!8a`VIANrK&EQZBe3W9yQ0Qe4`Hc9$ zNV(Fz6~075qaS^IsI<_mCXTvVnu_&0g*>uPzg5{4zKUxOL>F83UkCQp{MgBP_G*eRs)K%QP%g1GZ$P*@CSIJBrd$p}v zjSIA19rvXz85aFG-b?v+5X4llswet6GD$=v;o^!Rn6J!x%I16;%Oy{C1M5Q#helkR zlQ*#L-WPEa&;X=X1gb4{B=X<&g{2zW7Aus9ZHiH}D4O&TaXu}F zUb;AKi9U^6P1RkeY9KN@PXpqkVdVJ@YNQ0!r|GPUmm4(2UuU727T~EB;I&my_mi6& zXRuH_A|LF;?siA6??>mm`v!69#KgEiHR6nkIthGveJ}3V{J<)9_4Dj{TzehF!-uEW zKyP88DYUQ@`0b%!U&52XK>lvhFO#c#!lQzLPX5#6+;T+ls=>naV#Iw`rO?V^kPNDM zUhxF*AKz8CnZ1!2*py46Es>pkSgk%T4>ED0I{)-8Z&|&U{^HZST!+_To=)5|ICyyr zR>8hi;taqM}>?h~3>jATAO=32Ig$84M2HA?!P z4y1T*JlwhTjm~e@uHRF@&X5}iJHrC#$ladhoPeD1p7gb9JqY$3E3 zULiW&#O#>hl2i+y>e_@2Q$P^=xXF($LYE!^%x}}Zk2r)#ZE7H*fC>$qUVSQ$A&`l= z3fNcf`$EWie(`=~KI^u=?M{MM9TzNhb<=;pTKY8$^Ai44IFXzG3mw>#KsQd}0998I zKMoVq?-FWVo8~T*lVV)b>A*w>Csi(OUZKO(HbU`_%Hae=|J}`J62n*9v3Wlu z_QdYn)s1c~E-@Tm9IB%k>flEGjY2=;8h7J{`R$Nz45RE@bQ!tSLsmc5j@w=zQv+Vv zLiP!C62HVKeO%l~n~kR|InK-`f%#;aLv{lSF8!fw>v_=ex&xqm<&y)e0t&ULKnq#` zzbk^gMTd$4e${zR+qB|>^BQ^V#BDW+{w2(R?dYu9d-$e#`x3le^vtOL>l1%-3yb`( z->^Sjp2*63y;!;xo#dVfo(puO-FJ}jlgIaLI+kDDWPM8iE%?HiQ1Pxj*YRag**D;i zR_R~-3yf^@OfyUvUC`K{dg45G?$B@ipt_!*BPsLQj09aP*W9&`dTeGj8Um;t7h}WP zY3iCa1UpaaP_lsNfN;rV;!y2(G_;(b(XWY~YSzd6Sb(wVJEn&jcTz*>gkVbFni|e> zr{0mHos{1l8Xujl+(#uU9yeYf1x2)=kw4wlir+J&uP^tYmqfexp4n#mi1X}@*G7jR zOwwFM;yN!(s#h6i%)o+_1!AUTO;h_D!etu#??jjdU{!R{Ylk&15P z|DkR9{z5+>)~(h&2idcFw)gKv#5=vBCz=J>Ci@w;>pYG@G}ZvAhLAbprRSFNWZk%& z1Z#CKo6$d9uGGN=Z9x<2zg9hZm5&clI%V{;9i5CXF^m=xb~G;p4nU36;V5^`TI?{` zSH8zQ4gWj~=uq6u-{cJotIfOPWyCf3A#nN69f$9j5oS*4+2)=g@iQX?jo8CPf%oOG z6|mVs6h*P=EfW=VloJ}<8(5#5^d~$J@sX8+SE7H&d@Qg5n*UxyjNzv~(`N6nGmw|Gji+>~x2w_1d}l9!Nwl5)l1U19?|og;;xp1>sW| zMGC{7ZNBjm52TE|ObnAzVe{^pDeXa*h(bOil*p!02?3*g^lGFm087>~Ph=7PDmjYR z+G*G(^dUAHA&wmVHD+X2URAreE3mG#nvh4Yk=(~CZL)QaIC=~76pfp>iz_o8t(4S$ z0V4e#$a9i!~##DAbi=AdgYh^mKR^ZCfV2<1v?;FLp=2?1& zO5gROS(~ab{u5HOCbXp8ES^!fvQ?z9{V-TskV8uwGKgknBubt$^7Ix%v-xf}d}DJv z%1IDM(sZg$#J8-)D;aS%YTKXY0T>$k9u{ripB~X(o~Dx%0su3gy|^8GXYT`0vIQLJ zAu1K1oVF~KNdN`l_L(M!%iubFcTtKF?wdD8VJ)3*&V!Xrn-|)KQi>2h0w1#0Pd*d8 z6LFqp!QiFPuVWM%UPn@sd8=H;(yn#MvZM98M>E{3e9mDykAAHTq-;2jfVm};a zE@(lzpI}%l|GoqFronPQUBP(WC}xQ{#@cp0_`W0DIk$;CjEUx9FF9p%Tl}B#7Od+{ z)_$xM#@8j3E+C5j&{W-4e|_gosz3NKqP^(|7!~ zXX$4%ZhPk|Cz>w}(wQyfGDX6UE}BwVF&tpkidce$wKSc#Uy^C!8?;X>!8yZnm~2PN z-_zaBR$$_88z?J!6uI{gjvy%(GiUsr1hzfTk$F2c{%$>H0l1vR^m+z{4)WYH6vWyH z{le%hBl23$mx{v3g&_=`zP>d>PV}#JT}ce@bDq_R8}lk+ zixV-}OKEGmq!FE3%pY^#nNWJmobX)$T~40eZw(Ml z8^=Su@Ldw+qm4S$Uv^ExC2(AJ^jMB&K?wr*>E}?J2~fux{9KP6YeGg>e&>nSenyn) zbR!Oyh%gd~wZfE zbh-24P`=H66bOPJBO?~(1{C?z}kh9&t2%dlVGR6lgRGY|*9Gme|Hs+HuRa1;a;8$QD#Shnj_-T501{gQB zYnFVu8xEO!Br>uiE^#$^uI1%7TjmW1H3jv9bvv^Wi1e6&sPRFJt4H#3f1L=@Ho7}v zTFFn^I7@dTF3nYhN~^C*Y}PTBAJ~TuD~HPYL2d8jqET``YC4<0!x!u<&-Hr<2uW-z9nq*%4mKJ~z#k>q*sh59sm%5)#fr1h;T=A+U zUg4|y5VC~o!5!G)qaytogoyN6ZLTCf^`q^{jT7hWT7#cQy%$r*PweD`RD04k5J43wZjAJHDR>uf8}i z;p+S3CVWdY-klWsp*!TM4%mfg7mGALyD?O<9(V!NawRpzks7B9$?;tx zl7fJ|HwWzLT}%exMWcYF0Kgj$dvu`m4WR-;fyn*7mQca+ON;mx&Kx7x{pe#M_uV*w=MCO z(<z5TfgBQCvxyVa)bWxy zs7V^L1wdcO=t~iG;x54LPMXQw#=4%+uXXBpKVUTI99d@85i8n^n&eD5C=6e8C(o)_w z5idfCo#Nn)=oCDC$rfc6ed0V&B*@Ik#W2|eso(&TX+$;Xg@oE_jo`$X)KbS0L3M6DcBJ>$`+2M4 z7%3X^v8BmudJVb3_ltUe)a?w?!8Qt%T{q*+A{Y^ZtC7R$2W`oY;Nt&fM;@Uziy=mr zN`!RN0qx4O*Z~+H=AZYD?@8-7pT|LIOY_I|e#NA;R?%SF70Ii@GU!dH`R6QtF+teO zsL;#Jf)V}8Ra;w!-vU$C#k|KZdsqEC@oDBO!w2l8KY6_%@@UB0v3~aZ6H)Fy^p$zT zbGYgr^Y>K#3jLf$`Ekt4OjacG)TQ#k0;b>56_;wKC7rjXf<3<_V?HFz;8QebrGNV= zYt-aku6P6@5wZS_mDy8=w`MgH%Laq(IztD$&pIT>9k6gJ_}gSnJ7O-Dx%nV0gUf%c zXrbBi#a3?=4z53yX>ZimY=nH~hhAlC$4s4&v-pmQf0kGAy6y7F!*-=M9^ohhlMJ!_xZ-RgQD0e|FaHovjTbG_v5re~q0Z-LF29&Ro%LO5gYa6vod%uBK zxyzi6S}(@l#+9Sw4vt;f!v9{_vgJ@*Tx3t&h$kScMqJg85 zk5+iMV`L9+6K-8L8IeJ&w)XrMO#RtWh2pG*N5Tf?<5VicJ%s`+a=-_^If{v(_+EXv z>*pRma@W)EK}gK-{T#f~QEHZ1yS#Pvu(3eL_FI(G;brra9KTxfb^sG{Cpe_b`^P;O ztFL|Lt99*>^sLs#1Nx0WcN2=wkz;dPC-c_hi`;G@IsJbegUcJw!oXP%_uh{(sS*E( zKJ0e9exI_Sc-_Lm!dR?7o&W0^s&;?D?b?r{7blWZylF19xDCoL_v{ZzVGJW_#e;-W zFpI3j7SsDPNSsk@+&O$mFy;+#nwBx1mFb;2mw$1?R8Dj%K-bS|+f*F5P$HeL8}9J- z)cfr4jmv4R9rws5qTSJqxVt}{G#>*H*n>n`N%p=S0w198At)4GZ+k!U ziI|?&1qvi|Hs6n%z-T!)_|-D(CJtw`Ax496WGV{MlMRxM@(#v} z@FCAv^V}S`S|7-N_qxZvOE|S{!?*omDPu@MNnY9NnMj+6b731Hzr!fkF^8;SB?*;> zNFiy9qjSj~Z(xg5LPrQSbI`f)g2@la7R!(XD2Bv&nhUxL15goC6wYTPXxTy7^S%fC zvZec6R==QGDrFiqC%8y5F8`Swvd#9XOf5a*_5hZ))gK_2P7Y6 zqtNF@?+`R24Q@rXMd!Hn-ea_ALLzYmH^%<4z$Bo7ute!EyHW1pgi#eVCli9aYds*4iG zTlSmB2;i^y{m&Z`>dTYbGT)sf?g+N!H0Q9vkHU}T^v(J6ne7)^pX18DT)1i0cW<9c9Ve2a&b-29H)_l)iEht1e7Zwm^ ziLVMxOh(oQ6RHT1%gkBJzLq^ZWYha5rxuK7L&#o89C=E3@kO~r5Ov87KmJN^WG_K- z?1^;{V_7Ju6R&3_?HYp?p{XZVUsS{n#Y+Y*lLMEZ#>%&$GiA`4meZ<@;6i`alDrJ) zFAJF{8{DlRu>YYL+Fckjqfm)MzJVGgXYqMipA*o{8s5$7*8SQFNNVpoXhk#{n7vGH z?>I$DzGb7Hs}{_%1T5jLF&nn81r-fDlWwlhT0<0zNS77U;GAPJ3UxR2)DJ9k0xH1- zdQMs+PLiILR4ABMMZ+|(c#n3)XL=frTZ81p?aJ?NsfV>bLy1oKrU6}lJBYLBuXIqj zxY_KBV(B~ooI3Y&Ug3Y0hpoyheUXy}+=zm2&u4qvY*C1K#l==yFJiVyh>V#;YC!ZP z$6YLG7pzND`77|>QSmJ!1oclquhZ@L6&Y-pn1B|;KfUlwK0JlV7I?lw(&1%%bVagQ z1<~-v%I_@)lIX=Q3`rvq2DI%RUpADl)-V#%0VRQ@$M2Lx<|LHGbo$=pD?O>BKD8wD zO!HV^+S)!4H0y;hzqnwjATOPGTF7^wn14M(Q8m`jbivkk!O4@O z@;*eD2v>H&0Sl;02V6D}9zRgs2~fhylf})Gb=jKaGz02dV@PrnB!mu$H$2lYv3pM~ zU|GaGR~R;vAL4B-cH5>&SaXaSA~5h42&`}gRN?__FarI)eI5-Bj9~^_jK%J4NedV5 zD6nX;CA5DCO`_)iYO9ZyW2dD8G5a1ylNbu3&y?lpoxFjST~UP*QTRD5|V%8NvFOSCwM^O#3N}xBWmFDntDaX$;K*DIzPy@xbxzUAc#+ApjMKpkAsBj zEj2MwzJy1KGtKWq*7t`_KZo4!?yXr#9*`*yo9-fD-Sq_K#6MK)dWM1dqaPCf5&4bA zFe68F#=W(p;RtKHx2nf!kn8drX{f*dXsTJ++9M#5+m`&`k6qe-gs4p;_H_2Yu&uzc zF;wHB87ZzV?fsri89N zsb+D8WkOw`HdTUtk-T&o-c~Og2!`W7CN%g;y22HqxtG&B(as9rZVQ&jH_cUE0fqgJW;&fvf+Zv;z<=`V zEA{jnF7x9yyA{~`|7^c4?YNF{yN(!~gf3*?P-rl-xBKcb^0X6>$QZNyAZhS|{E?SG z-!Ow<=HZcSYQke`x0dYayW7v+^nPD>$?}w{yQu_J*y>Jt#9Ib@NwPWl%&w)~$_XdYIZ#uUw@21>Y zku!-Ma-`!^83qk9p|WPN<&Rr-DT4(^K^WM~90O34eLpWNa@zq@<(ngt%aATHPSf5uH!|Tqs8lHI0OTh6{)xc? z8@NdvS7KQR=F|u;hxy}B82PfX{PmUlyIrZrk}-n<_qPI5vU}R2UWZ&EZ&1c+95tSO z*facdC{_pKlir8i$|}ix-H0qL@{zx5Jn63y)uGD4G27=IKb^=0_{KP`aVc(R$a zd0t7o=}Z?1h(0L9(Cmw-v6`g0hKRA6bZ!c9aFx1rZh}HP~rKgN? z-E6cNvqNxRDFEVup1jg<1TVyZau;(`x8-}`1qi*-PYo+iaF(t zqA(M4`B%zh$AQg0KQhCUBGI5#n31dCX7N0Vc@-m5Qun{cwbO;Nbk*gdS($2_fahor zNfXDcj~*^PDUwl1w)HL9!)FIk%9flVA7Tc}A18kW*8)*jlx6H#op_MbsLj50w=v4zxhQ>6^P_bUE3Qaea&44&Vk&?;W$%I_ilcyxu&ZQ;#sW(shH1wd;eE@?0ZhW=}dB9S~BqWd&yjN zA!wTxQKuGB_E%#{-%5K*be|3_v;?iNjTO#{wdA1g6)^yxD`2K}vIpwlgBwj5=YUNR zl#TBZNyf#l{L||tuQ5W0wl@F|`iciTK^Ci1HdslRDjIPXO$~j88ln>5U0@AlE@13| z(9rNe6*Qq`Q!j;X>*txx&Ya*1p~-Q40rNN-HW*CL-gN$r&B!GARF zU6j2iKjj{+k8_x)d)$1qY?7;zmmn6>k7Zuu!r3!od+(yOnYB3bJZOIF| z-@cEL@^9hO9UL%~#aNx>S&J4;bEMUF41JZamkK64C|yWT8BRJbjmer|rRg{R(58;t zB`c234*yA%|Ay;F_w+NN+{D1*FTurHTnK(V0N>LN?b^LQb0hU9dGBH9dgKsl;eKji zY1i5sR&MfTH`E=rcxTropTjc~Y5JKp<&5XId;3`rH$1D$>LH z_~5(fNe3}XL4J*_lkAuK0%8wBw%SGxci!Rp`&dqg%!{s)l{B@tohbidL1x9oBq}WC z3~4;xol+%h#iR&JdMwU3I=ozcP=wr(erqB5gqNut{^HQi!K@YiRNLs}@=R&LW90b# zc-9zy$3u$rwO3S8YIHJoX!Cu$>wmCqqjSc3WH5!n%%R)6IFQI!(gam{#ZZ%9nj#>IpDeUPzU^3Yho>jHIEcqv_wzcYk{Cb)I2Gss1$1cGUy;fiJp5 zfs+TJgokNf%*6Ntqj&TC9-ZZ>DyBYm#M|I3t;&L=009C;%u(m0Ol1dbwxFufaLJ58 zD4$UQx^jV3^!q@{4%!+Kd5xR|&&V0OU-0Q+N^j1xD$3~F5yn||M$I1}wNjGw#5lq> zHImdcj6glCh}I{j7DOy%VwW#9_hib(f;EzHN~h2w%ATCE{4G|MDw~(epO&o9lrGbj zD*2K_Fbc3Hw=Y2b3P_Sk?pQb^G0xinjUx|U4Cpk7IR495F|Ht8KRNkV*>uzjSThGJ z{J1d$H$Hyy$`$$wuaNv;a%nZRs#6cmDd+k1)n6b){tFYUtsoQANO9=N`{(%u_Xo>1}qu0_ftXWRXw57|dCL8UByns1g@EYxmw72{ip9NE2O0?4le)!d?1q%VMtz3R2Gs!xI(HZF;`5*j8uRqvT=HAsRU zM0EhPO9D76hFd}cDe_6`+ZTxs&RmWd{d(O@A*0E^%tl^vCT@(}YQ!=~aclVt8 z<>Pst|0LJ!3BU4hzL?#K_-QKa9mFTE2@)&w(TLUtgT+;XQ&W8)5NgD@GOU+8WsSh- zWT5#R^*GRR(|T@A3l&|5xV4ZwFzh#8VcI!-o*h^0u=(U?eEu=P7JM z%eB>64A5%y&`K2oy_3;#sf7#EeNkc!_>khjQYsX)U(PIpk*K^c2ys03#JASx`NP#6 z`Sq{ClhYpmu7N@Dw|+0FKpiqFY=0O*ujg|{5q^@0NU@`tA;z!!_s*mKJzOD}qqy;o z0oKlAE;7$?jv8V|gLHyPKm6sX7I;NPLcPS5M-lYbB>x0>?PhLT{RKvWH4;6dz8kj6vNaM(qjs@95v1Z+ zt94j0%f{;kYuwHn9_xpPtN+7|I_zaat}|=52L~OTgqU$3cPqE=!bW3tW0|*n#1L{?4C{QlD=7hzK8Y`XSh9#G>Le$?R_OYKmotm! zc!ghLXesA_qh@)a6J@FtN&y5x7x7GXu`1Gn35xhf=y*rIp(4d2VRDg2v431lM~D(h zSn_3MTfM{_ZWBAHJ9i5kNI+4~&AmMgpvl3Xs}4BY#S$puz#kM*n1#0g!vV<^jgCXG+%{)QS{qB_&!Vu4n%_G6r>r&L!Fs_r@d@15B;&mbSum2&HaPG1o-qrXI4@)nwk0BVD3^4KVxc(-(6z@?U(xQ>Twb zWnxqmWn_NEI9r}-iZ&gCnPK(wP}*DnGegZ{+_|+9q_n4A7BpQE!Es4vbOJ%_CjE`@S7#C-?>ekx}zPy#y^*~u}7yk+DBpx{SV6@+jGa90` zq4)FvvpT4!xoIF53zGlP7#e8nBkI!$5S#l-F>vg9LAgak57QN|nh=j;y=hLh?L1L| zY)ql>G|`fBL`FasuXwhFv$&$0FM^-}L;ZfITT1mF{mtmFabmO?-@S_<3X-C6?|2gx z6TKa40PnXg42n9S4n^_Z2tnH)%SND|)|TCb5y-9@&j+&zBncCe`F@4X6AcOoWcUJn zSi`f^)_&eIed5~JlgJ$tdqFvRdl{43jJtn-{~@KBW#4bKXD;h~(n=-_2X{4y>lt!~ z2085v{ZoUx8joA({{|j>VGwk8K4J{HRlA4oRs7sOBZWY&LI+PmK{Q&c!Q#{{XC$j0 zH#BZXUtaSLuHY8DmjYj34s<-c-cra#>+*bSdmc2JBl#M!^NC~hHZR(Q`k-C|WkYlw zoO=;FQrAu%8oaIZQeIIerx4-gCm~2CF`JpGXJ#W4zx1G+)Kx+zlNS?}4le&AHmPG5B8uSfHRrl6Id_v!L8 z6U*n$Br~6kv}V!?O!ifqQB{^iFRJ&)y!C0nBvTztIOu)9WPPY@6PV58%#@o0pq5H1Q6xy z;hkitr>Deq=~u`&`H-(e7nF?srlP-rKG3;-*pC;XoGdYe=^N>1cmR^I8}@_!#*<^ie44q$@lR^y2XzUH>EcFtx88}Y#&7&xp{i$J`cFx}w-;e)HG;*>!tX99;=`@aNAJN0Piy*>VkZ$sDV zIZfT9sQnKw?iZxtxNz8cAlT~R@7Q?sm_tne4<(g)?0yBq!v@u;*jJYQ&4BHJ+`7Sz zS64s5P=%Irt!Ar-VH(iS2a58EdyO_y zM0p-_YY?f+PQ5{ixu`-RmH<;6D?UVw5A6t7B4~Wv_aD95;eBcTrn*shzVJ9F!XPX* z?*|F(K>mstUfJacb@0=_)xjllb+aA-8KLmjU7(2y2mL#Cysi==I^5IAH3V<)4 zvCrEMkV}dGrYYYXi4hb2Ok}DTcScrQs%Z16H^rz|?MJuH(@aga_czY{9AlhKNk89} zvOKU?-xQQIYNkoDNwI`j`MWK8lWyyLIqCt{@Bm}IikmrLFw7NRyw$`=YeU*n(S;s` zw{Hn9PzbLD5%TfzP0g*vuNN3zdtJzB`Es?1k4&xDba&&EJ*MtjZz^*?;@A{ld_aA~ z`Skq+L0Z(3T>Bc8w0GjbBx4s*Wp#B?Tv{PEe*OX01wh>Q{-^lR`>DV*9F7t)9JWJV zRHbxYJEOs^&-AF+>})7qu$Ng}1+h#AJbjXP(orI5FJn_FXr}H# z--!oNP2He&uTyTCH|+qgQ*LSA0WM^(Z>W1@v?wsg6oa0Dp<4+YnO2#pJawJ031cdf z@EeE<`q0=5Ewuy9p(vCpPPx*tXzi~rinwu6%%y%H=L}b^v{A5DVOW-_1kQpyM>zQ;LYMM@C9P_jm z+s)K) z=xPPxBa8h=K&g{3Rh>i;ZhY`P!BV6+p&#FX5{JXeU7k8p$e;3d7ag+hsns(wgC=Yq z$ffU3W`|gGR?y{heRm=2YQz*EK6)tiLuTR+%5~Fj5s>M1FxS$l|AnmRBkl>WTVGPR zka)_iz>YE8l2Up(`PUcvpf0GV--EyP)EB1v%|sv-O0*bL6g!iTZY@ZRjNSlqTd)R0!IO{DFxEf);)qke59K}ygk+yM`T(AhdOJ*OA$Lji(1s%EdB0gccRs~4lC@x}e z?XtQ`Qw{#T0xAkTKjr)05qkL`j}|R>qiPYNKrRQqRDa1eA-`!Py4a-~K3OXirKC^G z`U}O9@sd<8a`GE_RNp+UAKo?8{7X@z+2z)eFY@#4;>==IDY+{_C_@jB`YV{Q{=mLr zF^#LYE&m98TqlVZ`hZnG%dE*0JNf+s-XU?)DsK2J1jGHuSbkpB24NJ7EJl}k^5PGP z)qo9{af=`FwpGN`%2L#NipRAN>y*C;HWeDvp%kMGF2fV#jmR4~d`TP^Ts{1h3g^Hh zXbPiE*#54$YIss@wqn{iFDCW_@#}E%k>~D1?5X?RIBg=fi7#V#6dj(>M_~Ie!Y?vX z@B>NVO;$W~42;{3iK}22^r6;+j=Mmu^Ml~fNW-iLkzZ~t4rI>bBBtidIQ~fj{+t3W zgq1*3*+lWw?H=q^U%0b*;FO(^DSegtjNaEz65fB|_+TToNnnN#w5F^d<%Qs_Uxf5y zn9g&g-~ARE8|UKa)&P}&zH4dKvMgeYu*SGNj>>C7MV$7ehIWmH*3pI*E0Nu#lBLEZ z8zTaZx{u@*Y_sA2$F02xE^LTX=s2Rm_ z-0i~7+tqGfMdufc%mWV!FCk)qYpKG+{Cs60cedYsVmo|7!$(hmh_Vz8a{^>xkJ(Z(n(6OH?HgDB1S3XACm(nui~*Zth^ zbYgk+_p;Z!8LZ4eMu%bYhzcOxp`&L* zwcvS&ce6Q<9;{Tld@mMK)-VGc5yONY4Q5EKETCLecHzS56=bf9q^nE(_pOj=*N-=| zjouH2E~4*;Ua@Hi2HI7x$wUUes-0&WWj9?j#n?8p7_sy0S8Kl?)N?c-;2M`V zT_rJcQqm|=kuvN0ZlcCWStMU@aD zY}cJ#4O${L7l&Zpo{tq338n5IHwv^8i0FSR?^M+a%*gCDP)%vkZ)gt|JDr}|=bNP8 z6`DMqpJ9|FxAQ&{-JPB~t)Db+&n$0q7FZBp%508CoM5+%>AH&|V?Fu}7CbM$hVIt3 z0xmV8FoTEL2~G{IsK^+_`M|SAzr9av9&c!RlAEEKtT*%4_AFD?ot-#f$yZQzxdRzS z#c7_4vzsRa07^*F7lWEqGeN-aPT_G%J$elZ9lSzpSa(M2DJKx~e&+d_z?RQY>M^$f z_T+O1%P)vzJtPh7t*r=}6v=BOoWYOoN5sch|2&-cceVSSs(WT}$ zc$bg>B06#H*DXe*c-8E*`g-y$)ltd@yj#bSwi*B~l~3K9I;&UUoj$YLijH2O=f|SI zf2~5c$w`=iw`X8A%lqsKuKE!k(*cb5`A}3P@&L2y+0^&=ww8;p7+P*J9V_q7cRw#E zmaxb!=ezn2c$7}7o0Wy{>vAPCnI0$J%#s{ea!rO*MCRbwSFyd-(X%UU`b!IQ(u+^q zw|YIs#4C_RX2+^kP8&)kY%UtEuYiAMQ7i zS?NLDdiEvrtuf3-n9dmx%DT5RhAx*?J)@Skl+NSErWU-EbqRaQ2Ar`L^$o37V#Ort z)9;i5%d#0XC1d+h;s%u?9YwhsbL|}2nHc!LCi1{Ukzb(r?WUNuUPf5vNiWjC>Ez0y z(=oENcAMHS_EUkN9wkrPFFBb5gjy2ufNqiJ+hbDsYF|9I#(YkhA}E~Rv)Qj|@yIya>>m$mU2n8gwC##0!H8tsud-vpnf9A#EW)cc;5PzEV(I0*h zRf9uN&Bmy1A>Y#W?d{l~-{p7))RE&fgPp0+ocmF|7a@@hdc#{rCZ5Kh8#1HgYbKX} zw_!ocCB%=N0PJCG9FZ+Q)Qi6o-h*o<9`=Io-UcnokOW^LV~-2qw0;rU6`0blE^`58 zA4{T2h|h^!5sUaL?krbzH|oU)W*y%zb5YN=_&B(qPwuQ*_B1k(+@%;F1=g)G<2Y$- z*X05aro=;Jqq$N7a*NfvFX~Uq2E}p{P-Cf$Vp3GiBwVGRC}Qpl2L}X*&x3i>FQ#?t zSHf5TslKtOwp}E)T?N%a3>o6lSo-IP^I=tgyI!PiGsy)t_S}=#EcIJ$BfH{8f$*kF zKOCd}`*-ks-mcb58*Q`pLfi^<8@KKc^dzG;g}r^`Ygway@Qc;Zg|St)e!qb{>8LQe zXDo8DblK-#FomBKAuuGVU)Dsv@)n=XK|{G4FnyxE{*|56<@8#LHlg9>7FNxJ0?%gM zTSpU_UJg(_pesh2hF^MQ6;xD&0V-M9qU%z=%sB>LH`Wc0iymDiJ_=^WFBHu4E=WKG z4yIY(*I!00(7(HC=W7ai&;Wu`KMi@(2L9gy$dnR63nsW%2m#ny@3cu$eZJ6J)JJ}1 zL4E!^G!mb4`n0rp#DSh)pA&cF829a0&SaG%I*jp!x1W^Gad4F5>(a>({OOCQwgcj7 zDVaESqv0*k^8&SzOA(c+`O3}ogvzpIUxw?zZ;n@Y<9FZFDC2OOCdwOg#=x4!?#C8o^MWF$&REypH-Q5tjy}W7-@Iu#RkU49G|ftL04C> zTbgs-ocTA`%l~w0MBBDwUf>88r(;mG=v+rCl$fPs#hgQA{K;pcT{eH}WMu z@Mr4Wl|J!vQTyt?w9HKIGNMwcyNa2v@kT9Jc$C?Nh;?n~ZS;lF8tMOf0cJwZf8A8E zbdcclPG_Xu6y62Gf;s;i1zi6TJ^b1YNV7k?PJI%)B6;_!MaxWfQ2@Vyhmrk?2B4#7 zNWc{h0n+*nsAg&cRKQ+3{h@+IUybniyam;yhh$0!3?5%&EW}ejThYr)!A*1<=nA(O zfScPR%*AH)<_>crc9e;h_&S~67Ghz+*R zbmi^-oS!1Qdudx{{jAM=~w##+br6pK6uwTs)&;~X+fcM!UF4~*R-%Eo^z6_fSX84;9do-p~ z1&>Z?x9TVv6i`nzE9T`kiZoZUZIqUKs9`=))l0y-$C9W8K2J_a{jmnf2FKJTib7G3 z>gUUg3>`))20x`*LY{!U1EJ{ng>RDVG~gGiEBiQZ+nTtFj3O&m;t8^93NZV)0;tSRaX|7f|?i484A2{wxTTX<2m zAnJNM$P#>hrk3e*)RGPC2+=e+BJF=}Gu_yfm-DI=F}pA_xupCbsDS7gdz3kmQW#M0 zCtR<`K}jir#7TROb_F^;BRnB*LAn_|otMrms%G-vkXXFqaUUlldS+im)im=%4nS6` zWm!dL_3D^a!n=#Xn;{a57py;RJr;RIJ>txB2cY+i5%WJ3vluqtL-UB8n>4=6nrcCv zGKsBy!~z1@ZIn=L9LXPVIQ-yOOXh(qV^~x|QCBdSAGDzTn=njHh@}d=n3Vn_aqAC- z%jQ8=ift-GW*e7kb+dwQlY)xDdjaBBRaIdJwo##93pPDnoLF_Skg=68+An0^bP#O+BFL zDlAU==)gB7u+)wU-N^d8MOxkDut-8W%9RBeFNKttSewg3%8iGG2X%6?WFE@p^Ky4s zCmOXs*H+LDc2nmjK(FmEoda$w*jeixs}?_$@=eGbp$aN2v11DN>;CG0_dp_i^8*jgtDn~zO);6W&+k7UrTBKwsrldK=`_)A*Sa3 zxtd203XA@yg2LA(o6>WdJcAUmxjBKS-gWD(Cs7j|&pfxw6zyK-gDV#W;@unTDX_8T z)Id%MtS%)TAn7%~4*az%`VXr7&u|$L6+jEdy=QsP5xjZqR0&c(6WYYh=T3mtGJ6HE zt9AWFl&o^4Y1DNVK^>cv8bN!G_(OmG#M_|j=%|6?GL!)FDIuQ01l|!JbFxzHpxpF9 z;AjIMk5qUx+mPLF1@f>ste-_A^~Fn{w-rdxqD*>W}RPZGs!AzT9fvfap`s)=0WqtWy?GWUV*P^X2B>OY_O zOCBA#31hH#hZ7Yqr$7K%QmIsb{`I|H;pb6tAgCMg3}*FR^C0$BZ;PO)qnyx?rkS{r z2IbWNTat2ttzQN&V6WgIHzxG^8VrtQaVW-$d(8ZM6apf`_La+}ba2|LAE{kUP3%G^H3(Rx_LU9xms%TfY!DeJ)eP!!WG< z@M05pCR5ohR3cyU;x_5pJ->P1(b3@;7C5i<^4?SGn}_Oz#c$t2qT%53iG`VN9ulTD zz8rRrz+FL2_|=ByeIA!!BWLL56gN#^Z~Bp>Sk~Grs8#s*?SL7J#QgZATN~eUDy8Vo zE7pH?7#1!@17ihKwjJ3%UCS{rw4528jTWehUwj6D72= z`m^1d6!p^$1s3|7fvmJYhm=Q$Ld#R1R;FyJGIqp2iBBeUJYgFOhzd1O{4)8RH!6$~ zUrA;1P2A_odx^}>v#&=chDh+kkjID7pI1{x0QWQJ=C^i>+KR~?VN(Zyzi&aZKXhaD zocX7$jAjn;UR}uJbY!BoM|kAMgSA-r-TUyk$=#?46_fUxC|nM1SD1MnktG+Wce0k*5XzmWl?=|Y1_*zvqOrCgKbF|6_A&7av z{cuei%6%A0y%US-207d^mpVt`27ysr+B{HFSN$nK?H4Wu0f%%hUB~3i?*$4ltOwo( z9OA`bQ+EIT^DNrHoq@3SI=nJ~j& zm2IxbAs64ZE9E#pfc=iMqP8sJdQ<@Q71rL?;In3FK(4+Xy*?KbNcqyHPmA(ip{-}D z_bh)Mmmt*L7r4WaBg$6x=-&<9LuGZvZ>y8KevKb@6cMYO6-j75aRYgUeUCOrX_r0Bi zvD2w{rdAaC7lwq$`P<%lahhZsdpLdj?R<8fe?jY&My>iE+*@(t|~2(?vN;9Z30~lBFRh;;wt+Cv|%% zxh(GRGvUiZX?Ve6zo+O=Qw7QUF&XRfoO1QD^guwyr}SA1ap=1n`aWqc%v8ykd`Hg8gE1z%eY7{Af_~IB zt6Sz*dTG@I+2v*HE=7M_il%Ie`l~Aph)~pLbXwjGwf*ElFNC-jv>ym@cYF}&jlt>N zkPhQT)RN8Vn;2}W(Vt;|u+D_z#9ccw*C@WQP4I7y8tAl|C}psEd;@-Ahi6@A7Aqiu zxTvHN^XX|*P5$JH2?=K^lc1V|{>O{^UO^_7m;1jD)(Z5-*c$fD?nXU3LpH<~!U4Pb zzCz>)i7H(O;Vyxi6U{4!Dw`9zf?0J*`Ih)=&yNY<2f{hUMf^8G1B24}>o+JpGw?^% zW`H#`xtWS%m&KH!;QWFI1mEKBe6IwoY6mK1N}&A@_w`xw!$bV2Y~hP%y&(n8ZyPfBmzRM|R3c2=wF|{yl7;BhZE{|_*9-{i-F!wyl*{%|LYZb% z=L!}9R-iMlUv0f#ElGzDzh-+pUwb%xZVr9gOqXW(jLX2MX6^jnSXzdZ)Jl|dRz=-) zMay;N%XMV~b(zmxLpI+B9KIA|bO;erKk1NFHM7|_u-pBtuKev#8jWI`siW(Q0%{o; zP$#}Iq-r#&Z#moU*l{=E^5fH(jq8t9r=re{L&wW_?!Th2|2sSRt*U6<=KHcu(J+Zp z8AEo@sQjg_*CFZnUT0RpUOxhwyMkK(nSG8@fp;|K5>`ab7J+0>!i?IDQ%U?rO8q`m z8SR2Cz7NZ9+As~L=1t=TM-n!=|EyjiYUAh%V}=O7oQFjo3DlGh)(nIH@NY&l&|UWr zNZkRz&u^aQTd|-O1UT>8T|xTVwc@}BzxJLr45$^|VS`d&`|NgIJ7A-yZ+Az^!au+! zYx90^`5Iz7C>dhy|0*NJsp=}(7zv+?G;rmFWtuuVp$=y%roMRs@ZXhjRN^^#5GL(9 z&XWJ^zL8<3xY*#mRy=8cf}Y)#^OwyrE~lellkDgv_wwV+!OW(|NEd09Dc##pZR0T!R6umSX3_U_XP!}$bWqpyi<(H zotVk_W@ndY{rJ+US|O$#*`^Lus1E#s0?Oj|hFOT|B+Ei09A5$ti{2(X)eLan$&}g7!5F?wQ?`Ec z`bc?}=P-dLCB^x5*7c*{-+z&4UO11GU{?Uq;t|c=7}=|}=3Z%MF0~3EQ_!(#F$<9H z<3PZ>7tI7kTujDhvT=0{Po-Q8>UwHXb#clt7kZ5hPMy~H$qanTwB9S8@sFo16zFgG z5-b^FJfF^*!O)AsZNP-|7_=Ea{v{M1+1E6JQOiC*o`591nY`4%ksMx7e+n5<=v(m4gRL+oNGc1yn{5(-nP zhRznu2cB2oR81CnmPA)0sGq=e{R@Ms_pM2WsToajpF}8qP*AZB{vaSoix0)tBYlJy z(@U}-e7d4#ReK$Fb)x|D=;Mh@Rn1ENSI=Az-ua=6g`moX3p3m5PWCYqwC8)3=FPRP zoze-t4l5@2uP*1eCylTjfY+xym!j6oh9o_yCf2-#e74stBWt9@5A7x3EiKue*tvEe zPv+j|dGXE=s{cOqsL|yA13d!M^FzU7ruEHca*r=;e=s>A%77WjZ1*Te|L!svj~hAE zr1@Rwx1w}bm~63kbd((jhM?i)mJ>tP`w)uA*rdG!qxWMS6H?5s&X`ZrM)jf21ZalY zY|2wQpDS%clQMfgk=*vZS)_6uC^zIkD1`4eIo4V3ezsf<8sYgyN<1X-HU0n)n(Ds8 zV_g|4ecpZ*keM6Fw)~---AL|1QF5=9H$Awfq>VK;^O1S|e$esyg~P>>qco`<5YxaF8%^F(+yff zgHhkiw(5bKBU>wl@Y}_Up!2{(VvKtRtp1l2s!W@>Q(5@cHT3!b&C2^WBX5sGX~Ivg zRY%t$v_6A@PhH;S%N6{22Aibq8)YXc0C`QUG*@mCfWkuhdDnc)iK~=-VB6Q^8Ll_q z9xfd#yjJHQ1F8~2d;yNfuS3(OUv^2$uPh-5u_Nz^n^*DZU+V(4o~-vuKu0%@X-pI0 z|HE{D#5I0Aq`Vg0c>dxU%!Ziu3>k-&u&AR~-+hJEGw>XS zSf-6-(&ts;+QNiclzl*iTPb)hBhxDU$hh6zq#5iW!Mtjp!5g zJBqw1DW=sN?0SQZ#)axLNIt}W;{W8U#pm9~T9@bcD$7DRJ}k`AT*`)M3lQ>GGd4P6 zj6_u^W@nv1Wsdp9cx^p`8G?|kh^~6Y`98x^UP3Vv52n9V^*ag>#d8W$aYr_rxcGw&M5@H|y~Jn_BPRg<`v|p`+cuTO z`7#~7me`2vjC95~Zi6)tMs_x>xMD`$gkPn0ssz#L>5SE^NqV|(YD)V`OW$ht78rAF zkPE7ZLS79#zQWt2=>~?lQ#syTRsI#!`6a7%#4Kr$o9@@FxU*!_)48TT=fWkp=W`@C zgGLh~1A3OWUfOP@9G=|F@V#&Mt_X#b6ECEKKC5J<+S)cfzX7*5QB*CLeJ|^quJ0`z zNNe(-^r4W0jvqY70+`f0liylk{GsZm`2nQWH`{xCcHzA#9e*(KmfpsoyaI8-O&kLL zbHUN9x5vrBt`E==Lh5E#{|C0clf<{DCB#b#-#Y*PAu)xmR2ai!4~FhnYIKq3#|lH} zh#842kow;BWaA(wTD0z;-bt0R+eXd?XcPS+q(YIWYacb*I7DnnE~*8DTt#{U#f@c# zfz-#|K=0U>9}w!Njwv*4QZ;ulmiF+($qzV@Mhu7;IaoEo$e{az&q(m%7FJBI zsngVC8!dBzcB}N=;|vKun0FGDXwoP#UApu1_IUC@enk=eLKw$_axFUDhD&$*;5%yt ztlGxbhUP}_c;Ll!sEjnh3+ZxEi`^-+U8-l7m(LM5rpy*%DY?w=RS}dqCR^-AXD=gi zsF?}Y8L>O%moQLLGKB~DLOA`2+AkXYOp}cV5caGaO!lV@^PRxBi-mayYSMU`>ZwSYkT*~+Nvs}4lpc@hG_ZZitb-j`?PWH=DUYTud=8+BrsY{A|kLPKVa^MZ3GBr5S^#YNmO&+E7fsM z{mQzMni{!wApA;GQX@7>uo|4I#Le z%X|@D8C2bXCEqYW#MLEr8r#y+5lrTk{Ps5Z#Ho}opA7+)eqDobXVlR;KQ_GQ|1FqT^t+H9Di6 zS<%-n7wH@OKtCQUr9dPX(C=x!v%qD3rH>`gFZX5B-yrW%pgOVG4As>ULZucjderK z08{oz1Q=R>0x;_~t6f_E93RpRR@guZv6w&xNfmi+0(n?Z|5MhipA=G2;B|3_aIu)U za#Q%}Yuljn;?$r=!28U)Ex)saHpe`UG|E?nN*fp?~c7Z>gSl!S6=zbfB3V`hh+H_rV)&nLin@yDLi=!h|9B!E^7insWCyUGf z{;(`mn79>r0faqTy0o0WYpR>uS+^?Z)RKkRFpURMjZ?87t6??xdo}o3+1)x7bvCAv z%~bGy=!O^Nh-c?yuvb%&ui|JZskMD=>?Pzn#F|}dm^GUdNRYbC7IUbzsdiSZK3m>7 z{y2H+ZO5o0eOFR=HVn~3eYz3q_pL=b>zWVq7Gm-iGRR6Xah>y0R?BDPB@Hkqb6@qD z%TRB*rf;f^`_9f2k)Ywgw%yZcChM4;&9KKSAiglQY_5F#*mw_f!o!t#Cf6zNBtSwt zk?ev72a8%3Cq{KaUrIskmA_6toV#0_`2T0fFx42|y#Xh3qi)d9rv@WvSAt64JMTEk zXByJ#lRBMm-&Rk6GgbQe(&WQD(*n;~i9_){LF365Jd=Lxah_t&`o$yfFi2V$7DS&A4fy@3HM~>B0cW6wHAzR0sN@qMDFtf$qOkLppaeF~ zpa99xPlL09jm~DbMvbPl*~0)MbiP=%f(Q_0KJj$Xr3Qb7$6C$D`ekthcNgwO8dkxf z7)VCfCBEV>S1BqRkHnA`lpDt)7Hq_vWk}%%!_#U6_^8x$?bJXEtwxK5S_HSvkwR_M zrik&F8ltsPwqua!*p!_*6nH(A)>li&b1<#OW22_~#vqrTbt8!Az9bbnG?62Nq4pm^ zCNJz+uH~S}D5x8V|9puI0&*3CP5QM=t6QoyFVM!g|HF;ympRVgpo7f-p;wC+0-2dA zAX4EfM??nC03y6T&s94gB5`Rmc6PIH^pU{rV!7d=-7xJyxih?=;Om+WW`Kh)Jk(!S z;lJq%QKN8BV_n^h^z~L@>2I&cHkY)p=g(Cq*}{l4+#$+4#&4wJPX;f^G;tV;tsWsn zc0{S4q7r|8)F7r)B(;3(>;O-*V7(EES>l)8*G_)DfFnMB`Vs@xkxyUW%z&sg_W?Xd z+Uf?T6(W;mS>-Y@kVBG5MFLoB8xJzJ;si4Zy`P>kVzqWdV>t7yJ&7`7U1(9{s={dV)dV< zf=gK;%X2MMQeo1>n|F)JgYV!>hs_utq=IEgyU#}XHq*lZ-OXB3ix=-VrJIZ`7xY~9 zraut3!%QW7^agM8q`>WGynaqn{>~)!TO|WF#RF-7JyWIr-X{)zrXp1CFJ~PVcC1`4 zg7Iuhuj-EG%yE!OxV_YZTJqt?z@m=xIp;*hgN__WNP`Gyo{5NvS46?hv$MC4!6)iW z?BDKQf)mZ3Y5@~jK4#Xy0NWL9?_&47xp_D{t!%eXX|>@#<_t=DWC z9k&gzwIz}1d1mj(!yCu1|Mke3FWnt{pFoCh?eA;C7h}gXSnL$58Pg(ZM9xMVycAKd zq(jbuNg?<^SLdtlvjq}F^0E+ayAlfc--N~37?>RB`Q7@A8b zqAY!T?(F=(UVv6B+&u~c$VZdPdWuSXx(y$GK-Uv01MJZgURq-U#4H`nR5zICbC^z= zWh7ssDD0MoUmFcPK3L{mA~VICCJ@X~Ch2J7)!2X(#=49pIY@%X?4+e_Iw6z>{x+&t z>FF+WlrNskJqVqt$Lgt`8Dh!5iCUw+fISu!@MGH+qxhDqCWusyNrX^`)GiW}PICGT zQT;<_zOQK^;leh=#?Geae4o6)YznonbU)OTEZrzC6;g{_s4g`$b<)wgGn zV4)fy-EPhAezM5rDzc+;`UzO||0D{xF;V`1j!6ADDz*%6HmGXl6~3&=Of%y$Gc(3- zCG)$P#4&{Q3GBdo=dUYCTkmB94lo*L-GUy%0H@tz7}ISSy!zljSX7bFLCCTrytH*d z<%XJ~&M9r<{52gZ7uW|l=`r^$Z8fO><*zY10nF|(ZS8#PZtI0ji<<3j4@n)sm?c7> zdWQ79AmUum*@+=(`lX6H?r#`3?@R^Zub$=%ldoI)!PqkZzn62X@Sm<3 znU`4n)EGX3`nJ`Ts)?DKtJVgEV&KG+rfP_376e9I#bR51 znk7wl(b2uF4bP!L2L0V~b>#ZwBF^KUzJB8^t>AuIk4*sq$l*PIm`V2u*`8@qIt{52RnT z!PtKu`KTXmRJKZkt_?xikI=_f=%DN z&rQKpq8z7u&E1|eZ7}7Os^c-zGPUYGGk0oISH`L$`-hL|$*P#~Q2K-6Z-tuOgOWi= zY6W$Ba6R>WoA!t{dR^1 zTN%3x+gXyzyqrtDIPdH^Y4fxVxeTgQi)}!?t}Y$X#?Fk_-0io8M#fmuN>h?p;143^_rFc$6gpCjE}oed zl4z8ggueCk*(DGVohHWbhuKZp8vfP;Zk{{V?FX#6uBsS!fC5z?HgJ!Ce+BRf@%v7AplP{v z9Z6HjFIxTjXwN&&RSP<<0!X@^Y^@G7=y;gFeGy;l9FPcyP*F-II+noVtP1Or8o6FL_bKQ0B+@s40^m(N)_kH zxvb^Y-^+#UE6r#-l#Z2D^iydi`l6l1l4nbL3cYx`k(2ht`BZL)!>FW_c|Ia{Cq|^p z*aP`)GVDDOUo>oEd13k@=i#^i(*)?CKtMO*a(u^oD+oItn=>8rf`9;O(`C0V<6Gn z)35h!ztg||W}%sc8*g{w+2N&9zw@H*5uomoPe|1F)1LhUE}Mu0mYon>kfzmr_`zs= zKyQ3OKfL{1$hy3st?Mz2*YPmJGoF3CTj3!Mf5v;=^{MK}81ST&+JK!t*Y5d~;q74q zbW+=V6hYaIlY#y}UTsP+jfd|uvh=p+lYQ(tPXQ!&6EH}C4E_Cs6K3;z3&k{v8>P$w zHnTW&N_0PhG3}oX+f#)?l^~G33DN20q(CE*Ih6Y{_FMYY4H@UA=neR#VqMF{lS@Ym1;Xsa!QJ+m z?gb3}a3MA98TMutB;5-`bMIFmbsuS5@PzZY9E>!>6ShXGP61q)Ct5b> zC?Es#!Y{|_*o`JuQ}ij$D(vsdL1SNi*T-uKS=_fi<0padY(Idm-iUmcegT6b=)thC zt4DnB?+~n;E9;*dNT}Fvy1}x@tA0z-5`I^6qkcR7^!w0Qs$wLo0M7LWV=M%cDuutb z>44a>UwNfF&1^51RRpwW8rGxlB$UnNP879~KR;$=fv+egb|+@g`5t}#WE~ER`}9U@ z4gw`B3Asc3(P!s=>l?BBX}4o~!?{jcZk72>)!6BWRs597cP}hR7lh!)6#;#)h2qbm zyuaUm!Q-2P5M=T*N50B#U1D@t@$R1P_;dAO;o;4dgat9Dn#9%#b6v9w967eQZp^RM@QTGm?K{ng&9HW!|k49zZE z{bim%g-R+v_2jpRu)^+|?Z=4CoQ5Y@aEIZ6Z)bHR>AWLPW4Ltg=kfl7O;$+X_$Ur6lB~zvc&IVG!~Pw7NQd2>_2;t@kHmuMZb# zN&otBZw0tqMrMc&>+fJi8$Z45Y~Hyvom_AvaUP%1%d2;)b44A5#DkxxsVXI=mB)l6 zKUD=_q;P2F&%3IfY2rhNLj1`yS~6a&h<=*%>T2}f92X_fZo$~0Nq za4^mokJIT9qNjQi6@HpqvbEv(V=PzI`_cN$vTX8^!ReI2&X=`&f=sN3S|B&){wcIZ zoIv=<5zh;hypJt0-dyDQ=3+kYC!8T3N)&8Bc9j>`c(gnHGw48aQR*=T*EfVu324SVh%?E9( z*YatQ=+{URk0Kc2=qe3Mz-YCZc@v(vIOcIk_?h zsg=woHCN1v3LvxgNO#Yq-$h%zM@~fZYwfj~@uq|5f18xytdIF_#}3iOI=e7mt){DT zpe!uwFRT&>{3(M9Z5RUYSAKOWOb2x%HfTTxZKqKV8f{y$ONIs1mVdZjXDx31xMTMI z(juCB!AibI#K^4k{S>AlREpzciR8+c8d#?&Aw+M9TVb=L(qI&Az*)UWCSNC*OXh|l zXqcScyp7vXAwS(tXsU=w+J6yttA^G*N#K=U?plQ96~>_9AcoboIHPMZ(%;if%6k>U zWa%rNZ1AkLGuTU$NANvdf>SW?%bOLQ|FwoET&f@f*)b}g5FUk|?8ir#ZC-Jc+#s0I z4nm4r_b=^Tsf+aS&Ti5}+jqH801Mi62o-8&{sUn|&R<9T2<8Sk?cJyRi*hx;2BAXA zM6{gbTkOldMqdS5tsTkG9sen0Z>Edv=mpBO(FCQI>w8f|Ekh`Z$-sH|wqWoO^#Pb4 z1hZIYq7f`FSg&zXJvB4;k#%6+Pim{9+5}f34=VnC*sGIPlRQ9#LEmb?-~KB!TR5$3 z5QkK0pw%v#?E?${cy`v&C@HI?Qx+9voH5#P4ikpRORZ&-@J8bp@*s9@&RMU)_RZg< zYON01C3+yQhEE*^1|mS>(hs&$gd3%G#(MWnRrU0B6`~fTz@0OvX8AbP$eVc)&Xx3^ z=V}SFofviZ6Z+2%t4wHqbK#V7+mEo*Q6hj>Lt!{!AMW3u=FgSM+{1z*Leke`R%uJu z0;YPEkx~}`{lygnyE7^WM)dwCbxg;I;MJ7QQ>N;d(gY-&YGOLm@L#&N#YaljOR(%_|#Q0LNh0pp1nJ+UFhhX1@8Zlk;Fw9%lwn zsAGHfX78Uvv9wR23OnX&qd9@t9K43jf3|EAfpHaL%s^XO^_IK`LF7NJ-G#bZ>}`{0 z0&Q(RRc#rIxlGP(_eb8rmG|&E;QEdWv;RvD{%_CdKiwj}eHR-~9+lK!PTV3b@`rZgy?+;jblQxOOJ4No#lP6VUIKMYS9}lVG zp#@VC{K-wq74=Ww`*qbe0z=PCneS41@%-z*a`%I7)TDP??yYu*H>LV`K(OZGo*a*? zj(4K~Z4%}PD?f|}{@k#!hbH3dJmTb-CR%{LX_Q+;e02L)md`PNjmuvT{@W~2Fy+D03H;tBXcWm{>f(PsR` z=kQA*qV7{9L)QCZ&nB5IFn~oMAv-e7t)_*ws;;=YNCt))^h~yTZK%^6%Gh1Y zWJ_puxuciv{aCTIEgeXaE_urxMVD++cKZpZ>RAZfZEVZt&{Q2Y9OnuLs0bUh%$&Px zs`;bx0yHc%Gjnlq+x2!MKlAXQaPiy*1BtMxMd^-|nIC&?JB2);O1a$s6WIT~>yHAQ zSLx*c1S_oqiU#oWrPU12r*q>@ft>x>&>buI-*(;0%92n8JB^(Qv@gVLI}i?%TYgqx*X~AsDT}{XxPWkzpHfmllSE-13<;3xiFd+ z4a-=cLV$HlSn1vF5PR2gk{auH)7GTk?h2{e<&^OnG5`>W$?1t-cV5@JumGMz=jIKu z4!1oB*90VGlnps!B@Kios*dL+-@xl357jV=r;lHH7S>4^|(*<34d!fvta zEfm}X#mf*B;$zE!ueWrZ)^;3!V|}|^7rYgpu&oz<+L*uF=23jQwHvYyN5Y~S>|{UR z%?ZeD)Nd=+mZywF;#N~IhWPBC`f>7oXH<#co-(kKFWnf_wPQ^FmBZ1L-K5`CBegGGBh^q$lIx#tGP`K1=`C$ZiF4~NSgi-!#4f~l685b!GQT=*UA%bH%@U<9+gd}uCdcNlNO z%O$Jl4^KQ|3!YrH(G;7XA9*2vyV4)n*wQNs;bTZYaV#3N=qJhf+?jCl1sPkvYx&7Q z1oI75KtOc^yE(hLFXUrsKINZ|T%9+GIeVm$vr|LE7=W7AX}^g7Dj&z_6KZ)lXr) z_W6kW(ljJ53_-yYXHoq~&dG7ca_+7A#Jo(mgi-+$wdz0XT8U*mCbBwFNlCxSzVU15 z;>vDn-s@~#ba_9QWPStKN3SdEeNDia(Q{V=zF)!`M^FB6yrji9^M>ab){-luVISZhOOSsFJZQx@!X#`xVC#!}`j0p1agb7Y>Vpte{;N`Hi8sgY-a{IR5kg~z1 zTb=EaxspdNkyl#M2nS}7pIX;`CVS`kc#Uy|NrDlK-}lVF?v%etbr$Taz+a&ffW$8> zCbB+Bs&KD!1pB;O+&nC-f`Ddvsaz||V97jl9ys4U=IpbSO4kn5=`hLPQ-u0UUc(eM z4m&XuB*G#QfQz2~UIk3XiEIL9MV*g$!~OecXj$2F*Ma=M7FINRlU_#Uo*atjDC{56 z$c0Slt0xlr+ISP&gP08na~_Z}cE?Ys9h)4vlW#E*j*3jaHP|E$PuS{NvCRZPC|&_f z#ORZnMEN}&LQ9Q9=5qG7Qpm-(0v(jVid&pBC9M7orj_(o?=LU$k_XfFA>!dSP;$fm z|1=2y3p2GoivIt4=(|y9O}Lq1nZ6bx5@k4@ zIwr!f`QEe67<0DKn7RAo+27%O5&Idtd$GT*L|su2%>CMI5$Kz%L%-kn9I<&equWG_ z73CWrb~VZAf@FMUk;hkxZd_> zm9$bFCKi3lOnX?+pbr>1wFG&M{U6QBzZ~`d3}JvOGyX7{ZPmw7JW*s#=3L3ALMxA1CIc9VM;+POwI0$o!i7UpGiXV&`V`DJv(Q-h?0t2v}MXVChl*G z$?e_%X4qGFML|R&%;!n8yJ0sKlPke2E|ReddB}&I^P<4*%ivD?Mr!;x1n^pDKb+g!LA30 z9h9>hrZO!ge^|fxY(LYOT=42gk!-8_U>Y-4w9WxtGL?IN>!@}3w)~6*W|=sBeygWL z{V0d*0~m%g{1Qzwr;vmgyyRzOWVCK!;TJw<7Nqoru|S<^>{keXmMVVrk$yj%sJ9XS*d;vd z=~3him(p+&0Z45LJaK_Bmh7Bl`$tVIz}(1IWBrPuQVBsS#=Sov3yjdI4<=z;PvVX% zN|KP^T@!Mpv)7BQ1((bDj1uqrkE^`yf9ylDZrBG11W~@Yi{e9|SC@j4)s5gc(Wwgz zBNXMlsU@_LsXN&<7ZE)LU54t`#oLge;) z`j8ww8sYozN9Hh!_wZembm+hy)X3r#nhG4>^}%}n($2gV=kh}XJ+%SS&MsJn4H6Mv z_|A~eNE!N`;iW%gnUj7Q+hJiPZOi|CsQG4fsle7jPEpGipg5S?2P z|1=8?eCI~0Ftg)zc+iGacM!APUs$Ji{r#E8>5n{Y`)NTPnW;~zs)w(gOBIi|BJz{Q z=+!v8!U_fNmC)tAB$WfHZMMS@(BCL?wdpP7S}LvGwsb1D|08|hXZpC;1=)OcqFT&U z8PMm@%PVo&glxm6yG-S-l^#(*8HjRfd?7Er1|o`*^_vI&R~sMa;)JI48#iO4*qOJ0 zm^s~B6riIzd=s{EKJDvQBmuEAXZLrd885yANA3#c<&i*oc8^~}n$mxVMu-2U1^6$o z8$BRhrhItiJP)D~`Iv|D`VmloVxF1U2zbVMytRx8D@FxXd`NisOVF!Q1>&+T!QpD7 zk^4GIW+F)EAQr>yiVOCjid(zG`Kta2C3+HRzF^l~vF9^tm}pAD?9Bd8?VOwxVMon@ zQmXYPBPw`{dbsbWA`AP8z^}a*?zkAiyVjH0K%wGaPk1`nT#7!X9Bh0)E2MLCu87BW zOum+g-dV-q0Vq^+d|1@dls+VpIJo$*RLq6qLH2%yXKYC)W6YZLTXh*=O!hK3x2(}W zb^f`~Rsp36d0c&Qo-vdw|7~~t=MsF|3%@z8*))c`J!cxji0h>c?p6plB8F_sams!9 zouFD3_0A0!!ynmQ6P-eH{H&w*lLo^RrSp5S<>?i&;zj|10e7#6Vb}Pc{@L>neKt++ zUq%1JQ?@o;+@I>Ptky&@8Kpg~DJ&c?$!ILUP>6oJXZ?sz9`|utdI*YjDJb^Y z>VHwy{#Kg#YE1Cc&Wzw011>R)S2f$GN?pwxH7vkP<~d3_EivrfS;LB~+~&G#A7Xen zFORMtl7_KJ27LGLDEqF6FshAwv1fV(6a$p3#ZGlFLS91*di~t%BCr?Ee0>_u;eOEJ zw{eki+>6p!)ikP77U7gy^D{A5_h2HD627`2a;qZwf};2_rqIF>XVrWGP`GXLph4VZ z=IDSOiwb@U_TLXMd8AYg?UnzTZ}p`NWN=9?SMdBQ8pAOq2gV~C>`2eKM&_OZI3N0i z6qPC>IMD4cAp_s#;XKFvt<0jq`{~z%=M~;bB?%?P3cZ&49IK=Lc5o~li~}7Uo@@{K zs|BP`8H2O)#ceR2f`Yc%Djlr@PE8@Xw1Z6osO`gmo#cC~!98AVYwJx*aMwOIN$9hT zGErFCe03rVewj{fl42wu62JrH)mAozF0vV7`?psM?z2L;iMaR6S{Ejqh9ez6*sdV# zSZ^9Q2LGcKR6&y8y!$5sQ2rIowTINVa#F!-n)ZRsn|Xj>rRC*TUB+t0ZIfto!3bv3 zU_Usal_=xNHKfQJ&X24yrKe?Vii_J)f9C?ux?Nb-t`TVI1?cHd6N`p#L$HH)O%Xb* zJ9B!_y4aZiueKaeaOt+X@x9Ncd#|#HNaNc(!^m)VPaEPx1ef=h*42Xb1c+T>Y4><5 zTU83^RF4-_(cfs0?Np5yWD_(D4aSiCaWPUgTdq?n3g7Y-!|yo@IEE)%R0QMZDBxaD zln-TRHd%S!5?K!qW+`8S-zzl~y(YX$b`N%}YLM*g$96ws4lTqh+6*&O(^}EW%{r2f z>-(?}mg$rc0JWEs{y0ThrPzLuNQw>2W*KJD!0!h9sPbsK@IU3&ATDP}eR{Ri?vl*) z&=o)r<$73UJIa#@br)AB;@7V>+gdH>R@4eY-%cb<(8dWkHkAn#57D!N6)?)Xc!yaQ z!;jNWJm?NzNP#6FNFG~0Ek{=O}fS`GU(?oR2RFoCf+ ztaoRNcZ=fR56*XIjPGBzq=|6W&WT^LMo)0m&z}!y?N$Do^i#Kspu`LC89;KcI2MoIN1JA)khkV0TQ@Q-a z0d9^M90ha>4(!^U%!Fk*R$CX`($P+pl5qi_3sM_4Kj20NU=EqsuOIcst08>~qe$OKt zFDS>@JHaY1FOM(d7@W%RF!t%f+ys;r^i)*1hXKu7G7D>AQFfng8c6%hKnqki}iv34H5=@oNcZX%NfCw)&;M_`g0|T zjl;_*#(pH^ndaq_a)^Vs6W|FPx1tQD25C4>fc7LKGq!`ty8CK7A6E0 z!=^P`0Vj>hB(xF>e+ZDYd}(wX?*Ej9qLGU*5+ZsWz$n)tPWts!;EzTgTZ}L*Cly5E$A9_Fga;8~dU8|O-|&*BpS*H& zItzZD{;v;sI$xk|o%@M_YO(g0-N0)|4A+$6&;SJjw^YPdLY6W8ZyLFJrB40PPuJqe zVum|fOXUm(B*HHafwvd2m^vrcq8@VVW7OI~BB;mka5s-|H==yb{9~}vsD+;xoPD!O zr*q_rauSY<8JdNGbo697q_ZMf6w${EBGB}+{8P?lvi6q*C+D^=CEEj&FnF)~tf(wb zwo=~-KY?JqUq@sx*xHBXdZrE9r$%mO9sKFkMH6`(5`pVVWkzVBX%ulBtSQesHo(+W z4&30id`3sPMNM zZ$tFvDreM+3JjSS(e;Ff-2!qBWVHqaWvX_ugEq7V*oKqSd`}Ed#a!zb2vw?9ZosQh z@rj*|xbIjbceASG_s0MAA)^YVeqHB&p2Hk9C4wZs!)@Jzq3h+8CFt6l@Iz6X^N01Z z?-S-Dmjg&!AA*|>mz;1AnqUTxTo+2j68PiseBz#dv=Hz(IzY!Ukl%EJqJy;1rcSd< z-O*F0l5I2}_m6Djc2bN9ywV&PUxwMp#+c;_Bt8Q@`on>Ln6nwxv_BIa_#bm&JO8d^Dyv(&3JES8Lgum0U;J zxgM?)*X9?IY*7}BhbA4y-t@=fgq4!!S!Y3;(SXhVZX+v+heHVkhGN?~CR{%HvMCNDwOj{VpYyNf1yHsb1vO{waZrx&T(h4Uw;Li8m(J3lH zEBcNmRtJ0SG%b4(cE4@Ku7_KzO>b-?Yg`7M-&?48@R!inRNKyOg>~WZ%-NN0srR2R z{JRWzK2*T9LDfx;kpI|?Qqk-g>*e=yS8>?q{}_$DF}~MsaVI4IV?S)}L$NqIu1#E-8^g3ag)JaW(*VtH1s10&&PgCDjWJS}&07o^(6XbND2` z{JTT=!_^leiJMAVl*g-*sz02uV*Idi609i;?J_SfCbs#$_zhO5VVZCl=bHh3Oc)F@ zS?uv#dI)-s0&Zh7#1v7(&O=YPyoc*8(w~F2)ex9-Xtiw=6V{THbJs!b5Da?kWVaaa z3bL{~#+iK9axa^x&1H@UmD7^E?IqH>s#%@QuvOA9HFyt#)*j7~5ok%8^q)%70p7#r zsJE>*PS<<%aE+WJye8__Y`&K+6yqS6j3PE1CO!I32;r5QOt87nTXpgyzt0opxbLz$UwoYc~cnKLg8J zVZx>^t9Z!}#?v@KB1>*!fsU1lDx7L2x78_smHFRrVw4wuM9{>~22zG6w*bb0kMz;B z>{xN`yaHWl1ZRfU`^@3}jlI2x^F890VIdrRy(Vc_J#emk`#gK7JUm&T&Xd9nW}sHj z-N=tGHS`bHdprl5f%Lj4; zaLqC8h9yQ_4g%;A`mI=!A0UKhnSqy%Io(e!&7wAEVF!ERtwPt24p{$F7LPt!I)V0j zQDQlBPW;N`gkJ`^nobPu*upmXg#K26jPo5xu>B5mASZ5$_!(ot)LJ&dS_X&xV3Ce* zIbv%_H-xg6c>Zrl%4@U|F!(+aeEK&URtmcN{lO&geC#ps;CEglEdva>{0k7zOK!@L z5bJa$1>S7Vo6QY7Y@;wvUZv?l12Mp?0Sldgh6!l~d8+T=;W7RMsE%k`d6MQ?^O|3+$ZixBDfKQjD zZd{~HPmzj}QRHdeZ-qSD4p%klFH)plhZg2UU&?Q+qC4wOasRYjbx*xG324xazz-F8UxxBY3ZaP{Q#qh7XlBer+$)~7< z$dngTdEn4fvM_{Mvar; zyHShz?CHeVTn!V%R#5qP*VTMI0+$&MLy81Nbl4+h^GK(e5D#6R`lVRk7od?e>v}&0 zi|Ur;+8uRI?-WgM050!D4*z5$kkK9rQMu#Zrk0HdC zMy2OrQw2IB$w602e4C*+2M|&rDX3IuZzfpxC*a72PvU&zd5_YMwvxg}qg<`j7^SG{ zx>kxC!T_$;jkVND){Z3u3V7-oq#jO=>Np7i$6V|kLsKf<+Ow_--DZRqb^ zS8*W!lVx;8PT>}Z{uiImtbefA_B<>dpd}iWx5uaLSn_ml!T9n4jQxar@!Wbw_}X5z&~ii@JvH-Q_k24}2V)VGLejQ!jcZ zJr55QI;Z4?kM<-VT-z)O+@snX^-{<%_f9|X&3Wuzd$!ZzO1H^_KH6B4SbavBC4;8g z5|Bi+!pm27p6g`5up{pA{fvlIRIoiY{?Ej!X(H|7qNIgeSYsX!Jfz^;m}$=SnbBP$ z^_v=26`&bs{NxOrEnh!We+67JZS@zkF(*n{jusTYdTIjU&q zk)}V@LQ#&Nz$sPPeJ(~Ub|G29E;M$jQ0(dH{21xm^VnXhb>5j4aw^I>^2a%^(lOv< z1OmybD?Ck8iF^_K`6MFV^D-)rJ>x(&Ct$e${ozTC_mu)J6^RsuROosC+v9}~y4zB4 zn50kv7qRTS8uX|F4@E?DB!m_Xcz8L7+?G-eHsTZ%ovJ3Tec}aD44HCR4NYcpkW=*T zr!8iq7bN6`Ab!_d!v!)kVu)k#^UJeAfk|;#*nbS(ff(Ogx84|CJ-V$RYRT`N5W`yz zne$#70Uy9-{y1*CZnz9fEwzj}3mq|2L?&E^xE%AeG3Jm8#TeCFlq^>qraC}i8Ig{z z_yGQ|1m!X|Soi@vzl3buaU5FC*zlWa65+GZn_J3=@Saz=FioZ=Zn;D{q6#^VafG}X z=IrrZF^&f{-KrF;g(>4+p^v!nb18pX%HALup|wWR$uZq(4;QKE(VNH(cCYmJw#uMa z@cDUSWKWRoxct4%UHQG;b}%>X)isF5clRAdHo{KlJIo{@FS)UHl5Edpz_vecnitV_ z<2y>e-NB~))to=Ad-xBbH%Ncgsp5UoQ~HXdpZk|u$U}lR;HEvp`v&hZhRs?kw~89x z(&)dXR!tf)K~!*(+jh;O|C8*cZ>p!JP} znf(vyr^uXRb_vGW$yE)$xUDSIK(}NZx+S`N(tA>zfnT%cc}}!T^bdSKZLvf>juGbZ zFa=nVeDDL#=0(&;i~BzU_Aq}VH@Ge000cW83;}1>N6EBP%0F+nCUFnHG7C9t!bjQ>oDGgHu%}?5B zAF4qmk`={m_qQG5FOZ>ryFSr>KQ!+zN#48kmq5VI0)p*U8|=w1kF{Ia=7po9;SXUJcK@o`1aUN9zHyd<74Ha&wh}P+ZkLR2Cw<6s zlul|AsYsAjN$S*?@4ZvJE56v!Dzln+I$aOFs8#s9bsV|W5*Cg(i&y2lSM^h$#_T(h z53CjaUX2!b*lM$HZFIzX1^PW+Mc>=b!QWf&By{t-JXi!mUb}gH_u%uzJsG5euMrVK zUU3O@BLB1q;W&^c&NzOR%ge&>Xs0Orq`GmZ3wL>~)cpDk4(M;N(RCuz&G)J~%`YgJ$<+|dnTZmKE zN+Yoxhr@P~cZ5r8jLXWrt#^Jq;{yEoC<#J~M?ls5#=xSh?GFXXU&?`iM`a3@n){&Yk>V!y{~KVG72FbLLm!>4q?TajjLU9YHb?+%pq=kg|$~l&C*c&m`&PDC)b^ zRS61|E^b9E?#0NuDMb&$nq?LDA9?e^gC=kxxp8;G4-3+-3r7cx{~^=gMoN&)BZh!H zl3a{n7W50u>B4=aqKGe)`sU1QEkCCkOtfr+IH+Nh@90^+{n=l1#vzL_4_>Ds=69oGKdpw!fDi_KHF3^8(tcsq z%0{Vuk-6rHrndvOYy%VPcA)`xpT5(^!Q;mM4H?@Ftjftarf$kqEO@drF@ZwztwvEB zH@aCbfL?j(TPkD<$uaJJR?Znf9MC@PX??K!U$ZQtbQCs`Fi@g z6N11s>BRMd@SUnvggQ(On_i?0M+dL1CdAT3p>be9aW8fvHNo2p?qg9n3aVQyqO2;O z^SQ}xwnbjIQXcCqimt9z%Mn|Y`E0d9NtiS$Z?sv8&d6qpOe~=+#tnZx2M4XXr+k&Q z?8kuw7{1qXJHz8f_ZO|s`)w2O?O|MBy3{DQs;DN*V!1J^Y(@&k3(HB#R@{)}_wQ+GyhMl{woo0l+djP;Xk%-IrcqC@vIfN`52f(V1=zIU@M&vIs+gNy;WV&b z%m)yi>xFn6RjjApCk8)>3ideqa=oZi5hpcFaeBV?H#=mB8)AU7i-SxuH<|6Qq^R!<;7 zdx6f>BuVo5MBB0^zB*Dtd}hl%Dhu@vNkH_>TS#>4i%#&uC#}@GR4xJOjRoap2$O(N zhA?RU#rGVWlxd4b)xJ$H`+M7xyJhsIg2;^Q0WAE*nTK> zbHVCaokumG_7g}Dfm;#*R1x1OwkHy$;DjTW_5cksK{YBQv(N>cjCga|E**R`p}?q1 zxPC2LY2_oLqhc((WE6u#oK{}!@TYkTlErHwg;!tNv~NN{5lLLY6$3$*PYI2WRlDa- zfb5+Ws}0ucFdH7^@}TS9svtVtB}ibrW2n@hICWO-9l8e}$W3UXTYIJ?dhh#I?G`k# zTaoywVsBfbc&F1X^U}+pf8Z6}4(4%x;)_9e1UgN>23qfBp=@NQdjljP&h&}^iZVlt zCCABISP$a@oZ1B%<7)XvYv|8uEdd&UXcuVpO`@t6;7&z!<5g2B#mbjCry4fI?p-xS ziZy03I)(q}czBoQIy_CIhDE0qzOGtgBEgF@zrXvC_#cpJDWrnMQaxL^1NtQ-_dGZoWKaLP*ESsbQ3P)eWI zNd3k?&N<37LZecBWuP1SS3?94?B+PztsN?7KjXs1w5*9oTm%005sLlxk-LILTA%_lH2b+^^2D~VQ28TW! zc=2?uJ@49#XO@OpQ5;{l_nB%VIQ1af$l@v}GAaK{3jjWngc($lLIQOTCg6O-trnm) zSNTn*&!tti8MYoMyM~F+fl|TI`)$KQb`DzMMtaBMW+drW@g-}UdqqzDjTp0u7}kL= zt)qGx5!!M`JV21H4(}ODGws0?98p6Ar3SsOn(VK(2RuC{{3nCT+Iq{QybdPkwXq{m zU@_AfQo9>~^*)4yub@Q4IN;|}7m9fZOju2rOkZIZ3jPwExvJ(KQ%qK;kqPdXW?C7C zNNERWLPZ2N<%g`904G&eyOkqb5N3ZRk9FM-vgscL2~?6#eJy6v7-ym4cyf8W#0Gep zs7)*eL*GYkQF(vOG$@zyy$LBoVej>x7doNl24%vDm^Z@o(}@JfvJx&eD6eDdhT$5J z^^>VZ%4qCI{RfBzmqn7e_1A#gzVU}KY#Orxisw{OyxjH5#mH~b)jZi0u@)BbRgHp& z{N5ySY58gUI^x?6-{h5>a~y8y5h(`{1y?!Hh{)xP67TbuJa4b((+?s&!Ppdz{1im!Q>PO{X^+!>Pd@)cfndBfr1WFbKA^7P zgW7A5(KK5v7pXh#rQr*nVmYSdS-RgBKoB{9DQ(fcyO%}*fte8igfw(t@wR<_ zqiE^#8-?)A`l7lPYI&Z)*_K#);AUHVzLz6;Jv>SV-v5Yc!e6xBd%)%`y?}b^3x9X> zA2>?;mnPZuBQcCn@8OrC) zX|fza2@QM%mcf#UCke!HEYCRLU*W2@%BHz+L5ZNi?cC3noxwvSklhmv6v!P7T}~wq zlXNN%Ux>WX_~qfqo=>!!EqZp8B6n4W$NrRd?D(twXl zAc5{pj=Qn)>HZW5Ot2lkd+L<0cPrNDBNVnrC@M$2{RMm*z>pSw=;$~^_i`$UhjPfC z_E-yV-mSyaxoz$B^XaL(<`}wEr0^E~)kK>$gKD|g#_L-l%e)uML_O7HoQ9|7VBJb; z>VE=cANcJ%$r?}8TjflGKcZ~;V4^Z;ar_CJ&U*~@fm;&h%>g<8fw`Z$Hv!}2%Dgl+ z67T`VGCzH082OKH?&C|uzRl=1wM80F^<8b2wCJu?zCi$L-A8}^%O$2k>H)5GPUOA6 zCED>e_Z%`E=h!xThk(HM17s;B1gT^Ay#q8QvYnNi)(J-SeGTIZLiG&_rSG6RCfmP1 z!`n+rB#B0!Ajw>~&$f;`N|G-E%?%=qWEItUb}Io=JKC8{az5Wqf?U3nR2cxK`jvT|0aQGSj2%-HEj1}W`zYXsB zfDq*0P1(BnRGil z!%qB#M6_N4N$|sCT`$N_DQv50AF=~gZS=j>Zf#x-aV)Vk`Y#d@cHL4t6GnQP2fn>t zdvDu?sjVCPWN_TPNY+3%QJ#t zRmmYacP)0eT*v}cJdK11Rmy)zaWs7wa)=)EjyeY-xTE;DA4h@UyD%bp=KB8fU196F z^2B^WNBEJoH{PdbDd=SI=*x=(%MolkQM6cn$XA?gqhTh3sv!};?o*6H_86WHYpQ$& zM=t7{p#>z?u?1wT|78ToCrOlZ#9O6I+=VF7Z!t0*bhk>rA5gDb%8(N#g!FrpQKrQR zX?Dm{h91qV+KuYbnhn)g8QJQBq# zX^@Jj$~KK*|AxcnrGT@V(j^sDlIo^^*d$g*7B;3BQe+-4sWc_y6aWQ&uFFn)YCxa) zz*yRun;R&`ry4T_WZGVhd>^Yofhm@w)`0a7A_1V6=C#M87Ldbnil#KS&-)&d5aD}L z3af4`zj&AkD(Bf4ZJ~JpL{7#kh@uIt=|9=ZCX2oDl>Dw6%+p+#oMP+E)EDb=|;H zD{9RO;GO=WM_LdFGDm1UZXr;=F|Y|rm-uy-71aa}=d^BKYiCVTaMae9BRTaE;G+zvDaPjo#rsxk|RX zZ^Uiir^_@Hd5wZUUsz{qNjN@+#EbUu@hpj>`i9_vO*hp!ei)^8cx}eOc3_NVHPp(% z+|Jb8oTFkdvIo62Akd0m)BIXY=Xb_i@6GcqXkN53H@BUm7OkUA zy`o2)m3qv;R%%TRLVE17&nJh+5t`C(-tSYnCl*$0mRx-_jP&sNXIJj_)->OW8kjn_ zK)l|zFC#+26}n347dLi$omV8tLY$@tqiNL=LppK3Im_;t~7oScBX*3}p&!###0Xohf{E40r0DO@n*xZDNF1poV=jtu@TH^Haq%{OZyqXA-*=Q{uf^Jij&j5+=cSr(9qsELgkwo4LWB>TX6hUcU;HrjDmwY{bcs>uj(XgJS2~>*FKTqkwmYp; z`Wh(2VX)64t`pbr+CyO3^KA1oA~9cJf!a5Yg}3EG=ok12%Qw{D+VY|rOIZ zIACe_rC;?1bi}0|3RON-xr3c9@~S5nQ))0*tV>go-m-&YSdTN_vdDyrP#ei3uba`| zr>Z}Yeac?$kkyC+nVfb5KBOM4poa0^V!xLOnRNoSOiNP~lSlV>{eZVqVHNxC?kXrS zs@3@U&+#V-Ls~}`4%_>H_Zi|(4diyOZmRBG(EFB&+R(S$pkq!)L7eBuY7oN5t;$nS*_nH&5>8cfpB4`v0;3sjX(K=L zgSUZTuX#y4HpKqnIvj3mTO6U+GTm;koSpY?|0Ic|H#1XE4bQq-EOSuhR#O<>FRaU>&b5>}NAx6>{UP^*z;%L6t^FiI1>PGPscY&vYT?rM~(L$1>RT`;xYx#n) zKvX)FBi8|CWD)hxv|^!12FGpAgOnXV71b-x56Rb z_0;8_E1vEpn_stLDN7+~){kJ-AAV*SPk!CV+~E{yFcm~Kgo+RB9AKm4@TK5z6A2(D z(4G>K)dt0sb_1&s4ZE?hlKXpH{z@=E6i4*`RjAu|7kOma^2ckW4F|$C*w=1!)<0Mr zwy}M<_SuMU)vW=;@Or`rf_IC#O5!}dz~Og;a2mFcX((ve@&@xMtv!^X74$k1V(&mnL+FAlqvdt8(1{(V<>q@)wi%bEioC=fld$pMM#0pZ=tuej3`H5FLS%EHviP zBx~MLC9i`gKQozM{S6SL6NAUk^@vJjLLrmnFg1B+PU%gU5&w^>uZ(IV@SZKjO0gov ztrT~6DaGAgg1fti;_eie;!>bkakt{`5(w_WJ$d~8a^8DqKV_3IyXS6pX71cOl8>it zuq$z$y4`nLkL2Py3e=PcA!{8|;S^!Ht)=awFM{zuC4kP+p96#8;J~gOkV&ca=bjkI z6)6fPC}MkCn1t7>os^7MG_*Ga&=cv0XL_1Qpm5TO%Gv2@T&c`KcCXmWlutE&lo?H! z2Qr)Op^7EDBo`7oXTKrbiZX$JGCoTOY-cc`l&N96sC|y33@hLHo-g-8I6y}k0sMj| z`5jG8vyM(4@0XBfXno_@6l9J@Oj3`8VfK&%@3YMeORRbr#S10zkF*E()bzXNv6U|{=T;eqrE&D21e9+IX!r<|G;|P?2Q#u z8^75Ir+qHou7K*Z^?RXOKKni0tXck30b)L&5SZH>6*RrT*LWAGh=}#yjEaRg1m^WMi@S-d`v!cwhBBc!0ZCZ~ zG_5l~t(YMbzE7Vf=z;&BC-)igxz5!`jde_Wh z*6mgrsD-rjp-FzbmZK-%kXAcT_+=LlVdUn){np4EGZQJn=)TD6ew$!d@Us`_5hE{) zG@d>sy~rzn7!u*=Rg``3=Bf3=AUpaTq#KZ$v6mLBE8M&zn{TdguVpL;!i^=Q>nrj? z+yY>ouwTTaPXl#;-hST3L*=R}zi)?=V$u0Bu_M$g`EX072}GKMv?Em+R9b@Jnx83@FNNv&Y@hZtc!63sZ_&-sVFeM)G{0i#}>rEaN<**+m~TBIG_MGQko;1 zn6ky|(M);q(K~udHapZB8ZRdp{7ASM%~+vD=k)h9jdv2+_r2)tFw*~1 zFih3~g5fg}1!Cuenm^w^jsK8-yGeKd#=LKL&>tvwKfT9R_@H5pRWRMFY!gbJ$ya|r z9~xvXZ1-$^_S(&%A{d1cIN}>h$1O8f6I9dt(XWV9kwRygPK1`oq@G`;tOy%QGfzk~ ziSw#8pOIxE=1@g(|GGFGuA5&J(9nex@&ibC{Qh7=iGN8{sBI#;=8AbOk=^E4vzF$; zq=cO4CS2>e?7rj4vTS%?t``w_(2YzYg!CV5@Smg4Bfb}%_5p=Uy|q(PX%`euE~p<3 zG^`7)e>1cI6o}tRfBK{unkd2THhr(nL#ZOhxl&!Dd-NV9VD*FZpRSXiTlv3#kZU5- zS-Tay?^WXzdEp=pcoL0v+_)G_T+8o0bQxpwob?rAXGJIXe`FK^Xa)oQJwLdWN*b#q z935_)eR){Fp>u5C;}<%8L@f2L?N2ds9Y0SB&kWEY5aq-Bdj4UsD)5W9@+1FdWB{hu zBx$(*Ti`==$n(zLqcSD40Gf2lAm0~IXmF?pUD9v?6EjvfJvAToKh<)X@4vo}BM8Sq z7}MH0UdmEkn1gxH%ICW^K1bB2^6^Y2)ZeEax%}qsCah5nWJzohO(BY#tDu6I>ELxU zO>jRYa6Ff(5AdB$LQfz^0*EMaqZc|J90;(Xhh}Oo}K0)(>`PRQgm!c@qy9!X=cJoSEl!* z6))5cMDOnd5*t77elISZ7j!^=-meHGh|YFuBjxoV{+M8d366ix>vwYs?FVQ?YF_#N zHeMjoLQu~H$}zjmI=z*UnhJ3gu$~NVgvh_W_5I+gln#k67nkkKvbmUg3gILTxtWMchBUMZ*l4#1Z$Qe z9-4s{i#E!CAD%#>odt6fkL4F#O$vT*KQWvG<%>YsEmxOj3(*fQ@&kU&@w-uP1$SSA z;QycgxO#M-24}6n!2)nV&XogH)=BZ+R zg~$@7w6`|y>#<%;xPL0}Me{nv4IPp>{koV)=xG?`R~N=DJb#&gFteHInS$DX7WPj^0PB@2Iv=6Afl!&*v;7#*`AQw9j7NRb8_eEHOjQ zKsa7Pewaoh-lR^u81)eZW?6Uxawcq^_*|rX^f0pxLg*YT70@HfBqP>&cKVq(s|V#V(c&8L5*LZ8XIpuzdR&uRTH6_SHurjy)gmo2%Xcall~93_F3KQS{8t4!dVP0ZyJw zPbh6No2A}+GG7j|>!GtgfTM8mpckI$h`8u>+S)F`LBpqzwu$dkqVduRq9Q-mIz)P2 z1Mp4z68AEmpah(%sNb$WpVBxSBUX zppw!MeF+Jo+~{e${Nu(-fa@Kr+CVe?L&Z2K%+d--bkvMOUEyM-I;!vl-e_^}AV+zf zja4j{0Y#49pqHBmyv{RxN|UT)^YV#VN*OnQRI zG8(r`#uKu_5nHFWj?CO3PusfQf)|A2&)bATxBZL{Roe`qL@Xqb+^wUJy)2(#ETGJ<<|m{VWWz)<=xb|J!$LAJi}sTS~)tib2fv z=FvlsE9c?)wP-F?J>ss~+LasSt33o$Ye4`G``w=?{EbIjdjJtByWfA%u&~^AUA(9W zq+s{6yeJ{G=NKA$xqi+iig^LPwhwk|->=CQ$N8PMkxxx+t1^=*>kj>#JeaU}Qz|Ec znE%hfQ|Jjv2(cD~+^FNXI3W5y5bz_&0=bs)CB25g6Cx^1#clT9P)Itb==QVN}I96VAF$=SYg&}vj!4XD2aMwkk-r)IzOVeZfj+%@ZUR>6#w0Uhv+ZDfpAs=DL z(8dq4%~$pTzfXCYTx8n-|44YSca8>~Dph(R9Xp;yy>;s7Qjtt63gSnskD8`AxTdVw zFY`O(z`xzFV3)vC--h9fhOXAZoHi2S~>i2Ok}(XJqVX`?>%h|Qj0abW*J-O1%lqj!f8<$KJBjt`)0=^eZT>1oij zjnJ$Yb?J?4&rYXi#+)EoLuU%x8ZD`t|6j8%ZGkQV4+7Ni@c!#kEFFj=UthA_r{9ef z5@Knj7zk9&P$&Q;SOa~WlaeA6p_}l_h@}M05=Edi{V2Du4zY)|Jx8xC=eO#)LL=pX zXN>Y&C2Mpg>l>x{6QULB4XqQ>a`r63EZ(;%P8uXjh}EB>@Y~mvN5Hj%UraE_+-VJ~ z(}z^ArFd?Na|vfYL22a{qqdaByz{>I^J@(bsK)7%dZej~)9hQx&F-4>Wxa-`;cejS z*nh&GVlGD?Av;x9Px@BlL#o~1QcErVM7_JYPfiFjEVEh~>20T?MvuT_el`5cz^V-a z@*dc{U^n4uuJE&IYSi!i+9@=v6!!jr=yEy!FOes)28EnPVQ9OAQKeAs$K60J{y%|HPJ4U0nIi28WO zn~V7D!0Rz@np^6Q4CqHAEun;A{Dl5oPNaUj9%jh-CVD^8U4H~>UKj_(J5d=4zBHy@ zltCoi^1{qFLjM>H%SnL^7K+|c4g=FK0ZHw?1qP9l6%=m{UowjdHC!6nZCu0B5tT#+ z@s-UB=Y1Kn;~1raLI@x36+#E}n9Exh?SH)h?S~g`j8C5Ty^tRw4+nN;b-Io3p8pER zp$nF7gLFAB7n2aZZ>M9tPkOGsPqsL{U$+{(52o86u9Gk!3r_jqn42lc0P6MD_5w2R zvu{!Ml=J!wOjzOKBmv)l$;6YKeF$A)$o|58{BER2y|dXk?*dQgEHfA;rWUgBNpv6OZNb~{v4f}zmdrA*)srlw zodlN63ulYA*ig=`BghqGB!r5!GO%<2?Rj zAU0>2*Torx{RkD6u(n34LzZst!l|&U2!kbJWn6?@TMhC18GCj0?EDRC;Ln%0)I4fZ z5f*INe7HCNusBfMu1bZ$>ALlX-_aqdymL0LMJs_;cCjLV0)!fU8+A*(0^?6_%F0?U z#dp9qIKLQw9=59PvWJR_Ufg8-e~$smaJTN#uj{M*RPYJI4Loa0Kv3E}a$BA2E#`-q z>hBq&(P<-|zZD#UL~*RmC#~!V=ZEtx!vW!>tT6G=Z0G|mTKico0Y5)U5pDJ0a?~4M z5euHd!DhEjMkp#9FnQUz%;2XXRF5b&*1EW0oA47XeX<`!%fDdXxEEBoR2)j_RENwu z9C^R(LC^2(hEt6?VZRS)J@n94b|;5Oulq~$-sC^hD(JDJ*V1%{rq3A-TR`uyCA$i<0*RWcI4LWTy0D65+~Y0=zVhG6zqyYj!J~(?0G&+$ z6IDS5|4aM$>ts~mOjtIYIaE3x=E@$uONmQw<|0QuR67#IM?(c(sy?C{U5%M7LM~tt z466qK`&R!LcRankM{D0vx;fp14`}xo&QZPZx#*Prw+RtaqT4UBUK7e$<{=f`AMfkR z2MiqN=z8Dxq6YiH-=W^?_J9>eX76<)Z8r{a2@spD46>$ZztV0ivOcC-BSb?HpT)w(T0 znf0fF!Vz?;2SZa3jG-@WkM}EuBqj_u>!Y#cH;$%+a;)SUX%$!Wt`WYtIQ@@bHq_&9 zUOkq5B|^q=EX`kSr8em~G1B!R{e*&nzw5Sno6tXp)8z2*Y{yoh?6pa05E(XCp~aPR zx2aS+8JC-sK;1!Mx`83pW-9?%Wt%Gp>67nUr}Dci8{DQ)F59IIOliF8C~Q#8@0Qe% zHdHj$kBYjxvY*j#8t`lL`zTpYJPw_TFRN241nIGN9#%d6J>PKd>q7kEou1XNx7`g! z0k&c->q~`Z#p93*!+9-deqs?)tQTG2t9}H{c4Ol|m3ui!>K{ChGSi;r2W4q{Y?;nm zRkgKUTit#LJ9BAF;E7+g_+Ag*Zyg7b1CmNgI|s_)z2DJif-EO5eTcAIA0YpKhRefY zULOLVl#_<2nwq(6iQlKIv%-Fz1PAS(jJ-ki%#*jpyzV61v{*C*f1(k9av|~$^!5;K ze!T(P-X9ALU*CV`5qh;maO|@AaNhpP8JmS2{PqSgT@U4p&HohPf+Im7NKa%&>oo8% zLMbF-i%E005l0)~Wg31jqHw-oXx!!r@U0vOekf{LYA zhn=+d#hcw6E_{k0{yad)x#2<)LbwTEwAqQT?35AnNXTmDIMm?Hy`$Ta_QbU*?*j)* z?uyZVwKQ2D`^vT@Lhk+OE}fs3*HZ1rJKo_$0ELc|ybV=YQAJdL^Hlf{Kh`BymntiahYQgFu_MIAk^xnr*>KGNOnD4Dj7-6o|qptPNPyfE8!o1O}0tKUT zNZadp*w8Mz?l8!Fym5WK{rZcfG0~0%KfMZd&K#7U!}2??-^GBRo}3GV-Bnc0WMiYE zGAz?Kf0mrFbkch&DzL-c6`N;!vzGnDf0Xe0B@AscYJT#AAIR$fxh+xUhapnYSg&6T z9l&jLHgI}bM)+LtA9($--N+D@7u{Y~qc2PQnWNZrAW_OJ`8IIBxjlGP&GGjv5vzV9 ztcSDlsKfR-xka-Hh)&d&|KKP{ScJ6Ud_}0}Zh$p|QX;{EbM_?BF1cNp zmRG*lt?1o%QRzmW<$jy+H;$Klq+gST0uXtJQvIQ>+*w|8jyzdIOs4)*FcUS`ImgT> zhiU4b#85g`r-FMz%VbOFiGmES=vY*AyM)5JO`zQ9!NIP7X)!JvcT=LSF7-IT<7Zga z)Tj1yT$k+0WXPBgc5}{Gn-g6QPeU3zZNioIQQamO(Mf{A!Bh1y`R<^XKuNo1E(mZu zir5qN@3oUgO$O#KJTZIzYaRv&57hfw-s&n;ruJ>r^U*Khlsp2Tibk~2i87htDRIgC zB^>UeQq6H8^B?Y~Z&|YwSGDZ#+a%l0JE14({p9rUV&86xNXjBZL9JS^pz5qF8HcIV z79qSX{G&(HJZOBbjAE^3X(ZjsCqkTrF^w2Z?(wd26vz_Z-9kV@l`A>~)^JbcVz@g< z4dGK<*O#P`P!s{czj!>qrEg~ONBSWhY*ioyZu^XZ?c6z)lvya;2jT_pBa;LP3EL75 zml;E;HjNdMRSvsl&0h>lpmK&j5o{KOIN63OJ`V~6sU*U zxgV03Ij%Z;W1m5=jHd->%fM^qhA!C8R{lwU^=Cz&u3tmScK)`UdO9|D1c|YK6yv`dj zY~*LoM50Ra?r7NFdCnSmAZp`0wfermJ%GZ`6T@_|GQr!)TiM8ABZuZmkL$!Za&VgE z#^6vAgiDTO?mgqzDi~O00MWUK9@fJ2|NH!@u|FYj&G*V^U*}~rw@=Pk4P5qVEe4i59l91u^AkpW`rzc< zqK6MovIu3A*ChK%@iGc)ze*nMPfL%KX)C;P-m{?8MAfti9R^rF)>Lsn)kL=~w{_Ld zH`QK<9(K`{y&GV%y6-lE-NtO+Ph`?LRbH7i7$Qg4*OTX+f`A9<2~h2WVMf~~M)--W z)9i9qt+Hc4W|W>mk|M=WwC@RWb*0SjLnDYv*5gI@QcI1IIgyZD)>9FQy5ik0hAriD z+nT3GJ_Yw&Z;fSER{s{W0T8$9GmI|?V9hOr@qD(gs72RNwLy*!S&E|>265K~>7*da zy?r_Z5y#o>Q=Aj#gp`pz42nBj*c4!{rj7AN5p*_8{t$f`K6~Nw<;A+5!l~>KlK~q} zN0(ziNY4L_b#dw529x(*qoez@dH}|mEoT?5tlD(OOXctR+B_7?{Faz}-4*-m-8i(W zlmBo`e6uBqNf_}bA!Zgk7R)xPD?(YnVN+1?BP=AY>qX`Zr6AjKoiO7j&D!Hx8^#@$m$%qI(3_2 z{rbXmoBLxG++w>!x0WY4n=d!5@BkzoY~vX++4gkr5t-~#-&-FsK(-}%6PrVMOw>;- zB=p$nxrOhw+t&oP+y!+?I0{5O;#Fes3OB@%%3ow>vA2hJuJXS8`q#RDc-_Rq8yEsV zMl4Q%!9Q1WR{z98&sn&f$|YGNFh~ryGp(vPgK zEsd8zrnt*~@Y#L_k@ZZUPK-#e6Anjz0(nXRVKT!PzW(T~G#-TNG@Qnfhyn+rhl|9W zW_77)7AN1ckKmB6Uo$V2T* zMbY&PnU+Wz{PIHcWwbq)MC0H!fhS%W0|M&?g+bN&z+)(J?N8|ugW8{n04`IM;nTnA z_C^u)bZVk5zYaF=$E9g zb5<_yizExTtasG{Yr;IIPH^4jb9lPAX(dOfU&PrjOhJx-cCC=i*7nZhJ8AeA*m|c_ ziHTas#nvOKpyr+?KdNNRw)dp{rdp9&dVnhX!g&bN!wJ+Weu&%c(|znjuR7*S-(ZNN zA`@fvA0-n&@ohw4bDFBz9h;|KU8P6XJE%&MV{QT2gE%4|bs7$#S4<#)Y1@Z1K(A{n z*y6sz{fIr~ZM)H-J%0+z{`KLE1Io|b)a8!MV9gXs9fC4bhc=Y#X0yZlGrI8N0P!tW zxCh4*`K`aoD63Gcz;+?X8ttG}f=X%nPQX7zkT^%jeG9l-erOB+2ih1gA=4_50&0#> zbb$GNdnV~;+$|g1j8vJAxVi9JgK<4$#7|%KKQdxsuJ0nW5tAAg6UMTGTmCHO+vy{z zb$Pz~%m~_BjmG59<_m>y@T>FD`*bZerYp1~PV;hu<)qsYnHP^mv+9d>c}uYy@`DKl z)#U0Axh@fVJm9Co&OkV>l7rITOD%oQGj%*__kgSIoWZ0+PTwB-T;|=za3dE_;j`V4 zea@MU>OIz=c=%!5r9sbnEnU6UeYcnlA;zqXB%2RnxNgZ?< z!m^zsvS+eJZRG6@dzFK+E5v66T^nIrXV@K>d|yOQW2XW!!}nkby4o}i<&vS&k3dnM z z#YKtTJ6o5_ukhX8uVR8to&im$IHqt#3^5IbTsAnr!vSsF9V`j-c zXhy3madH;k0XJY1GcA@F;JiBi2jzF&4?DUzN%>dx`VwKMT223|7!w{$1kI!_Om24H zYhrrBJyTZt;kV37JPh^)U42ZF)qaGQRTE@%N$m_79=+_}yI<4?oeSg5P5AQ|wvYD| z?wH!Im6@(kA~F`~j1$uznFyy9_`v$@;z%w|;T|mRdacs%f1K>ytQJyOM<93k8A1fi z5MEoqg&)kL&f&>Ho3=f;uGNJ8;kM1$3*LMgm;Pv~|yadbJ<8A-j5d7*j-l6Gl z*))nbE-t_jrwH9Y{38iFvPQVYk1CBcL31Y^t&KAw^#ELhmET)eQkV>jkIoL&>7Kcp1lb-x;3u!7(4t#Hb<&zg{;~2fzXm2_z`uzlsfAT@~bu-OmrA@KGEo)av-2OIWi(>WnA78^nNN4+J zy8Cuc|8`RHJp}eUu#RPZZtp{rNyphH;#V_f_nYkL;wVzmZ2ypeISW3n=;68$qdN53D5NRmv4Z~y8H6-rx=zX0xk}E{`-e`wYvHbl@fJ1H9X#yF5eoK@A z67%XiU6aeNzDIy{TQw@d1?Z9RaHB4$ue?HpBaU+Nfa>?KN_y{~fcS^76lG zp4(JRDmUtH0(U2^W&3DVXy8mh(SLshE44P@M9#A7IeiQ5=u@uIkyNxl8wCp>Ld&rP zQ3yNJXK=0}jeMJV^ds+4Alk?;+n3!HdR1|X{39))ruK#+(%Ki?S==sLC0b(cz9-U5 z0#n25OrNXLIqL$xz(vr+BoeR|Nw3i%DZCIhqapGzKJVlS6Xbt(#jVOotZo>zC}N%k z7`UvQ$Jsl*+w}9l+}Me`7g6@VAN#O=Aay|+kG8ts({e^evb>RA?_relx)2ha`~}yL zbnG%%#Zu`sYG9jNugNApDGJ3w5&dDjmYZ-!izMc@K<8*^4~^Eb#%u3bfS08v&CRh> z^neG+IWUDB|1XlWuiLu=FN}Sy5t(TD!b8!2L}uavM)s?b`NM$&UGq`6=Hta|0W4u7 zoc!6s`Lnk%t@S2w+B{Q5?q*DW;Dgz&71YK>PUF*eyco=Ag>*AN3hfd3D>f>Y_hm*H zP^zpRuPj8R^>}9c=OvI?+j#~OZYh8x@8o_Wxbmz-fN`~>=W_TP*25Ty*Xf@Jz%$#R z!)qTJ`{U3-l1<~%Q4OPKc4yK&m>u20p-jj9gyj|YXCOOPvF*E#Dg9fGzwYz^8a{MG zh;3hf+LAN6{Sy6iZag8;6V1_Z{33Ei^%R1of;G{>8+=o+ys^Tpuc<=$?g|Wr$DC-8 zVXSokLV$g|7-VWT%=R8T^SxDvd<8Y>O&QIj18^5jS>+Yli+%Ejf_#^mnk9Xx1OcfE zs8|blgowuq?sVSs{O!COFRFKv=C~Rb3sGWPP2Mi%tK-sb@e+7pK|D>*;L->E9+=Dq z<*J8@1x-2U7Z%+G?>AP-c~7MJQ7tp-D)zYk8`+Iv(NLvuccD;eW!-;@db0KnmK3xi zjYFS%SfLq9#Xn*Dae@l*E7}n~-64speuRQw5;8=C-e3x-dQTf?JK@>z^EP>QlDI~n zjnv?2f#et&0a2r5)n<(Kk{|k4Lq;R}`D`V{VPlXv8H?0N3qgtTni*w)6)@gQj;)4X zOeS64Mb9zDzV&N8H}*P@z*gG~p4S(VW`R$GQck;T=5(H8kj;~57#Z!;x$9QL8IQg2 zmeL51mmII3t78!|ZL_c2_R=>;a{v4)+U|qJJ$jOL?7rdGY8y}+ecbJhwPiufI#|*& z@MLtcLWCsyYmxIW)H2hJ);|L=jPkO={c-gR=cx{NU~nxobw5DzD3e}70==iP zf+fy_7@|o|?m4Vn!sb9WXmd>`n>jJksGDQh6oi^>#-AvFJfd&eI6t=yf3n2)sgeSi z9QxAoE;^aoU}L|3+n#*$-ZwKHop|oBhEU#lH0aIB_}>7 zuEFTlzTRK_=5)rEI;o)&#dCw@li|G-ar$oT326V^!)f;BA@5vCmkoSY=L*98MLW2C z_v^Z;=)i=F5J&E3oh+9A8d)Bmp|Ls+qwvmYP3t7dZ-Ugd#CKv#homCJ+2buG+L}q1 ziErbR*tMUqSdE|Zt9LN9H=oA7R$hg!rM0U)OvnYe4kdJWJLgKJCN5qoW)+PpCtQC^u`d5h|X@?&Ge;AnuQ0oz)d?W9jf``pcA9oLIia9Sj zIR^RlaD1g{R6&ilr$W!Y+VnTl11$5p%4LGYx(2QRCIUgle=p5lC)_@X!(n?My8p}8 zSk!-4$uw|zaydy5B{Nl8A?Fh}j__P9&aaMNc#VB&%~5a5^b~9{QqpsVM;y3l!q46S zyBKk29@kuQ;r&b1;k=#V*J?7Z3i9zWUr~rukxGtW!m!i63Q7HXlBM65k;Xyh4Xku7 z!cE5Ahu?`?xeg)sa(*3Jf;*k7D9yL$bc^la&Jnx>Ql)*A=jJa zKlaL(^~hp4k!8H7kU-dPztUMw&R-{HAal1ntL8Mxd{tun1BVtzh+gkldc@RelJ#b1 z{XI}F*-!Q7HFG*kgsYHf>h!-ru7DMU0C7?@JzKNOB*c(XL~u4Z(#{s>pYr~am>{Ip zh2yUx_8<1qo?bTM3bBuW(E!QlPQiYZeU^+uKF>)YE`q)lXjV={>k9v9qg&dzP{7cg- zMx`MB!op>Y0)ofi+XhxUVb$xGOJ{KyLv@GrNOPRa3c)4My0zH+$ zuV9D-+zW3d0`y<8T46E^B<1R3eb1*GG^5g{I>h}_)n#7!fl65>L&ehR-UmTFuA6~! zXg1fpt(5R<>ETJ^Hh1>IwKN=|?F!8C$$K$lYg2EjdIi&UpXH;Vw4L!|w^3E@O+ z-a++J++(4#G`2eOH0@$?kA$?)oFPQvIL51YWQeh^T7)Vu@zPBavY2Se?v{q(B{f8a2qZ*3s-D*N>OL>P<; zOYaHSEPF|27o0;|bMX!`DTKgLe%b)AOTQ_@xz|!McBypNBf9cgH=MTKUU_{<=F;FO z{Nf|ZC^rVKdA1wJSXMGqRAu!>3Ad}<(pXEI>`+sj>*!LuN>^`Y^aPoK?!i=bkkF*H%u>$f~*ARt_hM1KRuI2U?SA*#7@|0r2mX$oq1pIxgAl zB<+NmiH=037>AxDZhks@Z@1^HLij&FrqGFYM!p$q%A|+!Z3@>C@fB~5F7A+YIsN(b zw#M5O7?Dcczb75VQuc=hg;K1~1vmq)2%Ws*`X;&1$1Lj@meUDXhG)HT;-wgm96El~ z3Xl|}%%~6@XI;AJv{j}V>bO>Xyb(`k{GixobO13TX6;geUq#j&D{AX<5<+KDn~|(6 zZ!5RAsD$2sEU&0yueHNxz9MUknBTs*^(1XERK1+ekdMF+ojuSh{R>dv66G+v6W@_I z4zwtYIaX-Ae{Mc#W7q86>OVO#Ml(+#vte11;|X7&iW8 zz^?!w+usUR{>HWcKAw)24F92x|Y`g zIz?Q37xl+}OL$2EC!fWid{idU=G)FLF7W!zI7+4MI_L0Dxvr3H;Ro2Ap86<0XR?0( z#nn!3sZFKMNNLa(5kGl@zj-``GvXf5WX;pHJ9OLWP~$IY!YcoxwyCubx3 zY9)k^cP?Rh8`^PM$X@M13^unIN(bE!p61>)+WzkUF|pI?=|WFGg4NQtHv3N{oyYg~ zXlV!0wA*EGD_Mj@!IesSCCnq!6HV5ohjRr#-5y0O>SGS)OHwiVfDu-Fj2Mlll)MYa zzrIO6RJ(p=KBDTuezrUlhlFht0aWqa=DI{oYPrUC`3@Tf_o)^ztQ>6@ z)O|kn<9`@Yp5hih%Q2sw>F3YQgyc>407PhAktf9LFt8BckcMd+>5FVLihRebGZ7H% z@EH;V>bJ}(Rpn%iy#eei_FuTLDfPFiQmTE0d$BwJ-VJrlx#e`>`PZ{Tfn)W`lxCeD zC(E!%iQ03b|1=IvFSvYVgZ(2__aP?goI^OHL7%pn4|=LTnOPo6#R!>jCfNzT&oLHC z@ki02D+6}Kt}y1VwF;5e*+UMnb2vr7+?Tm>fTw zXniY&R_9OB_&1cmKqn<;B8Bt5-qhb@k}Gxm8aXjLhc$$u0c)7ZC(YAyZ) z?a2xBt#yCkBb6Hgy*+x%l`BwsHH?c5gi4 zSy^%hs3p(i+4*%dTZzzjyh{GN1fPCmp~^NIPiOis<~>^X=>%FtgYJ93Dn>Ae`O}j7x4Dq zj)KIj0M;IniUK)M=m{JdjH!&b&U|MKkKd;Y9 z5B_keyfeqhKt&xka{p(Unl*v9VxC*4KXezAP`KzhxzRsPj(q?;LQ1k57nZ9-8nPuWUjFkx4Ie!9D*p z>;e^!`;!JXNtBaQvW=BpIzrXu4e0X@q|VW#T}uU0?i-LFQ~deZ`4f}#{akFbXb<*% zoGmXY79$?&c7f*unD;x@r}wy)v0LNkmj>N`V#lcEI1UobgAx$bzv39#KfGToZ9C08 zoC;fUl{@fr_RZs^#eZ!H3jeL6$!hF4qVz*pt!H5Hpo6$i!tuoy?~noRf!ZsR);a^QBi z|4la$D8~$+6!^ObRz?XE?-8}7EgCwfan-}x!KuOd>3<-y)t$Y{N&!Q{l5LLE*!;q= zEr6Y%nqhrgP_(@ylf|S}a*EpIC0L}%7b#QMOGJXM^@UyM#{kmPXIhg*_xc4RkO-YF@Z%3ex{M$^#q z@sFf`2Ka(47L&D!iA^et`eTp;-0jVKmNplE=iksj){(oI+@`OO=8xybxZs{<%O0|^ z2K#ZDEw#XcZKUt3Mr++J9A)4F|77g!P1Pptdbd3yhi3eHOfN$Sm<{&Vi;tIV-eM$RLE^hRCsnf#Y9a%XpS zs^J(MYfu+GbK`9$S261vr3m<;#V98dLdiRx@O8U$Bo3|AIXV_cj`$;vgHCqo)`o=i zE+LDzIbY$wzXQT^q-E~pyGKw2Me=YeXD+A4m;(80n}2Ags)zQyl`b?Gad_7j&t~4u zQN_Q`t1{X#tW00{3vp_&>GckT;XVP>0e>Go(Nc~!MM0oTKY(My z`KtJywy)u%31aXsQyOyJI@Tg=fwSU!5(#7f#YS}A*vPa2yyX-U1XZ-*D4l2{r`DiL zR%7{m80lsEtJf!fFv~9JwVO0Fc5`TXX6^BdJ1Y*J0Tm6D;aAiWifo%?#ubLisD3Dw zA+159i9Ewss*;>Pp=%6vUFk;G-(inle;m0!(-!Ga(Tq9IgM#KMh-pJk+Y6ljyojV9 zR}aZLy~RT})N#uW&+z5dununVp1}KqpSLp^r1Op z$#v*<-@1{NKc$oZDMb_q$#+^#nf#ikwBy7+zT3^K&BIz;J%G8b$!6&8k}StyJ!Njp z*VXjVnB|OQWaZ|bzpnW0Zw|@LQ;d~m92GpfZp<=qRQ;qYu#{irCe$i27IQN9+A6}G zah3E@?#X>O6(neIbtKnng~=c^(Rystn?~U``IM=SyBr38q3fc{9~_+{@q{P{Nxr)t zf9{FEaeHbIJTqg-QyN$}_sAl)Y499SiS9#oHV+OCmf>$z?b?;PE9M)|PhQExrDLX- zIUJglpe;<%A6)p#Pe~F5S2E?ECI~U*WxcJ%$8R|#O z?hV=?bhqW)gZ>Tl45a*I^|0z&-v}Mn{8Uj10t%=-6{$PAJlal#K1q;Yw{?jMow2O; zY-G3jKd*A91+C`SUL=sgRHVY*Ww&i!Aiws8p_+W(Fxq&=S?$)e;l;x$@k^cB;PaNL ze$yOPG_=B1bkRH77Jd38rXH+5KZI?H8D_Nk`lCZc1$$U$?-mv|MOo(lLq@FrmyEof z*Mn$iT6xqutoh!tzlU{VK}kbDZjm>Iuoro;xq@AWSQ)=ETxNZQz}^R@Q9i@qf!4N! z+n2m3xgI2jU){2JQ@5tgSy+ZHWkOG!G#1W4h|oREb|{fO`rpdqS8AG=I=jCgPVjIb z@r1#CSMSz2&HvStIFEUL%Q_2}FC~#|V-!v?zfri8$lLyL;))rJz5ZaMO2J5bPigAR zi`3pZxyDR1z5b0jBpiw0kn7h5D(hU?7_kCzxv{hidAuaDvwIFEGR>2lak41r(Y>q= z>L$_UG}cQm&fmn6T-c#w{~J&g7KP_hWVq69O)+@=?CPlc&Vtjwg>5PQ+ReX(1C&4O zBa%9GS@`|Io?PzW!7cCyrWKx#OJEHvZ`p0(jU8cFpdsIZ(=#Zr9z!27{TLHh#w*L4J(byKgVEgKPfm0IN@= zesPBbZW2_Q^ZesW175p}Olg3~e^H0Fh9Gh%DvJQgy50LjDDFP?J734=$!8-`5ui!# z#dS#F-eFxt4OWsPto(IC^6b7SGMO|>s6?x$dv@;r7kgn>bLrVLv=JR1Uo;As`BDsww?xW?v(6Qp=KPtM=Kg`hVnW zE98eTRL#w-)!9M*gtBm0ZR1grcND>%N3f`o#4D21WwZ&KlH$)T+tkn$)sNV(=zJ%&8mKQA;Z=XwHbYaO zLz{Q?zn<$V$<*)hSukoG9g5K3A(XCg@~vdKRNOHs zyMnFj8GYnvufS9!fWx87)&QATz24^YTxVbtKWX-%nO-*;vwNeDYByZ6wj$yAdeU;Q z>*C8YJ}@T!O@cO=L{`eld-_sxBIR1e6t$IyZN$B3=GP+ni1W5nn>{GFf+M<&j!$$h z|N8AMD#RKjf9>BYG!-*;EkvJvQV|RtyYA400YnB9W}VjgPn^VB)6w@nGBOBN)8VTI zILS{w>$!}AvcNxB3{V3SKxG&^FU7UGH%7`#0Q`9V(`@+Bgs<*xlatTT0M2@ zeAw3+Czb!h);Iq}+OTgg!e-lcZP<+6Hrs8sZ8zDr+iY*fWZSjbnwsq9?Y^Jq`Tq3& z0rSJm9G~+#be>o2)LFZu_1X8B*lTJoO*w-g7HBkIOj-L9<2v@>@FvV5O$XDw&mQl` zv*eqN9@wm2&_VW-_5*wqfjQx9#_4j)HTFGJRXFDin*$~;dd60tE+@{V6?$ogfstFC z)eJ|F9YmR2yV%G*al@1x8ZnO$39^lQs1H7>T{mb-kM~0|rPt5BfB}ZnYbNq%Kv&N8 zmN&rW zp2b_RTY1Z>-)#timRLfB8oF5~Yu?LCynA}NIPY_gpqqtlto-xSClhqOu4lzY(q`f9 zDs(v6leNxSkcG;-u*mwau&5Ms0_Vc|#`}8gyCZ+AtFPCJ6>S8ZmFl=zJ&MCELsT0- zkm?I@Q0JCDoCeXrxx5hsQATq3ew1;ec*zUT$07-$3Lo%ekdf<(;s1&V@+5Mvn8D%3 zV;K+?!YliMjsAK|AM9H)n*R)F(J5G1=Fp8!G~i#9ML&9)#QbCwx}B5vy}lP(Gz3Z` zggs&3QJa13i(J#8RuNU?Z1ze9{cg)_@{2p%BV5xS%`XSamE<)j7P;|DGOp3YfZ^{k zq4&mPOB>@COeUohf0sKJBZ{BeIOfX5|9Vbc#*_(GMd@t~JPjKA;z^v}+_%=d5#k`= z7_D4>)BVPwVDbr5=I*y~2j37412-{fY9;RbWX$cZp)u(`hL?knCW-Cj$=L1jkF_-7 zZcy&07FH#(6;uM|M=hlTn<=DLBsWa8wHuC--pz!DkJwqnq(x`#zS`N3NU6n9B}ttJrf_f?G36wRtuPqr&TLlre9lUdTuGC5vqMIUC9m+bFQi`O~skrk7dOq>p>N-0XRW{+0iRV!|E${yzTQ9~5@lnN9 zI9@X?P`98ajXr}xss8k-3#pKsey`%_3>l~2wXRRx<2bHzBhQ}2krw4oDiW{MGPCb>Qz zQ-gL?V2wXpgQbzVyqRVkJ9f4!HA|9UT$9Iz0Ncl)KkoWA0*ay*8( z^##{Y@95*1(nr8isZA7aXCMhVGP^-MszG}Z`r$J`AKm8h95urk=biP_JWkV^VA z!Ka9OAvZ!Ue?>h!9DVctR%jd#Gyxv8lOI1?XvBMcR=DZoz5=v7T|E--m~K>h0X0i= zhE@`!usFBd+P0J>F%vr=b*Lx#xcK zdwMgX!G|h~_)KCORa)-uY?LaNc}?M$<3wMEl03nvfQa*Ez?d&~7Gl%(4p)yho=_IT zoUjBTFVwNLjl1Ghn)=D9`{&Kd7@WHZta8{PamS08L2+Qae6ygh2bcaIJ9wm*RQ zW|^B`LLJTY(GV~q`7FBBjAF43^^sy$^RENtR9Pu#YMcm3zw zqUD|R^Dw)JPY_J#>eHf@s#H?F!!W}tjEKhS;l)v~?*0VMSpK65YpCtSbAbsMJ`k8G z-#@F=i(xB-re8A|n_I3y4?K$QTm_nCZ=$>*bXB`S=r1gacv@*U&z$dHLA>^!eS^=s zIEs~Rf-o*NFjo0d%=3++CvG<6{p6N3hvaS#%VM7-Top|h*%4k0 zXXF`W1f)(o0JFJRTkgD|ssy(a!4FPdFdEVtUP{wl@fkB?}zOoyodv{mMczOIdZG zUQEe%Cy*O*JCwKqmdq>iti4bo$ ztt%BF8*Z}Zl3(ZDlIcuT=P5}o8uoIweN@3nCb~^YUK&n4Ln%AowI;pKem=Ia&}F?) zgzEuhref@7y1KCKEUaeki_{odF9rq61jP3^&%jOOh0mEB7GOd$UDqU;*QBie9&P@w z{wSX9b9;}68p}h;9km~jqwn`*!e~U%;E~_<#1tz7cDTJFQdjI9(m5qfoasWD`HXY*m7w1& za=$k>jkX`4g@&@Jl^*vN!7Q7~4Z$1+LkE*J*U#O6o;e%B7Kcm2#%m@|2?!7M#ONE@ z@T%-S>_ocyqd(-0PrP`_lQ!~)wYpU=v$xTpDU&3rleS8F#=?k%KbE3iJV{O_9i)sR9T~ztH zYJ)c6BwHoScQiuYTCjcAxy%=)i3xTexlgEe^4(Ctr!}I6X=UYesrbf*HMs!=+wX1- zdpthXu6dSNYKqYA+wsuD5H$L4m8}eB@eEK5V;}SshZ+Bzwql_)UXJ+~3m)?x0}-9> zfy6c&pCg9#h-7^pX-|go7$aRwbJ5q0m^BmRy@56-uo+iOzq=DZn_mIzN5V+B0|^nm z7?GQ82^E%@_w8XSwzOt~)7i%Gk9-BN@j@3rtBJDvp2jUW<`#=hKI>?5dJ`vxc&6cf z>#Y#Q=sE|pIe|6}l2eiXb`q{Ixf$1l26gJ*5vp?A&km$}qj)*`*o@pa6)Bhw(6vJ2 zY+jcb&MS=4Aek+21>#!I1-kOKr7bM=)`X)qv~~3S;No-4n&3IJ9Uw~&3amyqB%1Kh z5zpMPYb>(w9l~ThA4*3O`f9XhmS*`)|4$|rbo}c(d-$o?!U7#nOqfPWW7h;*Vnmh1 z*jVUZhBP3ZO{;q%=FpL~Q)o*lU$^6)5JVw`Q%uFXooWzk-H4-WwRWjRN2MRk@W!xq zsk1M3k+k7%a%=Q(MArei)0UAXG`|ao8peBm_XnvNCByh5G~Y4-Oz6?F)mpc28Wpxq z6(Jpf#!?Ar7#ANB_f?%giEZx%_#)x3zreTiH#KkI>tNfKuO-Q9O)l`a<%Rmv5G8R| zO}4=E@a==)rk!@n@4F0)Rp`T}3xywtWsRq;9*16`9*r&m#q&08;%_HclM;3>8d}vAi&9^gGtg^wLmS&#Db8MZx4E(d}nf$ zvTfeLyP8Des_WD_Fk3j^0e0evUb5z4^QM`q$wlV>gfFfCo4@bjOBeOmmcg=D9F`4I zUT^t@_?MacXFd8?3Rkuh@sH2Xc;705XvqdsX`vqH?^SRV*BmaFR6&n}g*a$A?jgU) zq^HXv@F17x@}xVePdlni4~wL%zkIk7?6RUn0PE}TJ~w*IAl@=QjnRrbF((nlml)!3 z@eGzLXXp*dPKCm!tprejRSLu83bT7j+q(<2-)^#KEt~4dsPWxrUl)@`X~~98qPEDs z3}dLI)t-Mg@dZr!u3v{^Z;DBUUv8*qMq`b3KQCjm`*9-W4((335=K6x?X3Ld0s11& z4f#q`Jn_C<+}??3rCl$KW!uku8?5%nw)ykN{iig76_Jf+qvfPoO|wyX)i;Z3^rn%l z;bh5?cxmnv(=jwID|k;2d)sAHx#*WHRT8MV=V;D9Hp_lXjV@m-XQM;~+MK!ZmsdGIpyut-g}^MeQ1wTumNgP$H}0eA$+qvt8BPowXMc_oS$N{yNNMy9Si&)Y>1p-eB%4LP3ZDQ zilQXxl#_kKf(znag4&$p>q4*8I>n&xL5Z=<*_cuvBl2;vM{ zb+gHjG?X#Ph7Zc#Yk0UY1Zf1 zbR_$9Dc5pKy=y@d^oGNtr%_>Zz^Gm6n)$C}ul}E8$9br*98gIo;?q0S-4sGr)I`Opx-9L0+MJ(()Q(|^<WGvxC_6%X3s_Z~tV$NcqS zlzTGyZeOZy;0~YOs<}|8-XSt(4Do1PtUxRGzEQp>ow-I0abEUn!+`RrVj0Twna;BH zN?^F1>sDgw-z|gLFk~eBD@OsJ@f`O4GJm?x%s&SWO*QOQnG6=lIo5D0_J(ITg^-KS z0lHr4Tm`qd+hVmAPOC?P z@}on#+F1~0nv^c_6@QaIHm2iz#Ua)RL?S=jKZqB2CSZVm=oV&m(BgSu6lG~|Nf~5e zc!VSGQA-cY-jH)`gl1vlT&1iyQmc*DQED|+yBG+kQ1}y2JybiBaZ86h^K$K-W*udN zqL(tdpkg8ozyy*v-X?6Tw-d*v&pfPS%t_AgughCN4I?_@T?onoktGx&|n00 zhEj$$$@zL40~7MNa@md6f=|cqm9fZkl#??x3%n5>CgNk<{?`I}FZ_3d6RoUWJP8L= z@aj%vWGdV01m_1wHv}rn=|5g3SmrY@EUMJ&sBC<(il?mjb_2Or?A22}T#ss!Mjuiw z47o>xh0AtIDSfCAp{+qH#+-R25TUh0uZH<8VhKq4g(|@#|LY2inxSSkzv^?yEuZ0E zW44$Oslq`#?($V|c-t%1yQ1gHcHzdY>|5kUpxHH9#vL*+Xh6+979t%3_mj<$gpTXB z6nShXpZoFuB~0Dtq~A(4jr`-wWqhm@LiMYGOnP)X318$PIfcHiZ6}y{FD5a8WIGT0 zAu*(FuK4ZxrqMGL^tzXIRIoMJQuCl@DEsVrn{H}5Xes36%0Ritt);XxA5JLF;M)3J zKe!OfO~W$i+LSN<7dh=XW45IWF8Ty-wke0$&gyh}LL@h)xcGfk2BbD>Zq2mtI1p_f zv~&g)psU}}QJ!6$NmRE}d3?GnN}-=-vF|z^(y>`N-4=Vuw3};rbCNj$4UQbY(rM4X zZ#N?!xP5}F_~Qte`s^8(U5s9f!inSL(}L5eX*?f8H4+ad}-jg}UP+uIACs-fXRKy^Y5QNVuyM zf`rU*?#@(;9ygyiIIKdYPHu%>Pj4qKHa9=Ns%dCMlP^F@n@@kAY$<4^p%x*0V_HF{ zLL*m&Y+RBDzdWrteZvRrFFVw)W5yz7K!sU9?p@>CS-r1ZBwNDG8^`ZSetrXOdh6Zh zj|!|)Tkhmnit3y+p;oJSAwtNu-b+79$#8WNF8C-YAizxvee>=72Y_m%bpHJIYhLVp z-FIi+veAF@957A7ZBD>J9^re+c`*4L^2(JhcTH`d zMJoX_-~%uW>{Z+jmJLACs@LJx){z1Pe1n#9lI#Dg&@r6;0;hCEfwd>Pj`A`@?!&`2 znQPItZVq|5Mm&F@_X;S?B0QT~pc9%cIn7KH(2K=tN;yl^QlhOZ_+>fyF;>p{w$UIq z6@!nT|3P&zb0MK;ya8|0%QWv7@M}hM^wVWZ4h9Hc6D;LJ(6$H&b<^q^C6{J!ZQ)I1 zVW(MYWC&d+;yxhXFHA%lkXy`(+qe}zeor7`Du|xfsS0_1&6ICAc^{eoIN$TdWI7*v zOi~nya?cg>yv5tLnVdP=AdE)ddf;Y>AYY@4x*??*4Sm5CijjAw&^zwQX7A&vKNXLX zb@}3KA^pRfbSU-Fk#;3Mf1Y^HShK-6`<<5(Ip`ez_@2VIaY}UDxRCk=^bOlxuDP_v z8cXB4Gg^PY!SZK=6x^2{)Sp8`dD-0FUDHv1_1rG-!q??IXX7$8hm!t5@)E4Ew0U2$ zE{bwC3RNOfsN3yhVnkfIghTF5KKpWb`l3s@^ECe@gOW{HUx7=MX;ZX|pO-tg)(E{t*m;ec}^0n^f~J zX|oA?-M~OsGx2nBfxnl>2P1$nO^{AxL=@d(;i_CGMGvITcb4f7)fqDc5V~19+SK}K z)T^r<4QQ2jt`r`p=8#`_B2!8QU@rM-M+3{(#M+U|W5~&+^LhV`j_m#$9j$Z0PEmp@ zyjoPuma5lHPvq|-AHvPMP!9ED$JEU91MT(#S_*~z`Vx<@GA>| zrJM#S=3tulvER2U;*4R)M$Uqr?$(ohb5*!rW_2EY)4%;?&*Dk3e&4j-wV{=%Rrv@| z>>#4SX^rQa3=WW}c_x4H*xuL^d3JKn658eGyL`HyVr}z+jd@&7faC}rnnsJwyovJ8 z+jVo6VZg34ahsTlw>EyG$(k5(aM&yDjX)fFi}H1Wy1x`{8Xn-y&Kj}(QYzHm>{jH515%uV6!5S<6E_#9my~(`z8tq05*JU zWp+JtYc$Dh|0eLqrOtdM;OFkM!XMf+We2_bZ#M8xnzwlR27@yWuRQ2XQ1G2}5pl|> z$0L{Kdhq1lbm@CTc|7p8*+{!0+mz$Hol>EkvfQT=Gnv;-} zzq@U65fFOyPfa7ojSOjSODIDo-)SRErMf(wxB~TbzRw-n92r80cX#DCvEHwDpoFW3 z$uQE$JpLuIA%YMs=hj_uF4-+SSY3-0MhJnJNiWu#K*?gkqJ{U0t4_(48qZ5s`2y;q zBW3S?RjqdF5Pi_W=+za8ulwYf`ofmrY!h!sK>+rQoJ^RNKpM0nQFlZuR^(@L!;bf6 zFNrivf6RsWym`FL_x5Sy<(m!I@+oXZ`;QF^$wXmcE$^2=GyL=(q~!G)@elx-17&mN z)fmEi3R!bljw}m2=!Yd=sboRq6S>quXXX4M5L|GiXu`=^vsN>Ay9tAM@r1FwL`(S} z5oaYd@4n`t7PX+o zQBrYC=5*32Ibu|Vfrqpd)ASvFqm-@Kd>NaP$ z#FmJAdn2&p{&Pz$>8ISqVu?Qlj=xTZU8kxE6IYxTdqO^$5y){r)a8%4c^iW%&O;F5 z@Q1c7TX0JTr5FiJ4DHgwGv&x9pUTn*%#2oO@lXmh`d{Ab=^u4e(;E%SK9%|TbsF@D z$pbs@Q{*sE31Zk!ww4*FqHRSzX*w~`J>#h*LMaQ?E&Wh=pI%P2Z*uXVza1gQ_lsLJvW*?{Q^;b&P)~gW6bOcr~xRwQ?^*5om8&>HgYZbYB zUry)wNvseC7$H~vNcmEW zF^^5uo%R}t{G#sl<;iMs5ZKWl{t8?Dl=0<->+M%0x)G`8?;+s8mORqS625~=m8!Z| z$Z{PY9=j*b#lZO?s4TgELSUr7@VcK$hg8k)eTg0z_{{KRmI9&KW&|NRq$5S?%nk<) z;CTE7VCLA63~YMw)7M;N-ZDMM@99>+9VJA}7!hN?!v0DSGms+=r_zcg8wy~Sk8b-O zL9Z9+S~emOF_lr#pH|Y7Y)eW%(!XW=EqCu^B{i>@oG$3fcmz^0{;DkHksp*{jQ1@Z zp8k-IF1vVpi;`&n5}V%0Itc;k3|%H^AlUJ1kEBmPaPP822#kQpeF{@L6KLW`XuxWG zJYgobbex%&wjdFA>FI#07vN5VD#`MUFhyE}#UrE^IoIG)GirtpF3=6#v&MK{*J9Ya z0q%0*`*w2FC?8s|JUo3a;~!81-n1`#nZKQ4vASZZxuBZfe^ig z%`z#fJZ%bt%z@Zc67QgAEzhg)_1sas&nvB8v-+%xw{Grtdze8FY_?F}E>O(m0}}Hu z)ueWV5rNV5?gS>gAnQ!y?FV%d`$uQ^DWvnChsdDxy9}YCNEbqxW_wcc(YH@XMIQLY z)rdWfY*5dZ0(Uydgj!;xfCHbT=1Gym&Wu{-if|whe}Jj(kL~r>K0VE0kvtL+O@X>& z@o-6OvFlg2!K(o??ZSfx79gHblaqP_TX}vpaiA>A>udRC)%f_^04oMtKcsC(2LjQC9mg*{s~Gp=ghGNY(AJVv7)U-^wmpYR#p_G=i}c&;CIc9-nRFGJzOUCF zM2rJ0lzhK-@&FBog}TJov!j=)U_-7F^MdB5=l*3?Md!N0h};?l)i?Ja0H#I!2Y@UQ zJ-PVrA1TT>j3t2#&^-BwMpMdE813}3p9?&)zIPW=DLM9@4+UROSG0%{U1J``G5V>$ zf!_|rFT3VYrWSqRuCgKr`L4@rS@@x*-K&e2s>01X4-8mideYuLE&d@(bHX}G!JlSU zF3>2pzgSv2%vhs(L&JmDUa@qRZahbR1?W*4c~qy@J7evK|Jdy5N{TS%QKh3VPncI> zR#Ftyk{mgtkDw`|ll!WOZ|7!q%V1=aM0G@mSL5~>a0!JIYS@T5h}eI%ynbW85CI+W zc80WIe`EKVPToCLeDx1?ek9jGn>ZbZ#M=6GWTPg1vJyJ6J3+&tGvY>r!qd3uEbIG0 zTcsP&HEYw<&+SgkeK#c!CehvW&_xQA&>`y$;In{hMHME&jY^qw(n;a@nQte39KOqR ztV#u9o*0Em342`ZOMpkAD6H76k6$@lAgv6cgw8o^2exfqfgOV>5MA;%^Cw*nrs%dv zJHZ1A^)fBd#nzTP-(MOH+6j+%JNcqZ3vX_s)h4pL$00f$>k`@tG0BvK*&0O-n(TDOPjH-r$zgkJNtrfwHmg(Jtbnj4>`i68@gYz&{ltm-B%ZQ;w@OT<} zY7TN>X>Q~yILnHeWj`(NW^q2|a{cuK!dy1&gPr#+dNA#gvRDSEn2;jkg`vA zJnMw`giBK?48o+LP(^Hl17cNmwB3A0UH6Ia8rHM}yV>&tN3>BSa$r#Q<>{7WFGacz z*_Ba|h%~T?c)9J#ersO=dBQs-V*p?_`ApH$>La~@XZ)Mq7gdy%Ibx`tJ54{&JRo=n zik<4x6j`QDG2w<|Ge^$M;_k1z$DcjAKB6RQaiu@Y5*iU%ZG=Q7tztWBqhY%AWW4gR zrwjyvDY4yC0C;$V7sFIa<81Y>S9d!7Yg+vUAZ>Ry-?SxafTZ;m&tOruq4O%V;Bd>< z#vCLU>a7YEz;lMMNtUP8=6?0+g0pqg^`;KJfj}dttB%n>pY!TF-}Bb}2k$(=_lH-B z_l;c*+}wwt&I+z8Zf5qC>J`EYgNrUxw-JbdW!a>N>g4R{s3ig_jbF-iWi=1v;DYD@ zIMoa9H3G0aTam9S(6cd8o&SCCjy_s2f+ilZtdWLC0}17;)!~y6`uk|?6Dh^Dob<{4 zIJNEFaMBiTU=FLe_xz&X&JOvN(h((Po7FT?W>TVi-lA~0~ThHGd?y?~kQR=wbApN~(- z(uz-eWY-ho`#OHBPCS~iSmpvL1(%v3qkK4l7lzKnW#h+u8N7)B@SWkBoe7>lKir0( zzS}m7WIkV#Q%Faneqho~9Ch%a+dC1Yq?=&I!XHkXO*ME9GPt{#$2M;EVamc6c?xq- z?e@{;7#G>sCntof(bFu`<(L~>92$7f#r2;HB(}b~$~$+qDW&0xK9YalAAz^5zqQt} z*tKi*7ug?iz%sA(HX-vKvv>YXZFs_svQi}}VN16~cR|!IsN`QfX65dBB=s;H8z%pf zlJTsMUGBO&#xTxuPfY%v2RjcWIA_kcnI|7{NNLYVsa5eWv zS5Avui0MiM-+MD1VfeX0%;tO@VD9f34I&55FB8|a41DQNS@^@y@VVhpA8RmhG74UE zy~P%0J4i-4b#{0+fwew|mO$lcmg8>@OmHadO`XsFUI9VV<6uo`aO1RjwfTXyTSMdN zV{?$e#nKaEuW`ufE8-^(z0>jNBe+Pq^o@gwdU#%z^{OfvyBFp zqvs{*-Kukem;E{eOls~jT$`uQrqhlJ&4We4Ee1Tw-r3%=cyUt%2?Kvr?#xtb<4dc9g_hGd5W;8zB%nDS139I@qKF#_MWB%9+t zm^x9p=?F^pLXexL$x4)ftIKZV}ma+!8z38kp4Vn|Y;#SazAP=#dLA~sPu zll+8)sigDqH>|W~MRdC&#!0z!@Ep?9;Ds+d;NmxBkDNt7dc^YA2y|_gn8wfua>^Gg}VKlup}mJ?+jU4oZ&grNJq%V^gc4gxO^CC!d>BIl9Hv z3gpf^om$;bAQlyHWW&P@QE1 zaznZsd>Kt=_V~3b-T~q-@WS&q1NOPs`5antm7StSb3>=Wd%$kCQf^p5rvmC$ zGkq>ML+5RC&FR_cLd(5IxbFvE87B zHP*q30m*TbLxl)K@v%@3LQ0O0|{!v8aqq3IagPjjG2i)9QVDZR0T=rG{4kKtG_wG~0jOkd*8QOmNTS zD`4^Swb14^soN%ua#J`tUnY^W5atfu5!T?C{~n8M19#!c_cz*7svwFm210O=A?yw= zPGL-XXqd_`nOqJv62=h7_GeOKZ=FoVJGdv70v_L_?ro98MXlK}`+tlCUr&9X;e#Dn z*O2gTQ1rrj!FcFYhP&2|8oOt2`ttnXf(>8gD97;1Q;zieyqyt3wtipBlJo>?FG5o= ztFNz>mho8cdtqBpwO%y(xe4U5avQUs?&mP>BW%Q-6WEXW-XeFFb2 z%P}l*jL1%qC*-ikA3DC4#=T95uO{M8f6PldF=l^ON;(@?0juS}pVAY3PARLaPBPH0 zp?Nu#Ak>TA8eyVvXEVW+T5(pn~L9FU~U49z5P8 ze3;|Zvt4wqRECw7INhbu)%M*V5T&KV&WL)g?P%a<>g$AsG`G1U;8j{BC+N!dz8r7% zMF)bvOAhZrc5L7_eNWG|NI!VH=ytVxRyaH4KbL@8PnPWLQ>XQ~9d^Vary+-Lt!^4g zxPb)Q{;GoV!o>pcxwcrpl9> zbn#TdWz>T^?p=~utr=g~Cn2 zjPTpj-q5gS1a$azd9G2>beG<~C674)`gP%u|ksz_AIYlzP$ z5_6w&2zr?4719^<^^XK2{CZf=J)C2)|M=rgTH$V-yv2%Fio^ioE9kh%pVT#yCV&vu z<-sidDQD-mhIHd>H(GQQ{ZBwS^w!cacfOM2FQx)r0*UB6q2$32md<1<$AKEVUw-R_ zy3%&?FE=PY_5H64U;&>$SeE}V@-b@R$fME(K@jd}py$4qHq)|1&F_`Yv|O-BnP)+2 z0+T^i*?PXlGyB@?klkh$!P#OnL~+K4R)Z}ppOWEuwoyjQ#`Jqm*0eX%HiI+wqIsRo zx7}sIFVA-+xPRy(;w1|_M4nh9Y#ZE<>ux*1!X*6N)T-hB{Vo8Z&DkAj}g>T%32p_ge|xKW(`~J{!Ut zJP6712g$|GOT%CW{-xh{QsTROsc~8dbEn(Kh|h*!3|c?4vNRiM*!H(+$GW7WiC1TA zxB6om)6>Z(D#z0ou&I7{Lv5x<|5+-N;OOq~B{by=prph;Af*kLj|$`L46;1pe~H~i z$d}d|WI#wrFzWx{pERIJ7we{kc4&NY4a3Rz+_1iwIQ7gw*vGZruXVrP^D#7DM}F1f z=|Ps<_EZXZ2a;I2H_BY^Ar1`&Et*q@bVwdy*cqJ&B>VZb+IB1HjX9E zhre$(U`DI^YZW7};0mM|-u)VU5G1MpTV5b`@ms+xUyfzBb|d#^vF?1Pr;^>-6j^6h zpC(f14Z3ACdU+IEQ}Ae=%c)o{wv9#&T>dQi{#Lhw>H8we<`p%yz#aGBPzJh8^CH`b z=yE!2dcpcORe=Sm1r>rKyeWq=3%ClWq1iecg#3LJg%-*Hff4Koc0})ND-+F_jQ2fV z*2?&8?L5O_UOF_r%&_Gf#z4*}QpZ#MULh|8VCW#*OWr44Kw;1jKl!Bv&Tva?^H=Kj zkY!A=cEB;0|7+3T%+wX2^TGSAxg{ha|L~{loZMjRS3B0#^HAx3j0A7UxGWzR@G3T! zu;^ZS;>{NY{U3Sv`Ts`8kAGN8x`WR|XP$Q5>CV5u1G0+3m?}jhrEhY{?@|UMuuSSiK76_k;)Jx%Dt70KF84aB{%E;@u!2YP+an+L=NNcQtGHNQT^Kd+q5*F8(}l?eKv zoZ9FpINy-{wa^C+bbDIiLckEdleg>RQ!ZEwAl?J?cmeRkgOEb*Z;u4ng@HhZgE(Xxcvf-(}OT03_q%!z@?>yhJUQjH2}<=YY=@jFE@ z2P~OZ}VbG@*nIViu_|jk$RF|!v5m|n8}biLtzh+ zeKWJ6e=1&?#>#YznNm6hAAz}>8|%>49@z`KMrZ}D6}&22T;~L1u_XAIgfB9q>y6N_ z@^-)I5l35TU=}CCU=pa%E?U_$vw!7Qm+!A9O-QvW9amJyuKGGjCg= zE$f6)NQym>n{Dd*BNTfw4`BCiNUL}H^GBTyP4?v)*k?m=1zf%(c<>NU8L>_AxhO(A z(d#nkNcyD!6^A0ac(43)FHcVE zj_v;u0gR#%1l$`sVbPHgIwlVYR8qVhqbk*cx6Cd2#jj>`T>B|e9(@tH711VjVqEu< z&>s0PY_6X!GBOdFW{5m$KD2^oZ`Nu4Jb#36fT|#+0PW*EqJlCG^Y(QP%4~s&^oGZl zZv+$iXfM{;`zshxnk+`xv)k*rv?tyad$OMT2f{532;bHuUpZ)8a`{Rh$Hgu?zD%l2 z6kHx14u86!I_iqC&NM6j+V{hwJ|p1^X>XMFSKeXvC+LucAGofTVXI6gN2UyU zdPqasxIKRR08#u0;tT#?J+;IQmu;;q_xxw%rW>r(g*mSawLnYxYkj?8emJ7~fyi)o z)<*%)@58>fPk$NCeI`G3ELCZY<*{(h)SA_3^%8pKu~yW^0^RSlU1K7RK(!?!hzT3u zd*+iVyui`QEIscxtiitZqx)y4|KDhVE`D&_i=4@N{9kHc+~Vl!0PT>{i>=95-w2wB|Mp*z29`&-1K zeWOEay(t2n{zMHU84oxLVDUJX*VmL%pGH_h7crE{zxP@QL|m#f3fL!9WL@lyueH3L zOyNcl^on#sIXeuK6bSjEdf(T=hj>cE-)V~7;gOf(88LdVsTs>gCq$6Avy3QL`ni@y zTWUt1Aef{}O304nESs-x!lHN{f3V_J@B6k8k`81lKE&If()go)Jr$Jp&$lZoD9OY) z=%NC5egutXL=Hg@Tpu-~TRHN1D?!sKEOTB{_N|@LWFy>=sX;Ozia&s_nSkmkqM(;pkZ@jsF(j@==^m(BeX8Q zae+1~4t6kAqogqnaWZtJq%jk3QEed2DBh88^){t8h!I@doe4Q}Cn`#pJ4vCXn?y7P z{6GzcCOgxsHgU>{cT>1L7A*ZyGn+gWU@& z>OX?m@5o;S?V)B0N=vNMbs=6us1e=@NG7^MFMAc+thWk$u-abDlJ;p$=MvNh`)PJVp$bCL=@Rvf4>Fqr$ai zPLR$cHN8`zt*VKh;)GlIRFWetMbIcAo~i{kGQfIw3yS@gZ*%AB5p%GMBtpEMQ-;{Q z(9s$S79*cF#!ZQRQO+YPIKm;G(0zS0_4BL8W7FTGx!FkQ)$e!WFv8}il-`1=>vs4lBV|wFyh_MYgZnncbvJ7y^a2-|84~T|}r& zU&Jgux#<1vqDpaV)q>hNHG&4Hgnqg3L0d0c|7bS+*E!zAY3E{oml=H*(-X$qo>|ls z|7gqn^+bqIPvUb$=Anr}AW*@RURR_)GVJK%%;$XAxAty;@o5tz-GbJoVsTnU^|oDc zrjmz?pZb6F(>>6L)bt?}todL zA?4SH8fat3*aRc&YGM_uJ~4Cyr6PSBV}_II4dIP34^C@7yaKy@Z5FYJ{;Aw5FHhAJ zc~>h0Tom3I;UG~NwX7wLPLrW~|7|qfP!=m6yu_+X1hFN(vQj2FVe(TSk2(Te$4H8Sj@Zgf_g z{H4ab_wT;Zy|!@SkZ}Gg75X|^@Yku}7H_kivj;eN&uFrOL%lM#P{-xttuWa!e|vy- z^TTX&73?uSQBj^Ha!lA(eiCtauhL#_0@1DSZRHq^Fb7K# z6aut=ym3~-297dSXv0pN(N=P>u4Wl<)k?$ZBJx+ixW!9ObOLlNO+8hTd5& zG1%gS{>d4N_OyTsWBU+i-^ce-G(;c5-;vaAw|M=l=8It>O1r=}Y7qbK?E4m$V~+s3 zqs%IFb2ZF!Vm*tJPo)PjHa`%QeQ)&~7v+oB+~qodNVK^NYvjBZyzxgn(}4As7y z)9<`#1J!KN=hzuS9vTyo3j2P`J&I3jb3ARxOTlznp+MQ8+nh|2HSK=S?#kGnanYA` z`6*VM#gRzSo&bb?yWk)M0r8A8vZPh`j&Mwe&Q0_pX}Kg7nz~$TR_p24;HxhJE{h+N zLL_l-yM6dK%BimRm5`Tj&+xVl6Xb*s)ojv!U+$jmnR9I+{N5z}0I5`vn{swA^~4o7 zL;3CNLAjZe-g{tju`}1jH0sw*CtIhG!dS#e7nFgB=aMdE$^LEL^r&>4L!nNpr=G|$ zzquT@Ye4Eq?psJFb4aB}Y$?}9kBc7l#m$Ob4fE9O!r{k_49$t7!btBp-w@X0l5kmG zY3(Z$K^b88GoH}o3m^p$zFU$B`y8+Y!MTh6H>&@B(+$ao4bnvyrf#eI6Sd3drr2&s z^&X;~os&=oyhRHM{ufnXaj@UJ6h`|K&C>OfXyjDgG^y)1K)~F}UQQ`*#&DeuIsmQJqtBijWIj(ODn&Y>Ya~N)Uc8rLsYw@W4n@J4U`Y zIK$s~RI{AQQ_I-AP5)J2$3Uz>F4DCMW*47oZ~@=C!kIsnfy${d<1;jyRWq z@c!U(;1-n~VWbkx#V^glKY)3Ay@qjkdoar?m*1(QInUQR#rDnyBf{?XaUa3>jsZUa zqj%sGtqJ&07l@ikZRRo>O^F7W1r{KB%wm+HG(-y>>tm zo&=S}@F#YK+p!-T6p6q8D#fiOg!Ithc!jERuT~B1HKMze5dI-*sB>pAZc*n)Ac-v5 zFx{Jn!fR#ieT`GlRSo51x83glIPl)ua z2Bid)91Cs9M|TjNX?I<&FVmBghUFoP0rTl?>$?zl)DBkf?H3OEUkHC9!PF>B5_!xzPfP9DN<#E&*;TTqjRAV-Mv zM){X=J&wqZSfH8gz`gk^R{%-bn0Bg=rm+q5z0`Ppi2$?sG#j!D$gIcL;%q-(op@$h#qfy0{B zZW)m;1j(1>y*0P^qxj?Or1fnA;uUR)U{%>^e#6LC&`jq2^n5 zG2hSc&UGSx38Z|jn*I)nH9p$}7_of&RWr1IGUlNBzPF~TxRs;xOFhZ86nWJsOH}kywdG6Yt4S=hd?H5`F2QQJI8=0pf&kbqfw@ za*G4t=OOW4K9Zu`Hzo*W>!kz1?a%R?f#U3_@ibe-dJ=}eeecnI9)n=%OgHr6M8UOV zxZ2yk`=}k&4@9uXziV_o+Iv#}OytM$KeM)*kP6;eWeR=oTHJu5Z4q^i1L`TlSxCQX|0%bD&=7FR3qVQ)wUL)!BJV{VEVE2fv4T%)e1k00x#;i2OR8_xA7&LGVg337<= z#>(9J5Z==M={YC4eMd^?FUI%WPxzDtD2^;;h%XrnKFNoQD3kQwn6z%BiUbw%v5dVGN?~tnMmcHFk*vhE;ukRjVC)5h@UjGObptSmzZCm)b~yR5 zD3oKQa;LGL*1u}*m78(r04x;bsJ(c$qgV1T-JJ<#;AgUGPcORuqEw{C$1jjb<&dIF3x8Ct?``*>o*Lci^awD%VyVjkrnEVTt!0(wb z`Vw^X4M!^+Q07};!w}!ThWci_3$K`DnKe<|la%S@1I8y6lBso#!1uBUvUsT9-?wat z@KR&SI?_aBX%Dn8_pP^2Vm?Y`?+~@Kqg={@_mxZ+C@4nhq**`ZcF#x-&!5!gU0!U@ zW;o;>j#4bI&W4Q{9G~s24}M*L;A?Y4pan}f%MtbQh<|;WVE8FQo0xFfW1VhZXbf5c z3UMSjo$~L`Jku1#u&ZU6S&o}A+XDopgy$?`-j9(Z~2b^COMt5$bS?#XGpHCh`s|{`QgKm3W zt?eYd58)CsE%L7AT*W&VU5L@fxdz%&sIFf&FLm@#kJb>o4K;Vn*=g2Yt}MSjVKJS%nwir4<<)F7^*4nUbX>_sN&=6( z;G=P3%Qw-VF^enF(rp8P)z{#$lYizJdJP>i2ZuS2;k* zuf733u(J>79VNt0^90Gn(lMg?CyRe(Q1r&6 z?M);@{iKd>VJ4AMwY<v_X#FEKk-doj^0ADXGMEvW5aiqM7s=|KHU-1wu&=7gaf}4jQxUz;-y}@sr-zR{ip1a?sdoMn8r19Zj`#UDvYuc@_cvqi6Wg_jwDHAKB9U(wTE_T> zC9T$ml5KID2m+F*%xIvHgvY!P0iDLp5TBFHLsqWV{G>!>UqmpuzGL03iGE3dNo25d ze(;z4#C!`n%L?2v-Jm8b%yr=J?Am0bu{3i%*1={gw2k;?5kjUXtD3Oyhu&D?T^if* zAu?jSc)0PQoYWz2WZ7TV;KP~qj7vi3z!TC!3I6enW|_&GFfz2C5IyKZXPkAZb;OfQ zpYqe~<0xW6@?{ZDjBaA0w6<5}1C`*+P(ro8(8BhV%FW=1T_yj>r6q5Ueh5yO1?I+l zl$DY~Y>tyfst-1x_eg~VCc^m^#i!47I`3-;3%K{Yw0e+UwV&kqit>gLmanYbAF>ya zjiKex>q?o&gx1);FZAlhx6?e|kfMWcSVUQ}r}@$vSyE{%V?Y#LJuDWwmf72ceCXBl zfY`==wQ=24A7$X2eBK<5KWo^;j7_olGp=qGO6rpJk0#N-!Q)TY=*N*WXsF?9<6x!1 zy!Ug6dSTe)J9h!uZ(p@s`fTUk=tT4gkS5Kk-TM2#d<1+b^k9Cn&z6LKg;d+s250Ng*_*k5m`J-{Ve1^mFv%{G8gZZ{_+~3WIVx$?7%e{?`-v`&mI zW|471I-dL}S$1KuQ&G@j^D9mQ#a$D1igwD*6d(S&cX!bNiBYk6H2o5i712|kv`Odu zSPDO#jC?=rm{MuM5K(2P%YG0LHgC$KLP2A>HbWS?{RD%&H-2Ou-gP^E$Gv3fmz*FAj65ePPRpV&7YivOMUG%x zD9XX^K}jZ0qP(;kpV!-Pko~Dt_EP&i|LRySi&-MgV5xGP3+Hs~g)OlpA58?h2ASUL zftq7}Df`d3^}i+Nl}M|OgPc&wAEaZ&mgo?dj^#a1t5J9MYOx;}dj&!TV{l(P{>xL{W?n|@c>9X(Y3Yty@T*n(7W2KEi3{b zkkEKO`Qoe9-x-+h@438*-4H^W^Kj=t36M%2TD#4i4xX(97{eZ@G{#exzy zDoQAb*?gtWb974}bv2lP@*Qo#KR3>>MRy7`J>fbc{hJOGT9t-z3_)zI|KU7RmD^pl zohsfdTON8z!v&6_SK`%B@U0Vw_4YiwfM3zJ{8am-4LvRYzr6r`v|a}LrzbNaelj94 zBP0WZ3-DIy#DmQ&J$xcef#})N&9_&S`Z>k!pE(H`z>}`R1m>qFuD2a&JEtM4Y^t)8 zwgmN&Tx6pd1?;q`@r5KqwTZS&Di_a4d_fH);qw;neN%`*H^#yKmEdMc@i=#ARF^J? zX9w6>O4y*QUDz*8m}663Oc&MreHDI|kiX8DXI@y1Y#c56n8IQp<&PtQD`Zd}PJJ=%f9LiKZfs;wG8HD-i)6uEwp3gT|Fn;)_NHQKDq^NYhq?H4iU+O)tPvTzh3*fcdJe5~^T+OO#p- z7S+!LoP?%bVw^AKzatdCB8pdZlwmIO!L6=e3oKueb6urV@Z-(u z4K4oQ)~*nw2?a+{%~^qNx%=Ldm&tWoh=9Y5#6n-#MR*V& zKUEYSEUU3GBu;bm@I6T4gNrcNgD2pP1G0K+-&ygWu_-E+w(*Y0fM=Ox{M}f0sx90V zY&_qIW!W|B7{%f%vsQH{ zvC@*|HGY~~b+PB>b>u4!Zxf2qy0M1$-!t@o(*3{X|0_uBKY}}QPzFKX`g}~L-ln(L&jHfh~Xk40K%2llzJV(iGf`_wN# zXeD$pbsUFZ_pYs(FPaGP1>rQLy7Rn^3S{(;}1!i_&CBeYa)y+mSy@D9S$@m-M$G*~KEQ_I{mnsUzmx zg^ebo!bz*K_n(zG~Wswo)ai-*af2VAD;4JmT9Q1B*MYp5i~v(DjX zPV5OsLK9(E%c#iXru|+9#Bm4jLs->gj4NM1d?szkMCx(3ChpLVb$NJEAwcEjyc3xk z>MORI7jby=3AX;tY*=G6{8L*~1gsVCFP{BIvG)v$fi>trH(t<+OeIY9h0kGH(#9IV zCQX=URyC3RYv*wBSIbe3(t~S;17xIbGgrAL=!rtjW!z#~F^?6}c#W zETxQgR6=|c%Fq_s%r6wvJH|vBq7QP8HMGXmy<%DGHyOGY)=h~A;|2vz8yVXpWPR;+ zI(YeK#jc4#HE+?-?4C-vZ?+PBusefmU!xXR#+UbW=-&7P-LG;PIU^!i(OaqViE(+c z+$KEl?>m*hZK@Ycv7D)o<`?tF<$UMl#HM|FoE}|dN^!wVT+(XBaT5^BTLMV$-ThFO z8D#BP!fSdgp3gLZ`3NF+jtRLb=57+JO!SHGIU*)fJ&#w2 z>O!uzW1)&+n4e&&yBa;EPDm-W?}>*@vsj{4Qq0Yf zdSFXOJgkACjQ!3wBzZznhq0AwKEwb+Aez^G5p{`T6qJO=>sP(s@))(pbqdHhB7 zA474#AH7gj;pfipx#k(yryO^qCE4e^)hmat(_}Ec>v7|oYkk>u@&r|nEX9d@4x(g{ zh>{;-s7tw^I>Ue>m1w8zmk;*jW9|H_TCjBxetuQNI7)Q5`ir$2vc!YM+e3q6CeDUZVflGGujdHMd@#Bf@T*Uic z*BJ$$to()3pH^eT%fA~45_<6Q0i20cZ15$Xa-vDq*p|`VdYrP>H3NLcny^GWt`h|5 zpGiP!jXGe3e;?#}6z!GrMA<(6Jicn?(nHEfDiDwo+K*e16)eb#IBK^rzz;UP9qBu& z$FKEZW?u8c=rn~>_xY8O@1L#qpbl)1SxI=Qq?>e-4YHHNYQ6$;+&S@zNx2nFhHm`&K3 z!8~6vv5)veI*#0(FM9pH+82%;-v%*R$dpmm5Uj8=i17M>J8VD|`b2NQ2#9!6e5hIy zAtWVTcpNI{dGvYn|L`BQp6DC#Gt}g#%l4NT|N7&WTt%ycl+KdSGh|Hw^5Y%RA|O1s z7Y*Zh4sXUB>q5X{0=h~EYc*w7T{;RrVKrLH@2%4W+izMf`I#$2Bkux^RPDWGQn!t8|U>3n0ihvzeS6e^Lpr^IPWMqvfq-)Vtgl6P}Af>C#IW+QXgbKFQiVi z&6NeAfadhS8bb|B#KR&ctnh0#LM@uiTd(*uzeee6p53YF`6efClDT-0c_Wc3xCoGH zl}<4hzM!NujV?^#z_C2ja4N0tvsd#6VK+d^=9eQ#x~dX8Pl3UqeIN`vSgJSZI)JFWcj~~v;P%o z15N-{e|oRsrEZTSD93qHnDu$OGWq<%_fKCNAE?#de+hZwZT;Lg zc}+t?zqaeTC5At>x_q+HeZVtD<{Zvg;W{QbV8?1_ZYI6I_xapB% zW9-H*?R+euA{FA%03O>-z?OE>&3Mh|O&bwG7gU--HdK1Pp%MLE9gLATNt1Yi?6kNG zKzytW0J_P0GJQv~!~ylf@sW_rn7aZFq=$I%Ou;=@jwF@~R$4fN)x_hItTsfEA`l#r zJB&bc4%OwXs4>g;-lvqqN10*MKElf0?ee+%h%xh+WN!3LM&};4cuk)RyJd@CJa==V z`5`{%?T%crDvh{L!nzJjT@dDB(+jsLv{H^m_v$f}qtcZp!<7_F+D|bdEuNRqdJo!( zPstt>AOX`)^sPFUi%Lt@c6R*rF&qRn4ra-k>Qtb+CpMgXMwiu!|B3Q; zP!aAO0*1!%P7m3KUoFvD!QBagAIGn86h@#=%b?5F z@k?PA0vC`<1AVai4=<++5osqQ7X)huB$G7~g$r(mat8E$-nC~Q0XC%#sG}-uhuS%0 z8l5EXH}LHL94ZA}YXBTuMG|?NYgA@)&RMq&-x_?-StSH>?c>A*4=Q;};5wUi)GFVC zFIn~$jEIWdoPmBgg}o#8V~uLN?J4yV3Eg^lvRZr<-0F$1C=fW>wbrtdapNIYx@eZ$ ze8E?yr}TUrL+N-pAxZ|!8j!ZGCX;X6kc;Gm?_}Ugj^HFM+&^zpt0Wt13i~c#Zwk4d zb3SYgyJ*klaa_WNhEqQGGWlMnA8J3lqM2XfmZsYk(oBoqCA=O=@ZF0#FJ|EXD%DCE z?PRcv{NgzIqYl$cV7<>~2g}!(;yhj+00ucaXex~(xuMOB#Q7LF*r=B6LwUTQ?#M^E zKAC8Fer`B6&?N@j*(2)P+YDizGigjO_`{#(QIvuyPol~ymhuJM%&B8UhWydG7|gg3 zUEBVHPM&CY2NFuuLMqxUi{hXgpajVURN$;NnRd$66^Z;rahZcJ)B&?4E2Q%(*H;OJ zYs_;<6H&FjKcxnWk$VxX;_eW&o0MNV<1worA3a+}-|&m}w#DVIet82=(LM`bd3I}Z zsk!D~+ze+^@ylSSq}<-}1`qigWEKmqqxL&n))^Eoe;RNBiml=}wp{tbs$)Z|en zkE?H1m+>U0V`PK{*OhI~KeVi3v5|-Nb4CSn+T5D(kRLt!XW+Rf?Jw}-_#+^nLas)@^WbbKXx8x&v;GySN2t(84 z8l!ayJan+dWSz-5OMsDbj*|_??k%8DTa@)}|HbGJ!{Itl!x%=HTpFsZ1Z^kaNL2pL zxz!?>fvd#*ggtgH)okqR48meuX=nE@L@O*ssSLZ8=#`wWC8o7-E}hQC-6nuMp`m?S zyv>|xB4D8b@diuO3w`;IU2<)3Km@2Q!A38Szx{eFMGeK$8Qe+7xtE+>teQ=u$|38? zQfDMqIT+@OsO!SFV<}}$!mgvTIYKS~MpBi|36;;l|C<9|5})ni38}1~ zQq>>OHUfMd0Ge}!$ek`s75vqKPL$Ms4M}TGA0mD|#T6VyZ|~m*((w_AK2?1@@IanE zSNf278a&lD9os2hJv#utYAaWUQjen|Q89Ki^%Z2AF|qKy zN*amfYO7%~h4jJ^H3e#0lEWJ7*C<2Ui|)tsri;_Ur+Hs2Tf>2$HqfkKL*JDAOORR| zf%2!K%oavcA@q1Yj=r$Pg1DI_zY%^u3j|E%&`IOfd`f9?1F}jbfnzV#x}Cn%m#yys z7?{l->e(J5lmyRNmb4;Dsj#h?knlGNAh-dmKX7?X$n))2jgTIpq!3diJ8I)~H2;T{ zV!B9D`Ai|8PDT=Ud83!ndLJAhqq>C>toZv$)YXD^%U#FXy?-h`C0JnmS$18ys5@<- zRI`72<$G@jHu$_9KqAo<+cB7%LR)Ji2nK`lG#5; zP6_t)$AHe*-x(WhGbhV^J4F2f4x|Tjmxrba-vo$cH#;2rkHgJwPFU%&f}Nd}aqI-d z_Iw3K^d=ov_Z5fTDft_aCu@XWXr8ybwzzn1Pg)^iH-(DOAUMApRocN|08MX|D%7f_ zzgCviVFXz&@(NFPSM$*czP@qk@L8GjVjfE|P0KI!+c!uT8QG3@18nyP7+Q`DoT~Hl zq)pwSYK+yZcXmlh+rP8B+_Lt57jgd}HF@@tTHiXnj=8~wh9$5Fj^Q|x{A!zeBFQwD zX+y_H;ljbvM$@^MBF;@)yRk*wj}p!P|UGrv+PJgS09ksdD-IRII~+NQeE{>q44^lzZ?|6Cv%oyUqyc zf{Q<4WKJ2r^IWH$I_88UZKi?wJ+&>u9gD@p%5Qh=s`LF#@(S$eBe%xC+WRAt;Jx0T zU}Hr9{eotna#$1GlZ7tSHrJMLRb!c2Cy?IvkXOQsqtF+Cv)tMP>yrQ0uiwGh&J(|T zoZIAqG^s~6xl2C@fMQ)o6uTRv6J-pyLuhGMC&Ii``>06q`Rtd^EHoN+1b zV!LjVpWzxj+ninK+H~Bnv;=%Gl>|7l1@t!Hvbo|V2k3sIXv>;^D?iZUhIJi>N2mO) zcwskW=|EX|q|=b|jEDcxQSqkGOBJs2j=P0WcVFiyXO%Dg9&W4d*^?w1n;H$5H_2wl zcSX&cInm|$u`qM)Y~N9{W&W;f#=xUY4f|i1hJm{)fBfbTtQ{`bppUL&%`19Ulb$Dx zr`euqWmKl6ON4UMRjRuO9o3c4=h*XlOOu~T9XfH<{O?ZnekWDScS@)AzI4zohPS|*=u+r{xgdXKC2IF z_8sLz@=mO>JJZL%5`H0c4I76H-yj|l0&{wGL(fCnY?5}b7Y_M@~t#;6i$@x|$m z?dpmKi#O0vYBkJq9j*<|pohZ|)!?Q4ImU;cX#kgjzs=e0ybNT0R05lk#;U2hrxW$? zqv>w|e07lK_a_$O8PI%*6R%Y)xoG5Qh6|g4Aw;WV*ZTEx^B^Zncs|wK7mDS%J%xgf zH!%&%p6Qy)@(-)#Ko7(Fh9_ld4<7+avr)_6%BOp7*?$MDD-5tdzUqK?r0<4)r}(Br zp%jFBwu|EO3$6z@Qd2d4iOpq(4+aSJt?eaC1d_VflvefzgVAdc5|5{?k63N2F&9_( za>cZ2Srp2p#E{Cwu{zkLqIr5qL>;f(w3Lh4jTyJlgFHpOmy5%8=S9J>l6wJv$pD4? zLLh?NE@%t(M3`dWm2W*Uy13gpg04B6q0^SS^6HsB$lt21rjEon8Tp8T$zOQ{or%PfVIhsFya042=mNJ%A`&z_>!HTUbW2dXt6*i{ULC~`T-2`O6<1}@s<*WaMZ4b_ZF4@y2B zS;UNxDJ~pI#EjRd@-KaFf!o=xc3(2KcuyJG;c|@+(4*z5xcF^b=?xCc91P>DjSra@u;mC2o2r=)5O@%lyUaEo zng^eq>wa~ZpqEO|?Zg@HDuh6)`T*NfaA7~FdN3YO{a-_RLpx*q26Xx{wZ`HnN`c=R zr-&(}Yq@|8EGUmBt;cnYsiF@Z9=Q(}p^vdFzYxDP*qi3}tsbsw{0$ay|A56XN)Ro# z!<||nw77T0JNc+tn=#{(K~9tJ0}C3c{#{gh3RS5mThzssjNWvm#^|@Fim2&_;i!ju zu!MR`SNHi{c;bjMry;{GKX07*j;jUr?=*n!I?w03CBkAAL!&JPV=~NzTb0_YSIJ{T z)DvK0U%wQ#snbtV2pzVo)2CYx^!{0g=bcWNuYwxbzG$&c1~nRy*M(1PkXB>)5iCt` zMYX<<4w}53{(_p@hbtELey>x!VhZI;FuK*+v(oq#k zWpx9%Yp-z5?L}*by2<#{96pioqCPz9f%c4+wsTHi^%5tgyhXmyX}*LH!)Zeo6(OFn zk3!GS@@H4t=Bu5j1ZV8R{{8uP22`GV-tUmxB?o@mNnkEY&8&|GV{-*Ub$Y@&1O2`w zTxWIe)Cnav{9hUUT>hsz8tZg5KhRiTNktY$Geo&4rWMo2`vnBnl z?WAG8z^v9kcQnHk!vhZx?+6Ng^dz~!D}gP#UrPI^ZqMm9=2PM-tYEH$1bW@kl?xPc z9=H>fM$ax3gv#oL05-)fUym#K#l<~yoMSTnT5=#3ewJbafgHFx3=Y4MoH4V(H{$a9 z!p8+gpb|HP?1}0U`wb&e#LSD11!a`z)J9gxrQEgp5#-l)WMLET{2W{AoO>$yp9OkT znRiYXfME6@du1?-Hmhi30NM9?3vMO4*otXE^3sQ)bS+WxxJ)Fq{QQ&4G2qj>gKPhE z+k}+Ojnx=$@!4~m2%F^rCr!Vgm>i@%3cE9mPF7v^DR!UH--L!Q*7Cd+bsTT|7QdsL z&-0GCcMd=x{LCPX`dEH;s}1@^dA#ef@gS4ooVv6B>BuubMm>q#aQ5Vph3q;k_9ytf zc*N%SlEL3!zob8;0vY*{m|;LaTJh?pX<}N@V@sA_57|nnB`MkO&}y+dG1d0M9Q}rTeGVj+5Ei9eIg7vzlkmS?U_ujxHz5nzbfS$t~cLov^vFdPV-6{@?9- z;FHs}ru^ml*rUd0k&?I8V=cvf9oP?!LMJTo0kE`EKD2vbdM^e14Q~F^@QBa;T0ia( ztD$%MXi&pVD`A5nWbMmQcmkgX@4r*5lt0!0Uwa#scug4gA_HdOp59|^jRcEZb9f1{ z7R>!9|NAr6;1T`bLXH%Fx)-nrP^Nwnrl=|})uw#$7&5uCsx&%lxBD`@zY0}*DwzJ? zR47rjuipm3gPmI znJ_E6jr&(Z{&SxfnsV56J+NQUlcd!Tt5YH~ZvnTwARr=ewK{U_-n(jM((8t?xbc(| zK)B_>?$1-Xe%rzEH`xZ@0;_(mkyam%ubd}I2sv-5wtkodzr8Vq49j_I!dT_FeQQANFY0q-j-A*%6D z9{3T-zWx5c92WSuTfqLp2eoxVI7NiN#Qgsf9aRbl{NGMU{8NGa=f(Qusdo<3li&Kg z{|Wj(CJA^41J-}~+8cODosIweEQ+Tp Date: Wed, 21 Oct 2026 22:47:11 +0800 Subject: [PATCH 23/29] =?UTF-8?q?fix(build):=20=E5=AF=B9=E9=BD=90=E4=B8=8A?= =?UTF-8?q?=E6=B8=B8=20Go=201.27.3=20=E9=95=9C=E5=83=8F=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aad052f4fdca..0242d6f13efb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ # ============================================================================= ARG NODE_IMAGE=node:24-alpine -ARG GOLANG_IMAGE=golang:1.27.2-alpine +ARG GOLANG_IMAGE=golang:1.27.3-alpine ARG ALPINE_IMAGE=alpine:3.21 ARG POSTGRES_IMAGE=postgres:18-alpine ARG GOPROXY=https://goproxy.cn,direct From c591412c42ab195dacdede9eb19f9cbfee221144 Mon Sep 17 00:00:00 2001 From: apple050620312 Date: Fri, 23 Oct 2026 01:51:48 +0800 Subject: [PATCH 24/29] =?UTF-8?q?fix(openai):=20=E5=90=88=E5=B9=B6=20Codex?= =?UTF-8?q?=20overload=20=E8=BA=AB=E4=BB=BD=E4=B8=8E=E7=A9=BA=E6=B5=81?= =?UTF-8?q?=E5=AE=89=E5=85=A8=E9=87=8D=E8=AF=95=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/pkg/openai/request.go | 40 +++++++- ...quest_codex_identity_normalization_test.go | 78 ++++++++++++++++ .../service/openai_capacity_shed_test.go | 92 +++++++++++++++++++ .../internal/service/openai_codex_identity.go | 17 ++-- .../service/openai_codex_identity_test.go | 4 +- .../openai_codex_version_sync_service_test.go | 14 ++- .../service/openai_gateway_passthrough.go | 19 +++- .../service/openai_gateway_service.go | 6 +- .../service/openai_gateway_upstream_errors.go | 62 +++++++++++++ .../service/openai_oauth_passthrough_test.go | 2 +- .../service/openai_setup_token_compat_test.go | 5 +- .../openai_ws_forwarder_success_test.go | 4 +- .../service/setting_gateway_runtime.go | 8 +- backend/internal/service/settings_view.go | 2 +- .../src/i18n/locales/en/admin/settings.ts | 4 +- .../src/i18n/locales/zh/admin/settings.ts | 4 +- 16 files changed, 323 insertions(+), 38 deletions(-) create mode 100644 backend/internal/pkg/openai/request_codex_identity_normalization_test.go diff --git a/backend/internal/pkg/openai/request.go b/backend/internal/pkg/openai/request.go index 8ddf8d188d90..15d4bd14cb75 100644 --- a/backend/internal/pkg/openai/request.go +++ b/backend/internal/pkg/openai/request.go @@ -49,7 +49,7 @@ const codexOfficialClientFamilyPrefix = "codex " // 或在 bump context/codex 时同步补入本集合。 var codexOfficialClientOriginators = map[string]bool{ "codex_cli_rs": true, // CLI 默认 DEFAULT_ORIGINATOR - "codex-tui": true, // 交互式 TUI（连字符，真实流量占比最高） + CodexTUIOriginator: true, // 交互式 TUI（连字符，真实流量占比最高） "codex_vscode": true, // VSCode/Cursor 扩展 "codex_vscode_copilot": true, // 扩展 GitHub Copilot 集成模式 "codex_app": true, // 历史保留 @@ -259,11 +259,43 @@ func canonicalizeCodexOriginator(name string) string { return name } -// CodexCLIOriginator 是 codex-rs 客户端的历史默认 originator，保留用于兼容识别。 +// CodexCLIOriginator 是 codex-rs 客户端的默认 originator。 const CodexCLIOriginator = "codex_cli_rs" -// CodexDefaultOriginator 是网关默认使用的 Codex TUI originator。 -const CodexDefaultOriginator = "codex-tui" +// CodexTUIOriginator 是交互式 Codex TUI 使用的 originator。 +const CodexTUIOriginator = "codex-tui" + +// CodexDefaultOriginator 是网关默认发送的 Codex originator。 +// 历史 TUI 身份曾触发上游的身份风控分层，因此默认使用 CLI 身份；TUI 仍保留在官方身份集合中， +// 供入站识别和兼容归一化。 +const CodexDefaultOriginator = CodexCLIOriginator + +// NormalizeCodexClientIdentityToCLI 将已配对的 TUI 身份归一为 CLI 身份。 +// sub2api 历史默认 UA 使用 codex-tui；上游仓库的 #5198 记录了该身份会收到 +// server_is_overloaded，而相同请求改用 codex_cli_rs 可以成功。归一化必须同时改写 +// originator 和 User-Agent 首段，避免触发 #3901 的身份配对校验。 +// +// 版本、操作系统、架构和终端指纹保持不变；尾部 `(name; version)` 客户端标识组 +// 属于 TUI 身份时移除，因为真实 CLI UA 不携带该组。 +func NormalizeCodexClientIdentityToCLI(originator, userAgent string) (string, string, bool) { + if !strings.EqualFold(strings.TrimSpace(originator), CodexTUIOriginator) { + return originator, userAgent, false + } + + ua := strings.TrimSpace(userAgent) + slash := strings.IndexByte(ua, '/') + if slash <= 0 { + return CodexCLIOriginator, ua, true + } + + rest := ua[slash:] + if trailer := codexUATrailerName(ua); trailer != "" && IsCodexOfficialClientOriginator(trailer) { + if open := strings.LastIndex(rest, "("); open > 0 { + rest = strings.TrimRight(rest[:open], " ") + } + } + return CodexCLIOriginator, CodexCLIOriginator + rest, true +} // CodexUserAgentVersion 提取 Codex UA 的完整版本段，即 `{client}/{version} (...` 中的 version。 // 与 ParseCodexEngineVersion 的区别：后者只取三段数字用于引擎版本比较（会丢掉 -alpha.4 diff --git a/backend/internal/pkg/openai/request_codex_identity_normalization_test.go b/backend/internal/pkg/openai/request_codex_identity_normalization_test.go new file mode 100644 index 000000000000..a2d0b357da7f --- /dev/null +++ b/backend/internal/pkg/openai/request_codex_identity_normalization_test.go @@ -0,0 +1,78 @@ +package openai + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeCodexClientIdentityToCLI(t *testing.T) { + tests := []struct { + name string + originator string + userAgent string + wantOriginator string + wantUserAgent string + wantChanged bool + }{ + { + name: "TUI 完整身份", + originator: CodexTUIOriginator, + userAgent: "codex-tui/0.163.0 (Windows 11; x86_64) WindowsTerminal (codex-tui; 0.163.0)", + wantOriginator: CodexCLIOriginator, + wantUserAgent: "codex_cli_rs/0.163.0 (Windows 11; x86_64) WindowsTerminal", + wantChanged: true, + }, + { + name: "TUI 只有系统指纹", + originator: CodexTUIOriginator, + userAgent: "codex-tui/0.163.0 (Windows 11; x86_64) WindowsTerminal", + wantOriginator: CodexCLIOriginator, + wantUserAgent: "codex_cli_rs/0.163.0 (Windows 11; x86_64) WindowsTerminal", + wantChanged: true, + }, + { + name: "CLI 身份保持不变", + originator: CodexCLIOriginator, + userAgent: "codex_cli_rs/0.163.0 (Windows 11; x86_64) WindowsTerminal", + wantOriginator: CodexCLIOriginator, + wantUserAgent: "codex_cli_rs/0.163.0 (Windows 11; x86_64) WindowsTerminal", + wantChanged: false, + }, + { + name: "其他官方客户端保持不变", + originator: "codex_vscode", + userAgent: "codex_vscode/0.163.0 (Windows 11; x86_64) vscode (codex_vscode; 0.163.0)", + wantOriginator: "codex_vscode", + wantUserAgent: "codex_vscode/0.163.0 (Windows 11; x86_64) vscode (codex_vscode; 0.163.0)", + wantChanged: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + originator, userAgent, changed := NormalizeCodexClientIdentityToCLI(test.originator, test.userAgent) + require.Equal(t, test.wantOriginator, originator) + require.Equal(t, test.wantUserAgent, userAgent) + require.Equal(t, test.wantChanged, changed) + }) + } +} + +func TestNormalizeCodexClientIdentityToCLIIsPairedAndIdempotent(t *testing.T) { + originator, userAgent, changed := NormalizeCodexClientIdentityToCLI( + CodexTUIOriginator, + "codex-tui/0.163.0 (Windows 11; x86_64) WindowsTerminal (codex-tui; 0.163.0)", + ) + require.True(t, changed) + + pairedOriginator, pairedUserAgent, ok := PairCodexClientIdentity(userAgent) + require.True(t, ok) + require.Equal(t, originator, pairedOriginator) + require.Equal(t, userAgent, pairedUserAgent) + + againOriginator, againUserAgent, changedAgain := NormalizeCodexClientIdentityToCLI(originator, userAgent) + require.False(t, changedAgain) + require.Equal(t, originator, againOriginator) + require.Equal(t, userAgent, againUserAgent) +} diff --git a/backend/internal/service/openai_capacity_shed_test.go b/backend/internal/service/openai_capacity_shed_test.go index 086f7380e3c7..6737e42cbdf9 100644 --- a/backend/internal/service/openai_capacity_shed_test.go +++ b/backend/internal/service/openai_capacity_shed_test.go @@ -3,6 +3,7 @@ package service import ( "context" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -14,6 +15,7 @@ import ( "github.com/Wei-Shaw/sub2api/internal/pkg/openai" "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" ) // --- mock: 只记录临时不可调度写入，其余方法不应被调用 --- @@ -108,6 +110,33 @@ func TestOpenAIHTTPCapacityShedIsRequestScopedForOAuthAccounts(t *testing.T) { require.Zero(t, repo.tempUnschedCalls) } +func TestOpenAICapacityShedHonorsRetryAfter(t *testing.T) { + for _, test := range []struct { + name string + headers http.Header + body string + want time.Duration + }{ + {name: "http seconds", headers: http.Header{"Retry-After": []string{"6"}}, want: 6 * time.Second}, + {name: "http milliseconds", headers: http.Header{"Retry-After-Ms": []string{"6000"}}, want: 6 * time.Second}, + {name: "body seconds", body: `{"error":{"headers":{"retry-after":"7"},"message":"Our servers are currently overloaded. Please try again later."}}`, want: 7 * time.Second}, + {name: "body milliseconds", body: `{"response":{"error":{"headers":{"retry-after-ms":"7000"},"message":"Our servers are currently overloaded. Please try again later."}}}`, want: 7 * time.Second}, + {name: "bounded", headers: http.Header{"Retry-After": []string{"60"}}, want: openAIRequestScopedRetryMaxDelay}, + } { + t.Run(test.name, func(t *testing.T) { + failoverErr := newOpenAIUpstreamFailoverError( + http.StatusServiceUnavailable, + test.headers, + []byte(test.body), + "Our servers are currently overloaded. Please try again later.", + false, + ) + require.True(t, failoverErr.RequestScopedTransient) + require.Equal(t, test.want, failoverErr.SameAccountRetryDelay) + }) + } +} + // 上游降载的真实序列是「event: error → event: response.failed」。error 帧不算 // 客户端输出：若把它当首输出 flush，clientOutputStarted 被固化，随后的 failed // 事件就进不了 pre-output failover 分支，只能把致命错误原样转发给客户端。 @@ -218,6 +247,69 @@ func TestOpenAIStreamMetadataAndKeepaliveBeforeOverloadFailOver(t *testing.T) { } } +func TestOpenAIStreamEmptyContentBeforeFailureCanFailOver(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, payload := range []string{ + `{"type":"response.output_text.delta","delta":""}`, + `{"type":"response.reasoning_summary_text.delta","delta":""}`, + `{"type":"response.reasoning_text.delta","delta":""}`, + `{"type":"response.function_call_arguments.delta","delta":""}`, + `{"type":"response.custom_tool_call_input.delta","delta":""}`, + `{"type":"response.output_text.done","text":""}`, + `{"type":"response.reasoning_summary_text.done","text":""}`, + `{"type":"response.reasoning_text.done","text":""}`, + `{"type":"response.refusal.delta","delta":""}`, + `{"type":"response.refusal.done","refusal":""}`, + `{"type":"response.content_part.done","part":{"type":"output_text","text":""}}`, + `{"type":"response.reasoning_summary_part.done","part":{"type":"summary_text","text":""}}`, + } { + for _, passthrough := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/passthrough=%t", payload, passthrough), func(t *testing.T) { + svc := &OpenAIGatewayService{cfg: &config.Config{Gateway: config.GatewayConfig{MaxLineSize: defaultMaxLineSize}}} + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{}, + Body: io.NopCloser(strings.NewReader("data: " + payload + "\n\n" + + `data: {"type":"response.failed","response":{"status":"failed","error":{"code":"server_is_overloaded","message":"Our servers are currently overloaded. Please try again later."}}}` + "\n\n")), + } + account := &Account{ID: 1, Platform: PlatformOpenAI, Type: AccountTypeOAuth} + var err error + if passthrough { + _, err = svc.handleStreamingResponsePassthrough(c.Request.Context(), resp, c, account, time.Now(), "model", "model") + } else { + _, err = svc.handleStreamingResponse(c.Request.Context(), resp, c, account, time.Now(), "model", "model") + } + var failoverErr *UpstreamFailoverError + require.ErrorAs(t, err, &failoverErr) + require.True(t, failoverErr.RequestScopedTransient) + require.Empty(t, rec.Body.String()) + }) + } + } +} + +func TestOpenAIStreamReplayBoundaryPreservesContentAndToolCompletion(t *testing.T) { + for _, payload := range []string{ + `{"type":"response.output_text.delta","delta":" "}`, + `{"type":"response.function_call_arguments.delta","delta":"{}"}`, + `{"type":"response.custom_tool_call_input.delta","delta":"run"}`, + `{"type":"response.function_call_arguments.done","arguments":""}`, + `{"type":"response.custom_tool_call_input.done","input":""}`, + `{"type":"response.output_item.done","item":{"type":"function_call","arguments":""}}`, + `{"type":"response.output_item.done","item":{"type":"reasoning","encrypted_content":"ciphertext"}}`, + `{"type":"response.output_text.delta","delta":null}`, + `{"type":"response.output_text.delta"}`, + `{"type":"response.output_text.delta","delta":{}}`, + `{"type":"response.new_kind.delta","delta":""}`, + } { + eventType := gjson.Get(payload, "type").String() + require.True(t, openAIStreamDataStartsClientOutput(payload, eventType), payload) + } +} + func TestOpenAIStreamKeepaliveOnlyFailureDoesNotRecordFirstToken(t *testing.T) { gin.SetMode(gin.TestMode) for _, passthrough := range []bool{false, true} { diff --git a/backend/internal/service/openai_codex_identity.go b/backend/internal/service/openai_codex_identity.go index 09d1717c9adf..fbbba4a85722 100644 --- a/backend/internal/service/openai_codex_identity.go +++ b/backend/internal/service/openai_codex_identity.go @@ -32,7 +32,7 @@ func NormalizeCodexClientVersion(version string) string { return version } -// buildCodexCLIUserAgent 按版本号拼出规范 Codex TUI User-Agent。 +// buildCodexCLIUserAgent 按版本号拼出规范 Codex CLI User-Agent。 // UA 形态只在 codexCLIUserAgentSuffix 一处定义，避免多处拼装漂移。 func buildCodexCLIUserAgent(version string) string { if version = NormalizeCodexClientVersion(version); version == "" { @@ -43,7 +43,7 @@ func buildCodexCLIUserAgent(version string) string { // codexIdentityEnforcement 控制 enforceCodexIdentityHeaders 是否强制统一出站身份， // 由 gateway.disable_codex_identity_enforcement 在服务构造时取反发布。 -// 默认开启：上游在容量紧张时按客户端身份分优先级降载，被降载的请求会拿到 +// 默认开启：上游会按客户端身份进行风控分层，命中风险身份的请求会拿到 // HTTP 200 + 流内 server_is_overloaded，本次请求即失败；强制统一出口可确保没有 // 请求带着第三方或陈旧身份出站。关闭后退回「仅按最终 UA 配对 originator」的收口语义。 var codexIdentityEnforcement = func() *atomic.Bool { @@ -131,11 +131,11 @@ type codexOutboundIdentity struct { } // resolveCodexOutboundIdentity 由候选 User-Agent 推导自洽的出站身份。 -// candidateUA 为空时使用规范 User-Agent；推导不出官方身份时整体回退为规范 TUI 身份。 +// candidateUA 为空时使用规范 User-Agent；推导不出官方身份时整体回退为规范 CLI 身份。 // // 候选 UA（面板 / 账号级的管理员显式配置）只贡献客户端名与 OS / 架构 / 终端指纹， // 其自带的版本段一律用当前生效版本重建：一条填写于某个历史版本的 UA 否则会把出站身份 -// 永久钉死在陈旧版本上，绕过版本自动同步，落回上游优先降载的那一侧。 +// 永久钉死在陈旧版本上，绕过版本自动同步，落回上游身份风控更严格的一侧。 // 需要固定版本请填「Codex 客户端版本号」并关闭自动同步。 func resolveCodexOutboundIdentity(candidateUA string) codexOutboundIdentity { canonical := codexCanonicalUserAgent() @@ -149,9 +149,12 @@ func resolveCodexOutboundIdentity(candidateUA string) codexOutboundIdentity { originator, pairedUA, ok := openai.PairCodexClientIdentity(ua) if !ok { if originator, pairedUA, ok = openai.PairCodexClientIdentity(canonical); !ok { - originator, pairedUA = openai.CodexDefaultOriginator, codexCLIUserAgent + originator, pairedUA = openai.CodexCLIOriginator, codexCLIUserAgent } } + // 旧配置或账号级 UA 可能仍保存 codex-tui。该身份是 sub2api 默认出口曾经 + // 使用的风险指纹，必须与 UA 首段一起归一为 CLI，不能只替换 originator。 + originator, pairedUA, _ = openai.NormalizeCodexClientIdentityToCLI(originator, pairedUA) // 生效版本只有一个来源：规范身份（面板版本号 → 自动同步值 → 内置常量，见 // SettingService.GetOpenAICodexClientVersion）。UA 与 version 头由此同源派生。 version := codexClientVersionFromUA(canonical) @@ -206,8 +209,8 @@ func enforceCodexIdentityHeaders(h http.Header) { } // enforceCodexIdentityHeadersWithUA 强制统一 OAuth 出站身份：User-Agent / originator / version -// 一律改写为网关的规范身份，客户端自报身份不参与构造。上游在容量紧张时按客户端身份分优先级 -// 降载，被降载的请求会拿到 HTTP 200 + 流内 server_is_overloaded；统一出口可确保没有请求带着 +// 一律改写为网关的规范身份，客户端自报身份不参与构造。上游会按客户端身份进行风控分层， +// 命中风险身份的请求会拿到 HTTP 200 + 流内 server_is_overloaded；统一出口可确保没有请求带着 // 第三方或陈旧身份出站，也天然满足 originator 与 UA 首段配套的上游校验（issue #3901）。 // // overrideUA 是账号级自定义 User-Agent：管理员的显式配置仍然生效，但只贡献客户端名与 diff --git a/backend/internal/service/openai_codex_identity_test.go b/backend/internal/service/openai_codex_identity_test.go index 15768e5ea5e0..5580375e1ccb 100644 --- a/backend/internal/service/openai_codex_identity_test.go +++ b/backend/internal/service/openai_codex_identity_test.go @@ -174,8 +174,8 @@ func TestEnforceCodexIdentityHeadersWithAccountOverrideUA(t *testing.T) { enforceCodexIdentityHeadersWithUA(h, "codex-tui/0.125.0 (Mac OS X 14.0; arm64) iTerm") - require.Equal(t, "codex-tui", h.Get("originator")) - require.Equal(t, "codex-tui/0.200.1 (Mac OS X 14.0; arm64) iTerm", h.Get("user-agent")) + require.Equal(t, openai.CodexCLIOriginator, h.Get("originator")) + require.Equal(t, "codex_cli_rs/0.200.1 (Mac OS X 14.0; arm64) iTerm", h.Get("user-agent")) require.Equal(t, "0.200.1", h.Get("version")) }) } diff --git a/backend/internal/service/openai_codex_version_sync_service_test.go b/backend/internal/service/openai_codex_version_sync_service_test.go index b78cd8cc9435..76dbfe84d302 100644 --- a/backend/internal/service/openai_codex_version_sync_service_test.go +++ b/backend/internal/service/openai_codex_version_sync_service_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/Wei-Shaw/sub2api/internal/pkg/openai" "github.com/stretchr/testify/require" ) @@ -327,14 +328,14 @@ func TestGetOpenAICodexClientVersionFallsBackOnError(t *testing.T) { require.Equal(t, codexCLIVersion, svc.GetOpenAICodexClientVersion(context.Background())) } -// 规范 UA：面板未填完整 UA 时按当前生效版本号拼出标准 TUI 形态。 +// 规范 UA：面板未填完整 UA 时按当前生效版本号拼出标准 CLI 形态。 func TestGetOpenAICodexCanonicalUserAgentBuildsFromVersion(t *testing.T) { svc := NewSettingService(&codexVersionSettingRepoStub{values: map[string]string{ SettingKeyOpenAICodexClientVersionSynced: "0.200.1", }}, nil) require.Equal(t, - "codex-tui/0.200.1"+codexCLIUserAgentSuffix, + "codex_cli_rs/0.200.1"+codexCLIUserAgentSuffix, svc.GetOpenAICodexCanonicalUserAgent(context.Background()), ) } @@ -517,16 +518,19 @@ func TestGetOpenAICodexCanonicalUserAgentOutboundIdentity(t *testing.T) { SettingKeyOpenAICodexClientVersion: version.manual, SettingKeyOpenAICodexClientVersionSynced: version.synced, }}, nil) - wantUA := buildCodexCLIUserAgent(version.want) - wantOriginator := "codex-tui" + wantCanonicalUA := buildCodexCLIUserAgent(version.want) + wantUA := wantCanonicalUA + wantOriginator := openai.CodexDefaultOriginator if ua.client != "" { wantUA = ua.client + "/" + version.want + ua.suffix + wantCanonicalUA = wantUA wantOriginator = ua.client } + wantOriginator, wantUA, _ = openai.NormalizeCodexClientIdentityToCLI(wantOriginator, wantUA) SetCodexCanonicalUserAgentResolver(func() string { return svc.GetOpenAICodexCanonicalUserAgent(context.Background()) }) - require.Equal(t, wantUA, svc.GetOpenAICodexCanonicalUserAgent(context.Background())) + require.Equal(t, wantCanonicalUA, svc.GetOpenAICodexCanonicalUserAgent(context.Background())) identity := resolveCodexOutboundIdentity("") require.Equal(t, wantUA, identity.userAgent) require.Equal(t, wantOriginator, identity.originator) diff --git a/backend/internal/service/openai_gateway_passthrough.go b/backend/internal/service/openai_gateway_passthrough.go index e1da46f7ab05..90039f0522ff 100644 --- a/backend/internal/service/openai_gateway_passthrough.go +++ b/backend/internal/service/openai_gateway_passthrough.go @@ -1151,7 +1151,7 @@ func openAIStreamAddedEventStartsClientOutput(payload []byte, eventType string) default: return true } - case "response.content_part.added": + case "response.content_part.added", "response.content_part.done": part := gjson.GetBytes(payload, "part") if !part.Exists() || !part.IsObject() { return true @@ -1164,7 +1164,7 @@ func openAIStreamAddedEventStartsClientOutput(payload []byte, eventType string) default: return true } - case "response.reasoning_summary_part.added": + case "response.reasoning_summary_part.added", "response.reasoning_summary_part.done": part := gjson.GetBytes(payload, "part") if !part.Exists() || !part.IsObject() || strings.TrimSpace(part.Get("type").String()) != "summary_text" { return true @@ -1180,6 +1180,7 @@ func openAIStreamDataStartsClientOutput(data, eventType string) bool { if trimmed == "" { return false } + field := "" switch strings.TrimSpace(eventType) { case "response.failed": return false @@ -1191,8 +1192,20 @@ func openAIStreamDataStartsClientOutput(data, eventType string) bool { // （content_policy / invalid_request 等）维持原样转发，保留上游错误细节。 payload := []byte(trimmed) return !openAIStreamFailedEventShouldFailover(payload, extractOpenAISSEErrorMessage(payload)) - case "response.output_item.added", "response.content_part.added", "response.reasoning_summary_part.added": + case "response.output_item.added", "response.content_part.added", "response.reasoning_summary_part.added", + "response.content_part.done", "response.reasoning_summary_part.done": return openAIStreamAddedEventStartsClientOutput([]byte(trimmed), eventType) + case "response.output_text.delta", "response.reasoning_summary_text.delta", "response.reasoning_text.delta", + "response.function_call_arguments.delta", "response.custom_tool_call_input.delta", "response.refusal.delta": + field = "delta" + case "response.output_text.done", "response.reasoning_summary_text.done", "response.reasoning_text.done": + field = "text" + case "response.refusal.done": + field = "refusal" + } + if field != "" && gjson.Valid(trimmed) { + value := gjson.Get(trimmed, field) + return value.Type != gjson.String || value.String() != "" } return !openAIStreamEventIsMetadata(eventType) } diff --git a/backend/internal/service/openai_gateway_service.go b/backend/internal/service/openai_gateway_service.go index 91f9b646a4d0..a29ff0d1c6f7 100644 --- a/backend/internal/service/openai_gateway_service.go +++ b/backend/internal/service/openai_gateway_service.go @@ -33,7 +33,7 @@ const ( openaiPlatformAPIURL = "https://api.openai.com/v1/responses" openaiPlatformAPIInputTokensURL = "https://api.openai.com/v1/responses/input_tokens" openaiStickySessionTTL = time.Hour // 粘性会话TTL - // 与真实 Codex TUI 的 User-Agent 结构对齐： + // 与真实 Codex CLI 的 User-Agent 结构对齐： // {originator}/{version} ({OS} {OS_version}; {arch}) {terminal} // 缺少 OS/架构/终端后缀的形态易被上游指纹识别为非官方客户端。 // 该后缀是 UA 形态的唯一定义处，buildCodexCLIUserAgent 按运行时版本号复用它。 @@ -57,8 +57,8 @@ const ( openAICompactSessionSeedKey = "openai_compact_session_seed" openAIUpstreamEndpointContextKey = "openai_actual_upstream_endpoint" // codexCLIVersion 是网关对上游声明的 Codex 客户端版本，同时供 codexCLIUserAgent - // 与 version 头使用。上游 /backend-api/codex 在容量紧张时按客户端身份分优先级降载， - // 陈旧版本会被优先丢弃（HTTP 200 + 流内 server_is_overloaded）；非官方客户端配不出 + // 与 version 头使用。上游 /backend-api/codex 会按客户端身份进行风控分层， + // 陈旧版本可能被优先丢弃（HTTP 200 + 流内 server_is_overloaded）；非官方客户端配不出 // 官方身份时整体回退到本常量，因此它必须跟随官方 CLI 的当前发布版本， // 落后多个版本会让这些请求稳定落在被优先丢弃的一侧。 codexCLIVersion = "0.146.0" diff --git a/backend/internal/service/openai_gateway_upstream_errors.go b/backend/internal/service/openai_gateway_upstream_errors.go index 24900432843f..69d10bb23835 100644 --- a/backend/internal/service/openai_gateway_upstream_errors.go +++ b/backend/internal/service/openai_gateway_upstream_errors.go @@ -7,7 +7,9 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" + "time" "github.com/Wei-Shaw/sub2api/internal/config" "github.com/Wei-Shaw/sub2api/internal/pkg/logger" @@ -193,6 +195,63 @@ func isOpenAIRequestScopedCapacityShed(upstreamMsg string, upstreamBody []byte) (!gjson.ValidBytes(upstreamBody) && isOpenAICapacityShedMessage(string(upstreamBody))) } +const openAIRequestScopedRetryMaxDelay = 8 * time.Second + +func openAIRequestScopedRetryDelay(responseHeaders http.Header, responseBody []byte) time.Duration { + type retryAfterValue struct { + raw string + milliseconds bool + } + for _, value := range []retryAfterValue{ + {raw: responseHeaders.Get("Retry-After")}, + {raw: responseHeaders.Get("Retry-After-Ms"), milliseconds: true}, + {raw: gjson.GetBytes(responseBody, "error.headers.retry-after").String()}, + {raw: gjson.GetBytes(responseBody, "response.error.headers.retry-after").String()}, + {raw: gjson.GetBytes(responseBody, "error.headers.retry-after-ms").String(), milliseconds: true}, + {raw: gjson.GetBytes(responseBody, "response.error.headers.retry-after-ms").String(), milliseconds: true}, + } { + raw := strings.TrimSpace(value.raw) + if raw == "" { + continue + } + var delay time.Duration + var ok bool + if value.milliseconds { + delay, ok = parseOpenAIRequestScopedRetryDelayMilliseconds(raw) + } else { + delay, ok = parseOpenAIRequestScopedRetryDelay(raw) + } + if !ok || delay <= 0 { + continue + } + if delay > openAIRequestScopedRetryMaxDelay { + return openAIRequestScopedRetryMaxDelay + } + return delay + } + return 0 +} + +func parseOpenAIRequestScopedRetryDelay(raw string) (time.Duration, bool) { + if seconds, err := strconv.ParseFloat(raw, 64); err == nil && seconds > 0 { + return time.Duration(seconds * float64(time.Second)), true + } + if resetAt, err := http.ParseTime(raw); err == nil { + delay := time.Until(resetAt) + if delay > 0 { + return delay, true + } + } + return 0, false +} + +func parseOpenAIRequestScopedRetryDelayMilliseconds(raw string) (time.Duration, bool) { + if millis, err := strconv.ParseFloat(raw, 64); err == nil && millis > 0 { + return time.Duration(millis * float64(time.Millisecond)), true + } + return 0, false +} + func isOpenAIContextWindowError(upstreamMsg string, upstreamBody []byte) bool { match := func(text string) bool { lower := strings.ToLower(strings.TrimSpace(text)) @@ -336,6 +395,9 @@ func newOpenAIUpstreamFailoverError( RetryableOnSameAccount: retryableOnSameAccount || requestScopedCapacity, RequestScopedTransient: requestScopedCapacity, } + if requestScopedCapacity { + failoverErr.SameAccountRetryDelay = openAIRequestScopedRetryDelay(responseHeaders, responseBody) + } if isOpenAIRequestBodyTooLargeError(statusCode, upstreamMsg, responseBody) { failoverErr.RetryableOnSameAccount = false failoverErr.RequestScopedTransient = false diff --git a/backend/internal/service/openai_oauth_passthrough_test.go b/backend/internal/service/openai_oauth_passthrough_test.go index d871fcd901eb..c913bf0a94cb 100644 --- a/backend/internal/service/openai_oauth_passthrough_test.go +++ b/backend/internal/service/openai_oauth_passthrough_test.go @@ -1171,7 +1171,7 @@ func TestOpenAIGatewayService_OAuthLegacy_CompositeCodexUAUsesCodexOriginator(t _, err := svc.Forward(context.Background(), c, account, inputBody) require.NoError(t, err) require.NotNil(t, upstream.lastReq) - // 浏览器型复合 UA 被替换为默认 Codex TUI UA， + // 浏览器型复合 UA 被替换为默认 Codex CLI UA， // originator 随最终 UA 配套（issue #3901）。 require.Equal(t, DefaultOpenAICodexUserAgent, upstream.lastReq.Header.Get("User-Agent")) require.Equal(t, openai.CodexDefaultOriginator, upstream.lastReq.Header.Get("originator")) diff --git a/backend/internal/service/openai_setup_token_compat_test.go b/backend/internal/service/openai_setup_token_compat_test.go index 09dd2dae787e..cf02f2803c06 100644 --- a/backend/internal/service/openai_setup_token_compat_test.go +++ b/backend/internal/service/openai_setup_token_compat_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/openai" "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" "github.com/tidwall/gjson" @@ -212,7 +213,7 @@ func TestOpenAISetupTokenMessagesUsesCodexBridgeAndTurnState(t *testing.T) { require.Equal(t, chatgptCodexURL, upstream.requests[0].URL.String()) require.Equal(t, "Bearer setup-token-value", upstream.requests[0].Header.Get("Authorization")) require.Equal(t, "chatgpt-setup", upstream.requests[0].Header.Get("chatgpt-account-id")) - requireOpenAIMessagesCodexIdentity(t, upstream.requests[0], codexCLIUserAgent, "codex-tui") + requireOpenAIMessagesCodexIdentity(t, upstream.requests[0], codexCLIUserAgent, openai.CodexCLIOriginator) require.Empty(t, upstream.requests[0].Header.Get("x-codex-turn-state")) secondBody := []byte(`{"model":"claude-sonnet-4-5","max_tokens":16,"messages":[{"role":"user","content":"next"}],"stream":false}`) @@ -229,7 +230,7 @@ func TestOpenAISetupTokenMessagesUsesCodexBridgeAndTurnState(t *testing.T) { require.Equal(t, "turn_state_setup", upstream.requests[1].Header.Get("x-codex-turn-state")) require.Equal(t, generateSessionUUID(isolateOpenAIUpstreamSessionID(0, account, "stable-cache-key")), upstream.requests[1].Header.Get("session_id")) require.Empty(t, upstream.requests[1].Header.Get("conversation_id")) - requireOpenAIMessagesCodexIdentity(t, upstream.requests[1], codexCLIUserAgent, "codex-tui") + requireOpenAIMessagesCodexIdentity(t, upstream.requests[1], codexCLIUserAgent, openai.CodexCLIOriginator) } func openAISetupTokenCompatAccount(id int64) *Account { diff --git a/backend/internal/service/openai_ws_forwarder_success_test.go b/backend/internal/service/openai_ws_forwarder_success_test.go index 0401f59f55ef..8f4c7708772a 100644 --- a/backend/internal/service/openai_ws_forwarder_success_test.go +++ b/backend/internal/service/openai_ws_forwarder_success_test.go @@ -1024,9 +1024,9 @@ func TestOpenAIGatewayService_Forward_WSv2_OAuthHonorsAccountUserAgent(t *testin result, err := svc.Forward(context.Background(), c, account, body) require.NoError(t, err) require.NotNil(t, result) - require.Equal(t, "codex-tui", captureDialer.lastHeaders.Get("originator")) + require.Equal(t, openai.CodexCLIOriginator, captureDialer.lastHeaders.Get("originator")) require.Equal(t, - "codex-tui/"+codexCLIVersion+" (Mac OS X 15.1.0; arm64) iTerm.app", + "codex_cli_rs/"+codexCLIVersion+" (Mac OS X 15.1.0; arm64) iTerm.app", captureDialer.lastHeaders.Get("user-agent"), ) require.Equal(t, codexCLIVersion, captureDialer.lastHeaders.Get("version")) diff --git a/backend/internal/service/setting_gateway_runtime.go b/backend/internal/service/setting_gateway_runtime.go index ec6d21d502aa..4e91284c91b5 100644 --- a/backend/internal/service/setting_gateway_runtime.go +++ b/backend/internal/service/setting_gateway_runtime.go @@ -98,7 +98,7 @@ const antigravityUserAgentVersionErrorTTL = 5 * time.Second const antigravityUserAgentVersionDBTimeout = 5 * time.Second // DefaultOpenAICodexUserAgent 是 OpenAI Codex 默认 User-Agent，用于规避浏览器 UA 的质询。 -// 默认采用 codex-tui 身份，版本段随 codexCLIVersion 一起更新。 +// 默认采用 codex_cli_rs 身份，避免历史 codex-tui 出口触发 sub2api 的身份风控指纹。 const DefaultOpenAICodexUserAgent = codexCLIUserAgent // cachedOpenAICodexUserAgent 缓存 OpenAI Codex UA（进程内缓存，60s TTL） @@ -339,7 +339,7 @@ func (s *SettingService) GetOpenAICodexUserAgent(ctx context.Context) string { // GetOpenAICodexClientVersion 返回出站声明的 Codex 客户端版本号。 // 优先级：管理员在面板覆写的版本 → 自动同步到的官方最新稳定版 → 内置常量。 -// 上游在容量紧张时按客户端身份分优先级降载，陈旧版本会被优先丢弃，故该值需保持跟随官方发布； +// 上游会按客户端身份进行风控分层，陈旧版本可能被优先丢弃，故该值需保持跟随官方发布； // 自动同步让运维不必为了跟版本而发新版本。 func (s *SettingService) GetOpenAICodexClientVersion(ctx context.Context) string { fallback := codexCLIVersion @@ -500,11 +500,11 @@ func (s *SettingService) InvalidateClaudeCodeClientVersionCache() { } // GetOpenAICodexCanonicalUserAgent 返回出站规范 Codex User-Agent。 -// 未填面板 UA 时按当前生效的客户端版本号拼出标准 Codex TUI UA。 +// 未填面板 UA 时按当前生效的客户端版本号拼出标准 Codex CLI UA。 // // 面板 UA 只贡献客户端名与 OS / 架构 / 终端指纹，版本段一律用生效版本重建：该输入框是 // 唯一能改 UA 后缀的地方，但它填写于某个历史版本，逐字沿用会把出站身份永久钉死在陈旧 -// 版本上并绕过自动同步——而陈旧身份正是上游优先降载的那一侧。 +// 版本上并绕过自动同步——而陈旧身份正是上游风控更严格的那一侧。 // 需要固定版本请填「Codex 客户端版本号」并关闭自动同步。 func (s *SettingService) GetOpenAICodexCanonicalUserAgent(ctx context.Context) string { if s == nil { diff --git a/backend/internal/service/settings_view.go b/backend/internal/service/settings_view.go index bf2e840baaff..8535fb6b8b15 100644 --- a/backend/internal/service/settings_view.go +++ b/backend/internal/service/settings_view.go @@ -247,7 +247,7 @@ type SystemSettings struct { EnableClientDatelineNormalization bool // 是否对 Anthropic OAuth/SetupToken 请求体做客户端 dateline 归一化（默认 true） RewriteMessageCacheControl bool // 是否改写 messages[*].content[*].cache_control（默认 false） AntigravityUserAgentVersion string // Antigravity 上游 User-Agent 版本号；空值使用配置/默认值 - OpenAICodexUserAgent string // OpenAI Codex 上游完整 User-Agent；空值由 Codex 客户端版本号拼出标准 TUI UA + OpenAICodexUserAgent string // OpenAI Codex 上游完整 User-Agent；空值由 Codex 客户端版本号拼出标准 CLI UA OpenAICodexClientVersion string // 出站声明的 Codex 客户端版本号（管理员覆写）；空值跟随自动同步值 OpenAICodexClientVersionSynced string // 自动同步到的官方最新稳定版版本号（只读展示） OpenAICodexVersionAutoSyncEnabled bool // 是否启用 Codex 客户端版本号自动同步（默认 true） diff --git a/frontend/src/i18n/locales/en/admin/settings.ts b/frontend/src/i18n/locales/en/admin/settings.ts index a3075fc6014e..c056f3925bbc 100644 --- a/frontend/src/i18n/locales/en/admin/settings.ts +++ b/frontend/src/i18n/locales/en/admin/settings.ts @@ -548,8 +548,8 @@ export default { antigravityUserAgentVersionPlaceholder: '1.23.2', antigravityUserAgentVersionHint: 'Leave empty to use ANTIGRAVITY_USER_AGENT_VERSION or the built-in default 1.23.2; when set, the admin setting takes precedence.', openaiCodexUserAgent: 'OpenAI Codex UA', - openaiCodexUserAgentPlaceholder: 'codex-tui/0.146.1 (Ubuntu 22.4.0; x86_64) WindowsTerminal (codex-tui; 0.146.1)', - openaiCodexUserAgentHint: 'The full Codex User-Agent used for all outbound requests, for customizing the OS / arch / terminal fingerprint. Leave empty to build the standard codex-tui identity from the version below (recommended). If set, both the leading and trailing version declarations are synchronized to the version below, so the UA never stays pinned to the release entered here — under capacity pressure the upstream sheds load by client identity and drops stale or non-official identities first with server_is_overloaded.', + openaiCodexUserAgentPlaceholder: 'codex_cli_rs/0.146.1 (Ubuntu 22.4.0; x86_64) WindowsTerminal', + openaiCodexUserAgentHint: 'The full Codex User-Agent used for all outbound requests, for customizing the OS / arch / terminal fingerprint. Leave empty to build the standard codex_cli_rs identity from the version below (recommended). If set, both the leading and trailing version declarations are synchronized to the version below, so the UA never stays pinned to the release entered here — upstream identity and version risk controls may return server_is_overloaded for stale or non-official identities.', openaiCodexClientVersion: 'Codex client version', openaiCodexClientVersionPlaceholder: 'Leave empty to follow auto-sync', openaiCodexClientVersionHint: 'The Codex client version this gateway declares upstream, shared by the User-Agent and the version header. Leave empty to use the auto-synced latest stable release; setting a value pins it and stops following auto-sync.', diff --git a/frontend/src/i18n/locales/zh/admin/settings.ts b/frontend/src/i18n/locales/zh/admin/settings.ts index 14317bff8907..27b282b88dc8 100644 --- a/frontend/src/i18n/locales/zh/admin/settings.ts +++ b/frontend/src/i18n/locales/zh/admin/settings.ts @@ -541,8 +541,8 @@ export default { antigravityUserAgentVersionPlaceholder: '1.23.2', antigravityUserAgentVersionHint: '留空时使用 ANTIGRAVITY_USER_AGENT_VERSION 或内置默认值 1.23.2；填写后后台设置优先。', openaiCodexUserAgent: 'OpenAI Codex UA', - openaiCodexUserAgentPlaceholder: 'codex-tui/0.146.1 (Ubuntu 22.4.0; x86_64) WindowsTerminal (codex-tui; 0.146.1)', - openaiCodexUserAgentHint: '出站统一使用的完整 Codex User-Agent，用于自定义 OS / 架构 / 终端指纹。留空则按下方版本号拼出标准 codex-tui 形态（推荐）。填写后首段和尾部的版本号仍会被下方版本号同步覆盖，避免这条 UA 停在填写时的旧版本——上游在容量紧张时按客户端身份分优先级降载，陈旧或非官方形态的身份会被优先丢弃并回 server_is_overloaded。', + openaiCodexUserAgentPlaceholder: 'codex_cli_rs/0.146.1 (Ubuntu 22.4.0; x86_64) WindowsTerminal', + openaiCodexUserAgentHint: '出站统一使用的完整 Codex User-Agent，用于自定义 OS / 架构 / 终端指纹。留空则按下方版本号拼出标准 codex_cli_rs 形态（推荐）。填写后首段和尾部的版本号仍会被下方版本号同步覆盖，避免这条 UA 停在填写时的旧版本——上游会根据客户端身份和版本进行风控分层，陈旧或非官方形态可能返回 server_is_overloaded。', openaiCodexClientVersion: 'Codex 客户端版本号', openaiCodexClientVersionPlaceholder: '留空则跟随自动同步', openaiCodexClientVersionHint: '网关对上游声明的 Codex 客户端版本号，User-Agent 与 version 头同源使用。留空表示使用自动同步到的官方最新稳定版；填写后固定为该版本，不再跟随同步。', From 8286268047cbbca4e7f5fbb63ff0890c1ea37cf5 Mon Sep 17 00:00:00 2001 From: shaw Date: Fri, 23 Oct 2026 14:07:16 +0800 Subject: [PATCH 25/29] fix(deps): bump dompurify to 3.4.17 for GHSA-cg2g-xh64-v4q7 / GHSA-mrv2-3pqr-j5jr DOMPurify <=3.4.16 is affected by two advisories published 2026-10-22: - GHSA-cg2g-xh64-v4q7 (critical): SAFE_FOR_TEMPLATES bypass via split template expressions in attached shadow root content on DOM output modes - GHSA-mrv2-3pqr-j5jr (high): IN_PLACE hook removing a parent/sibling leaves detached subtree event handlers armed Bump the direct dependency and the pnpm override floor to 3.4.17 so the mermaid-transitive copy dedupes to the same patched version. Lockfile-only regen via pnpm 9 (matches CI); no other package changes. --- frontend/package.json | 4 ++-- frontend/pnpm-lock.yaml | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 177259daa3c6..fa12747fa719 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -23,7 +23,7 @@ "@vueuse/core": "^10.7.0", "axios": "^1.20.0", "chart.js": "^4.4.1", - "dompurify": "^3.4.14", + "dompurify": "^3.4.17", "driver.js": "^1.4.0", "file-saver": "^2.0.5", "marked": "^17.0.1", @@ -65,7 +65,7 @@ "js-cookie": "3.0.7", "form-data@<4.0.6": ">=4.0.6", "postcss@<8.5.18": ">=8.5.18", - "dompurify@<3.4.14": ">=3.4.14", + "dompurify@<3.4.17": ">=3.4.17", "source-map-js@<1.2.2": ">=1.2.2", "follow-redirects@<1.16.1": ">=1.16.1" } diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 5f02f3f2a8dc..d2714896a99f 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -8,7 +8,7 @@ overrides: js-cookie: 3.0.7 form-data@<4.0.6: '>=4.0.6' postcss@<8.5.18: '>=8.5.18' - dompurify@<3.4.14: '>=3.4.14' + dompurify@<3.4.17: '>=3.4.17' source-map-js@<1.2.2: '>=1.2.2' follow-redirects@<1.16.1: '>=1.16.1' @@ -38,8 +38,8 @@ importers: specifier: ^4.4.1 version: 4.5.1 dompurify: - specifier: ^3.4.14 - version: 3.4.14 + specifier: ^3.4.17 + version: 3.4.17 driver.js: specifier: ^1.4.0 version: 1.4.0 @@ -2327,8 +2327,8 @@ packages: resolution: {integrity: sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==} engines: {node: '>=6.0.0'} - dompurify@3.4.14: - resolution: {integrity: sha512-dVoH9z+MY+C9IilgGCk3YfFqjLi3fChm2OiKJMzh6axrJ5qwxqWaZamgmHrpv22CN/KdbZJuGEGgfQoL00LTdg==} + dompurify@3.4.17: + resolution: {integrity: sha512-B/pSlYVHQ58kqISD/yDjuJ1yfzI34m1gF4lAod2bwSRhiN4K5Cl3H8Af7hqB2PmTL911tdEudnnXn5FnhRoz0w==} driver.js@1.4.0: resolution: {integrity: sha512-Gm64jm6PmcU+si21sQhBrTAM1JvUrR0QhNmjkprNLxohOBzul9+pNHXgQaT9lW84gwg9GMLB3NZGuGolsz5uew==} @@ -6052,7 +6052,7 @@ snapshots: '@types/dompurify@3.2.0': dependencies: - dompurify: 3.4.14 + dompurify: 3.4.17 '@types/estree-jsx@1.0.5': dependencies: @@ -7001,7 +7001,7 @@ snapshots: dependencies: esutils: 2.0.3 - dompurify@3.4.14: + dompurify@3.4.17: optionalDependencies: '@types/trusted-types': 2.0.7 @@ -8102,7 +8102,7 @@ snapshots: d3-sankey: 0.12.3 dagre-d3-es: 7.0.14 dayjs: 1.11.20 - dompurify: 3.4.14 + dompurify: 3.4.17 katex: 0.16.45 khroma: 2.1.0 lodash-es: 4.18.1 From ac1b8f1271be0d674fa6f074403c1410f30a3a81 Mon Sep 17 00:00:00 2001 From: shaw Date: Fri, 23 Oct 2026 15:24:04 +0800 Subject: [PATCH 26/29] chore: update sponsors --- README.md | 5 +++++ README_CN.md | 5 +++++ README_JA.md | 5 +++++ assets/partners/logos/ooko.ai.png | Bin 0 -> 9566 bytes 4 files changed, 15 insertions(+) create mode 100644 assets/partners/logos/ooko.ai.png diff --git a/README.md b/README.md index 11955a37b6e9..38b70781cf8f 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,11 @@ Please read the following carefully before using this project: Thanks to Chenju for sponsoring this project! This is probably the only ad slot in this section that doesn't ask you to sign up for anything. Good open-source projects deserve to be paid for, so Chenju is chipping in a gift and raising a glass to the author. Find Chenju on X to chat about API relays, upstream providers, or just shoot the breeze. + +ooko.ai +🔑 Buying AI accounts at top prices | Gemini · Claude · GPT | Main and alt accounts all welcome, any channel wanted | Pay-per-usage supported | Seeking first-hand resources from major providers, I'll bring the KEY.
WeChat: Liuyou-zawd | Telegram: AACC8888888 + + ## Overview diff --git a/README_CN.md b/README_CN.md index 9064b68eacea..d34649c84556 100644 --- a/README_CN.md +++ b/README_CN.md @@ -159,6 +159,11 @@ 感谢 陈局 赞助了本项目！这大概是本栏唯一一个不让你注册的广告位。好的开源项目值得被付费，陈局来随个份子，敬作者一杯。X 上找 陈局，聊中转、聊上游、吹吹水。 + +ooko.ai +🔑 高价收 AI 账号｜Gemini · Claude · GPT｜大小号通收，各种渠道都要｜支持按量跑付｜寻一手大厂资源，我上 KEY 。
微信：Liuyou-zawd｜TG：AACC8888888 + + ## 项目概述 diff --git a/README_JA.md b/README_JA.md index 367cce6373db..58c5f379f79a 100644 --- a/README_JA.md +++ b/README_JA.md @@ -158,6 +158,11 @@ 陈局（Chenju）様、本プロジェクトへのご支援ありがとうございます！これはおそらく、この欄で唯一「登録しなくていい」広告枠です。良いオープンソースには対価が払われるべき——というわけで、陈局がご祝儀を包んで作者に一杯おごります。X で 陈局 を見つけて、中継サービスや上流の話、雑談でもどうぞ。 + +ooko.ai +🔑 AI アカウント高価買取｜Gemini · Claude · GPT｜メイン・サブアカウントすべて買取、チャネル問わず歓迎｜従量制での支払いに対応｜大手の一次リソースを求めています、KEY は私が用意します。
WeChat：Liuyou-zawd｜Telegram：AACC8888888 + + ## 概要 diff --git a/assets/partners/logos/ooko.ai.png b/assets/partners/logos/ooko.ai.png new file mode 100644 index 0000000000000000000000000000000000000000..8884583403521912e778111f71ac6f497238bab1 GIT binary patch literal 9566 zcmbtacRbX8{68ZuyE>5wb(J#X>=im(oLn4EB+1O)TbUVAC+qBymB`+tOP8*QBiWgk zy|UNu{r&s*c>MnP-Q!_AJ`X-CD(*VAHU;$eb7Agn0uJNF?F8bk2AE0h*| z?{4Dv20u=_YU4a05ax6LebGP&8Qc(v00ebM&CoYvInyr?h8*5)ez%{xk|mW$Cjl4G zriWo}(;&sHy^u6&7UoDAb1{q$XaAVb7N-%?$lTa1Bvhz5hF0w6hbghp7h)pcuG&5m z;3ywW%cuU$Y6}U0U9h6HluUc|s$|vn7)sRj!roKeQrha@R6_c<;^-N|P^!0ed>~;R zH7q8(w^9+Om{l1P3m;9A9`(UQ4!Ly+p%MIHD1#4cRucA1=Yv_Up*ub$bQIk9zy7$G z+2aBpF8)1&=VMh_@(L{jCB{y3k`eL>ZjVd0i^1BbgNKKDN92W`?d3IaikEeo5ll|@oOJbPX`tJHUYwOubS2>=SbxZhkI7*F!mrd(nds)Hj zn@_X3MM&%9y9$|bb-@IZ72(=8^iki-eyA$s_TZK_T+-(I-;2)NhB#cU(@6Yn0SM%_ zuoLt_@PiHsU7~~`RvJy2UN;MHnpj#Y`;jjD-mLMPbG-p9f{!M(=JKge?1FFL(bj3C zO97)c(}V9Hh{s&+*@Pi+$pgcPn%dYK!xJy}6(e8k{i96iIFh2AlVuZ|1jhjX}L7I7e<*UAnd1( zj}E?($@1|}ovJw*D7pyLE)Aqg#Ao5nNuQ2~amDfD)$!H~7cOv-_b+(e=Lo-|70(cs zOX3e%isJX)4^)Y?6pJ~wl0d`#CaJjOXM{&KpMpfxC6lONpM3Jv|~ z^<#S7ajfa3bz^V7)+t&VsAbG5f`Rf)`~EH~(N!T+cGQ(PT)UW|7gfGUuf`GX)!PO& zAI#JvfrHKslMF?a=&|EenR}8p;I_D7TJeWh(bW+VjB@c(Kk6RAHK52_{pIG038dDi zgO}9)RIVr36XE%L;M~ob#7NeWI;UoVoYwX8#$z{pFGLDL59^#pqF67^d0WjU*iR!& z!2D9im5*84Ce>K-Wd7E`7ZlE)I+en&?9T$} zLp(5Nu`rN8Geb81_F93%E~C3nPlYs|bs)~Y{Nmpd zC5!H5cas;ty4BSoDl8?u>}uv@S;DA(=`YVA&p>`}f2M;7#pM0?vaVFIs9nM&p z@UtbYBK0C*OZbkp!R@5xbX#mrH&u#|t3*a*B9oypc{v?;xEh6Jd#t}GLK-|uOrCDK zOs!VJrCz)K$qz2WHGcDb)~wWl0qwS{s0||fy{7$zr;jVSQTRgprm^ySsz*y?@xdWS z`&LX8T=mKotF=EDz08Y*ni?8*KS!TKpo@r3f~YDFsu%)&F5H@6gX*$xaWlv+h&Zvu z-9mwPX3sO$?87hdXINio%5zxf?p;swD_xP{{O`}I=dR@^bf+Th^GY4QmT}UrB;Hdw zm`isdoPYKz)U8$y0zu&!*e|s>4A@rjPRosC$cJPo2k03Y8KKV`rFT||KuelenyyPp zkr8NI@`Ud+C)$7}r_XXqCj%bQu6Nh5FX#LdTylX4g2$Ts+Ia(LsACSO_tT-U&Yy7? z5_D4~I#$^)8;s4mw;ilZwI9x+o5$E7+WWS8T@9vm+Ivq4m!j{OBxA7B?+vp{OJnj< z3rk8Jbavh*^}(uDuk!R=hB8CW_UljYDF%?G6)Xa0UeBMqt_K5KsjmuJKT_+vK5?=? z*shQne=UW*;@c$9IZx)Wke)f=bYE0V+{l@Qm_Ly`s%bpKw--rCIkENJQR(sRl1@4W4XtF!DE;mBUm5Z=ANc z9qWXqJi-0);**9#P7BryIoh==tMwu=U#djjl{#=oTH}h{A}t+d9^CEbxx@N}6j;7q zL(t42-tB>A#>JeEIxZ8c|P_fkF>+ajK$78g#_-AL-rtA8-Cx! zq~S>y{;f_*=i|zm0ycFaa@EN4!w-L!%gRJ>$(edo0E^jvb3TC4vxeV*M}ea7{}sm1 zd=jU5vp2cB`wYDz+Q4q>5?d6way+km;L`726I~lzdE03Sq+$JwATKF@@A1L*Di-VX zjlng&04sOsP>)H(gc&`3BEcQFI~jG!Z|1eDT%6>xNpjF+1efa3KZ69)xsXwV-F%tO z$*7xz1Rm*ytw9U&j~_{J`SY=9g(wW{CP!ImDQP-Q&h69hN-Ju2(B4bg@jrf-A|`%0 zGeUBNm`H9ety{l>cE+9(&MVBPT5=aGBG6JS`S-F_fY9;jQQlG7{yiPV#^>PJYY7!3 zNGFTnBz@NeumjwY+Xr+fJ0U#>N5S#aukND{8vBIPLEureU^m>0hf{*LHsb$9j~S`BBQUGziWEm=OBG@ zAyd`9uRx1G2sR%WlP45IP!8x`7%~bl2|sj<{LFPl8VIkvdqG|v zUf@Mp*>^GXCMfOd4x=3*OL3?3@UZ22c1AS*%v4U^X{8=wQxF~= zo7eO*S{`>+Wh&N_ic011aAf`&3c{+A+i2JJ>V-)yKj|A@yEw1$D=0}0s4fAqzE0w2 zjUvKPK*)#s9`6lhhaCH9YHDtF7JrEvC{Cgemw+*i`A+B5j}s~Y$qGHJ@L3+Nt!Udf zwY0QE@YIT^bL^IzH|u38%#H`FCc(Q_*>y7&-d0s@L~uPUQ%=bZjEZ39V!NbNT>6db z`uoc3uYeECTK&DS*rd`@1UjH~+>#iny!oB7x3@Pi&<3jas!nno_pQ9VJX$B`?D~2j zk*$Y7QLtT*Y$z=qUfjB*8VvTe8xo0pSivr9Giqg{iSZqvQhMw)ihea!60aYTxWJxpy$UhaojXMH=w2IURomu^E4L|8hd-IcL zl^G(y&7zkoF)7>br^FlI4`Z7-KBNvBmz&LR$>h&mXJjg<9f|PUJpL@9D{`Lvb?xwI zHH2Sku3Pj$ogLwe8*h(CQ~BypHMN8By3)?5M<@`^fH2E&QWb#owBeyyn3|^Jfr?|*0{|%Sz zB`POUzT$wcafLf_?{7YP6g$5u7Q4Jj1+_&rrIHYUbQjH{{j2CrdgPU;~IUAa~| zk4*Y<^Qw%v$=aN;MNp&5L{?s2SVY7@o#RkV;2s_sdz?M7YK1exVw=Z;J2_kXM1AG2 z!bJ~ns7PFAL*Y9f{Q-=yyX+Ejj!kQ-TZw{TWcj=5senMY>n9zaYm0`2tLHYawl&|& zeix?JJL$SKRPFPttjISDSQw$Pp9pk~a#~W7w99z*@p?i?gP9l>D<7Y(62$XOUSng< zJK$2t;jK6CKW~n1_VgSB6`|nSK!0^ckS;Z^I3putnv|fr@s;ICYn1vVCSP-Faj?Sj zwZ=F-w1i#N%pJG}tdWsl>o_qggjd=BJ1|94`0P(=SAE%^_HncVMEK8M0noe`wBG3u zDzNOWNOF_8NCIz(IrNWQaLX=P{3K%gP?z$p%ig%XjBc(4j`^}!q2@|+nSEjeei@?)YaCVhlWYd3g`DWsj+!q zC3N|&oJ#B9CAIF&9csfrN$4VOvhyj_cu>15O<2^qqwhqr^AA_sCTPQ_T;;r1Mr#B2 zHoku^^Y!*elSHssQ0_q59AMuW4K0V3mMZm(xsmu-g+d?@cYkr*Mu&Ltt-R7w1!3XM zpSc=VUq*3eW=oz5Pj_UnVRucAQRaiK8k|x$bD=?p*>N;p>F>SL(k^y ztaYy)4Ex3kS9gsbN`Vp{Zx@-6@}_S)x2%7`RoiIhV`?VjL~j3Ux9h2}Xm5M14ORc? z-`^%eK6UC8a71``8g*}KYHDk--CeGGB|wTLY`TGtkqN-URcL75=c?-JipIGG6OqUk z+!f8Z+hRe4LY0FhioTF%c_K!HObaQ)ZJtsv8kLBnq1$H;b^Rr+3i*g&#luy1{~TEy zZzL}`SlHOuSWZ6g%_n6j`BlZw6nX1Mk7+Zj3&nH*OI&7gvf86+;q~WZaHQpwRY~+s z`gLD=$2r;8KtIa&i%1^2ySwYFTqC(vnk>E6via|N!=Kehbb6FFv7%XfBW2%C zS_DqafGf<@iz+RZx!CM9QWp~w!|=@p0)eA|hmj5ch(JTW0?^=&d>*gbx={KK=w^FU z{LGK^laXh1JqUENHTxwcZYVu>BnBp}s;Vk1yjX^e>Mh-M2{{<~`nrBEFt~_P`E;-x z?A?^)Om`VR!X6A0#6wo>=TGm&KvS(SEH(;CfF?jC-WzwA<%MOFIHjY!`s; z%ai5i;?sNm^su+0qGsRo^_aDfdwU_d`gU{Q(?H1sj0;+>JyP&q;*RZvqnZvk$oxT` zUjV>*Hk(KG@>T}DyvQ|t@b}kpm2X2wThPHd+)9?5o8uww9`91&sIAD1VT-D7NSKGx`A3-qd9(cq$}xOhd7SIw**<$LHUX3hA+w>?}r zLNNl`L<#lW4?k?)iK_#8%Aay@h-v7BpN4+A4?8tr7siqITTShQBClQ%Whhr;1#4{n zlKGwN>*NV)#ipY`*UkZ5`_Zxs@MI~OnFi+ON8f}i+VR!>Y2eDkB-TDy>wM5)Rr^^g1ZZXiR&EzF$<{#~Efm0uieNU^Fl#*JsJ2qr3tsHUu z+Ad5!3;BcBao^ljw6>)-m|cn@3PNeg!*&GCU==jQ^nu9^Qj3{ut&9?BPY4w-G%`{- z-gRN)Q#y*!nL_(g>wY=A!<_Q*z>P$f?RKG||I7+=+6^ZGtjZCm2*u3YaaW!ImZjBR5VaB3F zZrs@NRpPim>H1`2rt9R8N(Dd^l=(LY##V}vJ%@nXs~EY@LouVkj%&w%r_o!~p}d0pa9X|&>E zl+~@yX*ZXljWK{oK1;tRU7>`+N|DW}Sn=PEOE}izLiSROJr^8x^R6R^bJ&#;vsL6K zkSdU?tkZL{w{Kh)fh8d)#|ShaBLH1Mx;l-ws1C>;r>hiyekoa#K2kB^ zRX?F(iiZ;=9A(l=OBvbEXDRsstiBHmd#!DPhocQRk@KT_eT4<=j|i|C0|T63(t$&= zbuS{qd-vzXzagx|4;sCEmJjC)T|2 zP1tMf8OK7C8_>>BK_n(ADJi%;0lXuuqUT@NuB&`JP&4F3cV+*7o(g<~!doonfcSQ< z*dE_YEksfN?QdCtm@JDbjrNe^1qyYz(yCLbM>u9`-521cftF6u6MY!+YF;|#ALil1 zw!!og`7UwDnzFrg^|PP&D|Q~6-pE@y)lG9> zOT7KiT?m$EPk?u6!~5PDi@#Zmjr$0*{B^-mqn=nL6a!ejqSOI~EGdoC41_eXl$wh6@fR`Pf1B1-PdwP~8od`6wTqYl5S{Magu*}h57HjiJC8MX&y+5! z^m~m>uZujW`!AbP%+#YGtmU_(R1aN>^fL8P*#O77X+Ia|W+{GroGO*Ht&P8P{ZxIy{ItnlH+B zo@i5!fxGzoEc{z&sRz6OcpZqTY7r|!p=D~d%@0q7Ogy}E=k0XF&K=AWphw0_+~O!n zB!*BJ0xYohxPBawApI27ynH-!6mif(_0RFa>bug?9D5#hZsT+gb!e^-L!oY3Ut5qO z*!gS+^%KXo)b32`7*E;Qro6}8oqTXEt4b46d?WC-ypp{4<%)-mL4u+rzNXZ(8T!NK zz|Th;mn=%YjG~oiLsKq?ur(D6F;p}bu5udxi|Guma_e?LKjN8t+;=E#0zX@CK z6(OIc&lRfF5o??Q5BjSnSw6i}LbrLW&8|0(Qlvw#28|+us*J_EAw+TZw)qINHOJhD0(|+9uGzt&&NT=c| zukpi&AoDQ94j|AN7={IFXsDZ|WWE2-K36}r&>l^B7;?1rnJPj-@Uf9V2s7bFu zt+_G_WNwynz~LHEFa4#{@Q_@w!%GGC87DtKzhYOc<>awT+tBaZWVg37n|mOCFhXMV z1)GG+8$PY>xQx_cIp~SnDvNZG`V?RmCNGet zSy<@*6ooOl@ydEHr2=pUnHlapQ38j<0Ra-UJ!EU!oe>THIqh~bEAG;=Gaj(+UTBB; z3A}E=O5^qOvFl3pKnH>D%BB)Omv;Yfcc!9C^Kuxi2R)Q!zid z+O)kaE(i^!p2D!K4A)*+NiPuMW?^XZUM3_Zy)V5P4icy=r5+vp9S(TUro8+zfFq9) zr~LB9Lh=1hFLNy#Hps`RseBkRmzpR5g*=*Q8_Zko8kRo92VNT%LPxJXHKWxHeF?{# z<}XH)XO9mzdl*@`CA^n@Kazk#Ufj8ypuGJXNHkDv4ysdKUL}>>k(1uds>i$2qoboK z5B&wt8w9FY05q8E36X*^XChzp}~&UIM#zs zu1#JkZ$H+Y*#ISqKW6vfAQP}^o%eA$*rROm}*6eA99p2>>aYP3fXC%P=k7!Ys}3=bS)skdbl?} z3|OuO&$hi+DkargT*ivR19`{!@x>i@`j5FS8^@8l<+>3~=h24BwpXczU2bM5R2QGm z==zYYu(?>u$a0M4Ra~-PmQVATQV+SROz5s=Bnk0@>!Gjxw>LMwURkTQ_*!L0u)$rE zl1j`guGXCY+qB}{Y+PniY4fAKWjwB|OcxKg?GD`NR$RT)d#;2B7CJ2#1-O5NkByP! z=H*YWT3VkbrHCEf#=ty>OW~-mTQ3}3@{d=?nge?C+QIA0B}TD>z6PYk=J?tW4?+R-`&0O)yG$ z;DT&@7=HKz9oKfWogT)(+PGBZ?A=Uu zBRi&VCZ69HbjX0t6{06T>-fAYy^H@nSHV2LWg|UBfk!(i#a)!$20a7$!Jg4#25Iy~ zdG{v(YyCBMRww@RKKlp6t@ap)U9$TU$dgz5Z&+aZ8%RE#F`UH=nrAoI1z|`x-~zG( zH|#N(GbFc{PV}5zv3D-(6-mj;0!epq=vv1kfU9LyRcB|Hvv&$uXF4>C&d$xu%*@R- znd#x_t3*I_0t8lc-d8u?7&nE*L1{E|#3x7w>Vm?ciG#|8%cy4=%Od1FL`*;1T@0*7 zEzC&vtHlFG$g{!{LLs2MAqxO_f#QbUoaFd~A#mgl7=0?qggfsVCK+l+iYIDa2B#5hM>a-Z}Sge(;@ zt1fUd^!bD7w)m--EAtaA|mC;_H6`T@PBPm`+7wOel>f7<|mYN zvh^9ZsQhl~)0mK}S59tRfK|!#Vhm~r-=Q*WqTr>4-#UAy1iFPC*PY?xlXp}S|K*%z zzkS<-H=HBQm)gW+den-@wuzXep{XfEFqGvzu0qhwM}?~0#!SrCt2QC;Z{6EN5c^0< zwf+^^_i&1{l5l#asil%L*4Ecob$A$v#0;mJ>*B-ngh0sp_N_JbZXD^+FawHr&>92t z+FhFr4W%{w+G;Z!(%Y`z%3(gp%*4dTuVO~+H)bh(*=si`i>5f&gRBJbqGE8ehevjX zkL5$5Ut^0~HrCeQIN7=0DxDvgVzaxeX{)zEBtqZuQ2(dX>UfY+kMym|`Xq=TXy3g! zBg@6fcRU$o9xCwY$AA6i#N}s_tul_k0qJq2kl)(-#|kgMX+fYtdT8F{IrPC&Bj$3- zt<+yS_!2!KNH8fr6Ew@27{Xx4f!)Ig!Smz0gJ}!5J07|I`-=ez#<9fx8Ta=HIBHUs zg*d$)xIJ_?xt5hz?n6`4{*+gOAeW7o*Vf19bhuxDeT9`Tu^S!y*CBs;EC)j_|E>&L z=q~|_T-^7-%H7@c^m%0O^V>N&IggfpSAy*Btq{-Znq}YfYo}FjzBMxR{996F!w1Yr ze@P-&W)v@Zgzs)(U@j*9spC`=T3JfA(8dIyPs+0q^^|F+3$+H?!Y=9r2 zHzj}=Z(H(S= z*+^-$y6@4+`pZ7%gT%=XV1Po(hJ~~a#Emg7dBmZPmchg5zQCb&9>^6m^TEx`kJR_P zzM^_eRZTw(Rt6x&1+SYhCgkNahFkNWDhe?Rw;&&#>lHJ* zpzpMvKj+%JiL z4DkeKX9ov$OibRJp5Rc(e3cf3XCZ!O z_ac!4Sg!3Yv{K)03%=Znf8s=Qw#+xSqu}%pp~Kpa6$DFR?eiXqPydf`q5m~obiyxN XHW>|3SFr;FQxFtV?@qb;qtO2WoZG(* literal 0 HcmV?d00001 From da25c2fca65c67d873db23d39fddbce70fc8b460 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 24 Oct 2026 11:18:22 +0800 Subject: [PATCH 27/29] docs: clarify official billing timer startup behavior --- .downstream/upstream-review-v0.2.18.md | 2 +- DOWNSTREAM.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.downstream/upstream-review-v0.2.18.md b/.downstream/upstream-review-v0.2.18.md index 88dc89b40b54..8917be875a00 100644 --- a/.downstream/upstream-review-v0.2.18.md +++ b/.downstream/upstream-review-v0.2.18.md @@ -19,7 +19,7 @@ | `frontend/package.json`, `frontend/pnpm-lock.yaml` | Use official v0.2.18 overrides, DOMPurify 3.4.17 and follow-redirects 1.16.1; remove redundant nanoid/Vue constraints. Preserve the independent xlsx 0.20.3 author-distribution dependency. | | `frontend/src/components/account/AccountUsageCell.vue` | Use the official Gemini/Vertex usage region and service-account quota capability. Remove the former common quota bars and loading-with-old-values condition in that region. | | `frontend/src/components/account/CreateAccountModal.vue`, `frontend/src/components/account/EditAccountModal.vue`, `frontend/src/components/admin/account/AccountActionMenu.vue` | Merge the imports used by official quota support and the independent model-sync/system-prompt features. Adopt official Vertex quota editing/reset behavior. | -| `frontend/src/views/admin/AccountsView.vue` | Platform/status return to official single-select localStorage persistence. Ignore old plural platform/status URL conditions; keep other filters and sensitive session state. Restore the official five-minute billing-rate refresh timer. | +| `frontend/src/views/admin/AccountsView.vue` | Platform/status return to official single-select localStorage persistence. Ignore old plural platform/status URL conditions; keep other filters and sensitive session state. Restore the official timer verbatim with `immediate: false`; it remains inactive, as explicitly selected after checking the library startup behavior. | These rows cover all 14 initial conflict files. Clean textual merges do not override these decisions: the final per-account identity, shared tool converter, diff --git a/DOWNSTREAM.md b/DOWNSTREAM.md index 16d351eda44d..b41ac6510689 100644 --- a/DOWNSTREAM.md +++ b/DOWNSTREAM.md @@ -30,7 +30,8 @@ Platform and status filters follow the official single-select controls and one filter state for both controls and requests. Legacy `platforms` / `statuses` URL conditions are ignored; the backend's existing plural-parameter API remains. Other filters retain their current URL or session storage. The official -five-minute upstream billing-rate refresh timer is enabled. +five-minute upstream billing-rate timer retains the official `immediate: false` +option and is not started by the page. Vertex service accounts use the official quota limits, progress and reset behavior. The Gemini/Vertex usage region uses the official template rather than From 2f725ae9e6db24c7833524dd3eec4c2ae2b91a33 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 24 Oct 2026 11:34:18 +0800 Subject: [PATCH 28/29] test: initialize Gemini compatibility service in image gate fixture --- backend/internal/handler/gemini_image_generation_gate_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/internal/handler/gemini_image_generation_gate_test.go b/backend/internal/handler/gemini_image_generation_gate_test.go index 048a7a7dd90c..3bc893a7d155 100644 --- a/backend/internal/handler/gemini_image_generation_gate_test.go +++ b/backend/internal/handler/gemini_image_generation_gate_test.go @@ -71,6 +71,7 @@ func runGeminiImageGateEndpoint(t *testing.T, ep geminiImageGateEndpoint, group t.Cleanup(billingCacheService.Stop) h := &GatewayHandler{ gatewayService: gatewayService, + geminiCompatService: service.NewGeminiMessagesCompatService(nil, nil, nil, nil, nil, nil, nil, nil, cfg), billingCacheService: billingCacheService, concurrencyHelper: NewConcurrencyHelper(service.NewConcurrencyService(&fakeConcurrencyCache{}), SSEPingFormatClaude, 0), maxAccountSwitches: 1, From b711f9c8976dbd3db058e91e3b4ebf48f97f57cf Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 24 Oct 2026 11:48:07 +0800 Subject: [PATCH 29/29] test: assert quota SQL errors with the existing pgx driver --- .../repository/usage_billing_repo_integration_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/internal/repository/usage_billing_repo_integration_test.go b/backend/internal/repository/usage_billing_repo_integration_test.go index 07971d08d987..9ca7ea03ceec 100644 --- a/backend/internal/repository/usage_billing_repo_integration_test.go +++ b/backend/internal/repository/usage_billing_repo_integration_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/google/uuid" - "github.com/lib/pq" + "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/require" "github.com/Wei-Shaw/sub2api/internal/service" @@ -534,14 +534,14 @@ func TestIncrementUsageBillingAccountQuota_StatementVariantsMatchFullStatement(t } if cfg.wantErr { - sqlState := func(accountID int64, daily, weekly bool) pq.ErrorCode { + sqlState := func(accountID int64, daily, weekly bool) string { tx, err := integrationDB.BeginTx(ctx, nil) require.NoError(t, err) defer func() { _ = tx.Rollback() }() _, err = incrementUsageBillingAccountQuota(ctx, tx, accountID, 2.5, daily, weekly) - var pqErr *pq.Error - require.ErrorAs(t, err, &pqErr) - return pqErr.Code + var pgErr *pgconn.PgError + require.ErrorAs(t, err, &pgErr) + return pgErr.Code } require.Equal(t, sqlState(reference.ID, true, true), sqlState(variant.ID, hints[0], hints[1])) return