feat: 鹈鹕测试次数与多模型并行

.downstream/native-domains.md

@@ -156,6 +156,10 @@ use the same local resolver, including the already integrated Cline/Command Code
 senders. No new provider protocol is introduced.
 
 `POST /admin/pelican-tests/tasks` returns 202 and starts a server-owned job.
+Optional `repeat_count` defaults to one; each unique account/model target produces
+that many independent results. A task permits at most 5000 expanded results.
+The composer shows combinations x repeats and preserves one effort per model.
+Each result carries `repeat_index` and `repeat_count` in summaries and details.
 The GET list, snapshot (`/:id`), paginated summaries (`/:id/results`), full single
 result (`/:id/results/:result_id`), and SSE (`/:id/events`) never start generation.
 Only `POST /:id/cancel` cancels a background job. Disconnecting observers or
@@ -171,7 +175,10 @@ complete administrator evidence. The page supports 2–4 same/cross-task compari
 filters, enlargement, downloads and parameter copying. Whole-task, single-result
 and failed-item retests directly create a new task with original model/effort
 parameters and budget. Cross-task selected retries create one task per source,
-preserving each source budget. New submissions retain UUID replay protection;
+preserving each source budget. Retests copy each selected result once, preserving
+its sample index and scene, with a fresh verification code; they never multiply
+by the task repeat count again. Parameter copying merges repeated model rows
+and restores the saved repeat count. New submissions retain UUID replay protection;
 history, browsing, reconnecting and comparison are read-only.
 New manual tasks select Pelican and/or candy (the UI defaults to both). Each
 result saves its chosen scene and a fresh visible verification code. The fixed
@@ -188,9 +195,12 @@ image input is unreviewed. Only all selected checks passing gives a passed verdi
 execution status is independent. Stage evidence retains raw bytes as Base64 in
 detail JSON. A single on-demand Chromium process renders network-blocked SVGs.
 
-All batches share ten model execution slots, with one request per account.
-Ordinary selected-account sends participate in normal business
-account capacity. Generation starts after preparation and admission; its default
+All batches share ten model request execution slots. Different models on one
+account may run concurrently, subject to the unchanged business account capacity.
+A process-wide account/model turn serializes the entire selected test flow,
+including repeats in sample order and matching models in other tasks. Waiting
+for a model turn, account capacity, or rendering holds no outbound execution slot.
+If account admission fails, its global slot is released before retrying admission. Generation starts after preparation and admission; its default
 budget is 600 seconds, configurable from 60 to 1800 seconds. Queue, preparation
 and generation durations are recorded separately. Observed business account slots
 renew every 30 seconds; lost capacity cancels the operation. Existing credential
@@ -200,7 +210,10 @@ usage and billing writes are suppressed, including background callbacks.
 Submission IDs are timestamped UUIDv7 values. Expired/future IDs are rejected and replay
 cannot dispatch again. Migration 275 owns the task/result tables; migration 276
 adds budgets, execution phases and actual invocation metadata in place, preserving
-old records and raw BYTEA output. Existing records retain their 90-second budget.
+old records and raw BYTEA output. Migration 281 adds sample indexes, defaults
+existing rows to one, and replaces the account/model unique key with a sample key.
+Single-run hashes remain compatible with existing submissions; omitted and
+explicit one-repeat requests are equivalent. Existing records retain their 90-second budget.
 Records expire 24 hours after task creation, immediately disappear from reads,
 and are removed by scoped minute cleanup. Administrators can explicitly delete a
 finished task or one of its results through `DELETE /tasks/:id` and

.github/workflows/downstream-verify.yml

@@ -55,10 +55,10 @@ jobs:
         run: |
           python3 ../.github/scripts/prepare_test_images.py
           go test -p=1 -tags unit ./internal/service ./internal/repository ./internal/handler ./internal/handler/admin ./internal/server/routes -run '^Test(Pelican|AccountObservation|ObservedAccountLease)' -count=1
-          go test -p=1 -tags integration ./internal/repository -run '^TestPelican(TestRepositoryIntegrationLifecycle|DeletionIntegration)$' -count=1
+          go test -p=1 -tags integration ./internal/repository -run '^TestPelican(TestRepositoryIntegrationLifecycle|DeletionIntegration|RepeatSamplesIntegration)$' -count=1
       - name: Pelican cancellation race test
         working-directory: backend
-        run: go test -p=1 -race -tags unit ./internal/service -run '^TestPelicanRunnerCancellationStopsQueuedAndInflightAndRetainsRawOutput$' -count=1
+        run: go test -p=1 -race -tags unit ./internal/service -run '^TestPelican(RunnerCancellationStopsQueuedAndInflightAndRetainsRawOutput|RepeatModel.*|AccountWaitReleases.*)$' -count=1
       - name: Account test keepalive lifecycle race tests
         working-directory: backend
         run: go test -p=1 -race -tags unit ./internal/service -run '^TestAccountTestKeepalive' -count=1

backend/internal/repository/pelican_background_repo.go

@@ -57,7 +57,7 @@ func (r *pelicanTestRepository) GetTaskSnapshot(ctx context.Context, id string)
 const pelicanResultSummaryColumns = `r.id,r.task_id,r.ordinal,r.account_id,r.account_name,r.platform,r.model_id,r.effort,r.status,
  (octet_length(r.raw_svg)>0 OR r.status='complete' AND octet_length(r.html)>0),
  (r.status='incomplete' AND t.error=convert_to('process stopped before the test finished; this task will not be replayed','UTF8')),
- r.started_at,r.finished_at,r.generation_started_at,r.duration_ms,r.queue_duration_ms,r.preparation_duration_ms,r.generation_duration_ms,t.expires_at,r.evaluation`
+ r.started_at,r.finished_at,r.generation_started_at,r.duration_ms,r.queue_duration_ms,r.preparation_duration_ms,r.generation_duration_ms,t.expires_at,r.evaluation,r.repeat_index,COALESCE((t.test_config->>'repeat_count')::integer,1)`
 
 func (r *pelicanTestRepository) ListTaskResults(ctx context.Context, id string, filter service.PelicanResultFilter) (*service.PelicanResultPage, error) {
 	if filter.Page < 1 {
@@ -122,7 +122,7 @@ func (r *pelicanTestRepository) ListTaskResults(ctx context.Context, id string,
 		item := &service.PelicanResultSummary{}
 		var evaluation []byte
 		if err := rows.Scan(&item.ID, &item.TaskID, &item.Ordinal, &item.AccountID, &item.AccountName, &item.Platform, &item.ModelID, &item.Effort, &item.Status,
-			&item.HasPreview, &item.Interrupted, &item.StartedAt, &item.FinishedAt, &item.GenerationStartedAt, &item.DurationMS, &item.QueueDurationMS, &item.PreparationDurationMS, &item.GenerationDurationMS, &item.ExpiresAt, &evaluation); err != nil {
+			&item.HasPreview, &item.Interrupted, &item.StartedAt, &item.FinishedAt, &item.GenerationStartedAt, &item.DurationMS, &item.QueueDurationMS, &item.PreparationDurationMS, &item.GenerationDurationMS, &item.ExpiresAt, &evaluation, &item.RepeatIndex, &item.RepeatCount); err != nil {
 			return nil, err
 		}
 		if len(evaluation) > 0 {

backend/internal/repository/pelican_repeats_repo_integration_test.go

@@ -0,0 +1,67 @@
+//go:build integration
+
+package repository
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/Wei-Shaw/sub2api/internal/service"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+func TestPelicanRepeatSamplesIntegration(t *testing.T) {
+	ctx := context.Background()
+	db := pelicanPrivateIntegrationDB(t, ctx)
+	repo := &pelicanTestRepository{db: db}
+	task := pelicanIntegrationTask(t, 42)
+	count := 3
+	task.TestConfig.RepeatCount = &count
+	first := task.Results[0]
+	first.RepeatIndex = 1
+	for i := 2; i <= 3; i++ {
+		row := *first
+		row.ID = uuid.NewString()
+		row.Ordinal = i
+		row.RepeatIndex = i
+		task.Results = append(task.Results, &row)
+	}
+	task.Total = 3
+	_, _, err := repo.Create(ctx, task)
+	require.NoError(t, err)
+	for _, row := range task.Results {
+		row.Status = "complete"
+		row.RawResponse = "RAW-SAMPLE-CANARY"
+		require.NoError(t, repo.SaveResult(ctx, row))
+	}
+	page, err := repo.ListTaskResults(ctx, task.ID, service.PelicanResultFilter{Page: 1, Size: 2})
+	require.NoError(t, err)
+	require.EqualValues(t, 3, page.Total)
+	require.Len(t, page.Items, 2)
+	require.Equal(t, 1, page.Items[0].RepeatIndex)
+	require.Equal(t, 2, page.Items[1].RepeatIndex)
+	require.Equal(t, 3, page.Items[0].RepeatCount)
+	encoded, err := json.Marshal(page)
+	require.NoError(t, err)
+	require.NotContains(t, string(encoded), "CANARY")
+	require.NotContains(t, string(encoded), "raw_response")
+	next, err := repo.ListTaskResults(ctx, task.ID, service.PelicanResultFilter{Page: 2, Size: 2})
+	require.NoError(t, err)
+	require.Equal(t, 3, next.Items[0].RepeatIndex)
+	full, err := repo.GetResult(ctx, task.Results[1].ID)
+	require.NoError(t, err)
+	require.Equal(t, 2, full.RepeatIndex)
+	require.Equal(t, 3, full.RepeatCount)
+	require.Equal(t, "RAW-SAMPLE-CANARY", full.RawResponse)
+	_, err = db.ExecContext(ctx, "INSERT INTO codex_gateway_borrow_test_results (id,task_id,ordinal,account_id,model_id,effort,status,repeat_index) VALUES ($1,$2,4,42,'gpt-6-astra','high','pending',2)", uuid.NewString(), task.ID)
+	require.Error(t, err, "one model cannot have the same sample number twice")
+	require.NoError(t, repo.FinishTask(ctx, task.ID, "complete", "", time.Now()))
+	require.NoError(t, repo.DeleteTaskResult(ctx, task.ID, full.ID))
+	_, err = repo.GetResult(ctx, full.ID)
+	require.ErrorIs(t, err, service.ErrPelicanTestNotFound)
+	page, err = repo.ListTaskResults(ctx, task.ID, service.PelicanResultFilter{})
+	require.NoError(t, err)
+	require.EqualValues(t, 2, page.Total)
+}

backend/internal/repository/pelican_test_repo.go

@@ -23,7 +23,7 @@ const pelicanTestResultColumns = `r.id, r.task_id, r.ordinal, r.account_id, r.ac
     r.upstream_model, r.effort, r.status, r.raw_answer, r.raw_response, r.raw_html, r.html, r.error,
     r.started_at, r.finished_at, r.duration_ms, t.expires_at, r.platform, r.actual_endpoint, r.actual_protocol,
     r.actual_transport, r.borrow_applied, r.queue_duration_ms, r.preparation_duration_ms, r.generation_duration_ms, r.generation_started_at,
-    r.evaluation,r.evidence,r.raw_svg,r.review_image`
+    r.evaluation,r.evidence,r.raw_svg,r.review_image,r.repeat_index,COALESCE((t.test_config->>'repeat_count')::integer,1)`
 
 type pelicanTestScanner interface{ Scan(...any) error }
 
@@ -61,7 +61,7 @@ func scanPelicanTestResult(row pelicanTestScanner) (*service.PelicanTestResult,
 		&result.UpstreamModel, &result.Effort, &result.Status, &answer, &rawResponse, &rawHTML, &html, &rawError,
 		&started, &finished, &result.DurationMS, &result.ExpiresAt, &result.Platform, &result.ActualEndpoint, &result.ActualProtocol,
 		&result.ActualTransport, &result.BorrowApplied, &result.QueueDurationMS, &result.PreparationDurationMS, &result.GenerationDurationMS, &generationStarted,
-		&evaluation, &evidence, &svg, &result.ReviewImage); err != nil {
+		&evaluation, &evidence, &svg, &result.ReviewImage, &result.RepeatIndex, &result.RepeatCount); err != nil {
 		return nil, err
 	}
 	result.RawAnswer, result.RawResponse, result.RawHTML, result.HTML, result.Error = string(answer), string(rawResponse), string(rawHTML), string(html), string(rawError)
@@ -135,13 +135,20 @@ func (r *pelicanTestRepository) Create(ctx context.Context, task *service.Pelica
 		return nil, false, err
 	}
 	for _, result := range task.Results {
+		if result.RepeatIndex == 0 {
+			result.RepeatIndex = 1
+		}
+		result.RepeatCount = 1
+		if task.TestConfig.RepeatCount != nil {
+			result.RepeatCount = *task.TestConfig.RepeatCount
+		}
 		evaluation, marshalErr := json.Marshal(result.Evaluation)
 		if marshalErr != nil {
 			return nil, false, marshalErr
 		}
 		if _, err = tx.ExecContext(ctx, `INSERT INTO codex_gateway_borrow_test_results
-            (id,task_id,ordinal,account_id,account_name,model_id,effort,status,platform,evaluation)
-            VALUES ($1,$2,$3,$4,$5,$6,$7,'pending',$8,$9)`, result.ID, stored.ID, result.Ordinal, result.AccountID, result.AccountName, result.ModelID, result.Effort, result.Platform, evaluation); err != nil {
+            (id,task_id,ordinal,account_id,account_name,model_id,effort,status,platform,evaluation,repeat_index)
+            VALUES ($1,$2,$3,$4,$5,$6,$7,'pending',$8,$9,$10)`, result.ID, stored.ID, result.Ordinal, result.AccountID, result.AccountName, result.ModelID, result.Effort, result.Platform, evaluation, result.RepeatIndex); err != nil {
 			return nil, false, err
 		}
 	}

backend/internal/repository/pelican_test_repo_integration_test.go

@@ -113,6 +113,13 @@ func pelicanPrivateIntegrationDB(t *testing.T, ctx context.Context) *sql.DB {
 	require.NoError(t, err)
 	_, err = privateDB.ExecContext(ctx, string(evaluationDDL))
 	require.NoError(t, err)
+	repeatDDL, err := migrations.FS.ReadFile("281_pelican_repeat_samples.sql")
+	require.NoError(t, err)
+	_, err = privateDB.ExecContext(ctx, string(repeatDDL))
+	require.NoError(t, err)
+	var legacyIndex int
+	require.NoError(t, privateDB.QueryRowContext(ctx, "SELECT repeat_index FROM codex_gateway_borrow_test_results WHERE id=$1", legacyResult).Scan(&legacyIndex))
+	require.Equal(t, 1, legacyIndex)
 	var executionMode string
 	var timeout int
 	var retention int64

backend/internal/repository/pelican_test_repo_test.go

@@ -29,9 +29,9 @@ func pelicanTaskTestRows(task *service.PelicanTestTask) *sqlmock.Rows {
 }
 
 func pelicanResultTestRows(result *service.PelicanTestResult) *sqlmock.Rows {
-	return sqlmock.NewRows([]string{"id", "task_id", "ordinal", "account_id", "account_name", "model_id", "upstream_model", "effort", "status", "raw_answer", "raw_response", "raw_html", "html", "error", "started_at", "finished_at", "duration_ms", "expires_at", "platform", "actual_endpoint", "actual_protocol", "actual_transport", "borrow_applied", "queue_duration_ms", "preparation_duration_ms", "generation_duration_ms", "generation_started_at", "evaluation", "evidence", "raw_svg", "review_image"}).AddRow(
+	return sqlmock.NewRows([]string{"id", "task_id", "ordinal", "account_id", "account_name", "model_id", "upstream_model", "effort", "status", "raw_answer", "raw_response", "raw_html", "html", "error", "started_at", "finished_at", "duration_ms", "expires_at", "platform", "actual_endpoint", "actual_protocol", "actual_transport", "borrow_applied", "queue_duration_ms", "preparation_duration_ms", "generation_duration_ms", "generation_started_at", "evaluation", "evidence", "raw_svg", "review_image", "repeat_index", "repeat_count"}).AddRow(
 		result.ID, result.TaskID, result.Ordinal, result.AccountID, result.AccountName, result.ModelID, result.UpstreamModel, result.Effort, result.Status, []byte(result.RawAnswer), []byte(result.RawResponse), []byte(result.RawHTML), []byte(result.HTML), []byte(result.Error), result.StartedAt, result.FinishedAt, result.DurationMS, result.ExpiresAt,
-		result.Platform, result.ActualEndpoint, result.ActualProtocol, result.ActualTransport, result.BorrowApplied, result.QueueDurationMS, result.PreparationDurationMS, result.GenerationDurationMS, result.GenerationStartedAt, nil, []byte{}, []byte{}, []byte{})
+		result.Platform, result.ActualEndpoint, result.ActualProtocol, result.ActualTransport, result.BorrowApplied, result.QueueDurationMS, result.PreparationDurationMS, result.GenerationDurationMS, result.GenerationStartedAt, nil, []byte{}, []byte{}, []byte{}, result.RepeatIndex, result.RepeatCount)
 }
 
 func pelicanRepoFixture(t *testing.T) *service.PelicanTestTask {
@@ -41,7 +41,7 @@ func pelicanRepoFixture(t *testing.T) *service.PelicanTestTask {
 	require.NoError(t, err)
 	task := &service.PelicanTestTask{ID: uuid.NewString(), ClientTaskID: clientID.String(), CreatedBy: 7, RequestHash: "hash", Status: "pending", Prompt: service.PelicanPrompt, CreatedAt: now, ExpiresAt: now.Add(service.PelicanTestTTL), Total: 1,
 		GenerationTimeoutSeconds: service.PelicanDefaultGenerationTimeoutSeconds, ExecutionMode: service.PelicanExecutionModeAccount}
-	task.Results = []*service.PelicanTestResult{{ID: uuid.NewString(), TaskID: task.ID, Ordinal: 1, AccountID: 42, AccountName: "paused target", Platform: "openai", ModelID: "gpt-6-astra", Effort: "high", Status: "pending", ExpiresAt: task.ExpiresAt, ReviewImage: []byte{}}}
+	task.Results = []*service.PelicanTestResult{{ID: uuid.NewString(), TaskID: task.ID, Ordinal: 1, RepeatIndex: 1, RepeatCount: 1, AccountID: 42, AccountName: "paused target", Platform: "openai", ModelID: "gpt-6-astra", Effort: "high", Status: "pending", ExpiresAt: task.ExpiresAt, ReviewImage: []byte{}}}
 	return task
 }
 
@@ -51,7 +51,7 @@ func TestPelicanTestRepositoryCreateAtomicAndReplayReadsWithoutWrites(t *testing
 	mock.ExpectBegin()
 	mock.ExpectQuery("INSERT INTO codex_gateway_borrow_test_tasks").WithArgs(task.ID, task.ClientTaskID, task.CreatedBy, task.RequestHash, task.Prompt, task.Total, task.CreatedAt, task.ExpiresAt, task.GenerationTimeoutSeconds, task.ExecutionMode, []byte("{}")).WillReturnRows(pelicanTaskTestRows(task))
 	result := task.Results[0]
-	mock.ExpectExec("INSERT INTO codex_gateway_borrow_test_results").WithArgs(result.ID, task.ID, 1, result.AccountID, result.AccountName, result.ModelID, result.Effort, result.Platform, []byte("null")).WillReturnResult(sqlmock.NewResult(0, 1))
+	mock.ExpectExec("INSERT INTO codex_gateway_borrow_test_results").WithArgs(result.ID, task.ID, 1, result.AccountID, result.AccountName, result.ModelID, result.Effort, result.Platform, []byte("null"), 1).WillReturnResult(sqlmock.NewResult(0, 1))
 	mock.ExpectCommit()
 	stored, replayed, err := repo.Create(context.Background(), task)
 	require.NoError(t, err)