diff --git a/.downstream/native-domains.md b/.downstream/native-domains.md index 770168bf81c4..f8e8bab1456a 100644 --- a/.downstream/native-domains.md +++ b/.downstream/native-domains.md @@ -176,24 +176,37 @@ filters, enlargement, downloads and parameter copying. Whole-task, single-result and failed-item retests directly create a new task with original model/effort parameters and budget. Cross-task selected retries create one task per source, preserving each source budget. Retests copy each selected result once, preserving -its sample index and scene, with a fresh verification code; they never multiply +its sample index and count, using the original fixed prompt; they never multiply by the task repeat count again. Parameter copying merges repeated model rows and restores the saved repeat count. New submissions retain UUID replay protection; history, browsing, reconnecting and comparison are read-only. -New manual tasks select Pelican and/or candy (the UI defaults to both). Each -result saves its chosen scene and a fresh visible verification code. The fixed -legacy prompt remains exclusive to the old `/tests` contract. The actual sending path, credentials, +New manual tasks select Pelican and/or candy (the UI defaults to both). Every +generation, including repeats and retests, sends the original prompt verbatim: +`创建一个HTML，内容是SVG绘制一个鹈鹕骑自行车的2D动画，你不需要任何测试`. +No scene, style, anatomy, color, verification code or format constraints are added. +Deprecated scene inputs remain accepted for replay compatibility but do not affect +new generation. Historical scene/nonce/prompt/verdict evidence is never rewritten. +New results use opaque HTML previews, including minimally wrapped bare SVG, and +retain inline scripts and original source. Saved works remain previewable after +candy or review failure. Existing standalone SVG works keep image previews. +The actual sending path, credentials, identity, model mapping, proxy, TLS and protocol remain owned by each account. No API Key, group policy, model score or fallback account is manufactured for a test. -Candy (90 seconds), SVG generation (the saved generation budget), rendering +Candy (90 seconds), HTML/SVG generation (the saved generation budget), rendering (30 seconds after renderer admission), and same-model visual review (180 seconds) run as distinct stages. Candy failure does not prevent generation. Review uses -four 960x640 frames at 0/0.23/0.51/0.79 of the sampled animation cycle, checks +four 960x640 frames at 0/0.23/0.51/0.79 of the sampled CSS/SMIL animation cycle. +Script-driven works use the same fractions of a three-second real-time observation +window and retain actual sampling times without rewriting scripts. Review checks pelican/bicycle/riding/motion, and keeps true/false/null judgments. Unsupported image input is unreviewed. Only all selected checks passing gives a passed verdict; execution status is independent. Stage evidence retains raw bytes as Base64 in -detail JSON. A single on-demand Chromium process renders network-blocked SVGs. +detail JSON. Generation checks extractable HTML/SVG rather than animation +declarations or a nonce. A single on-demand Chromium process renders HTML with +inline scripts; all network/local-file requests are blocked. Nested document +creation and page navigation terminate rendering with a stage error. +The browser is released after each render. All batches share ten model request execution slots. Different models on one account may run concurrently, subject to the unchanged business account capacity. diff --git a/.github/workflows/downstream-verify.yml b/.github/workflows/downstream-verify.yml index a7ad7f0fff6a..aaa16c3790ba 100644 --- a/.github/workflows/downstream-verify.yml +++ b/.github/workflows/downstream-verify.yml @@ -55,7 +55,7 @@ jobs: run: | python3 ../.github/scripts/prepare_test_images.py go test -p=1 -tags unit ./internal/service ./internal/repository ./internal/handler ./internal/handler/admin ./internal/server/routes -run '^Test(Pelican|AccountObservation|ObservedAccountLease)' -count=1 - go test -p=1 -tags integration ./internal/repository -run '^TestPelican(TestRepositoryIntegrationLifecycle|DeletionIntegration|RepeatSamplesIntegration)$' -count=1 + go test -p=1 -tags integration ./internal/repository -run '^TestPelican(TestRepositoryIntegrationLifecycle|DeletionIntegration|RepeatSamplesIntegration|OriginalHTMLGalleryIntegration)$' -count=1 - name: Pelican cancellation race test working-directory: backend run: go test -p=1 -race -tags unit ./internal/service -run '^TestPelican(RunnerCancellationStopsQueuedAndInflightAndRetainsRawOutput|RepeatModel.*|AccountWaitReleases.*)$' -count=1 diff --git a/backend/internal/repository/pelican_background_repo.go b/backend/internal/repository/pelican_background_repo.go index 743e56601d81..5a795e581a8b 100644 --- a/backend/internal/repository/pelican_background_repo.go +++ b/backend/internal/repository/pelican_background_repo.go @@ -55,7 +55,8 @@ func (r *pelicanTestRepository) GetTaskSnapshot(ctx context.Context, id string) // Keep this projection independent from pelicanTestResultColumns: // no raw_answer, raw_response, raw_html, html or complete error enters a page. const pelicanResultSummaryColumns = `r.id,r.task_id,r.ordinal,r.account_id,r.account_name,r.platform,r.model_id,r.effort,r.status, - (octet_length(r.raw_svg)>0 OR r.status='complete' AND octet_length(r.html)>0), + (octet_length(r.raw_svg)>0 OR octet_length(r.html)>0 AND (r.status='complete' OR COALESCE(r.evaluation->>'generation'='passed',FALSE))), + (octet_length(r.raw_svg)>0), (r.status='incomplete' AND t.error=convert_to('process stopped before the test finished; this task will not be replayed','UTF8')), r.started_at,r.finished_at,r.generation_started_at,r.duration_ms,r.queue_duration_ms,r.preparation_duration_ms,r.generation_duration_ms,t.expires_at,r.evaluation,r.repeat_index,COALESCE((t.test_config->>'repeat_count')::integer,1)` @@ -122,7 +123,7 @@ func (r *pelicanTestRepository) ListTaskResults(ctx context.Context, id string, item := &service.PelicanResultSummary{} var evaluation []byte if err := rows.Scan(&item.ID, &item.TaskID, &item.Ordinal, &item.AccountID, &item.AccountName, &item.Platform, &item.ModelID, &item.Effort, &item.Status, - &item.HasPreview, &item.Interrupted, &item.StartedAt, &item.FinishedAt, &item.GenerationStartedAt, &item.DurationMS, &item.QueueDurationMS, &item.PreparationDurationMS, &item.GenerationDurationMS, &item.ExpiresAt, &evaluation, &item.RepeatIndex, &item.RepeatCount); err != nil { + &item.HasPreview, &item.SVGPreview, &item.Interrupted, &item.StartedAt, &item.FinishedAt, &item.GenerationStartedAt, &item.DurationMS, &item.QueueDurationMS, &item.PreparationDurationMS, &item.GenerationDurationMS, &item.ExpiresAt, &evaluation, &item.RepeatIndex, &item.RepeatCount); err != nil { return nil, err } if len(evaluation) > 0 { diff --git a/backend/internal/repository/pelican_gallery_repo_integration_test.go b/backend/internal/repository/pelican_gallery_repo_integration_test.go index ecde29ba4b63..99156d73385f 100644 --- a/backend/internal/repository/pelican_gallery_repo_integration_test.go +++ b/backend/internal/repository/pelican_gallery_repo_integration_test.go @@ -68,3 +68,36 @@ func TestPelicanGalleryIntegration(t *testing.T) { require.NoError(t, err) require.EqualValues(t, 1, page.Total) } + +func TestPelicanOriginalHTMLGalleryIntegration(t *testing.T) { + ctx := context.Background() + db := pelicanPrivateIntegrationDB(t, ctx) + repo := &pelicanTestRepository{db: db} + task := pelicanIntegrationTask(t, 42) + _, _, err := repo.Create(ctx, task) + require.NoError(t, err) + row := task.Results[0] + row.Status = "failed" + row.RawResponse = "GEN-RAW-CANARY" + row.HTML = "" + row.Evaluation = &service.PelicanEvaluation{TestItems: []string{"pelican", "candy"}, Generation: "passed", Candy: "passed", Review: "error", Verdict: "error", Checks: map[string]bool{"document": true}} + require.NoError(t, repo.SaveResult(ctx, row)) + page, err := repo.ListTaskResults(ctx, task.ID, service.PelicanResultFilter{}) + require.NoError(t, err) + require.Len(t, page.Items, 1) + require.True(t, page.Items[0].HasPreview) + require.False(t, page.Items[0].SVGPreview) + encoded, err := json.Marshal(page) + require.NoError(t, err) + require.NotContains(t, string(encoded), "CANARY") + require.NotContains(t, string(encoded), "`, ``, ``, `x`} { - _, _, err = inspectPelicanSVG(bad, "") - require.Error(t, err) - } + } type pelicanEvaluationGenerator struct { @@ -73,9 +59,7 @@ func (g *pelicanEvaluationGenerator) GeneratePelicanInput(ctx context.Context, _ if len(input.Images) > 0 { return &PelicanResult{Status: "complete", RawAnswer: g.review}, nil } - start := strings.Index(input.Text, "本次校验码：") + len("本次校验码：") - nonce := strings.Split(input.Text[start:], "。")[0] - return &PelicanResult{Status: "complete", RawAnswer: fmt.Sprintf(`%s`, nonce)}, nil + return &PelicanResult{Status: "complete", RawAnswer: ``}, nil } type pelicanFixedRenderer struct{ err error } @@ -110,10 +94,10 @@ func TestPelicanEvaluationStagesKeepIndependentResults(t *testing.T) { e, err := newPelicanEvaluation(PelicanTestConfig{TestItems: tc.items}, PelicanTestTarget{}) require.NoError(t, err) result := &PelicanTestResult{ID: "result", TaskID: "task", AccountID: 1, ModelID: "model", Evaluation: e, ExpiresAt: time.Now().Add(time.Hour)} - persistedSVG := false + persistedWork := false repo.onSave = func(value *PelicanTestResult) error { - if value.Status == "running" && value.SVG != "" { - persistedSVG = true + if value.Status == "running" && value.HTML != "" { + persistedWork = true } return nil } @@ -121,8 +105,13 @@ func TestPelicanEvaluationStagesKeepIndependentResults(t *testing.T) { require.Equal(t, tc.want, e.Verdict) require.Len(t, g.calls, tc.calls) if tc.items[0] == "pelican" { - require.NotEmpty(t, result.SVG) - require.True(t, persistedSVG) + require.NotEmpty(t, result.HTML) + require.True(t, persistedWork) + require.Empty(t, result.SVG) + require.Empty(t, e.Scene) + require.Empty(t, e.Nonce) + require.Equal(t, map[string]bool{"document": true}, e.Checks) + require.Equal(t, PelicanPrompt, result.Evidence["generating"].Prompt) require.True(t, pelicanResultSummary(result, "").HasPreview) } if tc.calls == 3 { diff --git a/backend/internal/service/pelican_gallery.go b/backend/internal/service/pelican_gallery.go index 4a4180bf7b31..d3a9d284a9c6 100644 --- a/backend/internal/service/pelican_gallery.go +++ b/backend/internal/service/pelican_gallery.go @@ -62,6 +62,7 @@ func (r *PelicanTestRunner) Retest(ctx context.Context, createdBy int64, id stri } parameters := PelicanTestRequest{ClientTaskID: request.ClientTaskID, GenerationTimeoutSeconds: task.GenerationTimeoutSeconds, PelicanTestConfig: task.TestConfig} parameters.RetestKey = key + parameters.Scene = "" if len(parameters.TestItems) == 0 { parameters.TestItems = []string{"pelican"} } @@ -83,9 +84,6 @@ func (r *PelicanTestRunner) Retest(ctx context.Context, createdBy int64, id stri continue } target := PelicanTestTarget{AccountID: row.AccountID, ModelID: row.ModelID, Effort: row.Effort} - if row.Evaluation != nil { - target.Scene = row.Evaluation.Scene - } key := pelicanModelKey{row.AccountID, row.ModelID} if !seenModels[key] { parameters.Targets = append(parameters.Targets, target) diff --git a/backend/internal/service/pelican_gallery_test.go b/backend/internal/service/pelican_gallery_test.go index 014b188be379..47bbb8acdac8 100644 --- a/backend/internal/service/pelican_gallery_test.go +++ b/backend/internal/service/pelican_gallery_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestPelicanRetestPreservesSceneBudgetAndReplayIdentity(t *testing.T) { +func TestPelicanRetestUsesOriginalPromptAndPreservesBudgetReplayIdentity(t *testing.T) { runner, repo, catalog := newPelicanBackgroundFixture(t, func(context.Context, int64, string, string) (*PelicanResult, error) { return &PelicanResult{Status: "complete"}, nil }) @@ -21,11 +21,12 @@ func TestPelicanRetestPreservesSceneBudgetAndReplayIdentity(t *testing.T) { created, err := runner.Retest(context.Background(), 7, source.ID, request, catalog) require.NoError(t, err) require.Equal(t, 1200, created.GenerationTimeoutSeconds) + require.Equal(t, PelicanPrompt, created.Prompt) require.Equal(t, 1, created.Total) repo.mu.Lock() row := repo.tasks[created.ID].Results[0] - require.Equal(t, "雪山谷地", row.Evaluation.Scene) - require.NotEqual(t, "OLD-NONCE", row.Evaluation.Nonce) + require.Empty(t, row.Evaluation.Scene) + require.Empty(t, row.Evaluation.Nonce) require.Equal(t, "high", row.Effort) delete(repo.tasks, source.ID) repo.mu.Unlock() @@ -72,8 +73,8 @@ func TestPelicanRepeatRetestsDoNotMultiplySamples(t *testing.T) { for i, row := range repo.tasks[created.ID].Results { require.Equal(t, expected[i], row.RepeatIndex) require.Equal(t, 3, row.RepeatCount) - require.Equal(t, PelicanScenes[expected[i]], row.Evaluation.Scene) - require.NotEqual(t, "OLD", row.Evaluation.Nonce) + require.Empty(t, row.Evaluation.Scene) + require.Empty(t, row.Evaluation.Nonce) } }) } diff --git a/backend/internal/service/pelican_original_prompt_test.go b/backend/internal/service/pelican_original_prompt_test.go new file mode 100644 index 000000000000..ed6077b94f13 --- /dev/null +++ b/backend/internal/service/pelican_original_prompt_test.go @@ -0,0 +1,99 @@ +//go:build unit + +package service + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestPelicanOriginalPromptAndFormats(t *testing.T) { + const original = "创建一个HTML，内容是SVG绘制一个鹈鹕骑自行车的2D动画，你不需要任何测试" + require.Equal(t, original, PelicanPrompt) + frontend, err := os.ReadFile("../../../frontend/src/api/admin/pelicanTests.ts") + require.NoError(t, err) + require.Contains(t, string(frontend), "export const PELICAN_PROMPT = '"+original+"'") + for _, source := range []string{ + "", + "", + "", + } { + t.Run(source[:12], func(t *testing.T) { + g := pelicanParallelEvaluator{call: func(_ context.Context, _ int64, _ string, input PelicanModelInput) (*PelicanResult, error) { + if len(input.Images) > 0 { + return &PelicanResult{Status: "failed", Error: "review request failed", RawResponse: "REVIEW-RAW"}, nil + } + require.Equal(t, original, input.Text) + return &PelicanResult{Status: "complete", RawAnswer: source, RawResponse: "GEN-RAW"}, nil + }} + repo := newPelicanMemoryTestRepo() + runner := newPelicanTestRunner(repo, g, nil) + t.Cleanup(runner.Stop) + runner.renderer = pelicanFixedRenderer{} + e, err := newPelicanEvaluation(PelicanTestConfig{TestItems: []string{"pelican"}, Scene: PelicanScenes[0]}, PelicanTestTarget{Scene: PelicanScenes[1]}) + require.NoError(t, err) + result := &PelicanTestResult{ID: "work", TaskID: "task", AccountID: 1, ModelID: "model", Evaluation: e, ExpiresAt: time.Now().Add(time.Hour)} + repo.resultByID[result.ID] = result + runner.runEvaluation(context.Background(), result, time.Minute, nil) + require.Equal(t, "passed", e.Generation) + require.Equal(t, "error", e.Review) + require.Equal(t, "failed", result.Status) + require.Empty(t, e.Scene) + require.Empty(t, e.Nonce) + require.Empty(t, result.SVG) + require.Equal(t, source, result.RawHTML) + require.Contains(t, result.HTML, source) + require.Equal(t, original, result.Evidence["generating"].Prompt) + summary := pelicanResultSummary(result, "") + require.True(t, summary.HasPreview) + require.False(t, summary.SVGPreview) + preview := NewPelicanPreviewService(repo) + t.Cleanup(preview.Stop) + preview.DecorateSummary(summary) + require.True(t, strings.HasSuffix(summary.PreviewURL, "/index.html")) + cap := strings.TrimSuffix(strings.TrimPrefix(summary.PreviewURL, "/api/v1/pelican-tests/preview/"), "/index.html") + html, err := preview.Resolve(context.Background(), cap) + require.NoError(t, err) + require.Equal(t, result.HTML, html) + result.ExpiresAt = time.Now().Add(-time.Second) + _, err = preview.Resolve(context.Background(), cap) + require.ErrorIs(t, err, ErrPelicanPreviewUnavailable) + }) + } +} + +func TestPelicanOriginalPromptSceneReplayCompatibility(t *testing.T) { + repo := newPelicanMemoryTestRepo() + runner := newPelicanTestRunner(repo, nil, nil) + t.Cleanup(runner.Stop) + request := PelicanTestRequest{ClientTaskID: pelicanV7(t), PelicanTestConfig: PelicanTestConfig{TestItems: []string{"pelican"}, Scene: PelicanScenes[0]}, Targets: []PelicanTestTarget{{AccountID: 1, ModelID: "a", Scene: PelicanScenes[1]}}} + normalized, err := normalizePelicanTestRequest(request, time.Now()) + require.NoError(t, err) + task, _, err := runner.Create(context.Background(), 1, request) + require.NoError(t, err) + require.Equal(t, pelicanTestRequestHash(normalized), task.RequestHash) + require.Equal(t, PelicanPrompt, task.Prompt) + require.Empty(t, task.TestConfig.Scene) + require.Empty(t, task.Results[0].Evaluation.Scene) + repo.tasks[task.ID].Prompt = "historical prompt" + task.Results[0].Evaluation.Scene = "historical scene" + replayed, existing, err := runner.Create(context.Background(), 1, request) + require.NoError(t, err) + require.True(t, existing) + require.Equal(t, "historical prompt", replayed.Prompt) + require.Equal(t, "historical scene", replayed.Results[0].Evaluation.Scene) +} + +func TestPelicanOriginalPromptScriptDetection(t *testing.T) { + for _, source := range []string{"", "", ""} { + require.True(t, pelicanWorkHasScript(source)) + } + for _, source := range []string{"", "", ""} { + require.False(t, pelicanWorkHasScript(source)) + } +} diff --git a/backend/internal/service/pelican_preview.go b/backend/internal/service/pelican_preview.go index ed6d5e0bca0b..668e9744c38e 100644 --- a/backend/internal/service/pelican_preview.go +++ b/backend/internal/service/pelican_preview.go @@ -153,12 +153,18 @@ func (p *PelicanPreviewService) pruneLocked(now time.Time) { } } +// PelicanResultHasPreview separates saved generated works from later test failures. +// Legacy partial HTML remains unavailable unless generation completed. +func PelicanResultHasPreview(result *PelicanTestResult) bool { + return result.HTML != "" && (result.SVG != "" || result.Status == "complete" || result.Evaluation != nil && result.Evaluation.Generation == "passed") +} + // DecorateResult only adds an independently minted read capability. It makes no // upstream request and never turns incomplete HTML into a successful test. func (p *PelicanPreviewService) DecorateResult(result *PelicanTestResult) *PelicanTestResult { decorated := *result decorated.PreviewURL, decorated.PreviewExpiresAt = "", nil - if decorated.Status != "complete" && decorated.SVG == "" { + if !PelicanResultHasPreview(&decorated) && decorated.HTML != "" { decorated.PreviewUnavailable = "test " + decorated.Status return &decorated } @@ -241,7 +247,7 @@ func (p *PelicanPreviewService) Resolve(ctx context.Context, token string) (stri result, err = p.repo.GetResult(ctx, capability.ResultID) } finishedRead := p.now().UTC() - if err != nil || result == nil || result.ID != capability.ResultID || (result.Status != "complete" && result.SVG == "") || result.HTML == "" || !result.ExpiresAt.After(finishedRead) || !capability.ExpiresAt.After(finishedRead) { + if err != nil || result == nil || result.ID != capability.ResultID || !PelicanResultHasPreview(result) || !result.ExpiresAt.After(finishedRead) || !capability.ExpiresAt.After(finishedRead) { p.mu.Lock() delete(p.capabilities, token) p.mu.Unlock() @@ -256,7 +262,7 @@ func (p *PelicanPreviewService) DecorateSummary(summary *PelicanResultSummary) { } // Metadata is enough to issue a capability; no original answer is loaded. result := &PelicanTestResult{ID: summary.ID, Status: "complete", HTML: "preview", ExpiresAt: summary.ExpiresAt} - if summary.Evaluation != nil { + if summary.SVGPreview { result.SVG = "svg" } summary.PreviewURL = p.DecorateResult(result).PreviewURL diff --git a/backend/internal/service/pelican_renderer.go b/backend/internal/service/pelican_renderer.go index 6819d75a1428..be167ad55934 100644 --- a/backend/internal/service/pelican_renderer.go +++ b/backend/internal/service/pelican_renderer.go @@ -8,19 +8,49 @@ import ( "image" "image/draw" "image/png" + "strings" + "sync/atomic" "time" "github.com/chromedp/cdproto/cdp" "github.com/chromedp/cdproto/fetch" "github.com/chromedp/cdproto/network" + "github.com/chromedp/cdproto/page" "github.com/chromedp/chromedp" + "golang.org/x/net/html" ) var pelicanRenderSlot = make(chan struct{}, 1) +const pelicanRenderURL = "https://pelican.invalid/index.html" +const pelicanRenderCSP = "default-src 'none'; sandbox allow-scripts; script-src 'unsafe-inline'; style-src 'unsafe-inline'; img-src data: blob:; font-src data:; media-src data: blob:; connect-src 'none'; frame-src 'none'; child-src 'none'; worker-src 'none'; object-src 'none'; base-uri 'none'; form-action 'none'" + type pelicanChromiumRenderer struct{} -func (*pelicanChromiumRenderer) Render(parent context.Context, svg string) ([]PelicanImage, []byte, error) { +// Inspect source before execution: scripts can remove their own DOM nodes. +func pelicanWorkHasScript(source string) bool { + tokenizer := html.NewTokenizer(strings.NewReader(source)) + for { + kind := tokenizer.Next() + if kind == html.ErrorToken { + return false + } + if kind != html.StartTagToken && kind != html.SelfClosingTagToken { + continue + } + token := tokenizer.Token() + if token.Data == "script" { + return true + } + for _, attr := range token.Attr { + if strings.HasPrefix(strings.ToLower(attr.Key), "on") { + return true + } + } + } +} + +func (*pelicanChromiumRenderer) Render(parent context.Context, source string) ([]PelicanImage, []byte, error) { select { case pelicanRenderSlot <- struct{}{}: defer func() { <-pelicanRenderSlot }() @@ -29,43 +59,102 @@ func (*pelicanChromiumRenderer) Render(parent context.Context, svg string) ([]Pe } ctx, cancel := context.WithTimeout(parent, 30*time.Second) defer cancel() + _, document := ExtractPelicanHTML(source) + if document == "" { + return nil, nil, fmt.Errorf("render HTML: no extractable document") + } + realTime := pelicanWorkHasScript(document) options := append(chromedp.DefaultExecAllocatorOptions[:], chromedp.NoSandbox, chromedp.DisableGPU, chromedp.Flag("disable-dev-shm-usage", true), chromedp.WindowSize(960, 640)) alloc, stopAlloc := chromedp.NewExecAllocator(ctx, options...) defer stopAlloc() browser, stopBrowser := chromedp.NewContext(alloc) defer stopBrowser() - // Only this in-memory SVG is served; all other requests are rejected. + var served atomic.Bool + var mainFrame, blockedReason atomic.Value + block := func(reason string) { + blockedReason.Store(reason) + cancel() + } + renderError := func(err error) error { + if reason, ok := blockedReason.Load().(string); ok { + return fmt.Errorf("render HTML: %s", reason) + } + return fmt.Errorf("render HTML: %w", err) + } + // Fulfil the initial document once, in memory. Subresources and later + // navigations can never use a network or local file request. chromedp.ListenTarget(browser, func(event any) { - if request, ok := event.(*fetch.EventRequestPaused); ok { + switch event := event.(type) { + case *page.EventFrameAttached: + // CSP does not prevent about:srcdoc. Reject nested contexts at creation. + if served.Load() { + block("nested documents blocked") + } + case *page.EventFrameRequestedNavigation: + if served.Load() && event.FrameID == mainFrame.Load() && event.Disposition == page.ClientNavigationDispositionCurrentTab { + block("page navigation blocked") + } + case *page.EventFrameNavigated: + if served.Load() && event.Frame.ParentID == "" && event.Frame.URL != pelicanRenderURL { + block("page navigation blocked") + } + case *fetch.EventRequestPaused: go func() { - var err error - if request.Request.URL == "https://pelican.invalid/scene.svg" && request.ResourceType == network.ResourceTypeDocument { - err = fetch.FulfillRequest(request.RequestID, 200).WithResponseHeaders([]*fetch.HeaderEntry{{Name: "Content-Type", Value: "image/svg+xml"}, {Name: "Content-Security-Policy", Value: "default-src 'none'; script-src 'none'; style-src 'unsafe-inline'; img-src data:; base-uri 'none'"}}).WithBody(base64.StdEncoding.EncodeToString([]byte(svg))).Do(cdp.WithExecutor(browser, chromedp.FromContext(browser).Target)) + executor := cdp.WithExecutor(browser, chromedp.FromContext(browser).Target) + if event.Request.URL == pelicanRenderURL && event.ResourceType == network.ResourceTypeDocument && served.CompareAndSwap(false, true) { + _ = fetch.FulfillRequest(event.RequestID, 200).WithResponseHeaders([]*fetch.HeaderEntry{{Name: "Content-Type", Value: "text/html; charset=utf-8"}, {Name: "Content-Security-Policy", Value: pelicanRenderCSP}}).WithBody(base64.StdEncoding.EncodeToString([]byte(document))).Do(executor) } else { - err = fetch.FailRequest(request.RequestID, network.ErrorReasonBlockedByClient).Do(cdp.WithExecutor(browser, chromedp.FromContext(browser).Target)) + _ = fetch.FailRequest(event.RequestID, network.ErrorReasonBlockedByClient).Do(executor) } - _ = err }() } }) - var cycle float64 - err := chromedp.Run(browser, fetch.Enable(), chromedp.EmulateViewport(960, 640), chromedp.Navigate("https://pelican.invalid/scene.svg"), chromedp.Evaluate(`(() => { -const s=document.querySelector('svg');s.style.cssText+=';width:960px!important;height:640px!important;overflow:hidden!important;background:white!important'; -s.pauseAnimations();const durations=[...s.querySelectorAll('animate,animateTransform,animateMotion')].map(a=>{try{return a.getSimpleDuration()}catch{return 0}}); -document.getAnimations().forEach(a=>{a.pause();durations.push(a.effect.getTiming().duration/1000)}); -const usable=durations.filter(v=>Number.isFinite(v)&&v>=.5&&v<=10);return usable.length?Math.min(...usable):3; + var cycle float64 = 3 + err := chromedp.Run(browser, fetch.Enable(), chromedp.EmulateViewport(960, 640), chromedp.ActionFunc(func(ctx context.Context) error { + tree, err := page.GetFrameTree().Do(ctx) + if err == nil { + mainFrame.Store(tree.Frame.ID) + } + return err + }), chromedp.Navigate(pelicanRenderURL)) + if err == nil && !realTime { + err = chromedp.Run(browser, chromedp.Evaluate(`(() => { + const durations=[]; + document.querySelectorAll('svg').forEach(s=>{ + s.pauseAnimations(); + s.querySelectorAll('animate,animateTransform,animateMotion').forEach(a=>{try{durations.push(a.getSimpleDuration())}catch{}}); + }); + document.getAnimations().forEach(a=>{a.pause();durations.push(Number(a.effect.getTiming().duration)/1000)}); + const usable=durations.filter(v=>Number.isFinite(v)&&v>=.5&&v<=10); + return usable.length?Math.min(...usable):3; })()`, &cycle)) + } if err != nil { - return nil, nil, fmt.Errorf("render SVG: %w", err) + return nil, nil, renderError(err) } frames := make([]PelicanImage, 0, 4) sheet := image.NewRGBA(image.Rect(0, 0, 1920, 1280)) + samplingStarted := time.Now() for i, fraction := range []float64{0, .23, .51, .79} { seconds := cycle * fraction + if realTime { + timer := time.NewTimer(time.Until(samplingStarted.Add(time.Duration(seconds * float64(time.Second))))) + select { + case <-ctx.Done(): + timer.Stop() + return nil, nil, renderError(ctx.Err()) + case <-timer.C: + } + seconds = time.Since(samplingStarted).Seconds() + } else { + err = chromedp.Run(browser, chromedp.Evaluate(fmt.Sprintf(`document.querySelectorAll('svg').forEach(s=>s.setCurrentTime(%f));document.getAnimations().forEach(a=>{a.pause();a.currentTime=%f});`, seconds, seconds*1000), nil)) + if err != nil { + return nil, nil, err + } + } var data []byte - err = chromedp.Run(browser, chromedp.Evaluate(fmt.Sprintf(`document.querySelector('svg').setCurrentTime(%f);document.getAnimations().forEach(a=>{a.pause();a.currentTime=%f});`, seconds, seconds*1000), nil), chromedp.CaptureScreenshot(&data)) - if err != nil { - return nil, nil, err + if err = chromedp.Run(browser, chromedp.CaptureScreenshot(&data)); err != nil { + return nil, nil, renderError(err) } frame, decodeErr := png.Decode(bytes.NewReader(data)) if decodeErr != nil { diff --git a/backend/internal/service/pelican_renderer_test.go b/backend/internal/service/pelican_renderer_test.go index a1f78c1d8991..2c868e8cf641 100644 --- a/backend/internal/service/pelican_renderer_test.go +++ b/backend/internal/service/pelican_renderer_test.go @@ -5,10 +5,15 @@ package service import ( "bytes" "context" + "fmt" + "image/color" "image/png" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" + "sync/atomic" "testing" "github.com/stretchr/testify/require" @@ -18,22 +23,78 @@ func TestPelicanChromiumRenderer(t *testing.T) { if os.Getenv("PELICAN_RENDER_TEST") != "1" { t.Skip("executed once inside the candidate runtime image") } - svg := `FIXTURE` - frames, sheet, err := (&pelicanChromiumRenderer{}).Render(context.Background(), svg) - require.NoError(t, err) - require.Len(t, frames, 4) - for i, expected := range []float64{0, .46, 1.02, 1.58} { - require.InDelta(t, expected, frames[i].Time, .001) - img, decodeErr := png.Decode(bytes.NewReader(frames[i].PNG)) - require.NoError(t, decodeErr) - require.Equal(t, 960, img.Bounds().Dx()) - require.Equal(t, 640, img.Bounds().Dy()) + const svg = "" + const css = "" + const script = "" + for _, tc := range []struct { + name, source string + realTime bool + }{{"SMIL", svg, false}, {"HTML CSS", css, false}, {"HTML JavaScript", script, true}} { + t.Run(tc.name, func(t *testing.T) { + frames, sheet, err := (&pelicanChromiumRenderer{}).Render(context.Background(), tc.source) + require.NoError(t, err) + require.Len(t, frames, 4) + for i, fraction := range []float64{0, .23, .51, .79} { + if tc.realTime { + require.InDelta(t, 3*fraction, frames[i].Time, .5) + } else { + require.InDelta(t, 2*fraction, frames[i].Time, .001) + } + img, err := png.Decode(bytes.NewReader(frames[i].PNG)) + require.NoError(t, err) + require.Equal(t, 960, img.Bounds().Dx()) + require.Equal(t, 640, img.Bounds().Dy()) + if i > 0 { + require.NotEqual(t, frames[i-1].PNG, frames[i].PNG, "consecutive frames must capture motion") + require.Greater(t, frames[i].Time, frames[i-1].Time) + } + } + img, err := png.Decode(bytes.NewReader(sheet)) + require.NoError(t, err) + require.Equal(t, 1920, img.Bounds().Dx()) + require.Equal(t, 1280, img.Bounds().Dy()) + requirePelicanChromiumStopped(t) + }) } - require.NotEqual(t, frames[0].PNG, frames[1].PNG, "sampling must capture motion rather than repeat one screenshot") - img, err := png.Decode(bytes.NewReader(sheet)) - require.NoError(t, err) - require.Equal(t, 1920, img.Bounds().Dx()) - require.Equal(t, 1280, img.Bounds().Dy()) + t.Run("external resources blocked", func(t *testing.T) { + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _, _ = w.Write([]byte("document.body.style.background='red'")) + })) + defer server.Close() + source := fmt.Sprintf("", server.URL, server.URL, server.URL) + frames, _, err := (&pelicanChromiumRenderer{}).Render(context.Background(), source) + require.NoError(t, err) + require.Zero(t, requests.Load()) + img, err := png.Decode(bytes.NewReader(frames[0].PNG)) + require.NoError(t, err) + require.Equal(t, color.RGBA{R: 255, G: 255, B: 255, A: 255}, color.RGBAModel.Convert(img.At(400, 300)), "external script must not paint the work") + requirePelicanChromiumStopped(t) + }) + for name, source := range map[string]string{ + "inline nested document blocked": "", + "script-created nested document blocked": "", + } { + t.Run(name, func(t *testing.T) { + _, _, err := (&pelicanChromiumRenderer{}).Render(context.Background(), source) + require.ErrorContains(t, err, "nested documents blocked") + requirePelicanChromiumStopped(t) + }) + } + t.Run("page navigation blocked", func(t *testing.T) { + _, _, err := (&pelicanChromiumRenderer{}).Render(context.Background(), "") + require.ErrorContains(t, err, "navigation blocked") + requirePelicanChromiumStopped(t) + }) + peak, err := os.ReadFile("/sys/fs/cgroup/memory.peak") + if err == nil { + t.Logf("candidate cgroup memory peak: %s bytes", strings.TrimSpace(string(peak))) + } +} + +func requirePelicanChromiumStopped(t *testing.T) { + t.Helper() processes, err := os.ReadDir("/proc") require.NoError(t, err) for _, process := range processes { @@ -41,8 +102,4 @@ func TestPelicanChromiumRenderer(t *testing.T) { first := strings.SplitN(string(command), "\x00", 2)[0] require.False(t, strings.Contains(filepath.Base(first), "chromium"), "renderer leaked process %s", process.Name()) } - peak, err := os.ReadFile("/sys/fs/cgroup/memory.peak") - if err == nil { - t.Logf("candidate cgroup memory peak: %s bytes", strings.TrimSpace(string(peak))) - } } diff --git a/backend/internal/service/pelican_repeats_test.go b/backend/internal/service/pelican_repeats_test.go index 1f7a78fb6359..8f2252f2ef2f 100644 --- a/backend/internal/service/pelican_repeats_test.go +++ b/backend/internal/service/pelican_repeats_test.go @@ -7,7 +7,6 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "strings" "sync" "testing" "time" @@ -24,15 +23,14 @@ func TestPelicanRepeatExpansionValidationAndReplay(t *testing.T) { require.NoError(t, err) require.False(t, replayed) require.Equal(t, 6, task.Total) - ids, nonces := map[string]bool{}, map[string]bool{} + ids := map[string]bool{} for i, row := range task.Results { require.Equal(t, i%3+1, row.RepeatIndex) require.Equal(t, 3, row.RepeatCount) require.False(t, ids[row.ID]) ids[row.ID] = true - require.False(t, nonces[row.Evaluation.Nonce]) - nonces[row.Evaluation.Nonce] = true - require.Contains(t, PelicanScenes, row.Evaluation.Scene) + require.Empty(t, row.Evaluation.Nonce) + require.Empty(t, row.Evaluation.Scene) } _, replayed, err = runner.Create(context.Background(), 7, request) require.NoError(t, err) @@ -98,8 +96,10 @@ func TestPelicanRepeatModelFlowsParallelAndOrdered(t *testing.T) { stage = "reviewing" answer = "{\"checks\":{\"pelican\":true,\"bicycle\":true,\"riding\":true,\"motion\":true}}" } else { - nonce := strings.Split(strings.Split(input.Text, "本次校验码：")[1], "。")[0] - answer = fmt.Sprintf("%s", nonce) + if input.Text != PelicanPrompt { + return nil, fmt.Errorf("unexpected prompt: %q", input.Text) + } + answer = `` } mu.Lock() first := len(stages[model]) == 0 @@ -144,7 +144,7 @@ func TestPelicanRepeatModelFlowsParallelAndOrdered(t *testing.T) { } for _, row := range task.Results { require.Equal(t, "passed", row.Evaluation.Verdict) - require.NotEmpty(t, row.SVG) + require.NotEmpty(t, row.HTML) } } diff --git a/backend/internal/service/pelican_test_runner.go b/backend/internal/service/pelican_test_runner.go index 473a6778efdc..562c915fe153 100644 --- a/backend/internal/service/pelican_test_runner.go +++ b/backend/internal/service/pelican_test_runner.go @@ -324,9 +324,11 @@ func (r *PelicanTestRunner) Create(ctx context.Context, createdBy int64, request } } } + config := request.PelicanTestConfig + config.Scene = "" // Deprecated scene inputs remain in the replay hash, never in the new generation. now := r.now().UTC() task := &PelicanTestTask{ID: uuid.NewString(), ClientTaskID: request.ClientTaskID, - TestConfig: request.PelicanTestConfig, + TestConfig: config, CreatedBy: createdBy, RequestHash: pelicanTestRequestHash(request), Status: "pending", GenerationTimeoutSeconds: request.GenerationTimeoutSeconds, ExecutionMode: PelicanExecutionModeAccount, Prompt: PelicanPrompt, CreatedAt: now, ExpiresAt: now.Add(PelicanTestTTL), @@ -354,7 +356,6 @@ func (r *PelicanTestRunner) Create(ctx context.Context, createdBy int64, request return nil, false, evaluationErr } task.Results[len(task.Results)-1].Evaluation = evaluation - task.Prompt = PelicanScenePrompt } } stored, replayed, err := r.repo.Create(ctx, task) diff --git a/frontend/src/api/admin/pelicanTests.ts b/frontend/src/api/admin/pelicanTests.ts index 7461daab84e4..a5ec6240c9d7 100644 --- a/frontend/src/api/admin/pelicanTests.ts +++ b/frontend/src/api/admin/pelicanTests.ts @@ -41,7 +41,7 @@ export interface PelicanTaskBase { replayed?: boolean } -export const PELICAN_PROMPT = '创建独立的 SVG 鹈鹕骑自行车 2D 循环动画：橙色大嘴、蹼足踩踏板、翅膀保持平衡、车轮转动。按所选场景独立构图，在右下角显示本轮随机校验码。只返回 SVG，使用内联 CSS 或 SMIL，不使用脚本和外部资源。实际场景、校验码及完整题面保存在检测详情中。' +export const PELICAN_PROMPT = '创建一个HTML，内容是SVG绘制一个鹈鹕骑自行车的2D动画，你不需要任何测试' export function createPelicanClientTaskId(serverOffsetMs = 0): string { const bytes = crypto.getRandomValues(new Uint8Array(16)) diff --git a/frontend/src/components/admin/codex/PelicanResultCard.vue b/frontend/src/components/admin/codex/PelicanResultCard.vue index 35ac6c070569..bd3bc8115aaf 100644 --- a/frontend/src/components/admin/codex/PelicanResultCard.vue +++ b/frontend/src/components/admin/codex/PelicanResultCard.vue @@ -10,10 +10,10 @@

{{ text('账号', 'Account') }}：{{ result.account_name }} #{{ result.account_id }}

-

{{ text('场景', 'Scene') }}：{{ result.evaluation.scene }}

+

{{ text('场景', 'Scene') }}：{{ result.evaluation.scene }}

{{ result.model_id }} · {{ result.effort || t('admin.pelicanTests.normalDefault') }} · {{ (result.duration_ms / 1000).toFixed(1) }}s

{{ repeatLabel }}

-

{{ text('手动检测', 'Manual test') }}

+

{{ text('手动检测', 'Manual test') }}

diff --git a/frontend/src/components/admin/codex/PelicanTaskComposer.vue b/frontend/src/components/admin/codex/PelicanTaskComposer.vue index 7fbf22901bb0..956d2708d8e1 100644 --- a/frontend/src/components/admin/codex/PelicanTaskComposer.vue +++ b/frontend/src/components/admin/codex/PelicanTaskComposer.vue @@ -30,7 +30,7 @@