diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 00000000..4dbc0025 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,63 @@ +name: perf + +on: + push: + branches: [master, main] + paths: + - ".github/workflows/perf.yml" + - "SemiStep/**" + - "ConfigFiles/**" + - "global.json" + - "!**.md" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + perf-gates: + name: Perf Gates + runs-on: windows-latest + timeout-minutes: 30 + permissions: + contents: read + defaults: + run: + shell: bash + + steps: + - name: checkout + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: setup .NET + uses: actions/setup-dotnet@v5 + with: + global-json-file: global.json + + - name: build + run: dotnet build SemiStep/SemiStep.Tests/SemiStep.Tests.csproj -c Release + + # Dev-testbed byte baselines are not comparable here: an empty metrics document flips byte gates to + # record-only; invariant/ratio gates (thresholds in probe code) keep asserting. See Docs/perf/README.md, CI. + - name: neutralize byte baselines (telemetry-only on CI) + run: echo '{ "context": { "testbed": "ci-hosted" }, "metrics": {} }' > Docs/perf/baselines.json + + - name: run perf gates + run: dotnet run --project SemiStep/SemiStep.Tests/SemiStep.Tests.csproj -c Release --no-build -- -explicit only + + - name: collect actuals artifact + if: always() + run: | + mkdir -p perf-actuals + cp "$(cygpath -u "$TEMP")"/semistep-perf-actuals-*.json perf-actuals/ || echo "no actuals artifact produced" + + - name: upload actuals + if: always() + uses: actions/upload-artifact@v4 + with: + name: perf-actuals + path: perf-actuals/ + if-no-files-found: ignore diff --git a/.zed/tasks.json b/.zed/tasks.json index 45fdb242..40fe0919 100644 --- a/.zed/tasks.json +++ b/.zed/tasks.json @@ -61,4 +61,21 @@ "args": ["format", "SemiStep/SemiStep.slnx"], "cwd": "$ZED_WORKTREE_ROOT", }, + { + "label": "Perf Gates (explicit tests, Release)", + "command": "dotnet", + "args": [ + "run", + "--project", + "SemiStep/SemiStep.Tests/SemiStep.Tests.csproj", + "-c", + "Release", + "--", + "-explicit", + "only", + ], + "cwd": "$ZED_WORKTREE_ROOT", + "reveal": "always", + "use_new_terminal": true, + }, ] diff --git a/CLAUDE.md b/CLAUDE.md index d685a3a7..1264fe0f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,11 @@ dotnet test SemiStep/SemiStep.Tests/SemiStep.Tests.csproj --filter "FullyQualifi ``` Test traits: `[Trait("Component", "Core|Config|UI|Domain|Csv|S7")]`, `[Trait("Area", "")]`, -`[Trait("Category", "Unit|Integration|Performance")]` (Performance = env-gated measurement probes, skipped by default). +`[Trait("Category", "Unit|Integration|Performance")]` (Performance = xunit v3 explicit measurement probes, +`[AvaloniaFact(Explicit = true)]` / `[Fact(Explicit = true)]`, NOT run by default `dotnet test`; run them via +`dotnet run --project SemiStep/SemiStep.Tests/SemiStep.Tests.csproj -c Release -- -explicit only` +(`-method "*Name*"` selective); see `Docs/perf/README.md` for the full harness, gate hierarchy, and +re-baseline procedure). Invalid config test cases use an overlay pattern: copy `SemiStep.Tests/YamlConfigs/Standard/` to a temp directory and overlay only the differing files from `SemiStep.Tests/YamlConfigs/Invalid/{CaseName}/`. diff --git a/Docs/architecture/recipe-grid-surface.md b/Docs/architecture/recipe-grid-surface.md index ee3384bf..bf8f34fb 100644 --- a/Docs/architecture/recipe-grid-surface.md +++ b/Docs/architecture/recipe-grid-surface.md @@ -327,11 +327,13 @@ same recipe and churned gen0 on every mutation. The reductions land in three pla **Allocation gate.** The keep-attached panel is only half the recycle fix; the child-recycle half below ("Child recycled in place") lands the other half and carries the chain's first attached - after-trace. The live-app gc-verbose byte gate — realization-subtree allocation share must drop from - ~41–56% to <20%, with `Dictionary.Resize`, `CreateCompositionVisual`, and `StyleBase.Attach` / - `Setter.Instance` absent from the scroll path — is a manual step on the Release build against a - ~2100-step recipe and stays with the user (`CreateCompositionVisual` / `CreateSKFont` are - composition/Skia costs headless cannot see). Headless tests pin the keep-attached contract + after-trace. The realization-subtree allocation is now an automated black-box gate: the perf + harness (`SemiStep.Tests/Performance/`, see `Docs/perf/README.md`) asserts `FreshVisualInstances == 0` + across a scrolled round-trip and gates bytes per realized column against a committed baseline, so + the scroll-rebuild regression this fix removed is caught by an explicit test rather than by hand. + Only the composition/Skia share stays a manual step on the Release build against a ~2100-step recipe + (`CreateCompositionVisual` / `CreateSKFont` are costs headless cannot see, so felt smoothness on + real hardware stays the manual oracle). Headless tests pin the keep-attached contract (container reuse across scroll, no `DetachedFromVisualTree` on scroll, bounded `Children.Count`, focus-anchor deferral, selection round-trip). @@ -425,8 +427,8 @@ same recipe and churned gen0 on every mutation. The reductions land in three pla - **Measured result.** These figures were captured under the earlier `VirtualizingStackPanel` + pooled-rebind path; they measure the pooled-presenter reuse, not the recycle-in-place panel that - later replaced VSP (whose byte/gen0 allocation gate is still pending live-app measurement — see the - recycle-in-place panel note below). On the viewport-jump metric (one `ScrollIntoView(last)` frame + later replaced VSP (whose byte/gen0 allocation is now gated by the black-box perf harness — see the + recycle-in-place panel note above and `Docs/perf/README.md`). On the viewport-jump metric (one `ScrollIntoView(last)` frame after a round-trip, so it exercises container recycling), transposed bytes per realized column dropped from ~14.5x the canonical recycled-row cost to ~1.03x (WideParams, 36 cells/column) and from ~2.3x to ~0.69x (WithGroups, 5 cells/column); gen0/add fell from ~2.58 to ~0.17-0.25 (WideParams) and from @@ -443,6 +445,10 @@ same recipe and churned gen0 on every mutation. The reductions land in three pla ## Performance measurement discipline Each transposed-grid performance round is gated on measurement, not on felt lag or code review. +The lessons from these rounds are now pinned by the black-box perf harness +(`SemiStep/SemiStep.Tests/Performance/`); its commands, gate hierarchy, re-baseline flow, and +headless blind spots are documented in `Docs/perf/README.md`. + The rules: - **Open with a weighted trace.** Before touching code, capture a weighted CPU trace @@ -455,8 +461,9 @@ The rules: (for the selection fix: per-selection-event cost must not scale with N). Ship the after-trace with the change so the collapse is documented, not asserted. - **Keep a checked-in regression instrument.** `TransposedSelectionCostProbe` - (`SemiStep.Tests/Performance/`, env-gated `SEMISTEP_PROBE=1`, `Category=Performance`, skipped in - CI) holds selection size constant at S=200 while N grows (300 / 1200 / 4800) and asserts the + (`SemiStep.Tests/Performance/`, an xunit v3 explicit test (`Explicit = true`), `Category=Performance`, + not run by default and skipped in CI) holds selection size constant at S=200 while N grows + (300 / 1200 / 4800) and asserts the per-event cost at N=4800 stays within 3× of N=300. The fixed-S design isolates the `IndexOf`-in-N regression: select-all would make S=N and force O(N) even with the fix. Restoring the `IndexOf` scan makes the ratio return at ~7×–16× (linear in N at fixed S); the fix stays flat. This is the diff --git a/Docs/perf/README.md b/Docs/perf/README.md new file mode 100644 index 00000000..f68925db --- /dev/null +++ b/Docs/perf/README.md @@ -0,0 +1,228 @@ +# Performance Harness + +This directory holds the black-box performance harness for the recipe grid and the committed +baselines it gates against. The harness turns the July grid-perf work into checks that survive +refactoring: it drives the real headless views through their public surface and reads +framework-boundary signals (allocated bytes, fresh visual instances, GC counts, retained floor), +so a panel rewrite that preserves behavior keeps the gates green, while a regression that +reintroduces subtree rebuild or O(N) scanning turns them red. + +The probes live in `SemiStep/SemiStep.Tests/Performance/`; the reusable core (runner, signals, +drivers, baseline store) lives in `Performance/Harness/`. The baselines are `baselines.json` in +this directory. + +## Canonical commands + +The measurement facts are xunit v3 **explicit tests** (`Explicit = true`). A plain `dotnet test` +never runs them; they run only when the test executable is invoked with `-explicit only`. An +xunit v3 test project IS an executable, so `dotnet run` launches it and forwards the runner flags +after `--`. The command is identical in PowerShell, bash, and CI: + +```powershell +dotnet run --project SemiStep/SemiStep.Tests/SemiStep.Tests.csproj -c Release -- -explicit only +``` + +Run a single gate (or a group) by method-name pattern: + +```powershell +dotnet run --project SemiStep/SemiStep.Tests/SemiStep.Tests.csproj -c Release -- -explicit only -method "*PerAdd_ScalesFlat*" +``` + +`dotnet run` builds incrementally first; add `--no-build` to skip that when the Release build is +already current. The built runner at `SemiStep/Artifacts/bin/SemiStep.Tests/release/SemiStep.Tests.exe` +takes the same flags and is what `dotnet run` launches under the hood — the diagnostic-layer trace +capture (below) is the one place that must call it directly. + +IDE entry points for the same invocation: the Rider run configuration **Perf Gates** +(`SemiStep/.run/Perf Gates.run.xml`) and the Zed task **Perf Gates (explicit tests, Release)** +(`.zed/tasks.json`). Rider's unit-test tree shows explicit tests as "not run" on Run All by +design; use the run configuration, not the test gutter. + +The current probes and their gate methods: + +| Probe file | Method | What it gates | +| --- | --- | --- | +| `TransposedViewAllocationProbe` | `ScrollRoundTrips_CreateZeroFreshVisuals` | The headline invariant: a scrolled viewport creates 0 new visual instances after warmup. | +| `TransposedViewAllocationProbe` | `ViewportJump_BytesPerColumn_WithinParity_AndBaseline` | Bytes per realized column vs the soft baseline, plus the transposed/canonical parity ratio (hard cap). | +| `TransposedViewAllocationProbe` | `PerAdd_ScalesFlat_WithColumnCount` | Per-add bytes stay flat as column count grows (N=120 vs N=20 ratio). | +| `TransposedSelectionCostProbe` | `SelectionChangedCost_StaysFlatAsRecipeGrows` | Same-process selection time ratio at N=4800 vs N=300 (CPU-bound, allocation-neutral). | +| `GridRetentionProbe` | `Transposed_ScrollRetention_FlatFloor`, `Canonical_ScrollRetention_FlatFloor` | Retained-heap floor does not grow across N scroll cycles; bounded container survivors. | +| `CoreAllocationProbe` | `Report_PerAppend_CoreAllocation` | Core per-append allocation against the soft baseline (no view). | + +### Optional capture prefix (steadier byte telemetry) + +The soft byte baselines are the only signals sensitive to JIT tiering: background rejit and tier +transitions perturb allocation counts. When capturing or re-baselining byte numbers, disable +tiered compilation and dynamic PGO for that shell so the numbers settle: + +```powershell +$env:DOTNET_TieredCompilation='0'; $env:DOTNET_TieredPGO='0' +dotnet run --project SemiStep/SemiStep.Tests/SemiStep.Tests.csproj -c Release -- -explicit only +``` + +These knobs are scoped to the current PowerShell session. They matter only for the soft byte +tier. The hard gates (the `== 0` fresh-visuals invariant and the ratio gates) are tiering-immune +by construction, so a routine gate run does not need the prefix. + +## Gate hierarchy + +The gates are layered by reliability. The higher a tier sits, the less it depends on the machine +it runs on. + +1. **Invariants (exact, never re-baselined).** `FreshVisualInstances == 0` after a scrolled + round-trip. This is a reference-identity set-diff of the items-panel subtree before and after + the workload: any newly created control fails it. It is exact, cross-machine, and carries no + baseline entry in `baselines.json` — it is asserted directly in code. Swap the panel + implementation and it stays green; reintroduce subtree rebuild and it goes red. + +2. **Ratios and scaling (cross-machine, tolerance-gated).** These divide two measurements taken + in one process, so machine speed cancels out: + - **Per-add scaling** — per-add bytes at N=120 vs N=20 must stay flat (ratio ≤ 1.5). Catches + a per-add cost that grows with recipe length. + - **Transposed/canonical parity** — bytes per realized column for the transposed grid vs the + canonical grid. This is a **hard cap of 3.3x today**. The current worst case is ~3.0x, + driven by the `TransposedColumnCellsHost` + `TransposedColumnCellsPool` indirection still + present in production. That indirection is scheduled for deletion (a separate follow-up); + once the Host and pool are gone the cap tightens to **2.0x**. Until that deletion lands, the + live gate is 3.3x, not 2.0x. The 2.0x figure is the target, not the current assertion. + - **Selection time ratio** — the selection cost probe keeps its own `Stopwatch` and asserts a + same-process wall-clock ratio at N=4800 vs N=300 ≤ 3x. This is the one CPU-bound, + allocation-neutral gate; allocation signals cannot see the O(S·N) selection scan, and a + same-process time ratio is its only viable signal. + +3. **Soft byte baselines (telemetry with a hard cap).** Absolute byte metrics compared against + `baselines.json` with a 20% tolerance on the recorded value, plus a hand-set hard budget cap + per metric. The tolerance catches a step regression; the budget stops slow compounding drift + (see the baseline-vs-budget split below). + +**Absolute wall-clock milliseconds are never asserted.** They are machine-dependent and flaky by +construction. Same-process wall-clock *ratios* are permitted (tier 2) because dividing two +timings from one process cancels the machine's speed. + +## Baseline vs budget split + +Every soft byte metric in `baselines.json` carries two anti-drift levels: + +- **`value`** — the measured baseline. It moves on re-baseline and, with `tolerancePct`, catches a + step regression: the gate fails when `actual > value * (1 + tolerancePct/100)`. An improvement + beyond tolerance does not fail; it prints a "baseline is stale, consider re-baselining down" + advisory. +- **`budget`** — a hand-set absolute cap. The gate `actual <= budget` always applies, independent + of the tolerance band. Re-baseline promotion never rewrites the budget (the merge carries it + through verbatim), so it stops a sequence of small within-tolerance re-baselines from ratcheting + the ceiling upward forever. A missing or null budget fails the probe with "set the budget by + hand" guidance, and a `budget < value` is rejected at load as a config error. Raising a budget is + a deliberate PR edit that carries its own justification. + +In short: `value` drifts on re-baseline and catches per-step regressions; `budget` is a hard wall +that changes only by a hand edit with a stated reason. + +## Re-baseline procedure + +Re-baselining is a file copy. The actuals artifact that a probe run writes IS the proposed next +`baselines.json`. + +1. **Run the explicit suite.** Failures are expected when the numbers have drifted — that is the + point: see what changed before accepting it. + ```powershell + $env:DOTNET_TieredCompilation='0'; $env:DOTNET_TieredPGO='0' + dotnet run --project SemiStep/SemiStep.Tests/SemiStep.Tests.csproj -c Release -- -explicit only + ``` +2. **Copy the actuals artifact over the baselines.** The fixture writes + `%TEMP%/semistep-perf-actuals-.json`. A failing probe prints the exact `Copy-Item` + command with the resolved path; run it. +3. **Review the diff.** `git diff Docs/perf/baselines.json` — confirm the metric changes are the + ones you expected and nothing else moved. +4. **Commit with a stated reason.** A re-baseline commit must name why the numbers moved. + Legitimate reasons: a runtime or Avalonia bump, or a deliberate behavior change. "Tests were + red" is not a reason. +5. **Re-run to confirm green.** The suite must pass against the just-committed baselines. + +The written artifact is the FULL proposed baselines, not just the metrics you ran: the fixture +loads the current baselines and overlays only the measured metrics (values plus refreshed +context), carrying every unmeasured metric and every budget through untouched. So the copy is +safe even after a `-method`-filtered run that touched a single metric — the other metrics survive +verbatim. + +## CI + +`.github/workflows/perf.yml` runs the full explicit suite on every push to master (and on +manual `workflow_dispatch`), on a hosted `windows-latest` runner, separate from the regular +`ci` workflow so a perf failure is immediately distinguishable from a build/test failure. + +The committed `baselines.json` carries numbers captured on the dev testbed, and absolute-byte +values are not comparable across environments. The workflow therefore overwrites +`Docs/perf/baselines.json` with an empty metrics document before running. That flips every +byte gate to its designed record-only path (a metric absent from the baselines is reported, +not asserted), while the gates whose thresholds live in probe code keep asserting at full +strength on CI: + +- `FreshVisualInstances == 0` scroll invariant, +- per-add scaling ratio, transposed/canonical parity ratio, selection time ratio, +- retention flat-delta and bounded-survivor checks. + +The byte values measured on the runner are still collected: the actuals artifact is uploaded +as the `perf-actuals` workflow artifact on every run, pass or fail. That is the telemetry +trail. If those numbers prove stable across runs, promoting the byte tier to a hard CI gate +is a data-driven follow-up: commit the artifact as a `ci-hosted` baselines file (set budgets +by hand) and have the workflow copy it into place instead of the empty document. No test code +changes in either direction. + +Not PR-blocking by design: the perf job is a post-merge detector on master, not a merge gate. + +## Context policy + +The `context` block in `baselines.json` records the measurement environment, not the machine: + +- `runtime`, `avalonia`, `os` (family + arch, e.g. `win-x64`), and `capturedUtc`. +- `testbed` is a **role label** identifying the environment: `dev-primary` today, `ci-hosted` + later. + +There are **no hostnames, machine names, or usernames**. This repo is public, and hardware +identity is irrelevant to these metrics: allocated bytes depend on the runtime and the code, not +on the CPU. Cross-machine reliability comes from the invariant and ratio tiers, not from pinning +the box. + +## Headless blind spots + +The harness runs on the Avalonia headless platform. It has no Skia and no compositor, so a whole +class of costs is invisible to it: + +- `CreateSKFont` and text-layout / text-shaping cost. +- `CreateCompositionVisual` and composition-visual creation. + +These show up only in a live, on-hardware trace. The harness attributes what the eye finds and +narrows what reaches the eye, but it does not measure felt smoothness. **Felt smoothness on real +hardware stays the manual oracle**: after significant grid work, a human still checks a Release +build against a large recipe. The harness does not replace that check. + +## Diagnostic layer + +`TransposedScrollTraceScenario` and `scripts/perf/speedscope-shares.py` are **not gates** — they +assert nothing. They are the diagnosis kit for when a black-box gate trips and you need to find +where the cost went. + +`TransposedScrollTraceScenario.Drive_FixedWorkload_ForCpuTrace` drives the real transposed view +through a fixed workload (viewport jumps, add/remove steps, an execution-tick sweep) for a CPU +trace. It is explicit-gated like the rest; launch it under `dotnet-trace` in child-launch mode +against the Release test build. This is the one invocation that must call the built exe +directly: `dotnet-trace` traces only the process it launches, and `dotnet run` would put the +run host between the tracer and the tests: + +```powershell +$env:DOTNET_TieredCompilation='0'; $env:DOTNET_TieredPGO='0' +dotnet-trace collect --format Speedscope -o after.speedscope.json -- ` + SemiStep/Artifacts/bin/SemiStep.Tests/release/SemiStep.Tests.exe ` + -explicit only -method "*TransposedScrollTraceScenario*" +``` + +`scripts/perf/speedscope-shares.py` then reads the resulting Speedscope JSON and prints the +absolute inclusive time spent in a fixed set of frames (attach, styling, measure, realize, and +cell-host acquire-and-bind — `TransposedColumnCellsHost.AcquireAndBind`), plus whether the +attach/styling frames appear under a `Realize` stack. That +mechanism-presence check is what tells you whether a scroll is rebuilding subtrees rather than +recycling them. + +```powershell +python scripts/perf/speedscope-shares.py after.speedscope.json +``` diff --git a/Docs/perf/baselines.json b/Docs/perf/baselines.json new file mode 100644 index 00000000..16146026 --- /dev/null +++ b/Docs/perf/baselines.json @@ -0,0 +1,56 @@ +{ + "context": { + "runtime": "10.0.9", + "avalonia": "12.0.5", + "os": "win-x64", + "testbed": "dev-primary", + "capturedUtc": "2026-07-17T10:24:34.8905768Z" + }, + "metrics": { + "canonical.retention.floorBytes": { + "value": 123597672, + "tolerancePct": 20, + "budget": 250000000 + }, + "canonical.viewportJump.bytesPerColumn": { + "value": 86670.25, + "tolerancePct": 20, + "budget": 180000 + }, + "core.perAppend.bytes.n10": { + "value": 3816, + "tolerancePct": 20, + "budget": 8000 + }, + "core.perAppend.bytes.n100": { + "value": 5040, + "tolerancePct": 20, + "budget": 11000 + }, + "core.perAppend.bytes.n500": { + "value": 9936, + "tolerancePct": 20, + "budget": 20000 + }, + "transposed.perAdd.bytes.n120": { + "value": 275290, + "tolerancePct": 20, + "budget": 560000 + }, + "transposed.perAdd.bytes.n20": { + "value": 271918, + "tolerancePct": 20, + "budget": 560000 + }, + "transposed.retention.floorBytes": { + "value": 43662856, + "tolerancePct": 20, + "budget": 90000000 + }, + "transposed.viewportJump.bytesPerColumn": { + "value": 255439, + "tolerancePct": 20, + "budget": 290000 + } + } +} diff --git a/Docs/plans/completed/20260717-blackbox-perf-harness.md b/Docs/plans/completed/20260717-blackbox-perf-harness.md new file mode 100644 index 00000000..3a904762 --- /dev/null +++ b/Docs/plans/completed/20260717-blackbox-perf-harness.md @@ -0,0 +1,226 @@ +# Black-Box Performance Harness (driver + boundary signals + committed baselines) + +## Overview +- Formalize the July perf work (PRs #131-#138) into a black-box performance harness that survives refactoring. Today's five probes encode hard-won lessons but half of them are white-box: they name `TransposedColumnCellsHost`, count host re-attaches, and pin implementation details that the already-planned follow-up (deleting the Host/pool indirection) will legitimately break. +- The design is two stable contracts at the edges, with everything in between free to change: + - **Top: user actions.** `IRecipeGridDriver` — scroll, add/remove steps, select — implemented over the PUBLIC surface of the real views (ScrollViewer, view-model commands), the same entry points user input hits. No production interfaces added for testing (consumer-side interface, per project conventions); the only production concession is stable `x:Name` on anchor elements where missing. + - **Bottom: framework-boundary signals.** `PerfSignals` — allocated bytes, fresh visual instances (reference-identity set-diff of `GetVisualDescendants()`), GC collection counts, retained floor after full GC. No SemiStep type names in any assertion. +- The headline gate: **a scrolled viewport creates 0 new visual instances after warmup**. Swap the panel implementation and the gate stays green; reintroduce subtree rebuild and it goes red. +- Gate hierarchy (reliability order): + 1. **Invariants** (fresh visuals == 0) — exact, cross-machine, never re-baselined. + 2. **Ratios/scaling** (per-add bytes at N vs 2N flat; transposed vs canonical parity; selection wall-clock flat vs N at fixed S, same-process ratio) — cross-machine, tolerance-gated. Same-process wall-clock RATIOS belong to this tier: dividing two timings from one process cancels machine speed, so they discriminate CPU-bound regressions (like the O(S·N) selection scan) that allocation signals cannot see. + 3. **Absolute bytes** — telemetry against a committed baseline with soft tolerance (20%) plus a hand-set hard budget cap per metric. Re-baselining within budget is routine (runtime/Avalonia bumps); the budget stops compounding re-baseline drift and changes only by deliberate hand edit in a PR. + - ABSOLUTE wall-clock milliseconds are never asserted (machine-dependent, flaky by construction); same-process wall-clock ratios are permitted in the ratio tier. +- Gating: measurement facts are xunit v3 **explicit tests** (`Explicit = true`). Default `dotnet test`/CI does not run them; probes run via `SemiStep.Tests.exe -explicit only`. Probes are single-mode (measure → report to the actuals fixture → assert vs committed baselines) and read no environment variables. Re-baselining is a file copy: the actuals artifact is the proposed next `baselines.json` (see Technical Details). Production code is untouched. + +## Context (from discovery) +- Probes on master: `SemiStep/SemiStep.Tests/Performance/` — `TransposedViewAllocationProbe` (per-add sweep, viewport-jump, host-reattach counter), `TransposedSelectionCostProbe` (O(S·N) discrimination via same-process time ratio), `GridRetentionProbe`, `CoreAllocationProbe`, `TransposedScrollTraceScenario` (dotnet-trace diagnostic scenario). All five env-gated (`SEMISTEP_PROBE` / `SEMISTEP_TRACE_SCENARIO`), skipped by default, write reports to `%TEMP%`; no committed baselines, no gate-vs-baseline comparison. Env handling is test-project-only (zero SEMISTEP_ references in SemiStep.UI/Core). Separately, `SemiStep.Tests/UI/RecipeGrid/Transposed/TransposedViewportJumpTests.cs` is an ALWAYS-ON Integration test (no env gate) pinning the realized-container-count-stays-viewport-bound invariant in CI — it is NOT part of this migration and stays always-on. +- xunit v3 explicit-test support verified on the built runner: `SemiStep.Tests.exe -?` lists `-explicit on|off|only` with `off` (skip explicit) as default. Remaining to confirm in Task 1: `[AvaloniaFact(Explicit = true)]` compiles (property inherited from the v3 `FactAttribute`) and plain `dotnet test` reports explicit tests as not-run. Repo is on xunit.v3 3.2.2. +- Existing fixtures/harness: probes build the REAL headless views via `SemiStep.Tests.UI.Helpers` (window + config scenarios `WithGroups`/`WideParams`); the drivers adapt these helpers, they do not duplicate them. +- CI (`.github/workflows/ci.yml`): full `SemiStep.Tests` on windows-latest, Release. No CI change in this plan (Post-Completion option). +- `scripts/perf/speedscope-shares.py` exists (trace analyzer); `Docs/perf/` does not exist yet. +- Planned follow-up that motivates the migration: delete `TransposedColumnCellsHost` + pool (recorded in `Docs/plans/completed/20260716-transposed-child-recycle-fix.md` Post-Completion) — white-box host probes would break; black-box gates must not. + +## Development Approach +- **testing approach**: Regular (code first, then tests). +- The harness core (runner, signals, baseline comparer, fixture, drivers) gets ALWAYS-ON unit tests — fast, deterministic, part of the normal suite; only the measurement scenarios themselves are explicit-gated. +- Migration replaces white-box duplicates without weakening coverage: every lesson currently pinned by a probe must remain pinned by an equivalent-or-stronger black-box gate before the white-box assert is deleted. The old→new map in Technical Details is the contract. +- Every task fully green (`dotnet test`, full suite) before the next. +- Gates assert mechanisms; the felt smoothness on real hardware stays a manual oracle (headless sees no Skia/composition) and the README says so. + +## Testing Strategy +- **unit tests (always-on)**: runner detects a synthetically created control (`FreshVisualInstances > 0`) and reports 0 for a no-op workload; bytes measurement sane; baseline comparer pass/fail/tolerance/budget paths; fixture merge; driver smoke tests for BOTH driver implementations. +- **explicit scenarios (`Explicit = true`, `Category=Performance` trait for grouping)**: the migrated gates, each asserting an exact invariant or comparing against `Docs/perf/baselines.json`. Run: `SemiStep.Tests.exe -explicit only` (all) or `-explicit only -method "*Name*"` (selective). +- **diagnostic layer**: `TransposedScrollTraceScenario` + `speedscope-shares.py` are NOT gates — they are the diagnosis kit used when a black-box gate trips. Explicit-gated like the rest; launch: `dotnet-trace collect ... -- SemiStep.Tests.exe -explicit only -method "*TransposedScrollTraceScenario*"`. + +## Progress Tracking +- Mark `[x]` immediately. `➕` new tasks, `⚠️` blockers. + +## Solution Overview +- **Chosen**: consumer-side driver interface + scenario runner + boundary signals inside `SemiStep.Tests` (new `Performance/Harness/` folder); committed baselines with runtime/testbed context; re-baseline via copy-promotion of the fixture-merged proposed-baselines artifact. No new project: fixtures are the dominant reuse, and allocation-bytes APIs are GC-mode-insensitive, so nothing forces a runtimeconfig split. +- **Rejected**: + - Separate `SemiStep.PerfTests` project — no forcing function; revisit when BenchmarkDotNet or a pinned-GC runtimeconfig is actually needed. + - BenchmarkDotNet for these scenarios — wrong tool for windowed headless dispatcher scenarios (off-label per its maintainers); deferred as its own console project for Core micro-benchmarks (Post-Completion). + - JetBrains dotMemory Unit — deprecated 2026-05 with no replacement and no modern-.NET support; hand-rolled `GC.*` signals are the surviving mainstream option (precedented in `dotnet/runtime` no-alloc tests). + - Wrapper/runner script — build+run are two documented commands, comparer failure messages carry metric/baseline/actual, and promotion is a plain file copy; `scripts/perf/` keeps only the diagnostic `speedscope-shares.py`. + - Asserting absolute wall-clock — flaky by construction (same-process time ratios are fine, see gate hierarchy). +- Baselines live in `Docs/perf/baselines.json` (versioned, reviewed in PRs). A re-baseline commit MUST state the reason (runtime/Avalonia bump, deliberate behavior change). + +## Prior art & standard-practice alignment +- **Two-harness split is vendor-standard.** `dotnet/performance` runs BenchmarkDotNet for microbenchmarks and a separate purpose-built scenario harness for app-level scenarios. Our scenario runner + committed baselines, distinct from BDN, is the same shape. +- **Real-headless-control allocation measurement mirrors Avalonia's own** `tests/Avalonia.Benchmarks` (real controls in a `TestRoot` tree, layout pass, `[MemoryDiagnoser]`); the baseline/gate layer is what we add on top. +- **The `FreshVisualInstances == 0` reference-identity invariant is bespoke deliberately** — the question "what is a recycled column" is domain-specific; no off-the-shelf tool answers it, and it is the only signal immune to bytes, timing, GC nondeterminism, and SDK bumps. +- **`Explicit=true` gating is idiomatic** (official xunit v3 API, xunit#2518), with a known caveat: some runners mis-honor `Explicit` (microsoft/vscode-dotnettools#2261) — hence the Task 1 spike verifies actual skip behavior per runner. +- **Baseline-vs-budget split and testbed labels follow continuous-benchmarking practice**: Bencher models the measurement environment as a named `testbed` role; Lighthouse CI separates drifting goals from hard budgets. Full result-history tracking a la Bencher/github-action-benchmark is a deferred upgrade; until then the git history of `baselines.json` is the measurement history. + +## Technical Details +`Performance/Harness/PerfSignals.cs`: +- `sealed record PerfSignals(long AllocatedBytes, int FreshVisualInstances, int Gen0, int Gen1, int Gen2, long RetainedBytesAfterFullGc)`. + +`Performance/Harness/PerfScenarioRunner.cs`: +- `MeasureAsync(Visual snapshotScope, Func warmup, Func workload)` → `PerfSignals`. +- **`snapshotScope` is the items-panel subtree (the `ItemsPresenter` / items panel), not the whole `TopLevel`.** Walking the full root pulls in ScrollBar chrome and focus adorners, which would make the `== 0` invariant flaky on a stray +1/+2 of framework chrome; scoping to the realized-container subtree keeps `== 0` exact while still catching subtree rebuild (hundreds of instances). The driver exposes this scope. +- Sequence: run warmup → settle (`Dispatcher.UIThread.RunJobs`, `GC.Collect` x2 + `WaitForPendingFinalizers`) → snapshot visuals (`HashSet` with reference-equality comparer over `snapshotScope.GetVisualDescendants()`) → read `GC.GetAllocatedBytesForCurrentThread()` + collection counts → run workload → settle jobs → read deltas → fresh visuals = descendants not in snapshot. +- Warmup MUST reach steady-state peak realization (scroll the full measured range once) so the recycle pool is pre-filled; otherwise the first workload pass legitimately creates containers and `== 0` is unachievable. +- The snapshot pins visuals for the measurement window by design; `RetainedBytesAfterFullGc` is measured in a separate pass after the snapshot is dropped, so pinning cannot skew the floor. +- Two-point retention support: the runner also exposes `SampleRetainedFloorAsync()` (settle jobs → full blocking GC → `GC.GetTotalMemory(true)`) as a standalone call — the retention gate is a DELTA (floor sampled before and after N workload cycles); a single `MeasureAsync` window cannot produce it. +- Fixed workload only (iteration counts as parameters) — never fixed duration. + +`Performance/Harness/IRecipeGridDriver.cs` + `TransposedGridDriver.cs` + `CanonicalGridDriver.cs`: +- `ScrollToColumnAsync(int index)` (canonical: row), `AddStepsAsync(int count)`, `RemoveStepsAsync(int count)`, `SelectRangeAsync(int from, int to)`, `WaitForIdleAsync()`; a `TopLevel Root` property and the snapshot-scope accessor for the runner. +- Implemented over the existing test view builders (`SemiStep.Tests.UI.Helpers`) + public view surface: `ScrollIntoView`/ScrollViewer offset, surface mutation commands, selection model. Resolve controls by stable `x:Name` (add names in axaml only where missing). +- Parity: the same scenario body runs against both drivers; transposed-vs-canonical ratios come from one code path. + +`Performance/Harness/PerfBaselines.cs` + `Performance/Harness/PerfActualsFixture.cs` + `Docs/perf/baselines.json`: +- Schema: `{ "context": { "runtime", "avalonia", "os", "testbed", "capturedUtc" }, "metrics": { "": { "value", "tolerancePct", "budget" } } }`. Invariant gates (== 0) are asserted in code, not in the file. +- Context fields: `os` is family+arch (`win-x64`); `testbed` is a role label (`dev-primary`, later `ci-hosted`) identifying the measurement environment. No hostnames or usernames — the repo is public, and hardware identity is irrelevant to these metrics (allocated bytes depend on runtime+code, not CPU). +- Anti-drift, two levels: `value` is the measured baseline — moves on re-baseline, catches step regressions via `tolerancePct`; `budget` is a hand-set absolute cap — the gate `actual <= budget` always applies and promotion never writes it (the fixture merge carries it through verbatim). A missing/null budget fails the probe with "set the budget by hand" guidance; `budget < value` is rejected at load as a config error. Raising a budget is a deliberate PR edit with its own justification. +- Compare: fail if `actual > value * (1 + tolerancePct/100)` OR `actual > budget`; improvement beyond tolerance prints a "baseline is stale, consider re-baselining down" advisory (does not fail). +- Baseline-file location at runtime: the test exe runs from `SemiStep/Artifacts/bin/.../release/`, so `PerfBaselines` walks up from the assembly directory to the repo root (marker: `.git` dir or repo-root `global.json`; the `.slnx` lives one level down in `SemiStep/` and is NOT the root marker) and resolves `Docs/perf/baselines.json`. If not found: fail with paths searched + the exact capture-and-copy commands. +- Actuals artifact: probes report each measured metric to `PerfActualsFixture` (xunit v3 assembly fixture) via a thread-safe in-memory collector; the fixture writes `%TEMP%/semistep-perf-actuals-.json` once, on assembly disposal (single writer — safe under xunit test-class parallelization, fresh per process; the PID suffix prevents two concurrent test processes from clobbering each other, and failing probes print the exact path). +- The written artifact is the PROPOSED NEXT `baselines.json`: the fixture loads current baselines and overlays only the measured metrics (values + refreshed context); budgets, tolerances, and unmeasured metrics carry through untouched (a metric absent from baselines gets `"budget": null` — the field is always present, schema identical). `Copy-Item` promotion is therefore always safe, including after a `-method`-filtered run. +- Re-baseline flow: explicit run (failures expected when numbers drifted — that is the point: see what changed before accepting) → `Copy-Item` actuals over `Docs/perf/baselines.json` (failing probes print this exact command) → review git diff → commit with the reason → re-run to confirm green. + +Canonical commands (documented in `Docs/perf/README.md`): +- Run gates: `dotnet build SemiStep/SemiStep.slnx -c Release` then `SemiStep/Artifacts/bin/SemiStep.Tests/release/SemiStep.Tests.exe -explicit only` (`-method "*Name*"` selective). +- Capture runs (optional, steadies the soft byte telemetry; hard gates are tiering-immune): prefix with `$env:DOTNET_TieredCompilation='0'; $env:DOTNET_TieredPGO='0'` (.NET host knobs, scoped to that shell). + +Measurement determinism (absolute-byte baselines are only sound with these): +- SDK pinned via the repo-root `global.json` (10.0.100). Its `rollForward: latestFeature` can shift the runtime within the feature band and thus allocation shapes — acceptable because absolute bytes are soft-tolerance telemetry, and a re-baseline on an SDK bump is expected and documented. +- Tiered compilation / dynamic PGO disabled for capture runs via the documented prefix (also in the dotnet-trace launch): background rejit and tier transitions perturb allocation counts. Only the soft byte telemetry depends on this; invariant and ratio gates are tiering-immune by construction. +- Workstation GC x64 (the app default); do not measure under server GC. +- Warmup + forced blocking `GC.Collect` around the measurement window (in the runner) tames JIT-warmup allocations and GC nondeterminism. + +Gating: +- `Explicit = true` on every measurement fact; `Category=Performance` trait kept for grouping/reporting. All `SEMISTEP_PROBE`/`SEMISTEP_TRACE_SCENARIO` reads are deleted during migration — five files carry them today (the five probes). `TransposedViewportJumpTests` reads no env var and stays always-on, untouched. + +Old→new gate map (Task 4-5 contract; nothing deleted before its replacement is green): +- host re-attach counter (white-box) → `FreshVisualInstances == 0` on scroll round-trips (catches ANY newly-created control). CAVEAT: fresh-visuals==0 catches subtree REBUILD (new instances), not a re-attach of an EXISTING instance without rebuild. That second lesson stays pinned by the `DetachedFromVisualTree`-assertion contract tests (`TransposedChildRecycleTests`, `TransposedColumnsPanelContractTests`, `TransposedColumnsPanelItemsChangedTests`) — which live in `UI/`, not `Performance/`, and are not migrated or deleted by this plan. +- viewport-jump bytes/column (absolute) → same metric via runner, baseline-gated at 20% + transposed/canonical parity ratio ≤ 2.0x (cross-machine). +- per-add sweep across seeds → scaling gate: per-add bytes at N=120 vs N=20 ratio ≤ 1.5 (flat-growth invariant), absolute values telemetry. +- selection discrimination → unchanged mechanism: same-process `Stopwatch` wall-clock RATIO at N=4800 vs N=300 (fix ~flat, restored regression ~an order of magnitude over the 3x limit — the probe already records these). The timing stays local to the probe, OUTSIDE `PerfSignals` (the runner measures allocation/visual signals only); the driver supplies the actions (`SelectRangeAsync`). This is the one CPU-bound, allocation-neutral gate, and the time-ratio is its only viable signal. +- retention floor → PRIMARY gate is the flat-delta invariant (retained floor after N cycles − floor at start ≈ 0, machine-independent); the absolute floor is baseline-gated telemetry only (a 20% tolerance on ~100 MB would hide a slow leak, so it cannot be the primary gate). The probe also carries a weak-reference bounded-container-survivor check and the transposed-vs-canonical layer isolation — BOTH are preserved in the migrated probe (the survivor check as-is, layer isolation via running the same scenario on both drivers). +- `CoreAllocationProbe` → no driver (Core API is already black-box); adopts the baseline mechanism only. + +## What Goes Where +- **Implementation Steps** (checkboxes): harness core, drivers, baseline store, probe migration, initial baseline capture, docs. +- **Post-Completion** (no checkboxes): CI guard and optional nightly CI job; BenchmarkDotNet tier for Core micro-benchmarks if ever wanted; the manual on-hardware smoothness oracle. + +## Implementation Steps + +### Task 1: PerfSignals + PerfScenarioRunner (measurement core) + +**Files:** +- Create: `SemiStep/SemiStep.Tests/Performance/Harness/PerfSignals.cs` +- Create: `SemiStep/SemiStep.Tests/Performance/Harness/PerfScenarioRunner.cs` +- Create: `SemiStep/SemiStep.Tests/Performance/Harness/PerfScenarioRunnerTests.cs` + +- [x] SPIKE (first, cheap, de-risks the gating design): confirm `[AvaloniaFact(Explicit = true)]` compiles and behaves — a scratch explicit test is NOT run by plain `dotnet test` (reported not-run/skipped, suite green) and IS run by `SemiStep.Tests.exe -explicit only`. Verify the skip against the actual CI invocation (`dotnet test`), not just the exe (runner-specific bugs exist where `Explicit` is not honored — microsoft/vscode-dotnettools#2261), and check once that Rider run-all does not execute explicit tests (probes take tens of seconds each). If `AvaloniaFact` does not surface the v3 `Explicit` property, fall back to a thin `PerfFactAttribute : AvaloniaFactAttribute` setting it. Record the outcome here. **OUTCOME:** `[AvaloniaFact(Explicit = true)]` compiles as-is on xunit.v3 3.2.2 — `AvaloniaFactAttribute` inherits the v3 `Explicit` property; NO `PerfFactAttribute` fallback needed. Verified empirically on the built exe (Debug): plain `dotnet test --filter` reports the explicit test `Пропущен`/skipped with the suite green (0 counted); `SemiStep.Tests.exe -explicit only -method "*Scratch*"` runs it (Total: 1, Failed: 0, Skipped: 0); the exe default (`-explicit off`) reports it `Not Run: 1`. So `dotnet test` (the CI invocation) honors the skip. Rider run-all behavior is not automatable from this environment — manual check pending, but the underlying xunit-v3 discovery honors `Explicit` for `dotnet test`, and Rider uses the same VSTest/xunit path. +- [x] Implement `PerfSignals` record and `PerfScenarioRunner` (`MeasureAsync` sequence per Technical Details; `SampleRetainedFloorAsync` standalone; fixed workload only). +- [x] Write ALWAYS-ON unit tests (`[AvaloniaFact]`): no-op workload → `FreshVisualInstances == 0` and near-zero bytes; workload adding a `TextBlock` INSIDE the snapshotScope subtree → `FreshVisualInstances == 1`, and one added OUTSIDE the scope → 0 (pins the scoping); workload allocating a known array → `AllocatedBytes` at least that size. +- [x] Write error-path test: workload throwing propagates and does not corrupt subsequent measurement on the same runner. +- [x] Run tests — must pass before Task 2. + +### Task 2: IRecipeGridDriver + both drivers + +**Files:** +- Create: `SemiStep/SemiStep.Tests/Performance/Harness/IRecipeGridDriver.cs` +- Create: `SemiStep/SemiStep.Tests/Performance/Harness/TransposedGridDriver.cs` +- Create: `SemiStep/SemiStep.Tests/Performance/Harness/CanonicalGridDriver.cs` +- Modify: view axaml ONLY if an anchor element lacks a stable `x:Name` (record which, if any) — NONE added: both anchors already had stable names (`x:Name="StepListBox"` on the transposed view, `x:Name="RecipeGrid"` on the canonical view). No production axaml touched; the snapshot scope resolves from a realized container's visual parent, needing no extra anchor. + +- [x] Implement the interface and both drivers over the existing `SemiStep.Tests.UI.Helpers` view builders and public view surface (no production interfaces; resolve by `x:Name`; expose the runner's snapshot scope). +- [x] Write ALWAYS-ON smoke tests for BOTH drivers: `ScrollToColumnAsync` changes the realized range/viewport offset; `AddStepsAsync`/`RemoveStepsAsync` changes column (row) count; `SelectRangeAsync` is reflected in the selection model; `WaitForIdleAsync` drains dispatcher jobs. +- [x] Run tests — must pass before Task 3. + +### Task 3: Baseline store + actuals fixture + +**Files:** +- Create: `SemiStep/SemiStep.Tests/Performance/Harness/PerfBaselines.cs` +- Create: `SemiStep/SemiStep.Tests/Performance/Harness/PerfActualsFixture.cs` +- Create: `Docs/perf/baselines.json` (placeholder with context schema, empty metrics) +- Create: `SemiStep/SemiStep.Tests/Performance/Harness/PerfBaselinesTests.cs` + +- [x] Implement `PerfBaselines` load/compare (tolerance gate, budget gate, budget>=value load validation, stale-baseline advisory, repo-root file resolution) and `PerfActualsFixture` (thread-safe in-memory collector; writes the merged proposed-baselines artifact once on assembly disposal; no promotion code — promotion is a documented file copy). +- [x] Write ALWAYS-ON unit tests: within-tolerance pass; over-tolerance fail naming metric/baseline/actual; improvement prints advisory but passes; over-budget fails even within baseline tolerance; `budget < value` rejected at load; missing metric fails with capture-and-copy guidance naming the exact commands; missing/null budget fails with "set the budget by hand" guidance; fixture merge — measured metrics overlaid, unmeasured metrics and all budgets carried through untouched, absent budget written as explicit null; concurrent metric reports from parallel test classes all land in the artifact. +- [x] Run tests — must pass before Task 4. + +### Task 4: Migrate scroll/recycle gates to black-box scenarios + +**Files:** +- Modify: `SemiStep/SemiStep.Tests/Performance/TransposedViewAllocationProbe.cs` (rewrite on the harness; delete the white-box host re-attach counter) +- Modify: `SemiStep/SemiStep.Tests/Performance/TransposedScrollTraceScenario.cs` (only if reusing the driver is trivial; otherwise leave for Task 5's gating swap — it is diagnostics, not a gate) + +- [x] Rewrite the scroll gate black-box: warmup round-trips, then fixed round-trips via `ScrollToColumnAsync`; assert `FreshVisualInstances == 0`. **PASSES** (`ScrollRoundTrips_CreateZeroFreshVisuals`, WideParams, 420 cols, 20↔220, 20 round-trips → freshVisuals=0). The headline invariant is proven falsifiable-ready and green. +- [x] Rewrite viewport-jump: bytes/realized-column via the runner against baseline (20% tolerance) + transposed/canonical parity ratio (honest hard cap, see ✅ below) using the same scenario body on both drivers. Byte-baseline compare wired as record-only (empty baselines); parity ratio is a HARD assert. **PASSES.** +- [x] Rewrite per-add sweep as the scaling gate (N=120 vs N=20 per-add ratio ≤ 1.5), absolute values recorded as telemetry. **PASSES** (`PerAdd_ScalesFlat_WithColumnCount`, ratio well under 1.5). +- [x] Delete the replaced white-box asserts ONLY after the new gates pass; keep report output (`%TEMP%` + console). Convert the file's facts to `Explicit = true` and delete its `SEMISTEP_PROBE` reads. White-box host-reattach counter deleted; three facts now `[AvaloniaFact(Explicit = true)]`; no `SEMISTEP_PROBE`/`SkipUnless` remain in the file. +- [x] Run the migrated probes (`SemiStep.Tests.exe -explicit only -method "*AllocationProbe*"`) AND the full normal suite — must pass before Task 5. Full normal suite green (1437 passed, 0 failed, explicit probes not run). Migrated probes: all three facts PASS. + +➕ Registered `[assembly: AssemblyFixture(typeof(PerfActualsFixture))]` in `SemiStep.Tests/AssemblyAttributes.cs`; probe class constructor-injects `PerfActualsFixture` and calls `Report(...)` for every measured byte metric. Confirmed the empty-guard in `Dispose` keeps a plain `dotnet test` clean (probes skipped → nothing reported → no artifact written). Added `PerfBaselines.Contains(metric)` to drive the assert-or-record helper (records-only when the metric is absent, hard-asserts once Task 6 captures it). + +✅ **Parity gate resolved — honest cap now, tighten later (user decision).** The transposed column rebind measures deterministically ~2.95–3.01x a canonical row on current master (both jumps recycle cleanly: `FreshVisualInstances == 0` on both; ~3x holds per-cell too, 7096 vs 2406 bytes/cell, so it is not a window/denominator artifact). Root cause is the `TransposedColumnCellsHost` + `TransposedColumnCellsPool` indirection, still present in production (Overview §Context line 22 schedules its deletion as the motivating follow-up); the 2.0x target is achievable only after that deletion. Per the decision, the parity assert is a **HARD cap set to ≤ 3.3x** (~10% headroom over the 3.01x worst-case) so it catches a regression above ~3.3x today, with an inline comment (`ParityRatioTargetAfterHostPoolDeletion = 2.0`) recording that the cap tightens to 2.0x once the Host + pool are deleted. Scope unchanged: Host/pool NOT deleted here, that stays the scheduled follow-up. + +### Task 5: Migrate selection, retention, and Core probes + +**Files:** +- Modify: `SemiStep/SemiStep.Tests/Performance/TransposedSelectionCostProbe.cs` +- Modify: `SemiStep/SemiStep.Tests/Performance/GridRetentionProbe.cs` +- Modify: `SemiStep/SemiStep.Tests/Performance/CoreAllocationProbe.cs` +- Modify: `SemiStep/SemiStep.Tests/Performance/TransposedScrollTraceScenario.cs` (gating swap only) + +- [x] Selection probe: drive the actions via `SelectRangeAsync`; KEEP its own `Stopwatch` same-process time-ratio as the gate (per the old→new map — this metric is CPU-bound and allocation-neutral, `PerfSignals` cannot express it; do not route the measurement through the runner). **PASSES.** View built via `TransposedGridDriver.CreateAsync`; the fixed 200-column tail range is selected via `driver.SelectRangeAsync` (setup, outside the timed window); the timed single-index toggle uses the driver's exposed index-based `Selection` (Deselect/Select), which keeps the O(N) item→index lookup and the dispatcher pump OUT of the stopwatch. Measured ratio N=4800/N=300 = **0.57x** (flat/decreasing), hard-asserted ≤ 3.0x. +- [x] Retention probe: two-point flat-delta via `SampleRetainedFloorAsync()` before/after N cycles — the PRIMARY gate is delta ≈ 0; the absolute floor is baseline-gated telemetry only. Preserve the existing weak-reference bounded-container-survivor check and the transposed-vs-canonical layer isolation (both drivers). **PASSES.** Floor sampled before/after 150 round-trips on BOTH drivers; measured per-round-trip delta transposed = **-551 bytes**, canonical = **3,121 bytes** (both flat, guard 100,000). Absolute floor reported as record-only telemetry (`{driver}.retention.floorBytes`). Survivor check preserved and now hard-asserted (viewport-bound realization: transposed 16, canonical 32 realized/survivors, both ≪ 300 steps). +- [x] Core probe: no driver; adopt baseline compare only. **PASSES.** Per-append bytes at N=10/100/500 reported to the actuals fixture (record-only until Task 6); `bytes > 0` kept as the hard sanity assert. Converted to `[Fact(Explicit = true)]`. +- [x] Gating sweep: convert all remaining measurement facts to `Explicit = true` and delete every `SEMISTEP_PROBE`/`SEMISTEP_TRACE_SCENARIO` read (`grep -rn "GetEnvironmentVariable(\"SEMISTEP"` must return zero hits in the repo — the plain SEMISTEP_ string legitimately survives in comments). Update the trace scenario's header comments (its documented launch command changes). **DONE.** All five probes' facts are `Explicit = true`; `GetEnvironmentVariable("SEMISTEP` returns ZERO hits repo-wide. Trace scenario launch comment updated to `dotnet-trace collect ... -- SemiStep.Tests.exe -explicit only -method "*TransposedScrollTraceScenario*"` (env-var form deleted). +- [x] Run all migrated probes (`-explicit only`) + full suite — must pass before Task 6. **DONE.** Explicit probes all green (selection, retention x2, core, allocation x3, trace); full `dotnet test` = 1437 passed, 0 failed, explicit probes not-run; `dotnet format --verify-no-changes` clean. + +### Task 6: Initial baseline capture + +**Files:** +- Modify: `Docs/perf/baselines.json` (first real capture) + +- [x] Capture initial baselines by walking the documented flow exactly as a user would: Release build → full explicit run with the capture prefix (`$env:DOTNET_TieredCompilation='0'; $env:DOTNET_TieredPGO='0'`) → probes fail on the empty baselines and print the capture-and-copy command → run it → hand-set each metric budget (README guidance: round up generously from the initial value, ~1.5-2x, or derive from an acceptance criterion such as the 2x-canonical parity) → review the diff → commit `Docs/perf/baselines.json` (context: runtime, Avalonia 12.0.5, win-x64, testbed `dev-primary`, date) with the reason "initial capture" in the message. **DONE.** Captured **9 metrics** (canonical/transposed viewportJump.bytesPerColumn, transposed perAdd.bytes.n20/n120, canonical/transposed retention.floorBytes, core.perAppend.bytes.n10/n100/n500). Budgets hand-set with ~1.5-2x headroom over each measured value; the **transposed.viewportJump.bytesPerColumn budget = 290,000 (~3.3x the 86,670 canonical baseline)** — the honest parity cap from Task 4, not the deferred 2.0x target. Context: runtime `10.0.9`, Avalonia `12.0.5`, `win-x64`, testbed `dev-primary`, capturedUtc `2026-07-17`. No budget null, no budget < value. +- [x] Verify the gate loop end-to-end: re-run the full explicit suite → all green against the just-committed baselines; temporarily inflate one metric tolerance-breakingly in a scratch copy to confirm the failure message names metric/baseline/actual and the copy command (do not commit the scratch). **DONE.** Re-run: all explicit probes green (0 failed) against the populated baselines. Falsifiability: deflated `transposed.viewportJump.bytesPerColumn` value 255439→1000 in a swap-then-restore copy; the failure printed `Metric 'transposed.viewportJump.bytesPerColumn' regressed: actual=255445.5 exceeds baseline 1000 +20% (limit 1200)` plus the full capture-and-copy command. Real baselines.json restored (`git status` clean of the scratch edit). +- [x] Verify merge-promotion safety on a filtered run: run ONE probe via `-method`, confirm the actuals artifact still contains ALL baseline metrics (unmeasured ones carried through) with only the measured one updated. **DONE.** Filtered `*PerAdd_ScalesFlat*` run: the actuals artifact carried ALL 9 metrics; only the two perAdd values refreshed (275290→275806, 271918→271592), the other 7 metrics + every budget carried through verbatim. +- [x] Run full suite — must pass before Task 7. **DONE.** `dotnet test` = 1437 passed, 0 failed, 8 explicit probes skipped. + +### Task 7: Docs/perf/README.md + architecture pointer + +**Files:** +- Create: `Docs/perf/README.md` +- Modify: `Docs/architecture/recipe-grid-surface.md` (short pointer from the perf-lessons section to Docs/perf) + +- [x] Write README: the canonical commands (Release build + `SemiStep.Tests.exe -explicit only`; optional capture prefix for byte telemetry), gate hierarchy (invariants → ratios → soft byte baselines; absolute wall-clock never asserted, same-process time ratios permitted), re-baseline procedure (explicit run → copy actuals over baselines (command printed by failing probes) → review diff → commit with reason → re-run green; legit reasons: runtime/Avalonia bump, deliberate behavior change; the artifact is the full proposed baselines, so the copy is safe even after filtered runs), the baseline-vs-budget split (tolerance catches steps, budget stops compounding drift; budgets change only by hand edit with justification), the context policy (testbed role labels, no machine identity), headless blind spots (no Skia/composition — `CreateSKFont`/`CreateCompositionVisual` only visible in live traces; on-hardware smoothness stays the manual oracle), and the diagnostic layer (trace scenario + speedscope-shares.py — what to reach for when a gate trips). **DONE.** `Docs/perf/README.md` written; commands/method-names verified against the actual probe files; parity documented as the honest 3.3x hard cap today → 2.0x aspiration. +- [x] Update the architecture doc pointer; retire any statements this plan obsoletes (e.g. "allocation gate pending live-app measurement" where the black-box gate now stands). **DONE.** Three obsolete statements retired in `recipe-grid-surface.md`: the "manual step ... stays with the user" allocation gate now cites the automated black-box gate (only composition/Skia share stays manual); "byte/gen0 allocation gate is still pending live-app measurement" now points to the standing harness; and `TransposedSelectionCostProbe` re-described from env-gated `SEMISTEP_PROBE=1` to `Explicit = true`. Pointer to `Docs/perf/README.md` added to the perf-discipline section. +- [x] Run `dotnet format SemiStep/SemiStep.slnx --verify-no-changes`. **DONE.** Clean, exit 0, no changes. + +### Task 8: Verify acceptance criteria +- [x] Every lesson from the old probes maps to a passing new gate (walk the old→new map; no coverage lost). **DONE — all 6 lessons mapped to a green gate:** + | Old lesson (probe) | New black-box gate | Result | + | --- | --- | --- | + | host re-attach counter (white-box) | `TransposedViewAllocationProbe.ScrollRoundTrips_CreateZeroFreshVisuals` → `FreshVisualInstances == 0` | PASS (freshVisuals=0) | + | ↳ CAVEAT: re-attach of an EXISTING instance w/o rebuild | UI contract tests `TransposedChildRecycleTests`, `TransposedColumnsPanelContractTests`, `TransposedColumnsPanelItemsChangedTests` | all 3 files present, NOT migrated/deleted/weakened | + | viewport-jump bytes (absolute) | `ViewportJump_BytesPerColumn_WithinParity_AndBaseline` → runner metric + 20% baseline + hard parity ratio ≤3.3x | PASS | + | per-add sweep | `PerAdd_ScalesFlat_WithColumnCount` → N=120 vs N=20 ratio ≤1.5 | PASS | + | selection discrimination | `TransposedSelectionCostProbe` → same-process time ratio ≤3x (N=4800/N=300) | PASS | + | retention floor | `GridRetentionProbe` Transposed+Canonical → two-point flat-delta primary + weak-ref survivor + both-drivers layer isolation | PASS (both drivers) | + | `CoreAllocationProbe` | baseline compare only (assert-or-record) | PASS | +- [x] `FreshVisualInstances == 0` scroll gate red-tests correctly: temporarily break recycling in a scratch worktree (or revert the `TransposedStepListBox` override locally) and confirm the gate FAILS, then restore — the gate must be proven falsifiable, not just green. **DONE — proven falsifiable.** Made `ClearContainerForItemOverride` call `base.ClearContainerForItemOverride(...)` (discards the recyclable subtree), Release-rebuilt, ran `-method "*ScrollRoundTrips*"` → **freshVisuals=108, gate FAILED** as expected. `git restore`d `TransposedStepListBox.cs` (clean), rebuilt, re-ran → **freshVisuals=0, gate GREEN**. The break was never committed. +- [x] Normal `dotnet test` run (no arguments, no env vars) does not execute any explicit scenario and stays green — CI unaffected. `grep -rn "GetEnvironmentVariable(\"SEMISTEP"` over the repo returns nothing. **DONE.** `dotnet test` (no args/env) = **1437 passed, 0 failed**; all 8 explicit probes reported `Пропущен`/skipped (not executed). `grep GetEnvironmentVariable("SEMISTEP` over `SemiStep/` → **zero hits** (the only repo-wide hit is this plan file's own documentation of the grep string). +- [x] Full suite + explicit run (`SemiStep.Tests.exe -explicit only`) both green. **DONE.** Normal suite = 1437 passed, 0 failed. Explicit run (`-explicit only`) = Total 1445, **Failed 0**, Not Run 1437 → all **8 explicit probes green**. Working tree clean except this plan file (no stray break, no scratch files). + +### Task 9: Final documentation +- [x] Update `CLAUDE.md` test section: the Performance trait note becomes "explicit tests (xunit v3 `Explicit = true`), not run by default; run via `SemiStep.Tests.exe -explicit only`; see Docs/perf/README.md". +- [x] Move this plan to `Docs/plans/completed/`. (harness moves the plan after all phases — not moved here per exec protocol) + +## Post-Completion +*Items requiring external action or future decisions — no checkboxes* + +**CI guard (cheap, in the main workflow):** assert the default `dotnet test` does not run any explicit scenario (suite count/time must not include them) — protects against a runner regression silently pulling the slow probes into every PR run (the vscode-dotnettools#2261 bug class). + +**Optional nightly CI job (deferred decision):** a scheduled GitHub Actions job running the deterministic tier via `SemiStep.Tests.exe -explicit only`, windows-latest, Release, with the capture prefix, once baselines prove stable for a few weeks. Bytes are reproducible on hosted runners with the SDK pinned via `global.json`; absolute wall-clock never gates, and the selection time-ratio is same-process and machine-independent, so runner noise is irrelevant. Not PR-blocking initially. + +**BenchmarkDotNet tier (only if Core micro-benchmarks are wanted):** separate `SemiStep.Benchmarks` console project with `[MemoryDiagnoser]` and `[Benchmark(Baseline=true)]` + ResultsComparer, mirroring the `dotnet/performance` micro tier and `Avalonia.Benchmarks`; `CoreAllocationProbe` moves there rather than being duplicated. + +**The manual oracle stays:** felt smoothness on real hardware (Release RIE, large recipe) remains a human check after significant grid work — the harness narrows what reaches the eye and attributes what the eye finds; it does not replace it. diff --git a/SemiStep/.run/Perf Gates.run.xml b/SemiStep/.run/Perf Gates.run.xml new file mode 100644 index 00000000..4ac7d148 --- /dev/null +++ b/SemiStep/.run/Perf Gates.run.xml @@ -0,0 +1,23 @@ + + + + diff --git a/SemiStep/SemiStep.Tests/AssemblyAttributes.cs b/SemiStep/SemiStep.Tests/AssemblyAttributes.cs index a4bcec54..0f133d8e 100644 --- a/SemiStep/SemiStep.Tests/AssemblyAttributes.cs +++ b/SemiStep/SemiStep.Tests/AssemblyAttributes.cs @@ -1,3 +1,10 @@ -using Xunit; +using SemiStep.Tests.Performance.Harness; + +using Xunit; [assembly: CollectionBehavior(DisableTestParallelization = true)] + +// Assembly-wide actuals collector for the explicit performance probes. Constructed once per test process +// and disposed at assembly teardown; on a normal `dotnet test` run the explicit probes do not execute, so +// nothing is reported and the empty-guard in Dispose writes no artifact (CI stays clean). +[assembly: AssemblyFixture(typeof(PerfActualsFixture))] diff --git a/SemiStep/SemiStep.Tests/Performance/CoreAllocationProbe.cs b/SemiStep/SemiStep.Tests/Performance/CoreAllocationProbe.cs index 9f6fd560..132b3a5d 100644 --- a/SemiStep/SemiStep.Tests/Performance/CoreAllocationProbe.cs +++ b/SemiStep/SemiStep.Tests/Performance/CoreAllocationProbe.cs @@ -1,61 +1,61 @@ -using SemiStep.Core.Recipes; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; + +using SemiStep.Core.Recipes; using SemiStep.Tests.Core.Helpers; using SemiStep.Tests.Helpers; +using SemiStep.Tests.Performance.Harness; using Xunit; namespace SemiStep.Tests.Performance; -// Manual measurement tool, NOT part of the normal suite (Fact.Explicit). Run it directly: -// dotnet test SemiStep/SemiStep.Tests/SemiStep.Tests.csproj --filter "FullyQualifiedName~CoreAllocationProbe" -// It reports the transient bytes a single Core append allocates (RecipeSession.Apply -> -// RecipeAnalyzer.Analyze -> TimingCalculator/RecipeSnapshot) at growing recipe sizes. The per-append -// figure scaling with N is the O(N)-per-mutation / O(N^2)-per-build churn that the Core tasks of -// Docs/plans/20260714-transposed-grid-allocation-reduction.md reduce. Re-run it before/after each -// Core task and compare against the recorded baseline. Results are written to -// %TEMP%/semistep_core_probe.txt. -// -// gcdump A/B protocol for the UI tasks (retained heap; needs a running RELEASE app): -// dotnet-gcdump collect -p -o .gcdump ; dotnet-gcdump report .gcdump -// Diff type totals (DynamicResourceExpression / StyleClassActivator / StyleInstance / -// CompositionVisual / PropertyTextCellViewModel) between the transposed and canonical grids at -// 200 steps. For the recycling churn gate (Task 7) use dotnet-counters (gen0 count, allocation -// rate) or a dotnet-trace GC-events capture while scrolling/adding, not gcdump. +// Core per-append allocation gate. No driver: the Core API is already black-box, so this probe measures +// it directly and adopts only the baseline-compare mechanism. Per-append bytes scaling with N is the +// O(N)-per-mutation churn signal. [Trait("Category", "Performance")] [Trait("Component", "Core")] public sealed class CoreAllocationProbe { private const int WarmupSize = 50; - [Fact] - public async Task Report_PerAppend_CoreAllocation() + private static readonly int[] _sizes = { 10, 100, 500 }; + + private readonly ITestOutputHelper _output; + private readonly PerfActualsFixture _actuals; + + public CoreAllocationProbe(ITestOutputHelper output, PerfActualsFixture actuals) { - Assert.SkipUnless( - Environment.GetEnvironmentVariable("SEMISTEP_PROBE") == "1", - "Measurement probe: set SEMISTEP_PROBE=1 to run."); + _output = output; + _actuals = actuals; + } + [Fact(Explicit = true)] + public async Task Report_PerAppend_CoreAllocation() + { var (_, session, _) = await CoreTestHelper.BuildAsync("WithGroups"); - // Warm the JIT and analysis paths so the measured samples are steady-state. MeasureSingleAppend(session, WarmupSize); - var sizes = new[] { 10, 100, 500 }; + var baselines = PerfBaselines.Load(); var lines = new List(); - foreach (var size in sizes) + foreach (var size in _sizes) { var bytes = MeasureSingleAppend(session, size); Assert.True(bytes > 0, $"expected a positive per-append allocation at N={size}"); lines.Add($"N={size,4} per-append = {bytes,12:N0} bytes"); + PerfMetricGate.AssertOrRecord( + _actuals, baselines, _output, PerfMetricNames.CorePerAppendBytes(size), bytes); } var report = string.Join(Environment.NewLine, lines); - var path = Path.Combine(Path.GetTempPath(), "semistep_core_probe.txt"); - File.WriteAllText(path, report); + _output.WriteLine(report); + File.WriteAllText(Path.Combine(Path.GetTempPath(), "semistep_core_probe.txt"), report); } - // Resets the session, grows the recipe to seedSize steps, then measures the thread-local bytes - // allocated by exactly one more append at recipe size == seedSize. private static long MeasureSingleAppend(RecipeSession session, int seedSize) { session.Reset().EnsureSuccess("probe reset"); diff --git a/SemiStep/SemiStep.Tests/Performance/GridRetentionProbe.cs b/SemiStep/SemiStep.Tests/Performance/GridRetentionProbe.cs index aa5c769d..d13611a7 100644 --- a/SemiStep/SemiStep.Tests/Performance/GridRetentionProbe.cs +++ b/SemiStep/SemiStep.Tests/Performance/GridRetentionProbe.cs @@ -1,46 +1,23 @@ -using System.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; -using Avalonia; -using Avalonia.Controls; using Avalonia.Headless.XUnit; -using Avalonia.Threading; -using Avalonia.VisualTree; -using SemiStep.Tests.UI.Helpers; - -using SemiStep.UI.RecipeGrid; -using SemiStep.UI.RecipeGrid.Transposed; +using SemiStep.Tests.Performance.Harness; using Xunit; namespace SemiStep.Tests.Performance; -// Memory-retention diagnostic, NOT part of the normal suite. Run it directly: -// SEMISTEP_PROBE=1 dotnet test SemiStep/SemiStep.Tests/SemiStep.Tests.csproj \ -// --filter "FullyQualifiedName~GridRetentionProbe" -// -// Question it answers: does scrolling the recipe grid back and forth ACCUMULATE retained managed -// memory (a leak), or is the live-app sawtooth just normal transient gen0 churn that a full GC -// reclaims? A leak shows the post-full-GC floor growing roughly linearly with the number of scroll -// round-trips; no leak shows the floor flat within GC noise regardless of how many round-trips ran. -// -// It runs the same workload against BOTH surfaces to isolate the layer: -// - transposed: StepColumns are virtualized horizontally; each realized column builds control-layer -// presenters/editors on top of the shared StepColumnViewModel/ParameterCellViewModel. -// - canonical : RecipeRows are virtualized vertically in the DataGrid over the same Core row model. -// If BOTH floors grow -> retention is in the shared/Core layer (the row/cell view models). If ONLY the -// transposed floor grows -> it is view-specific to the transposed control stack. If BOTH are flat -> -// not a leak. +// Memory-retention gate. Leak vs sawtooth: a leak shows the post-full-GC floor growing roughly linearly +// with scroll round-trips; a floor flat within GC noise is transient churn, not a leak. // -// Soundness guards: -// - The floor is read only AFTER a forced, blocking, compacting gen2 collection plus a finalizer -// drain, done twice, so nothing collectible is counted. -// - Transposed cell view models are built lazily per column on first realization and then held by -// their column for the surface lifetime. The warmup force-realizes EVERY column's cells so that -// lazy creation (legitimate, bounded, one-time) cannot masquerade as growth during the measured -// workload. The floor baseline therefore models a user who has already scrolled the whole recipe. -// - No per-realization container/VM is held in a strong local across a floor read; only the window and -// surface stay rooted (legitimately), plus WeakReferences that cannot root anything. +// The SAME scenario runs against BOTH drivers to localize a failure: both floors grow -> the shared/Core +// layer (row/cell view models); only the transposed floor grows -> the transposed control stack; both +// flat -> no leak. [Trait("Category", "Performance")] [Trait("Component", "UI")] [Trait("Area", "RecipeGrid")] @@ -51,180 +28,105 @@ public sealed class GridRetentionProbe private const int TransposedWindowWidth = 1400; private const int CanonicalWindowWidth = 1600; private const int WindowHeight = 800; - private const int WarmupRoundTrips = 4; private const int CheckpointRoundTrips = 75; private const int WorkloadRoundTrips = 150; - // Generous guard: over 150 round-trips a flat floor jitters by a few hundred KB from GC - // nondeterminism, so a real linear leak (tens of KB per round-trip -> multiple MB total) clears - // this bar while normal noise does not. The probe's job is the number; this only trips on a gross leak. - private const long RetainedPerRoundTripGuardBytes = 100_000; + // Flat-delta guard. Over 150 round-trips a genuinely flat floor moves only a few thousand bytes per + // round-trip from GC nondeterminism (measured: transposed ~-1.4 KB, canonical up to ~6 KB), so this + // ~20 KB bar sits ~3x above the largest observed |delta| yet still below a real linear leak of tens of + // KB per round-trip (-> multiple MB total), which trips it. Normal noise does not. + private const long RetainedPerRoundTripGuardBytes = 20_000; + + // Absolute viewport-derived cap on control-layer survivors, independent of how many containers were + // realized at the endpoint (bounding by that count is a tautology - survivors are a subset of it). A + // healthy recycling pool keeps roughly a viewport-worth alive (measured: transposed ~16, canonical ~32); + // this cap sits above the larger with headroom and far under the 300-step recipe, so it trips only if the + // control stack roots well past a viewport (a leak that keeps scrolled-history containers alive). + private const int MaxSurvivingContainers = 64; private readonly ITestOutputHelper _output; + private readonly PerfActualsFixture _actuals; - public GridRetentionProbe(ITestOutputHelper output) + public GridRetentionProbe(ITestOutputHelper output, PerfActualsFixture actuals) { _output = output; + _actuals = actuals; } - [AvaloniaFact] - public async Task Transposed_ScrollRetention_Report() + [AvaloniaFact(Explicit = true)] + public async Task Transposed_ScrollRetention_FlatFloor() { - Assert.SkipUnless( - Environment.GetEnvironmentVariable("SEMISTEP_PROBE") == "1", - "Measurement probe: set SEMISTEP_PROBE=1 to run."); + await using var driver = await TransposedGridDriver.CreateAsync( + ConfigName, SeedSteps, TransposedWindowWidth, WindowHeight); - var result = await MeasureTransposedAsync(); + var result = await MeasureRetentionAsync(driver); Emit("transposed", result); - AssertLooseGuard(result); + AssertFlatDelta("transposed", result); + AssertBoundedSurvivors("transposed", result); + + // Absolute floor is telemetry only: a 20% tolerance on a ~40 MB floor would hide a slow leak, so the + // flat-delta invariant and the survivor cap above are this probe's hard gates. Report it, never fail on it. + PerfMetricGate.RecordAdvisory( + _actuals, PerfBaselines.Load(), _output, PerfMetricNames.TransposedRetentionFloorBytes, result.Floor1); } - [AvaloniaFact] - public async Task Canonical_ScrollRetention_Report() + [AvaloniaFact(Explicit = true)] + public async Task Canonical_ScrollRetention_FlatFloor() { - Assert.SkipUnless( - Environment.GetEnvironmentVariable("SEMISTEP_PROBE") == "1", - "Measurement probe: set SEMISTEP_PROBE=1 to run."); + await using var driver = await CanonicalGridDriver.CreateAsync( + ConfigName, SeedSteps, CanonicalWindowWidth, WindowHeight); - var result = await MeasureCanonicalAsync(); + var result = await MeasureRetentionAsync(driver); Emit("canonical", result); - AssertLooseGuard(result); - } + AssertFlatDelta("canonical", result); + AssertBoundedSurvivors("canonical", result); - private static async Task MeasureTransposedAsync() - { - var fixture = new UIFixture(); - await fixture.InitializeAsync(ConfigName); - try - { - fixture.SeedRecipe(SeedSteps); - var surface = fixture.CreateTransposedSurface(); - surface.Initialize(); - var view = new TransposedRecipeGridView { DataContext = surface }; - var window = new Window { Width = TransposedWindowWidth, Height = WindowHeight, Content = view }; - window.Show(); - Dispatcher.UIThread.RunJobs(); - - var listBox = view.FindControl("StepListBox")!; - var lastIndex = surface.StepColumns.Count - 1; - - void ToEnd() - { - listBox.ScrollIntoView(lastIndex); - SetHorizontalOffset(listBox, double.MaxValue); - } - - void ToStart() - { - listBox.ScrollIntoView(0); - SetHorizontalOffset(listBox, 0); - } - - // Realize every column's lazy cells so the baseline floor already contains the full, - // bounded shared-VM population; only transient control-layer churn can move the floor after this. - ForceAllTransposedCells(surface); - - var sharedVmCount = CountTransposedCellVms(surface); - - var result = RunWorkload( - ToEnd, - ToStart, - () => listBox.GetRealizedContainers().Cast()); - - window.Close(); - return result with { SharedVmCount = sharedVmCount }; - } - finally - { - await fixture.DisposeAsync(); - } + PerfMetricGate.RecordAdvisory( + _actuals, PerfBaselines.Load(), _output, PerfMetricNames.CanonicalRetentionFloorBytes, result.Floor1); } - private static async Task MeasureCanonicalAsync() + // Mid-point floor sample so linearity is observed directly rather than assumed. + private static async Task MeasureRetentionAsync(IRecipeGridDriver driver) { - var fixture = new UIFixture(); - await fixture.InitializeAsync(ConfigName); - try - { - fixture.SeedRecipe(SeedSteps); - var surface = fixture.CreateCanonicalSurface(); - surface.Initialize(); - var view = new CanonicalRecipeGridView { DataContext = surface }; - var window = new Window { Width = CanonicalWindowWidth, Height = WindowHeight, Content = view }; - window.Show(); - Dispatcher.UIThread.RunJobs(); - - void ToEnd() - { - SetVerticalOffset(view, double.MaxValue); - } - - void ToStart() - { - SetVerticalOffset(view, 0); - } - - // The DataGrid holds one RecipeRowViewModel per step, all built eagerly at seed time, so the - // shared-VM population is already complete; there is no lazy per-cell VM to force. - var sharedVmCount = surface.RecipeRows.Count; - - var result = RunWorkload( - ToEnd, - ToStart, - () => view.GetVisualDescendants().OfType().Cast()); - - window.Close(); - return result with { SharedVmCount = sharedVmCount }; - } - finally - { - await fixture.DisposeAsync(); - } - } + var runner = new PerfScenarioRunner(); + var lastIndex = driver.ItemCount - 1; - // Drives the round-trip workload and reads the post-full-GC floor at three checkpoints so linearity - // is observed directly rather than assumed. realizedContainers supplies the currently-realized - // containers at the far endpoint for the control-layer survivor probe. - private static RetentionResult RunWorkload( - Action toEnd, - Action toStart, - Func> realizedContainers) - { - for (var i = 0; i < WarmupRoundTrips; i++) + // Realize every item once so the bounded, one-time lazy population is already paid before floor0; + // only transient control-layer churn can move the floor after this. + for (var index = 0; index <= lastIndex; index++) { - RoundTrip(toEnd, toStart); + await driver.ScrollToColumnAsync(index); } - var floor0 = Floor(); + await driver.ScrollToColumnAsync(0); + + var floor0 = await runner.SampleRetainedFloorAsync(); for (var i = 0; i < CheckpointRoundTrips; i++) { - RoundTrip(toEnd, toStart); + await RoundTripAsync(driver, lastIndex); } - var floorMid = Floor(); + var floorMid = await runner.SampleRetainedFloorAsync(); for (var i = CheckpointRoundTrips; i < WorkloadRoundTrips; i++) { - RoundTrip(toEnd, toStart); + await RoundTripAsync(driver, lastIndex); } - var floor1 = Floor(); + var floor1 = await runner.SampleRetainedFloorAsync(); - // Control-layer survivor probe: realize the far endpoint, weakly reference every realized - // container, then park at the near endpoint (unrealizing them) and force a full GC. Survivors are - // what the control stack still roots after unrealize. A recycling pool keeps a BOUNDED set alive; - // only a count that would grow with N indicates a control-layer leak (the flat floor is the primary - // signal for that). - toEnd(); - Dispatcher.UIThread.RunJobs(); - var endContainers = realizedContainers().Select(container => new WeakReference(container)).ToList(); + // Survivors = containers the control stack still roots after the viewport is parked away; only + // weak references outlive this scope, so the probe itself roots nothing. + await driver.ScrollToColumnAsync(lastIndex); + var endContainers = driver.RealizedContainers + .Select(container => new WeakReference(container)) + .ToList(); var realizedAtEnd = endContainers.Count; - toStart(); - Dispatcher.UIThread.RunJobs(); + await driver.ScrollToColumnAsync(0); - var floorFinal = Floor(); + var floorFinal = await runner.SampleRetainedFloorAsync(); var containerSurvivors = endContainers.Count(reference => reference.IsAlive); endContainers = null; @@ -234,62 +136,13 @@ private static RetentionResult RunWorkload( Floor1: floor1, FloorFinal: floorFinal, RealizedAtEnd: realizedAtEnd, - ContainerSurvivors: containerSurvivors, - SharedVmCount: 0); - } - - private static void RoundTrip(Action toEnd, Action toStart) - { - toEnd(); - Dispatcher.UIThread.RunJobs(); - toStart(); - Dispatcher.UIThread.RunJobs(); - } - - private static long Floor() - { - GC.Collect(2, GCCollectionMode.Forced, blocking: true, compacting: true); - GC.WaitForPendingFinalizers(); - GC.Collect(2, GCCollectionMode.Forced, blocking: true, compacting: true); - GC.WaitForPendingFinalizers(); - return GC.GetTotalMemory(forceFullCollection: true); - } - - private static void ForceAllTransposedCells(TransposedRecipeGridSurface surface) - { - foreach (var column in surface.StepColumns) - { - _ = column.Cells.Count; - } + ContainerSurvivors: containerSurvivors); } - private static int CountTransposedCellVms(TransposedRecipeGridSurface surface) + private static async Task RoundTripAsync(IRecipeGridDriver driver, int lastIndex) { - return surface.StepColumns.Sum(column => column.Cells.Count); - } - - private static void SetHorizontalOffset(ListBox listBox, double x) - { - var scrollViewer = listBox.GetVisualDescendants().OfType().FirstOrDefault(); - if (scrollViewer is null) - { - return; - } - - var clamped = x >= scrollViewer.Extent.Width ? scrollViewer.Extent.Width : x; - scrollViewer.Offset = new Vector(clamped, scrollViewer.Offset.Y); - } - - private static void SetVerticalOffset(Control view, double y) - { - var scrollViewer = view.GetVisualDescendants().OfType().FirstOrDefault(); - if (scrollViewer is null) - { - return; - } - - var clamped = y >= scrollViewer.Extent.Height ? scrollViewer.Extent.Height : y; - scrollViewer.Offset = new Vector(scrollViewer.Offset.X, clamped); + await driver.ScrollToColumnAsync(lastIndex); + await driver.ScrollToColumnAsync(0); } private void Emit(string surfaceLabel, RetentionResult result) @@ -298,14 +151,19 @@ private void Emit(string surfaceLabel, RetentionResult result) var perRoundTrip = deltaTotal / (double)WorkloadRoundTrips; var firstHalf = result.FloorMid - result.Floor0; var secondHalf = result.Floor1 - result.FloorMid; - var verdict = Math.Abs(perRoundTrip) < RetainedPerRoundTripGuardBytes ? "FLAT (no leak)" : "GROWING (leak)"; + // Verdict must track the gate (which fails only on positive growth past the guard): a floor that + // shrank past the guard is not a leak, so it reads FLAT rather than GROWING. + var verdict = perRoundTrip >= RetainedPerRoundTripGuardBytes + ? "GROWING (leak)" + : perRoundTrip <= -RetainedPerRoundTripGuardBytes + ? "SHRANK (no leak)" + : "FLAT (no leak)"; var lines = new List { $"surface = {surfaceLabel}", $"config / seed steps = {ConfigName} / {SeedSteps}", - $"round-trips (warm/half/full)= {WarmupRoundTrips} / {CheckpointRoundTrips} / {WorkloadRoundTrips}", - $"shared VM count (rooted) = {result.SharedVmCount:N0}", + $"round-trips (half/full) = {CheckpointRoundTrips} / {WorkloadRoundTrips}", $"floor @0 round-trips = {result.Floor0,15:N0} bytes", $"floor @{CheckpointRoundTrips,-3} round-trips = {result.FloorMid,15:N0} bytes (+{firstHalf,13:N0} vs @0)", $"floor @{WorkloadRoundTrips,-3} round-trips = {result.Floor1,15:N0} bytes (+{secondHalf,13:N0} vs @{CheckpointRoundTrips})", @@ -325,13 +183,27 @@ private void Emit(string surfaceLabel, RetentionResult result) File.AppendAllText(path, report + Environment.NewLine); } - private static void AssertLooseGuard(RetentionResult result) + private static void AssertFlatDelta(string label, RetentionResult result) { var perRoundTrip = (result.Floor1 - result.Floor0) / (double)WorkloadRoundTrips; Assert.True( perRoundTrip < RetainedPerRoundTripGuardBytes, - $"Post-GC floor grew {perRoundTrip:N0} bytes per scroll round-trip, above the " + - $"{RetainedPerRoundTripGuardBytes:N0} guard: this indicates a retention leak, not transient churn."); + $"{label}: post-GC retained floor grew {perRoundTrip:N0} bytes per scroll round-trip over " + + $"{WorkloadRoundTrips} cycles (floor {result.Floor0:N0} -> {result.Floor1:N0}), above the " + + $"{RetainedPerRoundTripGuardBytes:N0} flat-delta guard: this is a retention leak, not transient churn."); + } + + private static void AssertBoundedSurvivors(string label, RetentionResult result) + { + Assert.True( + result.RealizedAtEnd < SeedSteps, + $"{label}: {result.RealizedAtEnd} containers realized at the endpoint of a {SeedSteps}-step recipe; " + + "virtualization must keep realization viewport-bound, well under the full count."); + Assert.True( + result.ContainerSurvivors <= MaxSurvivingContainers, + $"{label}: {result.ContainerSurvivors} container survivors exceed the {MaxSurvivingContainers} " + + "viewport-derived cap after parking the viewport away; the control stack is rooting more than a " + + "recycling pool's bounded viewport-worth (a control-layer retention leak)."); } private readonly record struct RetentionResult( @@ -340,6 +212,5 @@ private readonly record struct RetentionResult( long Floor1, long FloorFinal, int RealizedAtEnd, - int ContainerSurvivors, - int SharedVmCount); + int ContainerSurvivors); } diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/CanonicalGridDriver.cs b/SemiStep/SemiStep.Tests/Performance/Harness/CanonicalGridDriver.cs new file mode 100644 index 00000000..f6db6954 --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/CanonicalGridDriver.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +using Avalonia; +using Avalonia.Controls; +using Avalonia.Threading; +using Avalonia.VisualTree; + +using SemiStep.Tests.Core.Helpers; +using SemiStep.Tests.UI.Helpers; + +using SemiStep.UI.RecipeGrid; + +namespace SemiStep.Tests.Performance.Harness; + +// Drives the real canonical DataGrid view over its public surface. Symmetric with TransposedGridDriver +// so one parity scenario body runs against both. +public sealed class CanonicalGridDriver : IRecipeGridDriver +{ + private const int DefaultWindowWidth = 1200; + private const int DefaultWindowHeight = 400; + + private readonly UIFixture _fixture; + private readonly CanonicalRecipeGridSurface _surface; + private readonly DataGrid _dataGrid; + private readonly Window _window; + private readonly Visual _snapshotScope; + + private CanonicalGridDriver( + UIFixture fixture, + CanonicalRecipeGridSurface surface, + DataGrid dataGrid, + Window window, + Visual snapshotScope) + { + _fixture = fixture; + _surface = surface; + _dataGrid = dataGrid; + _window = window; + _snapshotScope = snapshotScope; + } + + public TopLevel Root => _window; + + public Visual SnapshotScope => _snapshotScope; + + public int ItemCount => _surface.RecipeRows.Count; + + public IReadOnlyList SelectedIndices => _surface.SelectedStepIndices; + + public IReadOnlyList RealizedIndices => _dataGrid + .GetVisualDescendants() + .OfType() + .Select(row => row.DataContext) + .OfType() + .Select(row => _surface.RecipeRows.IndexOf(row)) + .Where(index => index >= 0) + .OrderBy(index => index) + .ToList(); + + public IReadOnlyList RealizedContainers => _dataGrid + .GetVisualDescendants() + .OfType() + .Cast() + .ToList(); + + public static async Task CreateAsync( + string configName = "WithGroups", + int stepCount = 60, + int windowWidth = DefaultWindowWidth, + int windowHeight = DefaultWindowHeight) + { + var fixture = new UIFixture(); + await fixture.InitializeAsync(configName); + fixture.SeedRecipe(stepCount); + + var surface = fixture.CreateCanonicalSurface(); + surface.Initialize(); + + var view = new CanonicalRecipeGridView { DataContext = surface }; + var window = new Window { Width = windowWidth, Height = windowHeight, Content = view }; + window.Show(); + Dispatcher.UIThread.RunJobs(); + + var dataGrid = view.FindControl("RecipeGrid") + ?? throw new InvalidOperationException( + "CanonicalRecipeGridView is missing its RecipeGrid anchor."); + + var snapshotScope = ResolveRowsPresenter(dataGrid); + + return new CanonicalGridDriver(fixture, surface, dataGrid, window, snapshotScope); + } + + public Task ScrollToColumnAsync(int index) + { + _dataGrid.ScrollIntoView(_surface.RecipeRows[index], null); + return WaitForIdleAsync(); + } + + public Task AddStepsAsync(int count) + { + for (var i = 0; i < count; i++) + { + _fixture.Coordinator.AppendStep(RecipeTestDriver.WaitActionId); + } + + return WaitForIdleAsync(); + } + + public Task RemoveStepsAsync(int count) + { + for (var i = 0; i < count && _surface.RecipeRows.Count > 0; i++) + { + _fixture.Coordinator.RemoveStep(_surface.RecipeRows.Count - 1); + } + + return WaitForIdleAsync(); + } + + public Task SelectRangeAsync(int from, int to) + { + _dataGrid.SelectedItems.Clear(); + for (var index = from; index <= to; index++) + { + _dataGrid.SelectedItems.Add(_surface.RecipeRows[index]); + } + + return WaitForIdleAsync(); + } + + public Task WaitForIdleAsync() + { + Dispatcher.UIThread.RunJobs(); + return Task.CompletedTask; + } + + public async ValueTask DisposeAsync() + { + _window.Close(); + Dispatcher.UIThread.RunJobs(); + await _fixture.DisposeAsync(); + } + + // Resolved from a realized row's visual parent so it stays correct regardless of the DataGrid + // template's chrome. + private static Visual ResolveRowsPresenter(DataGrid dataGrid) + { + var row = dataGrid.GetVisualDescendants().OfType().FirstOrDefault() + ?? throw new InvalidOperationException( + "Seed at least one step so the DataGrid realizes a row for the snapshot scope."); + + return row.GetVisualParent() + ?? throw new InvalidOperationException("Realized row has no visual parent presenter."); + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/CanonicalGridDriverTests.cs b/SemiStep/SemiStep.Tests/Performance/Harness/CanonicalGridDriverTests.cs new file mode 100644 index 00000000..8f25832e --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/CanonicalGridDriverTests.cs @@ -0,0 +1,69 @@ +using System.Threading.Tasks; + +using Avalonia.Headless.XUnit; + +using Xunit; + +namespace SemiStep.Tests.Performance.Harness; + +[Trait("Component", "UI")] +[Trait("Area", "Performance")] +[Trait("Category", "Integration")] +public sealed class CanonicalGridDriverTests +{ + [AvaloniaFact] + public async Task ScrollToColumn_ChangesRealizedRange() + { + await using var driver = await CanonicalGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.ScrollToColumnChangesRealizedRange(driver); + } + + [AvaloniaFact] + public async Task AddSteps_IncreasesRowCount() + { + await using var driver = await CanonicalGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.AddStepsIncreasesItemCount(driver); + } + + [AvaloniaFact] + public async Task RemoveSteps_DecreasesRowCount() + { + await using var driver = await CanonicalGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.RemoveStepsDecreasesItemCount(driver); + } + + [AvaloniaFact] + public async Task SelectRange_IsReflectedInSelectionModel() + { + await using var driver = await CanonicalGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.SelectRangeIsReflectedInSelectionModel(driver); + } + + [AvaloniaFact] + public async Task WaitForIdle_DrainsDispatcherJobs() + { + await using var driver = await CanonicalGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.WaitForIdleDrainsDispatcherJobs(driver); + } + + [AvaloniaFact] + public async Task SnapshotScope_IsItemsPanelSubtree_NotWholeRoot() + { + await using var driver = await CanonicalGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.SnapshotScopeIsItemsPanelSubtreeNotWholeRoot(driver); + } + + [AvaloniaFact] + public async Task RealizedContainers_ReflectViewport() + { + await using var driver = await CanonicalGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.RealizedContainersReflectViewport(driver); + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/IRecipeGridDriver.cs b/SemiStep/SemiStep.Tests/Performance/Harness/IRecipeGridDriver.cs new file mode 100644 index 00000000..0835a8d4 --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/IRecipeGridDriver.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +using Avalonia; +using Avalonia.Controls; + +namespace SemiStep.Tests.Performance.Harness; + +// User actions driven over the PUBLIC view surface, so one parity scenario body runs against either driver. +public interface IRecipeGridDriver : IAsyncDisposable +{ + // The window hosting the view; the runner reads it as the TopLevel of the measured tree. + TopLevel Root { get; } + + // The items-panel subtree the runner snapshots visuals over. A strict subset of Root. + Visual SnapshotScope { get; } + + // Step count as the surface projects it (transposed columns / canonical rows). + int ItemCount { get; } + + // Ascending step indices the selection model currently holds, as both views feed the surface. + IReadOnlyList SelectedIndices { get; } + + // Ascending step indices whose containers are realized right now (the virtualized viewport range). + IReadOnlyList RealizedIndices { get; } + + // Snapshot of the container controls realized right now. The retention survivor probe weak-references + // these, parks the viewport away, and counts how many the control stack still roots after unrealize. + IReadOnlyList RealizedContainers { get; } + + // Transposed drives to a column; canonical drives to the row at the same index. + Task ScrollToColumnAsync(int index); + + Task AddStepsAsync(int count); + + Task RemoveStepsAsync(int count); + + Task SelectRangeAsync(int from, int to); + + Task WaitForIdleAsync(); +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/PerfActualsFixture.cs b/SemiStep/SemiStep.Tests/Performance/Harness/PerfActualsFixture.cs new file mode 100644 index 00000000..b73fe06f --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/PerfActualsFixture.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Threading; + +namespace SemiStep.Tests.Performance.Harness; + +// xunit v3 assembly fixture: a thread-safe collector each probe reports measured metrics to. On assembly +// disposal it writes %TEMP%/semistep-perf-actuals-.json ONCE: the PROPOSED NEXT baselines.json. The +// PID suffix keeps concurrent test processes from clobbering each other. +public sealed class PerfActualsFixture : IDisposable +{ + private readonly ConcurrentDictionary _measured = new(StringComparer.Ordinal); + + private int _disposed; + + // Live read-only view, not a snapshot. + public IReadOnlyDictionary CollectedMetrics => _measured; + + // Last write wins for a repeated name. + public void Report(string metricName, double value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(metricName); + _measured[metricName] = value; + } + + public static string DefaultOutputPath() + { + return Path.Combine(Path.GetTempPath(), $"semistep-perf-actuals-{Environment.ProcessId}.json"); + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + if (_measured.IsEmpty) + { + return; + } + + var currentBaselinesJson = PerfBaselines.TryReadBaselinesJson() ?? EmptyBaselinesDocument; + var proposed = PerfBaselines.MergeIntoBaselines(currentBaselinesJson, _measured, DateTime.UtcNow.ToString("O")); + var outputPath = DefaultOutputPath(); + File.WriteAllText(outputPath, proposed); + Console.WriteLine($"[perf] proposed baselines written: {outputPath}"); + } + + private const string EmptyBaselinesDocument = + "{ \"context\": { \"runtime\": null, \"avalonia\": null, \"os\": null, \"testbed\": null, " + + "\"capturedUtc\": null }, \"metrics\": {} }"; +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/PerfBaselines.cs b/SemiStep/SemiStep.Tests/Performance/Harness/PerfBaselines.cs new file mode 100644 index 00000000..2fad737a --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/PerfBaselines.cs @@ -0,0 +1,271 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text.Json; + +namespace SemiStep.Tests.Performance.Harness; + +// No xunit dependency by design: a probe turns a non-Passed result into an assertion failure, keeping +// the comparer pure. +public sealed record BaselineComparison(bool Passed, string Message); + +// Thrown at load when the committed baselines file violates a structural invariant that no re-baseline may +// silently repair (a budget below its own baseline value). Distinct from a compare failure, which is data +// drift and surfaces through BaselineComparison. +public sealed class BaselineConfigException : Exception +{ + public BaselineConfigException(string message) + : base(message) + { + } +} + +// Loads Docs/perf/baselines.json and gates measured metrics against it; value/tolerance vs hand-set +// budget: see Docs/perf/README.md, "Baseline vs budget split". +public sealed class PerfBaselines +{ + // For a metric measured but absent from the committed file; budget stays null so the probe fails + // until a human sets it. + private const double DefaultTolerancePct = 20.0; + + private const string BaselinesRelativePath = "Docs/perf/baselines.json"; + + // The exact commands a failing probe prints so re-baselining is copy-paste. The concrete path is + // printed by PerfActualsFixture on the failing run; the pattern here names where to look. + private const string CaptureAndCopyGuidance = + "To capture and promote a fresh baseline:\n" + + " 1. dotnet run --project SemiStep/SemiStep.Tests/SemiStep.Tests.csproj -c Release -- -explicit only\n" + + " 2. Copy-Item \"$env:TEMP\\semistep-perf-actuals-.json\" \"Docs/perf/baselines.json\"\n" + + " (the failing run prints the concrete actuals path)"; + + private static readonly JsonSerializerOptions _jsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }; + + private readonly Dictionary _metrics; + + private PerfBaselines(Dictionary metrics) + { + _metrics = metrics; + } + + public static PerfBaselines Load() + { + var path = ResolveBaselinesPath(); + return Parse(File.ReadAllText(path)); + } + + // Lets a probe pick hard-assert vs record-only (see PerfMetricGate). + public bool Contains(string metricName) + { + return _metrics.ContainsKey(metricName); + } + + public IReadOnlyCollection MetricNames => _metrics.Keys; + + public static PerfBaselines Parse(string json) + { + var document = JsonSerializer.Deserialize(json, _jsonOptions) + ?? new BaselinesDocument(); + + var metrics = new Dictionary(StringComparer.Ordinal); + foreach (var (name, dto) in document.Metrics ?? new Dictionary()) + { + if (dto.Budget is double budget && budget < dto.Value) + { + throw new BaselineConfigException( + $"Metric '{name}' has budget {Format(budget)} below its baseline value {Format(dto.Value)}. " + + "A budget is the absolute cap and must be >= value; fix Docs/perf/baselines.json by hand."); + } + + metrics[name] = new Metric(dto.Value, dto.TolerancePct, dto.Budget); + } + + return new PerfBaselines(metrics); + } + + public BaselineComparison Compare(string metricName, double actual) + { + if (!_metrics.TryGetValue(metricName, out var metric)) + { + return new BaselineComparison( + false, + $"Metric '{metricName}' is not present in Docs/perf/baselines.json. {CaptureAndCopyGuidance}"); + } + + if (metric.Budget is not double budget) + { + return new BaselineComparison( + false, + $"Metric '{metricName}' has no budget set (baseline value {Format(metric.Value)}). " + + "You must set the budget by hand in Docs/perf/baselines.json: round up generously from the " + + "value (~1.5-2x) or derive it from an acceptance criterion."); + } + + if (actual > budget) + { + return new BaselineComparison( + false, + $"Metric '{metricName}' over budget: actual={Format(actual)} exceeds hard budget {Format(budget)} " + + $"(baseline value={Format(metric.Value)}). {CaptureAndCopyGuidance}"); + } + + var upperLimit = metric.Value * (1.0 + (metric.TolerancePct / 100.0)); + if (actual > upperLimit) + { + return new BaselineComparison( + false, + $"Metric '{metricName}' regressed: actual={Format(actual)} exceeds baseline {Format(metric.Value)} " + + $"+{Format(metric.TolerancePct)}% (limit {Format(upperLimit)}). {CaptureAndCopyGuidance}"); + } + + var lowerLimit = metric.Value * (1.0 - (metric.TolerancePct / 100.0)); + if (actual < lowerLimit) + { + return new BaselineComparison( + true, + $"Metric '{metricName}' improved past tolerance: actual={Format(actual)} is under baseline " + + $"{Format(metric.Value)} -{Format(metric.TolerancePct)}% (limit {Format(lowerLimit)}). " + + "The baseline is stale; consider re-baselining down."); + } + + return new BaselineComparison( + true, + $"Metric '{metricName}' within tolerance: actual={Format(actual)}, baseline={Format(metric.Value)}, " + + $"tolerance={Format(metric.TolerancePct)}%, budget={Format(budget)}."); + } + + // Budget is emitted even when null so the schema stays identical and a Copy-Item promotion is always + // safe; unmeasured metrics and budgets carry through verbatim. + public static string MergeIntoBaselines( + string currentBaselinesJson, + IReadOnlyDictionary measured, + string capturedUtc) + { + var document = JsonSerializer.Deserialize(currentBaselinesJson, _jsonOptions) + ?? new BaselinesDocument(); + + var context = document.Context ?? new ContextDto(); + context.CapturedUtc = capturedUtc; + + var merged = new SortedDictionary(StringComparer.Ordinal); + foreach (var (name, dto) in document.Metrics ?? new Dictionary()) + { + merged[name] = new MetricDto + { + Value = dto.Value, + TolerancePct = dto.TolerancePct, + Budget = dto.Budget + }; + } + + foreach (var (name, value) in measured) + { + if (merged.TryGetValue(name, out var existing)) + { + merged[name] = new MetricDto + { + Value = value, + TolerancePct = existing.TolerancePct, + Budget = existing.Budget + }; + } + else + { + merged[name] = new MetricDto + { + Value = value, + TolerancePct = DefaultTolerancePct, + Budget = null + }; + } + } + + var result = new BaselinesDocument + { + Context = context, + Metrics = new Dictionary(merged, StringComparer.Ordinal) + }; + + return JsonSerializer.Serialize(result, _jsonOptions); + } + + // Reads the committed baselines file, or null when it cannot be resolved (so a caller can still emit an + // actuals artifact against an empty document instead of throwing on assembly disposal). + public static string? TryReadBaselinesJson() + { + try + { + return File.ReadAllText(ResolveBaselinesPath()); + } + catch (FileNotFoundException) + { + return null; + } + catch (DirectoryNotFoundException) + { + return null; + } + } + + public static string ResolveBaselinesPath() + { + var searched = new List(); + var root = PerfRepoRoot.Find(searched); + if (root is null) + { + throw new FileNotFoundException( + $"Could not locate the repo root (marker: a '.git' entry or a repo-root global.json). Searched:\n" + + $" {string.Join("\n ", searched)}\n{CaptureAndCopyGuidance}"); + } + + var path = Path.Combine(root, BaselinesRelativePath.Replace('/', Path.DirectorySeparatorChar)); + if (!File.Exists(path)) + { + throw new FileNotFoundException( + $"Repo root found at '{root}' but the baselines file is missing. Expected:\n {path}\n" + + CaptureAndCopyGuidance); + } + + return path; + } + + private static string Format(double value) + { + return value.ToString("0.######", CultureInfo.InvariantCulture); + } + + private sealed record Metric(double Value, double TolerancePct, double? Budget); + + private sealed class BaselinesDocument + { + public ContextDto? Context { get; set; } + + public Dictionary? Metrics { get; set; } + } + + private sealed class ContextDto + { + public string? Runtime { get; set; } + + public string? Avalonia { get; set; } + + public string? Os { get; set; } + + public string? Testbed { get; set; } + + public string? CapturedUtc { get; set; } + } + + private sealed class MetricDto + { + public double Value { get; set; } + + public double TolerancePct { get; set; } + + public double? Budget { get; set; } + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/PerfBaselinesTests.cs b/SemiStep/SemiStep.Tests/Performance/Harness/PerfBaselinesTests.cs new file mode 100644 index 00000000..fff5a396 --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/PerfBaselinesTests.cs @@ -0,0 +1,263 @@ +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Tasks; + +using FluentAssertions; + +using Xunit; + +namespace SemiStep.Tests.Performance.Harness; + +[Trait("Category", "Unit")] +[Trait("Component", "UI")] +[Trait("Area", "Performance")] +public sealed class PerfBaselinesTests +{ + private const string OneMetricJson = """ + { + "context": { "runtime": "net10", "avalonia": "12.0.5", "os": "win-x64", "testbed": "dev-primary", "capturedUtc": null }, + "metrics": { + "scroll.bytes": { "value": 1000, "tolerancePct": 20, "budget": 2000 } + } + } + """; + + // Budget sits inside the tolerance band (1000 +20% = 1200, budget 1100): the hard cap bites before the + // soft tolerance does. + private const string TightBudgetJson = """ + { + "context": { "runtime": "net10", "avalonia": "12.0.5", "os": "win-x64", "testbed": "dev-primary", "capturedUtc": null }, + "metrics": { + "scroll.bytes": { "value": 1000, "tolerancePct": 20, "budget": 1100 } + } + } + """; + + private const string NullBudgetJson = """ + { + "context": { "runtime": "net10", "avalonia": "12.0.5", "os": "win-x64", "testbed": "dev-primary", "capturedUtc": null }, + "metrics": { + "scroll.bytes": { "value": 1000, "tolerancePct": 20, "budget": null } + } + } + """; + + private const string BudgetBelowValueJson = """ + { + "context": { "runtime": "net10", "avalonia": "12.0.5", "os": "win-x64", "testbed": "dev-primary", "capturedUtc": null }, + "metrics": { + "scroll.bytes": { "value": 1000, "tolerancePct": 20, "budget": 500 } + } + } + """; + + // Two committed metrics: one budgeted, one with a null budget. Used to prove the merge carries both + // budget shapes through untouched. + private const string TwoMetricJson = """ + { + "context": { "runtime": "net10", "avalonia": "12.0.5", "os": "win-x64", "testbed": "dev-primary", "capturedUtc": "2020-01-01T00:00:00Z" }, + "metrics": { + "scroll.bytes": { "value": 1000, "tolerancePct": 20, "budget": 2000 }, + "add.bytes": { "value": 500, "tolerancePct": 10, "budget": null } + } + } + """; + + private const string EmptyMetricsJson = """ + { + "context": { "runtime": "net10", "avalonia": "12.0.5", "os": "win-x64", "testbed": "dev-primary", "capturedUtc": null }, + "metrics": {} + } + """; + + [Fact] + public void Compare_WithinTolerance_Passes() + { + var baselines = PerfBaselines.Parse(OneMetricJson); + + var comparison = baselines.Compare("scroll.bytes", 1100); + + comparison.Passed.Should().BeTrue(); + } + + [Fact] + public void Contains_ReturnsTrue_WhenMetricPresent() + { + var baselines = PerfBaselines.Parse(OneMetricJson); + + baselines.Contains("scroll.bytes").Should().BeTrue(); + } + + [Fact] + public void Contains_ReturnsFalse_WhenMetricAbsent() + { + var baselines = PerfBaselines.Parse(OneMetricJson); + + baselines.Contains("absent.metric").Should().BeFalse(); + } + + // Reads the real committed file on purpose: the file<->constants drift is exactly what it guards. + [Fact] + public void CommittedBaselines_MetricNames_MatchProbeMetricConstants() + { + var baselines = PerfBaselines.Load(); + + baselines.MetricNames.Should().BeEquivalentTo( + PerfMetricNames.All, + "the committed baselines.json must carry exactly the metrics the probes report; a mismatch means a " + + "probe metric was renamed/dropped or a baseline entry drifted, silently disabling a gate"); + } + + [Fact] + public void Compare_OverTolerance_Fails_NamingMetricBaselineAndActual() + { + var baselines = PerfBaselines.Parse(OneMetricJson); + + var comparison = baselines.Compare("scroll.bytes", 1300); + + comparison.Passed.Should().BeFalse(); + comparison.Message.Should().Contain("scroll.bytes"); + comparison.Message.Should().Contain("1000"); + comparison.Message.Should().Contain("1300"); + } + + [Fact] + public void Compare_ImprovementBeyondTolerance_Passes_WithStaleBaselineAdvisory() + { + var baselines = PerfBaselines.Parse(OneMetricJson); + + var comparison = baselines.Compare("scroll.bytes", 500); + + comparison.Passed.Should().BeTrue(); + comparison.Message.Should().Contain("stale"); + } + + [Fact] + public void Compare_OverBudget_Fails_EvenWithinBaselineTolerance() + { + var baselines = PerfBaselines.Parse(TightBudgetJson); + + var comparison = baselines.Compare("scroll.bytes", 1150); + + comparison.Passed.Should().BeFalse(); + comparison.Message.Should().Contain("budget"); + comparison.Message.Should().Contain("1100"); + } + + [Fact] + public void Parse_BudgetBelowValue_RejectedAtLoad() + { + var act = () => PerfBaselines.Parse(BudgetBelowValueJson); + + act.Should().Throw() + .WithMessage("*scroll.bytes*"); + } + + [Fact] + public void Compare_MissingMetric_Fails_WithCaptureAndCopyGuidance() + { + var baselines = PerfBaselines.Parse(OneMetricJson); + + var comparison = baselines.Compare("absent.metric", 5); + + comparison.Passed.Should().BeFalse(); + comparison.Message.Should().Contain("absent.metric"); + comparison.Message.Should().Contain( + "dotnet run --project SemiStep/SemiStep.Tests/SemiStep.Tests.csproj -c Release -- -explicit only"); + comparison.Message.Should().Contain("Copy-Item"); + } + + [Fact] + public void Compare_NullBudget_Fails_WithSetBudgetByHandGuidance() + { + var baselines = PerfBaselines.Parse(NullBudgetJson); + + var comparison = baselines.Compare("scroll.bytes", 900); + + comparison.Passed.Should().BeFalse(); + comparison.Message.Should().Contain("set the budget by hand"); + } + + [Fact] + public void Merge_OverlaysMeasured_CarriesUnmeasuredAndBudgetsThrough_WritesAbsentBudgetAsNull() + { + var measured = new Dictionary + { + ["scroll.bytes"] = 1234, + ["new.metric"] = 4200 + }; + + var proposed = PerfBaselines.MergeIntoBaselines(TwoMetricJson, measured, "2026-01-01T00:00:00Z"); + + using var document = JsonDocument.Parse(proposed); + var root = document.RootElement; + var metrics = root.GetProperty("metrics"); + + var scroll = metrics.GetProperty("scroll.bytes"); + scroll.GetProperty("value").GetDouble().Should().Be(1234); + scroll.GetProperty("tolerancePct").GetDouble().Should().Be(20); + scroll.GetProperty("budget").GetDouble().Should().Be(2000); + + var add = metrics.GetProperty("add.bytes"); + add.GetProperty("value").GetDouble().Should().Be(500); + add.GetProperty("tolerancePct").GetDouble().Should().Be(10); + add.GetProperty("budget").ValueKind.Should().Be(JsonValueKind.Null); + + var fresh = metrics.GetProperty("new.metric"); + fresh.GetProperty("value").GetDouble().Should().Be(4200); + fresh.TryGetProperty("budget", out var freshBudget).Should().BeTrue(); + freshBudget.ValueKind.Should().Be(JsonValueKind.Null); + + var context = root.GetProperty("context"); + context.GetProperty("testbed").GetString().Should().Be("dev-primary"); + context.GetProperty("capturedUtc").GetString().Should().Be("2026-01-01T00:00:00Z"); + } + + // The assembly runs serially (parallelization disabled), so probes do not actually report concurrently; + // this exercises the collector's defensive thread-safety anyway - it must stay correct if that changes. + [Fact] + public async Task ConcurrentReports_AreThreadSafe_AllLandInTheArtifact() + { + const int ClassCount = 8; + const int MetricsPerClass = 50; + var fixture = new PerfActualsFixture(); + + var reporters = new List(); + for (var classIndex = 0; classIndex < ClassCount; classIndex++) + { + var localClassIndex = classIndex; + reporters.Add(Task.Run( + () => + { + for (var metricIndex = 0; metricIndex < MetricsPerClass; metricIndex++) + { + fixture.Report( + $"class{localClassIndex}.metric{metricIndex}", + (localClassIndex * 1000) + metricIndex); + } + }, + TestContext.Current.CancellationToken)); + } + + await Task.WhenAll(reporters); + + fixture.CollectedMetrics.Should().HaveCount(ClassCount * MetricsPerClass); + + var proposed = PerfBaselines.MergeIntoBaselines( + EmptyMetricsJson, + fixture.CollectedMetrics, + "2026-01-01T00:00:00Z"); + + using var document = JsonDocument.Parse(proposed); + var metrics = document.RootElement.GetProperty("metrics"); + + for (var classIndex = 0; classIndex < ClassCount; classIndex++) + { + for (var metricIndex = 0; metricIndex < MetricsPerClass; metricIndex++) + { + metrics.TryGetProperty($"class{classIndex}.metric{metricIndex}", out _) + .Should().BeTrue($"class{classIndex}.metric{metricIndex} should be present"); + } + } + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/PerfMetricGate.cs b/SemiStep/SemiStep.Tests/Performance/Harness/PerfMetricGate.cs new file mode 100644 index 00000000..15702ef5 --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/PerfMetricGate.cs @@ -0,0 +1,47 @@ +using Xunit; + +namespace SemiStep.Tests.Performance.Harness; + +internal static class PerfMetricGate +{ + public static void AssertOrRecord( + PerfActualsFixture actuals, + PerfBaselines baselines, + ITestOutputHelper output, + string metricName, + double actual) + { + actuals.Report(metricName, actual); + + if (!baselines.Contains(metricName)) + { + output.WriteLine( + $"[perf] {metricName}={actual:N0} recorded (no baseline yet; gates once captured)."); + return; + } + + var comparison = baselines.Compare(metricName, actual); + output.WriteLine($"[perf] {comparison.Message}"); + Assert.True(comparison.Passed, comparison.Message); + } + + public static void RecordAdvisory( + PerfActualsFixture actuals, + PerfBaselines baselines, + ITestOutputHelper output, + string metricName, + double actual) + { + actuals.Report(metricName, actual); + + if (!baselines.Contains(metricName)) + { + output.WriteLine( + $"[perf][advisory] {metricName}={actual:N0} recorded (telemetry only, never gates)."); + return; + } + + var comparison = baselines.Compare(metricName, actual); + output.WriteLine($"[perf][advisory] {comparison.Message}"); + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/PerfMetricNames.cs b/SemiStep/SemiStep.Tests/Performance/Harness/PerfMetricNames.cs new file mode 100644 index 00000000..0f60de90 --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/PerfMetricNames.cs @@ -0,0 +1,48 @@ +using System.Collections.Generic; + +namespace SemiStep.Tests.Performance.Harness; + +// Single source of truth for every baseline metric name the probes report. Both the probes and the +// always-on drift guard reference these constants, and the drift guard asserts this set equals the metric +// keys in Docs/perf/baselines.json. That closes the silent-downgrade hole: renaming a metric here (without +// re-capturing the baseline) or dropping an entry from the committed file fails the normal suite instead of +// quietly turning that gate into a record-only no-op. +internal static class PerfMetricNames +{ + public const string TransposedViewportJumpBytesPerColumn = "transposed.viewportJump.bytesPerColumn"; + public const string CanonicalViewportJumpBytesPerColumn = "canonical.viewportJump.bytesPerColumn"; + public const string TransposedPerAddBytesN20 = "transposed.perAdd.bytes.n20"; + public const string TransposedPerAddBytesN120 = "transposed.perAdd.bytes.n120"; + public const string TransposedRetentionFloorBytes = "transposed.retention.floorBytes"; + public const string CanonicalRetentionFloorBytes = "canonical.retention.floorBytes"; + public const string CorePerAppendBytesN10 = "core.perAppend.bytes.n10"; + public const string CorePerAppendBytesN100 = "core.perAppend.bytes.n100"; + public const string CorePerAppendBytesN500 = "core.perAppend.bytes.n500"; + + public static readonly IReadOnlyList All = new[] + { + TransposedViewportJumpBytesPerColumn, + CanonicalViewportJumpBytesPerColumn, + TransposedPerAddBytesN20, + TransposedPerAddBytesN120, + TransposedRetentionFloorBytes, + CanonicalRetentionFloorBytes, + CorePerAppendBytesN10, + CorePerAppendBytesN100, + CorePerAppendBytesN500, + }; + + // The Core probe measures per-append bytes at a swept recipe size; each size maps to a fixed constant so + // a swept name never drifts from the committed baseline key. + public static string CorePerAppendBytes(int size) + { + return size switch + { + 10 => CorePerAppendBytesN10, + 100 => CorePerAppendBytesN100, + 500 => CorePerAppendBytesN500, + _ => throw new KeyNotFoundException( + $"No core per-append metric constant for recipe size {size}; add one to PerfMetricNames."), + }; + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/PerfRepoRoot.cs b/SemiStep/SemiStep.Tests/Performance/Harness/PerfRepoRoot.cs new file mode 100644 index 00000000..5eabb43b --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/PerfRepoRoot.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace SemiStep.Tests.Performance.Harness; + +// Locates the repository root by walking up from the test assembly directory (the exe runs from +// SemiStep/Artifacts/bin/.../release/). The root is marked by a `.git` entry or a repo-root global.json; +// the .slnx one level down in SemiStep/ is NOT the marker. Records every directory it visits so the caller +// can report the searched path list on failure. Returns null when no marker is found. +internal static class PerfRepoRoot +{ + public static string? Find(List searched) + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + searched.Add(directory.FullName); + var gitEntry = Path.Combine(directory.FullName, ".git"); + var globalJson = Path.Combine(directory.FullName, "global.json"); + if (Directory.Exists(gitEntry) || File.Exists(gitEntry) || File.Exists(globalJson)) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + return null; + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/PerfScenarioRunner.cs b/SemiStep/SemiStep.Tests/Performance/Harness/PerfScenarioRunner.cs new file mode 100644 index 00000000..97ac4232 --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/PerfScenarioRunner.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +using Avalonia; +using Avalonia.Threading; +using Avalonia.VisualTree; + +namespace SemiStep.Tests.Performance.Harness; + +// snapshotScope MUST be the items-panel subtree, never the TopLevel: window chrome (scrollbars, focus +// adorners) adds stray visuals that make the FreshVisualInstances == 0 invariant flaky. +// +// warmup MUST reach steady-state peak realization (scroll the full measured range once) so the recycle +// pool is pre-filled; otherwise the first workload pass legitimately creates containers and == 0 is +// unachievable. Workload is fixed (iteration counts), never fixed duration. +public sealed class PerfScenarioRunner +{ + public async Task MeasureAsync(Visual snapshotScope, Func warmup, Func workload) + { + await warmup(); + SettleAndCollect(); + + var snapshot = new HashSet(snapshotScope.GetVisualDescendants(), ReferenceEqualityComparer.Instance); + + var gen0Before = GC.CollectionCount(0); + var bytesBefore = GC.GetAllocatedBytesForCurrentThread(); + + await workload(); + Dispatcher.UIThread.RunJobs(); + + var allocatedBytes = GC.GetAllocatedBytesForCurrentThread() - bytesBefore; + var gen0 = GC.CollectionCount(0) - gen0Before; + + var freshVisualInstances = snapshotScope + .GetVisualDescendants() + .Count(descendant => !snapshot.Contains(descendant)); + + return new PerfSignals(allocatedBytes, freshVisualInstances, gen0); + } + + // Standalone floor sample for the two-point retention gate: the retention invariant is a DELTA + // (floor before N cycles vs floor after), and a single MeasureAsync window cannot produce it. + public Task SampleRetainedFloorAsync() + { + Dispatcher.UIThread.RunJobs(); + FullBlockingCollect(); + return Task.FromResult(GC.GetTotalMemory(forceFullCollection: true)); + } + + private static void SettleAndCollect() + { + Dispatcher.UIThread.RunJobs(); + FullBlockingCollect(); + } + + private static void FullBlockingCollect() + { + GC.Collect(2, GCCollectionMode.Forced, blocking: true, compacting: true); + GC.WaitForPendingFinalizers(); + GC.Collect(2, GCCollectionMode.Forced, blocking: true, compacting: true); + GC.WaitForPendingFinalizers(); + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/PerfScenarioRunnerTests.cs b/SemiStep/SemiStep.Tests/Performance/Harness/PerfScenarioRunnerTests.cs new file mode 100644 index 00000000..c8d0cf6f --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/PerfScenarioRunnerTests.cs @@ -0,0 +1,161 @@ +using System; +using System.Threading.Tasks; + +using Avalonia.Controls; +using Avalonia.Headless.XUnit; +using Avalonia.Threading; + +using FluentAssertions; + +using Xunit; + +namespace SemiStep.Tests.Performance.Harness; + +[Trait("Category", "Unit")] +[Trait("Component", "UI")] +[Trait("Area", "Performance")] +public sealed class PerfScenarioRunnerTests +{ + // The no-op workload only allocates whatever a single idle Dispatcher.RunJobs costs, orders of + // magnitude below the deliberate 8 MiB array workload; this ceiling separates the two without + // tracking exact idle overhead. + private const long NearZeroBytesCeiling = 500_000; + + private const int WorkloadArrayBytes = 8 * 1024 * 1024; + + [AvaloniaFact] + public async Task NoOpWorkload_ReportsZeroFreshVisuals_AndNearZeroBytes() + { + var scope = new StackPanel(); + scope.Children.Add(new TextBlock { Text = "seed" }); + var window = ShowWindow(scope); + + var runner = new PerfScenarioRunner(); + var signals = await runner.MeasureAsync(scope, NoOp, NoOp); + + signals.FreshVisualInstances.Should().Be(0); + signals.AllocatedBytes.Should().BeLessThan(NearZeroBytesCeiling); + + window.Close(); + } + + [AvaloniaFact] + public async Task WorkloadAddingChildInsideScope_ReportsOneFreshVisual() + { + var scope = new StackPanel(); + var window = ShowWindow(scope); + + var runner = new PerfScenarioRunner(); + var signals = await runner.MeasureAsync( + scope, + NoOp, + () => + { + scope.Children.Add(new TextBlock { Text = "added" }); + return Task.CompletedTask; + }); + + signals.FreshVisualInstances.Should().Be(1); + + window.Close(); + } + + [AvaloniaFact] + public async Task WorkloadAddingChildOutsideScope_ReportsZeroFreshVisuals() + { + var scope = new StackPanel(); + var sibling = new StackPanel(); + var root = new StackPanel(); + root.Children.Add(scope); + root.Children.Add(sibling); + var window = ShowWindow(root); + + var runner = new PerfScenarioRunner(); + var signals = await runner.MeasureAsync( + scope, + NoOp, + () => + { + sibling.Children.Add(new TextBlock { Text = "added" }); + return Task.CompletedTask; + }); + + signals.FreshVisualInstances.Should().Be(0); + + window.Close(); + } + + [AvaloniaFact] + public async Task WorkloadAllocatingArray_ReportsAtLeastThatManyBytes() + { + var scope = new StackPanel(); + var window = ShowWindow(scope); + + var runner = new PerfScenarioRunner(); + var signals = await runner.MeasureAsync( + scope, + NoOp, + () => + { + var payload = new byte[WorkloadArrayBytes]; + GC.KeepAlive(payload); + return Task.CompletedTask; + }); + + signals.AllocatedBytes.Should().BeGreaterThanOrEqualTo(WorkloadArrayBytes); + + window.Close(); + } + + [AvaloniaFact] + public async Task ThrowingWorkload_Propagates_AndLeavesRunnerUsableForNextMeasurement() + { + var scope = new StackPanel(); + scope.Children.Add(new TextBlock { Text = "seed" }); + var window = ShowWindow(scope); + + var runner = new PerfScenarioRunner(); + + await Assert.ThrowsAsync( + () => runner.MeasureAsync( + scope, + NoOp, + () => throw new InvalidOperationException("workload boom"))); + + var signals = await runner.MeasureAsync(scope, NoOp, NoOp); + signals.FreshVisualInstances.Should().Be(0); + signals.AllocatedBytes.Should().BeLessThan(NearZeroBytesCeiling); + + window.Close(); + } + + [AvaloniaFact] + public async Task SampleRetainedFloor_ReturnsPositiveAndStableFloor() + { + var runner = new PerfScenarioRunner(); + + var first = await runner.SampleRetainedFloorAsync(); + var second = await runner.SampleRetainedFloorAsync(); + + first.Should().BeGreaterThan(0); + second.Should().BeGreaterThan(0); + + var drift = Math.Abs(second - first) / (double)first; + drift.Should().BeLessThan( + 0.25, + "two back-to-back floor samples with no workload between them must be stable within GC noise"); + } + + private static Task NoOp() + { + return Task.CompletedTask; + } + + private static Window ShowWindow(Control content) + { + var window = new Window { Width = 400, Height = 400, Content = content }; + window.Show(); + Dispatcher.UIThread.RunJobs(); + return window; + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/PerfSignals.cs b/SemiStep/SemiStep.Tests/Performance/Harness/PerfSignals.cs new file mode 100644 index 00000000..31e832f5 --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/PerfSignals.cs @@ -0,0 +1,8 @@ +namespace SemiStep.Tests.Performance.Harness; + +// Runtime/visual-tree facts only: no SemiStep type names by design, so a gate built on these survives +// any refactor of the panel implementation. +public sealed record PerfSignals( + long AllocatedBytes, + int FreshVisualInstances, + int Gen0); diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/RecipeGridDriverScenarios.cs b/SemiStep/SemiStep.Tests/Performance/Harness/RecipeGridDriverScenarios.cs new file mode 100644 index 00000000..ebd8dc98 --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/RecipeGridDriverScenarios.cs @@ -0,0 +1,100 @@ +using System.Linq; +using System.Threading.Tasks; + +using Avalonia.Threading; +using Avalonia.VisualTree; + +using FluentAssertions; + +namespace SemiStep.Tests.Performance.Harness; + +// Parity scenario bodies: every assertion is expressed against IRecipeGridDriver only, so the exact +// same body runs against the transposed and the canonical driver. The two smoke-test classes are thin +// wrappers that build their driver and call in here, guaranteeing both orientations stay symmetric. +internal static class RecipeGridDriverScenarios +{ + public const int SeededStepCount = 60; + + public static async Task ScrollToColumnChangesRealizedRange(IRecipeGridDriver driver) + { + var before = driver.RealizedIndices; + before.Should().NotBeEmpty("the seeded grid must realize a viewport of containers"); + + await driver.ScrollToColumnAsync(SeededStepCount - 1); + + var after = driver.RealizedIndices; + after.Should().NotBeEmpty(); + after.Should().NotEqual( + before, + "scrolling to the far end must realize a different range of containers"); + after.Max().Should().BeGreaterThan( + before.Max(), + "the far target index must become realized after the scroll"); + } + + public static async Task RealizedContainersReflectViewport(IRecipeGridDriver driver) + { + await driver.WaitForIdleAsync(); + + var containers = driver.RealizedContainers; + containers.Should().NotBeEmpty("a seeded grid must realize container controls for the survivor probe"); + containers.Count.Should().BeGreaterThanOrEqualTo( + driver.RealizedIndices.Count, + "every realized index maps to a realized container, so containers cannot be fewer than indices"); + } + + public static async Task AddStepsIncreasesItemCount(IRecipeGridDriver driver) + { + var before = driver.ItemCount; + + await driver.AddStepsAsync(5); + + driver.ItemCount.Should().Be(before + 5, "appending steps must grow the projected item count"); + } + + public static async Task RemoveStepsDecreasesItemCount(IRecipeGridDriver driver) + { + var before = driver.ItemCount; + + await driver.RemoveStepsAsync(5); + + driver.ItemCount.Should().Be(before - 5, "removing steps must shrink the projected item count"); + } + + public static async Task SelectRangeIsReflectedInSelectionModel(IRecipeGridDriver driver) + { + await driver.SelectRangeAsync(2, 6); + + driver.SelectedIndices.Should().Equal( + 2, 3, 4, 5, 6); + } + + public static async Task WaitForIdleDrainsDispatcherJobs(IRecipeGridDriver driver) + { + var executed = false; + Dispatcher.UIThread.Post(() => executed = true); + + executed.Should().BeFalse("the posted job must stay queued until the dispatcher is drained"); + + await driver.WaitForIdleAsync(); + + executed.Should().BeTrue("WaitForIdleAsync must drain queued dispatcher jobs"); + } + + public static Task SnapshotScopeIsItemsPanelSubtreeNotWholeRoot(IRecipeGridDriver driver) + { + driver.SnapshotScope.Should().NotBeSameAs((object)driver.Root); + + var scopeDescendants = driver.SnapshotScope.GetVisualDescendants().Count(); + var rootDescendants = driver.Root.GetVisualDescendants().Count(); + + rootDescendants.Should().BeGreaterThan( + scopeDescendants, + "the snapshot scope is the items-panel subtree, a strict subset of the window"); + scopeDescendants.Should().BeGreaterThan( + 0, + "the items-panel subtree must contain the realized containers"); + + return Task.CompletedTask; + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/TransposedGridDriver.cs b/SemiStep/SemiStep.Tests/Performance/Harness/TransposedGridDriver.cs new file mode 100644 index 00000000..86b1f237 --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/TransposedGridDriver.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +using Avalonia; +using Avalonia.Controls; +using Avalonia.Controls.Selection; +using Avalonia.Threading; +using Avalonia.VisualTree; + +using SemiStep.Tests.Core.Helpers; +using SemiStep.Tests.UI.Helpers; +using SemiStep.Tests.UI.RecipeGrid.Transposed; + +using SemiStep.UI.RecipeGrid.Transposed; + +namespace SemiStep.Tests.Performance.Harness; + +// Drives the real transposed view over its public surface: ScrollIntoView on the StepListBox, the +// ListBox selection model, and the coordinator mutation commands. +public sealed class TransposedGridDriver : IRecipeGridDriver +{ + private const int DefaultWindowWidth = 560; + private const int DefaultWindowHeight = 800; + + private readonly UIFixture _fixture; + private readonly TransposedRecipeGridSurface _surface; + private readonly TransposedStepListBox _stepListBox; + private readonly Window _window; + private readonly Visual _snapshotScope; + + private TransposedGridDriver( + UIFixture fixture, + TransposedRecipeGridSurface surface, + TransposedStepListBox stepListBox, + Window window, + Visual snapshotScope) + { + _fixture = fixture; + _surface = surface; + _stepListBox = stepListBox; + _window = window; + _snapshotScope = snapshotScope; + } + + public TopLevel Root => _window; + + public Visual SnapshotScope => _snapshotScope; + + public int ItemCount => _surface.StepColumns.Count; + + public IReadOnlyList SelectedIndices => _surface.SelectedStepIndices; + + public IReadOnlyList RealizedIndices + { + get + { + var containers = _stepListBox.GetRealizedContainers() ?? Enumerable.Empty(); + return containers + .Select(container => container.DataContext) + .OfType() + .Select(column => _surface.StepColumns.IndexOf(column)) + .Where(index => index >= 0) + .OrderBy(index => index) + .ToList(); + } + } + + public IReadOnlyList RealizedContainers => + (_stepListBox.GetRealizedContainers() ?? Enumerable.Empty()).ToList(); + + // The index-based selection model the transposed view feeds. Exposed so the selection-cost probe can + // toggle a single index inside its stopwatch window without pumping the dispatcher (index-based, so no + // O(N) item->index lookup lands in the timed region). + public ISelectionModel Selection => _stepListBox.Selection; + + public static async Task CreateAsync( + string configName = "WithGroups", + int stepCount = 60, + int windowWidth = DefaultWindowWidth, + int windowHeight = DefaultWindowHeight) + { + var fixture = new UIFixture(); + await fixture.InitializeAsync(configName); + fixture.SeedRecipe(stepCount); + + var surface = fixture.CreateTransposedSurface(); + surface.Initialize(); + + var view = new TransposedRecipeGridView { DataContext = surface }; + var stepListBox = view.FindControl("StepListBox") + ?? throw new InvalidOperationException( + "TransposedRecipeGridView is missing its StepListBox anchor."); + stepListBox.UseTransposedColumnsPanel(); + + var window = new Window { Width = windowWidth, Height = windowHeight, Content = view }; + window.Show(); + Dispatcher.UIThread.RunJobs(); + + var snapshotScope = ResolveItemsPanel(stepListBox); + + return new TransposedGridDriver(fixture, surface, stepListBox, window, snapshotScope); + } + + public Task ScrollToColumnAsync(int index) + { + _stepListBox.ScrollIntoView(index); + return WaitForIdleAsync(); + } + + public Task AddStepsAsync(int count) + { + for (var i = 0; i < count; i++) + { + _fixture.Coordinator.AppendStep(RecipeTestDriver.WaitActionId); + } + + return WaitForIdleAsync(); + } + + public Task RemoveStepsAsync(int count) + { + for (var i = 0; i < count && _surface.StepColumns.Count > 0; i++) + { + _fixture.Coordinator.RemoveStep(_surface.StepColumns.Count - 1); + } + + return WaitForIdleAsync(); + } + + public Task SelectRangeAsync(int from, int to) + { + var selectedItems = _stepListBox.SelectedItems + ?? throw new InvalidOperationException("StepListBox exposes no SelectedItems collection."); + + selectedItems.Clear(); + for (var index = from; index <= to; index++) + { + selectedItems.Add(_surface.StepColumns[index]); + } + + return WaitForIdleAsync(); + } + + public Task WaitForIdleAsync() + { + Dispatcher.UIThread.RunJobs(); + return Task.CompletedTask; + } + + public async ValueTask DisposeAsync() + { + _window.Close(); + Dispatcher.UIThread.RunJobs(); + await _fixture.DisposeAsync(); + } + + // Resolved from a realized container's visual parent so it stays correct regardless of the ListBox + // template's chrome. + private static Visual ResolveItemsPanel(TransposedStepListBox stepListBox) + { + var container = stepListBox.GetRealizedContainers()?.FirstOrDefault() + ?? throw new InvalidOperationException( + "Seed at least one step so the items panel realizes a container for the snapshot scope."); + + return container.GetVisualParent() + ?? throw new InvalidOperationException("Realized container has no visual parent panel."); + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/Harness/TransposedGridDriverTests.cs b/SemiStep/SemiStep.Tests/Performance/Harness/TransposedGridDriverTests.cs new file mode 100644 index 00000000..b127ccb6 --- /dev/null +++ b/SemiStep/SemiStep.Tests/Performance/Harness/TransposedGridDriverTests.cs @@ -0,0 +1,91 @@ +using System.Threading.Tasks; + +using Avalonia.Headless.XUnit; + +using FluentAssertions; + +using Xunit; + +namespace SemiStep.Tests.Performance.Harness; + +[Trait("Component", "UI")] +[Trait("Area", "Performance")] +[Trait("Category", "Integration")] +public sealed class TransposedGridDriverTests +{ + [AvaloniaFact] + public async Task ScrollToColumn_ChangesRealizedRange() + { + await using var driver = await TransposedGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.ScrollToColumnChangesRealizedRange(driver); + } + + [AvaloniaFact] + public async Task AddSteps_IncreasesColumnCount() + { + await using var driver = await TransposedGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.AddStepsIncreasesItemCount(driver); + } + + [AvaloniaFact] + public async Task RemoveSteps_DecreasesColumnCount() + { + await using var driver = await TransposedGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.RemoveStepsDecreasesItemCount(driver); + } + + [AvaloniaFact] + public async Task SelectRange_IsReflectedInSelectionModel() + { + await using var driver = await TransposedGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.SelectRangeIsReflectedInSelectionModel(driver); + } + + [AvaloniaFact] + public async Task WaitForIdle_DrainsDispatcherJobs() + { + await using var driver = await TransposedGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.WaitForIdleDrainsDispatcherJobs(driver); + } + + [AvaloniaFact] + public async Task SnapshotScope_IsItemsPanelSubtree_NotWholeRoot() + { + await using var driver = await TransposedGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.SnapshotScopeIsItemsPanelSubtreeNotWholeRoot(driver); + } + + [AvaloniaFact] + public async Task RealizedContainers_ReflectViewport() + { + await using var driver = await TransposedGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + await RecipeGridDriverScenarios.RealizedContainersReflectViewport(driver); + } + + // Selection is the index-based accessor the selection-cost gate toggles inside its stopwatch window; + // smoke-cover that a deselect/select through it is reflected in the surface selection. + [AvaloniaFact] + public async Task Selection_ToggleReflectedInSelectionModel() + { + await using var driver = await TransposedGridDriver.CreateAsync( + stepCount: RecipeGridDriverScenarios.SeededStepCount); + + await driver.SelectRangeAsync(2, 6); + driver.SelectedIndices.Should().Contain(4); + + driver.Selection.Deselect(4); + await driver.WaitForIdleAsync(); + driver.SelectedIndices.Should().NotContain(4); + + driver.Selection.Select(4); + await driver.WaitForIdleAsync(); + driver.SelectedIndices.Should().Contain(4); + } +} diff --git a/SemiStep/SemiStep.Tests/Performance/TransposedScrollTraceScenario.cs b/SemiStep/SemiStep.Tests/Performance/TransposedScrollTraceScenario.cs index e309294e..c4753abd 100644 --- a/SemiStep/SemiStep.Tests/Performance/TransposedScrollTraceScenario.cs +++ b/SemiStep/SemiStep.Tests/Performance/TransposedScrollTraceScenario.cs @@ -15,28 +15,9 @@ namespace SemiStep.Tests.Performance; -// Agent-runnable cpu-trace scenario for the transposed-grid recycle work. NOT part of the normal suite. -// It drives the REAL headless transposed view (production TransposedColumnsPanel item panel) through a -// FIXED workload so the acceptance metric is ABSOLUTE inclusive time for the same work, captured before -// the child-recycle fix (Task 0 baseline) and after it (Task 4). Fixed iteration counts, NOT a fixed -// duration: a fixed-duration loop would let faster code do more iterations and erase the A/B signal. -// -// Run it directly under dotnet-trace (child-launch mode, Release test build), env-gated so the normal -// suite skips it. PowerShell (the child inherits the parent environment): -// $env:SEMISTEP_TRACE_SCENARIO='1' -// dotnet-trace collect --format Speedscope -o before.speedscope.json -- \ -// SemiStep/Artifacts/bin/SemiStep.Tests/release/SemiStep.Tests.exe \ -// -method "*TransposedScrollTraceScenario*" -// -// The three phases mirror what the app actually does to the panel: -// (a) viewport jumps — 300 round-trips between two columns 200 apart; each jump recycles a full -// viewport of containers, which is the scroll-phase rebuild the fix targets. -// (b) change-step-quantity — append then remove 50 steps, the panel churn on step-count edits. -// (c) execution-tick sweep — walk IsCurrentStep/IsPastStep across 200 steps (RecipeActive ticks), -// exercising the binding traffic that hits idle subtrees once the child is kept alive. -// -// The host re-attach count (fresh TransposedColumnCellsHost instances built across a scripted scroll) is -// asserted separately by TransposedViewAllocationProbe.Report_HostReattach_IsZeroAfterFix. +// Diagnostics, not a gate: drives the real headless transposed view through a fixed workload for a CPU +// trace and asserts nothing (see Docs/perf/README.md, Diagnostic layer). Fixed iteration counts, never a +// fixed duration: a fixed-duration loop lets faster code do more iterations and erases the A/B signal. [Trait("Category", "Performance")] [Trait("Component", "UI")] [Trait("Area", "RecipeGrid")] @@ -45,9 +26,7 @@ public sealed class TransposedScrollTraceScenario private const string ConfigName = "WideParams"; private const int WindowWidth = 1400; - // ~2100 columns is the real-scale recipe the user reports. Seeding is O(N^2) in the surface - // (per-append loop-depth rescan), so this is the slowest single step; if it proves prohibitive the - // scenario records the largest N that still keeps the traced run >= 20s (see the plan's Task 0 note). + // ~2100 columns is the real-scale recipe; seeding is O(N^2) in the surface, so the slowest single step. private const int ScenarioColumns = 2100; // Fixed workload counts. Tuned so the untraced baseline run is >= 20s wall-clock on current code. @@ -64,13 +43,9 @@ public TransposedScrollTraceScenario(ITestOutputHelper output) _output = output; } - [AvaloniaFact] + [AvaloniaFact(Explicit = true)] public async Task Drive_FixedWorkload_ForCpuTrace() { - Assert.SkipUnless( - Environment.GetEnvironmentVariable("SEMISTEP_TRACE_SCENARIO") == "1", - "Trace scenario: set SEMISTEP_TRACE_SCENARIO=1 to run."); - var fixture = new UIFixture(); await fixture.InitializeAsync(ConfigName); try diff --git a/SemiStep/SemiStep.Tests/Performance/TransposedSelectionCostProbe.cs b/SemiStep/SemiStep.Tests/Performance/TransposedSelectionCostProbe.cs index c437aacc..f131f14e 100644 --- a/SemiStep/SemiStep.Tests/Performance/TransposedSelectionCostProbe.cs +++ b/SemiStep/SemiStep.Tests/Performance/TransposedSelectionCostProbe.cs @@ -1,62 +1,34 @@ -using System.Diagnostics; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; -using Avalonia.Controls; using Avalonia.Headless.XUnit; -using Avalonia.Threading; -using SemiStep.Tests.UI.Helpers; - -using SemiStep.UI.RecipeGrid.Transposed; +using SemiStep.Tests.Performance.Harness; using Xunit; namespace SemiStep.Tests.Performance; -// Selection-cost regression guard for TransposedRecipeGridView.OnSelectionChanged. -// -// UNLIKE the report-only TransposedViewAllocationProbe, this probe ASSERTS a ratio (deliberate -// departure): it is the checked-in instrument that catches a re-introduction of the O(S*N) IndexOf -// scan the fix removed. It is still env-gated out of CI (SEMISTEP_PROBE=1), because the measurement -// is wall-clock and seeding 2100 steps is slow. -// -// Design that isolates the regression: -// - The selection size S is held CONSTANT (a fixed tail range) while N grows across 300 / 1200 / -// 4800. Select-all would make S = N and force O(N) even with the fix, so it cannot tell a flat -// handler from a linear one. A fixed-S tail range is what exposes IndexOf-in-N: the old handler -// summed IndexOf over the selected items, each ~N deep, so its per-event cost grew linearly in N; -// the fix reads Selection.SelectedIndexes (O(S)) and stays flat. -// - The driven operation is a toggle of one selected tail column off then on, issued through the -// INDEX-based selection model (Selection.Deselect(index) / Selection.Select(index)). Deselect -// then re-select of a tail index raises SelectionChanged (RemovedItems / AddedItems), so it -// routes through OnSelectionChanged synchronously. The index-based API is used ON PURPOSE instead -// of SelectedItems.Remove/Add: the latter resolve item->index via an O(N) IndexOf over the source -// collection INSIDE the timed window, injecting an N-growing harness cost that both adds noise and -// mimics the very regression under test. Index-shifting inserts are deliberately NOT used: they -// raise IndexesChanged, which SelectingItemsControl does not surface as SelectionChanged. -// - CRITICAL: only the selection mutations are inside the stopwatch. The Dispatcher.RunJobs() -// re-render (layout + cell realization) is kept OUT of the timed region. That render floor is a -// large, N-independent, GC-noisy cost that swamped the handler in the first cut of this probe -// (its fixed-S baseline was non-monotonic: 191 / 314 / 125 us, proving the handler sat below the -// floor). With the render excluded, what remains in the timed window is the selection-model -// diff plus OnSelectionChanged itself, so the O(S) fix and the O(S*N) regression separate cleanly. -// - S is 200 (not 100) so the regression's S*N comparison count is unmistakable at large N, and -// the per-event cost stays measurably above stopwatch noise. -// - Per-event cost is a MEDIAN of repeated runs to absorb GC/JIT jitter. -// - Before ANY measurement is recorded, the full measured path is exercised once for EVERY N -// (a discarded warmup pass over the whole step-count set). Median-of-runs cancels within-fixture -// jitter but not a fixture-wide cold-JIT/GC skew, and the baseline N is measured first in a fresh -// fixture; the warmup pass puts the baseline and the largest-N fixture on equal JIT footing so the -// ratio's denominator is not inflated by first-run warmup. +// CPU-bound, allocation-neutral gate: PerfSignals (bytes / fresh visuals) cannot express it, so the +// measurement is NOT routed through PerfScenarioRunner. It stays a same-process Stopwatch wall-clock +// RATIO local to this probe (dividing two timings from one process cancels machine speed), the only +// signal that discriminates the O(S*N) IndexOf scan the fix removed. // -// Assertion: per-op cost at N=4800 <= 3x per-op cost at N=300. The fix stays near 1x (flat in N once -// the render floor is removed); reintroducing the StepColumns.IndexOf scan makes it grow ~16x (linear -// in N at fixed S), far past the 3x guard. Discrimination was verified by temporarily restoring the -// IndexOf scan and confirming this probe FAILS; see the plan for the recorded fix/regression numbers. +// S is held CONSTANT (a fixed tail range) while N grows: select-all would make S = N and hide a linear +// handler; S=200 keeps the regression's S*N cost above stopwatch noise. Index-shifting inserts are not +// used: they raise IndexesChanged, which SelectingItemsControl does not surface as SelectionChanged. [Trait("Category", "Performance")] [Trait("Component", "UI")] [Trait("Area", "RecipeGrid")] public sealed class TransposedSelectionCostProbe { + private const string ConfigName = "WithGroups"; + private const int WindowWidth = 1200; + private const int WindowHeight = 800; private const int SelectionSize = 200; private const int WarmupToggles = 8; private const int TogglesPerRun = 40; @@ -72,13 +44,9 @@ public TransposedSelectionCostProbe(ITestOutputHelper output) _output = output; } - [AvaloniaFact] + [AvaloniaFact(Explicit = true)] public async Task SelectionChangedCost_StaysFlatAsRecipeGrows() { - Assert.SkipUnless( - Environment.GetEnvironmentVariable("SEMISTEP_PROBE") == "1", - "Measurement probe: set SEMISTEP_PROBE=1 to run."); - var lines = new List(); var perOpByStepCount = new Dictionary(); @@ -87,12 +55,12 @@ public async Task SelectionChangedCost_StaysFlatAsRecipeGrows() // N is on equal footing. Results are discarded. foreach (var stepCount in _stepCounts) { - await MeasurePerEventCostAsync(stepCount); + await MeasurePerEventMicrosecondsAsync(stepCount); } foreach (var stepCount in _stepCounts) { - var perOpMicroseconds = await MeasurePerEventCostAsync(stepCount); + var perOpMicroseconds = await MeasurePerEventMicrosecondsAsync(stepCount); perOpByStepCount[stepCount] = perOpMicroseconds; lines.Add( $"N={stepCount,5} S={SelectionSize,4} per-selection-event(median) = {perOpMicroseconds,10:N2} us"); @@ -117,79 +85,53 @@ public async Task SelectionChangedCost_StaysFlatAsRecipeGrows() $"This is the O(S*N) IndexOf regression the fix removed. Report:{Environment.NewLine}{report}"); } - // Returns the median (across RunsForMedian) per-event wall-clock cost in microseconds of one - // OnSelectionChanged invocation, with a fixed 200-column tail selection over a recipe of stepCount. - private static async Task MeasurePerEventCostAsync(int stepCount) + private static async Task MeasurePerEventMicrosecondsAsync(int stepCount) { - var fixture = new UIFixture(); - await fixture.InitializeAsync(); - try - { - fixture.SeedRecipe(stepCount); - - var surface = fixture.CreateTransposedSurface(); - surface.Initialize(); - var view = new TransposedRecipeGridView { DataContext = surface }; - var window = new Window { Width = 1200, Height = 800, Content = view }; - window.Show(); - Dispatcher.UIThread.RunJobs(); - - var stepListBox = view.FindControl("StepListBox")!; - var selection = stepListBox.Selection; - - // Fixed tail range [N-200 .. N-1], selected through the index-based model (no item->index - // lookup): positioned so the old IndexOf scan was ~N deep per selected item. - var rangeStart = stepCount - SelectionSize; - for (var index = rangeStart; index < stepCount; index++) - { - selection.Select(index); - } + await using var driver = await TransposedGridDriver.CreateAsync( + ConfigName, stepCount, WindowWidth, WindowHeight); + + // Fixed tail range [N-200 .. N-1], established through the driver: positioned so the old IndexOf + // scan was ~N deep per selected item. This setup is outside every timed window. + var rangeStart = stepCount - SelectionSize; + await driver.SelectRangeAsync(rangeStart, stepCount - 1); + + var selection = driver.Selection; + var toggledIndex = stepCount - 1; - Dispatcher.UIThread.RunJobs(); + for (var i = 0; i < WarmupToggles; i++) + { + selection.Deselect(toggledIndex); + selection.Select(toggledIndex); + } - var toggledIndex = stepCount - 1; + await driver.WaitForIdleAsync(); - for (var i = 0; i < WarmupToggles; i++) + var perOpSamples = new List(RunsForMedian); + for (var run = 0; run < RunsForMedian; run++) + { + var stopwatch = Stopwatch.StartNew(); + for (var i = 0; i < TogglesPerRun; i++) { + // Deselect then re-select raises SelectionChanged synchronously twice, so the handler + // runs inside this timed window. Driving the index-based model keeps the O(N) item->index + // lookup that SelectedItems.Remove/Add would inject OUT of the measurement. The dispatcher + // pump (the re-render floor) stays OUT of the timed window too, on purpose. selection.Deselect(toggledIndex); selection.Select(toggledIndex); } - Dispatcher.UIThread.RunJobs(); + stopwatch.Stop(); - var perOpSamples = new List(RunsForMedian); - for (var run = 0; run < RunsForMedian; run++) - { - var stopwatch = Stopwatch.StartNew(); - for (var i = 0; i < TogglesPerRun; i++) - { - // Deselect then re-select raises SelectionChanged synchronously twice, so the handler - // runs inside this timed window. Driving the index-based model keeps the O(N) item->index - // lookup that SelectedItems.Remove/Add would inject OUT of the measurement. RunJobs (the - // re-render floor) stays OUT of the timed window too, on purpose. - selection.Deselect(toggledIndex); - selection.Select(toggledIndex); - } - - stopwatch.Stop(); - - // Drain the deferred re-render outside the measured window so it never counts toward the - // handler cost and the dispatcher queue does not grow across runs. - Dispatcher.UIThread.RunJobs(); - - // Two SelectionChanged events per toggle (deselect, then select). - var events = TogglesPerRun * 2; - perOpSamples.Add(stopwatch.Elapsed.TotalMilliseconds * 1000.0 / events); - } + // Drain the deferred re-render outside the measured window so it never counts toward the + // handler cost and the dispatcher queue does not grow across runs. + await driver.WaitForIdleAsync(); - window.Close(); - - return Median(perOpSamples); - } - finally - { - await fixture.DisposeAsync(); + // Two SelectionChanged events per toggle (deselect, then select). + var events = TogglesPerRun * 2; + perOpSamples.Add(stopwatch.Elapsed.TotalMilliseconds * 1000.0 / events); } + + return Median(perOpSamples); } private static double Median(List samples) diff --git a/SemiStep/SemiStep.Tests/Performance/TransposedViewAllocationProbe.cs b/SemiStep/SemiStep.Tests/Performance/TransposedViewAllocationProbe.cs index 335013a2..a9a4f473 100644 --- a/SemiStep/SemiStep.Tests/Performance/TransposedViewAllocationProbe.cs +++ b/SemiStep/SemiStep.Tests/Performance/TransposedViewAllocationProbe.cs @@ -1,408 +1,260 @@ -using System.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; -using Avalonia; -using Avalonia.Controls; using Avalonia.Headless.XUnit; -using Avalonia.Threading; -using Avalonia.VisualTree; using FluentAssertions; -using SemiStep.Tests.Core.Helpers; -using SemiStep.Tests.UI.Helpers; - -using SemiStep.UI.RecipeGrid; -using SemiStep.UI.RecipeGrid.Transposed; +using SemiStep.Tests.Performance.Harness; using Xunit; namespace SemiStep.Tests.Performance; -// Manual measurement tool, NOT part of the normal suite. Run it directly with the real headless view -// realized in a window so the layout/realization pass is included in the measurement: -// SEMISTEP_PROBE=1 dotnet test SemiStep/SemiStep.Tests/SemiStep.Tests.csproj \ -// --filter "FullyQualifiedName~TransposedViewAllocationProbe" -// -// It answers the ONE routing question that a gcdump cannot: does the per-add allocation churn GROW -// with the existing step count N? A gcdump forces a full GC before capturing, so it only ever shows -// retained/live objects — it structurally erases the transient per-add churn and the layout CPU cost, -// which is exactly what "тормозит / 50-100 MB per column" describes. This probe measures the -// thread-local bytes allocated by one AppendStep INCLUDING the RunJobs layout pass that realizes the -// new column, at growing N, for both the transposed and the canonical view. -// -// Two scenarios are covered: -// - WithGroups (5 cells/column) — the small baseline the prior analysis decomposed. -// - WideParams (~36 cells/column, several combo + many text editors) — real-scale, where the -// per-column realization cost and the viewport-jump cost dominate. -// -// Beyond the per-add sweep it also measures the "viewport jump": seed N, realize the horizontal start, -// then measure the allocation of a single ScrollIntoView(last) that jumps the viewport start->end. -// This reproduces the real "add step while scrolled far away" the app performs (auto-scroll to the -// inserted step realizes a full viewport of columns in one dispatcher frame), which the one-column -// per-add shift does NOT. -// -// Reading the result (written to %TEMP%/semistep_ui_probe.txt): -// - transposed per-add roughly CONSTANT across N -> churn is one column's realization; the -// 50-100 MB lives in render/composition (headless-invisible) -> capture a Rider Timeline snapshot. -// - transposed per-add GROWS with N -> an append re-touches all realized columns/cells; -// that is the bug, and it is fixable in managed layout/binding code. -// - canonical per-add stays flat while transposed climbs -> confirms the transposed-view-specific -// regression the user reports ("на конвенциальной таблице проблем нет"). -// - viewport-jump per-realized-column is the primary success metric: the target is <= ~2x the -// canonical recycled-row cost once the container-reuse fix lands. +// Black-box scroll/recycle gates over framework-boundary signals. Explicit tests: commands and the gate +// hierarchy live in Docs/perf/README.md. [Trait("Category", "Performance")] [Trait("Component", "UI")] [Trait("Area", "RecipeGrid")] public sealed class TransposedViewAllocationProbe { + // Scroll gate: enough columns to virtualize in a 1400px window, scrolled ~200 apart per round-trip. + private const string ScrollConfig = "WideParams"; + private const int ScrollWindowWidth = 1400; + private const int ScrollWindowHeight = 800; + private const int ScrollColumns = 420; + private const int ScrollLowColumn = 20; + private const int ScrollHighColumn = 220; + private const int ScrollRoundTrips = 20; + + // Viewport-jump: a single far jump (start -> end) after a warmup round-trip, measured on both drivers so + // the transposed/canonical parity ratio comes from one scenario body. + private const string JumpConfig = "WideParams"; + private const int JumpSeed = 120; + private const int TransposedWindowWidth = 1400; + private const int TransposedWindowHeight = 800; + private const int CanonicalWindowWidth = 1400; + private const int CanonicalWindowHeight = 800; + + // The parity ratio rides a ~10% margin, so each side is measured as a median of a few jumps to absorb + // GC/JIT jitter rather than trusting a single shot. + private const int ParitySampleCount = 3; + + // ~10% over the measured ~3.0x worst case while TransposedColumnCellsHost + pool remain; tightens to + // 2.0x after their deletion (see Docs/perf/README.md, gate hierarchy). + private const double ParityRatioLimit = 3.3; + private const double ParityRatioTargetAfterHostPoolDeletion = 2.0; + + private const string ScalingConfig = "WideParams"; + private const int ScalingWindowWidth = 1400; + private const int ScalingWindowHeight = 800; + private const int ScalingSmallSeed = 20; + private const int ScalingLargeSeed = 120; private const int WarmupAppends = 6; private const int MeasuredAppends = 12; - - // Host-reattach probe: enough columns to scroll ~200 apart in a 1400px window. - private const string HostReattachConfig = "WideParams"; - private const int HostReattachWindowWidth = 1400; - private const int HostReattachColumns = 420; - private const int HostReattachLowColumn = 20; - private const int HostReattachHighColumn = 220; - private const int HostReattachRoundTrips = 20; - - private static readonly int[] _seedSizes = { 20, 60, 120 }; - - // Narrow window for WithGroups keeps the horizontal panel virtualizing; the wide config uses a - // wider window so a viewport jump realizes a realistic ~20-25 columns in one frame. - private static readonly ProbeScenario[] _scenarios = - { - new(Label: "WithGroups", ConfigName: "WithGroups", WindowWidth: 560), - new(Label: "WideParams", ConfigName: "WideParams", WindowWidth: 1400), - }; + private const double ScalingRatioLimit = 1.5; private readonly ITestOutputHelper _output; + private readonly PerfActualsFixture _actuals; - public TransposedViewAllocationProbe(ITestOutputHelper output) + public TransposedViewAllocationProbe(ITestOutputHelper output, PerfActualsFixture actuals) { _output = output; + _actuals = actuals; } - [AvaloniaFact] - public async Task Report_PerAdd_ViewAllocation() + [AvaloniaFact(Explicit = true)] + public async Task ScrollRoundTrips_CreateZeroFreshVisuals() { - Assert.SkipUnless( - Environment.GetEnvironmentVariable("SEMISTEP_PROBE") == "1", - "Measurement probe: set SEMISTEP_PROBE=1 to run."); - - var lines = new List(); - - foreach (var scenario in _scenarios) - { - foreach (var seed in _seedSizes) - { - var sample = await MeasureAsync(scenario, seed, transposed: true); - lines.Add( - $"transposed {scenario.Label,-11} N={seed,4} per-add(+realize) = {sample.BytesPerAdd,13:N0} bytes " + - $"gen0/add = {sample.Gen0PerAdd:F2}"); - } + await using var driver = await TransposedGridDriver.CreateAsync( + ScrollConfig, ScrollColumns, ScrollWindowWidth, ScrollWindowHeight); - foreach (var seed in _seedSizes) + var runner = new PerfScenarioRunner(); + var signals = await runner.MeasureAsync( + driver.SnapshotScope, + warmup: async () => { - var sample = await MeasureAsync(scenario, seed, transposed: false); - lines.Add( - $"canonical {scenario.Label,-11} N={seed,4} per-add(+realize) = {sample.BytesPerAdd,13:N0} bytes " + - $"gen0/add = {sample.Gen0PerAdd:F2}"); - } - - lines.Add(await DescribeRealizedColumnAsync(scenario, 60)); - - var jump = await MeasureViewportJumpAsync(scenario, 120); - var perColumn = jump.RealizedColumns == 0 ? 0 : jump.TotalBytes / jump.RealizedColumns; - lines.Add( - $"viewport-jump {scenario.Label,-11} N=120 total = {jump.TotalBytes,13:N0} bytes " + - $"realized-cols = {jump.RealizedColumns,3} per-realized-col = {perColumn,11:N0} bytes"); - - lines.Add(string.Empty); - } - - var report = string.Join(Environment.NewLine, lines); - _output.WriteLine(report); - File.WriteAllText(Path.Combine(Path.GetTempPath(), "semistep_ui_probe.txt"), report); - } - - // Host re-attach gate: during a scripted ~200-column scroll after warmup, counts how many FRESH - // TransposedColumnCellsHost instances get built. Pre-fix each recycle discards and rebuilds the host - // subtree, so ~36 new hosts appear per round-trip (the pre-fix baseline recorded in the plan). Post-fix - // the child is recycled in place, hosts persist, and steady-state scroll builds zero new hosts. - // - // Discriminating metric = new host instances after warmup, NOT AttachedToVisualTree on pre-realized - // hosts: pre-fix the host instance is REPLACED (not re-attached), so subscribing AttachedToVisualTree on - // the warmed-up hosts fires 0 either way. attachFirings is reported for context only. The public - // AttachedToVisualTree subscription (same technique the contract test uses for DetachedFromVisualTree) - // still catches any re-attach of a persisted host, which post-fix must also be 0. - [AvaloniaFact] - public async Task Report_HostReattach_IsZeroAfterFix() - { - Assert.SkipUnless( - Environment.GetEnvironmentVariable("SEMISTEP_PROBE") == "1", - "Measurement probe: set SEMISTEP_PROBE=1 to run."); - - var fixture = new UIFixture(); - await fixture.InitializeAsync(HostReattachConfig); - try - { - fixture.SeedRecipe(HostReattachColumns); - var surface = fixture.CreateTransposedSurface(); - surface.Initialize(); - var view = new TransposedRecipeGridView { DataContext = surface }; - var window = new Window { Width = HostReattachWindowWidth, Height = 800, Content = view }; - window.Show(); - Dispatcher.UIThread.RunJobs(); - - var listBox = view.FindControl("StepListBox")!; - - // Warm one round-trip so both scroll endpoints have realized before the seen-set snapshot. - listBox.ScrollIntoView(HostReattachHighColumn); - Dispatcher.UIThread.RunJobs(); - listBox.ScrollIntoView(HostReattachLowColumn); - Dispatcher.UIThread.RunJobs(); - - var seenHosts = new HashSet(ReferenceEqualityComparer.Instance); - var attachFirings = 0; - var newHostInstances = 0; - - void Discover() + // Scroll the full measured range once so the recycle pool reaches steady-state peak + // realization; end at the low endpoint so the workload's final realized set matches the + // snapshot instance-for-instance when recycling is clean. + await driver.ScrollToColumnAsync(ScrollHighColumn); + await driver.ScrollToColumnAsync(ScrollLowColumn); + }, + workload: async () => { - foreach (var host in listBox.GetVisualDescendants().OfType()) + for (var i = 0; i < ScrollRoundTrips; i++) { - if (seenHosts.Add(host)) - { - newHostInstances++; - host.AttachedToVisualTree += (_, _) => attachFirings++; - } + await driver.ScrollToColumnAsync(ScrollHighColumn); + await driver.ScrollToColumnAsync(ScrollLowColumn); } - } + }); - // Seed the seen-set with warmed-up hosts; these are not scroll-driven rebuilds. - Discover(); - newHostInstances = 0; + Report( + "semistep_scroll_gate.txt", + $"scroll round-trips: freshVisuals={signals.FreshVisualInstances} roundTrips={ScrollRoundTrips} " + + $"allocatedBytes={signals.AllocatedBytes:N0} gen0={signals.Gen0}"); - for (var i = 0; i < HostReattachRoundTrips; i++) - { - listBox.ScrollIntoView(HostReattachHighColumn); - Dispatcher.UIThread.RunJobs(); - Discover(); - - listBox.ScrollIntoView(HostReattachLowColumn); - Dispatcher.UIThread.RunJobs(); - Discover(); - } - - window.Close(); - - var report = - $"host-reattach after-fix: newHostInstances = {newHostInstances} " + - $"attachFirings(tracked) = {attachFirings} roundTrips = {HostReattachRoundTrips} " + - $"new-hosts/roundtrip = {(double)newHostInstances / HostReattachRoundTrips:F1}"; - _output.WriteLine(report); - File.WriteAllText(Path.Combine(Path.GetTempPath(), "semistep_host_reattach_after.txt"), report); - - newHostInstances.Should().Be( - 0, - "the recycle-in-place fix reuses host instances, so steady-state scroll builds no fresh hosts"); - attachFirings.Should().Be( - 0, - "persisted hosts stay attached across scroll, so AttachedToVisualTree must not re-fire"); - } - finally - { - await fixture.DisposeAsync(); - } + signals.FreshVisualInstances.Should().Be( + 0, + "a scrolled viewport must recycle containers in place after warmup; any fresh visual is a subtree " + + "rebuild regression"); } - // Counts the live editor controls a single realized transposed column instantiates, to size the - // always-live-editor cost: every ComboBox/TextBox is a full control, a TextBlock is cheap. - private static async Task DescribeRealizedColumnAsync(ProbeScenario scenario, int seed) + [AvaloniaFact(Explicit = true)] + public async Task ViewportJump_BytesPerColumn_WithinParity_AndBaseline() { - var fixture = new UIFixture(); - await fixture.InitializeAsync(scenario.ConfigName); - try + JumpSample transposed; + await using (var driver = await TransposedGridDriver.CreateAsync( + JumpConfig, JumpSeed, TransposedWindowWidth, TransposedWindowHeight)) { - fixture.SeedRecipe(seed); - var surface = fixture.CreateTransposedSurface(); - surface.Initialize(); - var view = new TransposedRecipeGridView { DataContext = surface }; - var window = new Window { Width = scenario.WindowWidth, Height = 800, Content = view }; - window.Show(); - Dispatcher.UIThread.RunJobs(); - - var listBox = view.FindControl("StepListBox")!; - var container = (ListBoxItem)listBox.ContainerFromIndex(0)!; - var descendants = container.GetVisualDescendants().ToList(); - var comboBoxes = descendants.OfType().Count(); - var textBoxes = descendants.OfType().Count(); - var textBlocks = descendants.OfType().Count(); - - window.Close(); - - return $"one realized column {scenario.Label}: cells={surface.StepColumns[0].Cells.Count} " + - $"ComboBox={comboBoxes} TextBox={textBoxes} TextBlock={textBlocks}"; + transposed = await MeasureViewportJumpMedianAsync(driver); } - finally + + JumpSample canonical; + await using (var driver = await CanonicalGridDriver.CreateAsync( + JumpConfig, JumpSeed, CanonicalWindowWidth, CanonicalWindowHeight)) { - await fixture.DisposeAsync(); + canonical = await MeasureViewportJumpMedianAsync(driver); } + + var transposedPerColumn = transposed.BytesPerColumn(); + var canonicalPerColumn = canonical.BytesPerColumn(); + + Report( + "semistep_viewport_jump.txt", + $"viewport-jump transposed: total={transposed.Bytes:N0} realized={transposed.RealizedColumns} " + + $"per-column={transposedPerColumn:N0}\n" + + $"viewport-jump canonical: total={canonical.Bytes:N0} realized={canonical.RealizedColumns} " + + $"per-column={canonicalPerColumn:N0}"); + + var baselines = PerfBaselines.Load(); + PerfMetricGate.AssertOrRecord( + _actuals, baselines, _output, PerfMetricNames.TransposedViewportJumpBytesPerColumn, transposedPerColumn); + PerfMetricGate.AssertOrRecord( + _actuals, baselines, _output, PerfMetricNames.CanonicalViewportJumpBytesPerColumn, canonicalPerColumn); + + // Both denominators must be non-zero. Guarding only canonical would let a transposed regression that + // stops realizing columns pass: transposedPerColumn would be 0, so parityRatio 0 <= cap trivially. + transposed.RealizedColumns.Should().BeGreaterThan( + 0, + "the transposed jump must realize a viewport of columns; zero realized is a virtualization regression, " + + "not a passing parity"); + canonical.RealizedColumns.Should().BeGreaterThan( + 0, + "the canonical jump must realize a viewport of rows for a meaningful parity denominator"); + transposedPerColumn.Should().BeGreaterThan( + 0, + "a zero transposed per-column numerator means the jump allocated nothing to realize its viewport; " + + "that is a broken measurement, not parity"); + + var parityRatio = transposedPerColumn / canonicalPerColumn; + _output.WriteLine($"[perf] viewport-jump parity ratio (transposed/canonical) = {parityRatio:F2}"); + + parityRatio.Should().BeLessThanOrEqualTo( + ParityRatioLimit, + $"a recycled transposed column rebind must stay within the honest {ParityRatioLimit}x cap of a " + + $"recycled canonical row; tighten this to {ParityRatioTargetAfterHostPoolDeletion}x once " + + "TransposedColumnCellsHost + pool are removed"); } - private static async Task MeasureAsync(ProbeScenario scenario, int seed, bool transposed) + [AvaloniaFact(Explicit = true)] + public async Task PerAdd_ScalesFlat_WithColumnCount() { - var fixture = new UIFixture(); - await fixture.InitializeAsync(scenario.ConfigName); - try - { - fixture.SeedRecipe(seed); - - Control view; - Action realize; - if (transposed) - { - var surface = fixture.CreateTransposedSurface(); - surface.Initialize(); - var transposedView = new TransposedRecipeGridView { DataContext = surface }; - view = transposedView; - // Scroll the ListBox's OWN (horizontal) scroll viewer to the end so the freshly - // appended column realizes. The outer UserControl scroll viewer is vertical and would - // leave the new column virtualized off-screen, measuring nothing. - realize = () => ScrollListBoxToEnd(transposedView, surface.StepColumns.Count); - } - else - { - var surface = fixture.CreateCanonicalSurface(); - surface.Initialize(); - view = new CanonicalRecipeGridView { DataContext = surface }; - realize = () => ScrollToEnd(view); - } - - var window = new Window { Width = scenario.WindowWidth, Height = 800, Content = view }; - window.Show(); - Dispatcher.UIThread.RunJobs(); - - // Warm the JIT, the template-application, and the virtualization paths so the measured - // samples are steady-state and the first-realize JIT cost is not charged to add #1. - for (var i = 0; i < WarmupAppends; i++) - { - AppendAndRealize(fixture, realize); - } - - var gen0Before = GC.CollectionCount(0); - var bytesBefore = GC.GetAllocatedBytesForCurrentThread(); - - for (var i = 0; i < MeasuredAppends; i++) - { - AppendAndRealize(fixture, realize); - } - - var bytes = GC.GetAllocatedBytesForCurrentThread() - bytesBefore; - var gen0 = GC.CollectionCount(0) - gen0Before; - - window.Close(); - - return new Sample(bytes / MeasuredAppends, (double)gen0 / MeasuredAppends); - } - finally - { - await fixture.DisposeAsync(); - } + var smallSeedPerAdd = await MeasurePerAddAsync(ScalingSmallSeed); + var largeSeedPerAdd = await MeasurePerAddAsync(ScalingLargeSeed); + + var scalingRatio = smallSeedPerAdd == 0 ? 0 : (double)largeSeedPerAdd / smallSeedPerAdd; + + Report( + "semistep_per_add_scaling.txt", + $"per-add N={ScalingSmallSeed}: {smallSeedPerAdd:N0} bytes\n" + + $"per-add N={ScalingLargeSeed}: {largeSeedPerAdd:N0} bytes\n" + + $"scaling ratio (N={ScalingLargeSeed}/N={ScalingSmallSeed}) = {scalingRatio:F2}"); + + var baselines = PerfBaselines.Load(); + PerfMetricGate.AssertOrRecord( + _actuals, baselines, _output, PerfMetricNames.TransposedPerAddBytesN20, smallSeedPerAdd); + PerfMetricGate.AssertOrRecord( + _actuals, baselines, _output, PerfMetricNames.TransposedPerAddBytesN120, largeSeedPerAdd); + + smallSeedPerAdd.Should().BeGreaterThan( + 0, + "appending a column must allocate something for its realization; a zero denominator is a broken " + + "measurement"); + + scalingRatio.Should().BeLessThanOrEqualTo( + ScalingRatioLimit, + "per-add allocation must stay flat with the existing column count; growth means an append " + + "re-touches every column in the recipe, not just the viewport-realized ones"); } - // Measures the allocation of a single far viewport jump (horizontal start -> end) after a warmup - // round-trip, so the measured jump reuses recycled containers — the steady-state the app hits when - // it auto-scrolls to a step added while the viewport is far from the insertion point. - private static async Task MeasureViewportJumpAsync(ProbeScenario scenario, int seed) + private static async Task MeasureViewportJumpMedianAsync(IRecipeGridDriver driver) { - var fixture = new UIFixture(); - await fixture.InitializeAsync(scenario.ConfigName); - try - { - fixture.SeedRecipe(seed); - var surface = fixture.CreateTransposedSurface(); - surface.Initialize(); - var view = new TransposedRecipeGridView { DataContext = surface }; - var window = new Window { Width = scenario.WindowWidth, Height = 800, Content = view }; - window.Show(); - Dispatcher.UIThread.RunJobs(); - - var listBox = view.FindControl("StepListBox")!; - var lastIndex = surface.StepColumns.Count - 1; - - // Warm the jump path with one full round-trip so the measured jump exercises container - // recycling, not first-time realization. - listBox.ScrollIntoView(lastIndex); - Dispatcher.UIThread.RunJobs(); - ScrollListBoxToStart(listBox); - Dispatcher.UIThread.RunJobs(); - - var bytesBefore = GC.GetAllocatedBytesForCurrentThread(); - listBox.ScrollIntoView(lastIndex); - Dispatcher.UIThread.RunJobs(); - var bytes = GC.GetAllocatedBytesForCurrentThread() - bytesBefore; - - var realizedColumns = listBox.GetRealizedContainers().Count(); - - window.Close(); - - return new JumpSample(bytes, realizedColumns); - } - finally + var samples = new List(ParitySampleCount); + for (var i = 0; i < ParitySampleCount; i++) { - await fixture.DisposeAsync(); + samples.Add(await MeasureViewportJumpAsync(driver)); } + + return samples.OrderBy(sample => sample.BytesPerColumn()).ElementAt(ParitySampleCount / 2); } - // Appends one step and scrolls the new column/row into view so the realization/layout pass for it - // is included in the measurement, mirroring the app auto-scrolling to a freshly added step. - private static void AppendAndRealize(UIFixture fixture, Action realize) + // One far viewport jump (start -> end) after a warmup round-trip, so the measured jump reuses recycled + // containers - the steady-state the app hits when auto-scrolling to a step added far from the viewport. + private static async Task MeasureViewportJumpAsync(IRecipeGridDriver driver) { - fixture.Coordinator.AppendStep(RecipeTestDriver.WaitActionId); - Dispatcher.UIThread.RunJobs(); - realize(); - Dispatcher.UIThread.RunJobs(); + var lastIndex = driver.ItemCount - 1; + var runner = new PerfScenarioRunner(); + var signals = await runner.MeasureAsync( + driver.SnapshotScope, + warmup: async () => + { + await driver.ScrollToColumnAsync(lastIndex); + await driver.ScrollToColumnAsync(0); + }, + workload: () => driver.ScrollToColumnAsync(lastIndex)); + + return new JumpSample(signals.AllocatedBytes, driver.RealizedIndices.Count); } - private static void ScrollListBoxToEnd(TransposedRecipeGridView view, int count) + // Per-add bytes at a fixed seed: warm the append+realize path, then measure a fixed run of appends, each + // followed by a scroll to the new last column so the realization/layout pass is charged to the add. + private static async Task MeasurePerAddAsync(int seed) { - var listBox = view.FindControl("StepListBox"); - if (listBox is null || count == 0) - { - return; - } + await using var driver = await TransposedGridDriver.CreateAsync( + ScalingConfig, seed, ScalingWindowWidth, ScalingWindowHeight); - listBox.ScrollIntoView(count - 1); + var runner = new PerfScenarioRunner(); + var signals = await runner.MeasureAsync( + driver.SnapshotScope, + warmup: () => AppendAndRealizeAsync(driver, WarmupAppends), + workload: () => AppendAndRealizeAsync(driver, MeasuredAppends)); - var scrollViewer = listBox.GetVisualDescendants().OfType().FirstOrDefault(); - if (scrollViewer is not null) - { - scrollViewer.Offset = new Vector(scrollViewer.Extent.Width, 0); - } + return signals.AllocatedBytes / MeasuredAppends; } - private static void ScrollListBoxToStart(ListBox listBox) + private static async Task AppendAndRealizeAsync(IRecipeGridDriver driver, int count) { - var scrollViewer = listBox.GetVisualDescendants().OfType().FirstOrDefault(); - if (scrollViewer is not null) + for (var i = 0; i < count; i++) { - scrollViewer.Offset = new Vector(0, 0); + await driver.AddStepsAsync(1); + await driver.ScrollToColumnAsync(driver.ItemCount - 1); } } - private static void ScrollToEnd(Control view) + private void Report(string fileName, string report) { - var scrollViewer = view.GetVisualDescendants().OfType().FirstOrDefault(); - if (scrollViewer is not null) + _output.WriteLine(report); + File.WriteAllText(Path.Combine(Path.GetTempPath(), fileName), report); + } + + private readonly record struct JumpSample(long Bytes, int RealizedColumns) + { + public double BytesPerColumn() { - scrollViewer.Offset = new Vector(0, scrollViewer.Extent.Height); + return RealizedColumns == 0 ? 0 : (double)Bytes / RealizedColumns; } } - - private readonly record struct Sample(long BytesPerAdd, double Gen0PerAdd); - - private readonly record struct JumpSample(long TotalBytes, int RealizedColumns); - - private sealed record ProbeScenario(string Label, string ConfigName, int WindowWidth); }