From d2fab1e08f231fe2e23540f3b25b2650cd416d09 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 09:03:09 +0000 Subject: [PATCH 1/5] Add a Public231 fast-screen before full C1 Smoke still comes first. A 60-request screen can drop a tip when both p50 and p95 are at least 2% or 0.5 ms worse than the current primary, so a clear regression does not spend a full three-round C1. KEEP stays on the full dual gate. Co-authored-by: Hongsheng Liu --- .agents/skills/new-model-integration/SKILL.md | 11 +- .../skills/optimize-remeasure-gate/SKILL.md | 81 ++++- .agents/skills/public231-c1/SKILL.md | 122 ++++++- benchmarks/README.md | 21 ++ benchmarks/bench.py | 321 ++++++++++++++++-- tests/benchmarks/test_bench.py | 235 +++++++++++++ 6 files changed, 732 insertions(+), 59 deletions(-) diff --git a/.agents/skills/new-model-integration/SKILL.md b/.agents/skills/new-model-integration/SKILL.md index 6a9e839a..d95a8938 100644 --- a/.agents/skills/new-model-integration/SKILL.md +++ b/.agents/skills/new-model-integration/SKILL.md @@ -87,10 +87,13 @@ that: do not invent a C1 result. CPU contract tests are not a latency baseline. Further latency work stays on this model's draft branch. Run it under [optimize-remeasure-gate](../optimize-remeasure-gate/SKILL.md): paste the C1 -p50/p95, `device`, and `compute capability` first, and do not stack the next -tip commit until that tip is remeasured with the same protocol on the same -named hardware. Incomplete evidence is INCONCLUSIVE. Quality and contract -checks stay green. Each tip states its RSI phase (bring-up/scheduling, +p50/p95, `device`, and `compute capability` first. After that primary +exists, smoke and the fast-screen run before another full C1; a clear +regression is dropped there and is not a KEEP. Do not stack the next tip +commit until that tip has been dropped by the screen or remeasured with +full C1 on the same named hardware. Incomplete evidence is INCONCLUSIVE. +Quality and contract checks stay green. Each tip states its RSI phase +(bring-up/scheduling, parallelism/comms, kernel, or launch/post) and which alternatives were considered versus the primary; without that it is not merge-clean. Feed each delta back to the agent before the next edit. diff --git a/.agents/skills/optimize-remeasure-gate/SKILL.md b/.agents/skills/optimize-remeasure-gate/SKILL.md index 6dd861f8..92d75c98 100644 --- a/.agents/skills/optimize-remeasure-gate/SKILL.md +++ b/.agents/skills/optimize-remeasure-gate/SKILL.md @@ -1,6 +1,6 @@ --- name: optimize-remeasure-gate -description: Gate continuous optimization after a measured Public231 C1 baseline, including Cloud Agent and coding-agent loops. Use when a new tip must not land until that tip is remeasured with the same protocol on the same recorded device and compute capability. Every paste includes those two fields. The documented production target is H200 / SM90. Each tip names its RSI phase and the primary change versus alternatives considered, or it is not merge-clean. Incomplete evidence is INCONCLUSIVE. The pull request stays draft until a separate owner yes. +description: Gate continuous optimization after a measured Public231 C1 baseline, including Cloud Agent and coding-agent loops. Use when a new tip must not land until smoke passes and a fast-screen either drops a clear regression or the tip is remeasured with full C1 on the same recorded device and compute capability. Every paste includes those two fields. The documented production target is H200 / SM90. Each tip names its RSI phase and the primary change versus alternatives considered, or it is not merge-clean. Incomplete evidence is INCONCLUSIVE. A fast-screen cannot KEEP a tip. The pull request stays draft until a separate owner yes. --- # Optimize only against a remeasured tip @@ -35,15 +35,16 @@ Those are the primary figures. Per-round values stay attached so a pooled number can be checked. A recalled or rounded figure, a single round, or a percentile averaged across rounds is not a baseline. -## One tip, then remeasure +## One tip, then screen, then full C1 A tip is one optimization commit on the draft branch. Do not add another -commit on top of it until that tip has its own Public231 C1 remeasure. The -gate requires the same measurement protocol and the same named hardware as -the baseline. It does not require the SKU to be H200, L20X, SM90, or sm_89. +commit on top of it until that tip has either been dropped by the fast-screen +or has its own full Public231 C1 remeasure. The full gate requires the same +measurement protocol and the same named hardware as the baseline. It does not +require the SKU to be H200, L20X, SM90, or sm_89. -Same measurement protocol means warmup 5, concurrency 1, three measured -rounds, the same manifest, and the same clock +Same measurement protocol for that full remeasure means warmup 5, concurrency +1, three measured rounds, the same manifest, and the same clock (`post_to_body_json_and_schema_validation`). Same named hardware means the same recorded `device` string and the same recorded `compute capability`. The remeasure SHA must be the tip SHA. A measurement of an older commit does @@ -54,13 +55,51 @@ cache policy, thinking treatment, server shape). State any necessary difference. Do not reset the comparison by changing the clock, the quantile method, or the named hardware. +## Fast-screen before full C1 + +Follow [public231-c1](../public231-c1/SKILL.md). Smoke must pass before the +screen. Run one fast-screen: 60 requests, seed 231, concurrency 1, warmup 5, +bucket quotas 0–127:10, 128–255:10, 256–511:8, 512–1023:8, 1024–2047:8, +2048+:16. Compare its p50 and p95 to this model's current full-C1 primary, +on the same device, compute capability, and clock. + +The StartLux Instant primary named for this gate, until a full-C1 KEEP +replaces it, is runtime revision +`51f80ad681c8f7b1be9b22b5e700bef34e33534c` (`51f80ad`), pooled p50 7.338 ms, +pooled p95 51.816 ms. Clef and pplx-decider keep the primaries recorded in +their recipes and in the latency leaderboard. Do not reuse the StartLux +Instant pair for those models. A primary moves only when a later full C1 +KEEPs. + +Early DROP only when both Δp50 and Δp95 are worse. A metric is worse when +its delta (tip minus baseline) is ≥ +2% of that baseline or ≥ +0.5 ms. +Otherwise, including a mixed result, run full C1. +`python benchmarks/bench.py run --fast-screen` records `drop`, `full_c1`, +or `inconclusive`. It does not record a KEEP. + +A `drop` is a regression. Do not run full C1. Do not stack the next commit +on that tip. Do not adopt the screen p50/p95 as the next baseline. A +negative screen delta is not a gain and is not a KEEP. An `inconclusive` +screen stops the loop. + +KEEP, merge, and ready-for-review require the full C1 dual gate: pooled p50 +and pooled p95 both strictly lower, pooled 2048+ p95 strictly lower, and no +quality collapse under the rules in the next section. The +2% / +0.5 ms +band does not make a full-C1 tie a KEEP. + ## INCONCLUSIVE stops the loop Mark the tip **INCONCLUSIVE** and do not stack the next commit when any of these are true: - the baseline p50/p95 were not recorded before the edit -- fewer than three measured rounds finished +- smoke did not pass before the latency screen +- the fast-screen was skipped on a tip that already has a full-C1 primary +- the fast-screen decision is `inconclusive` +- a KEEP, a gain, or a new primary is claimed from fast-screen numbers +- the fast-screen decision was `full_c1` but fewer than three measured + rounds finished +- a completed `drop` is treated as a KEEP or as the next baseline - `evidence_complete` is not true, or warmup failed - any measured request failed, was incomplete, or was unattempted - `device` or `compute capability` is missing on the baseline or the tip @@ -76,7 +115,8 @@ these are true: INCONCLUSIVE is not a win and not a regression. Do not drop failures, relax a tolerance, or add undeclared rounds to produce a preferred delta. A recorded `device` other than H200, or a `compute capability` other than SM90, is not -by itself INCONCLUSIVE. +by itself INCONCLUSIVE. A fast-screen `drop` is a regression. It is not +INCONCLUSIVE, and it is not a reason to relax the rule that produced it. ## Quality and contract stay green @@ -98,16 +138,22 @@ stack past it. Before the agent proposes the next change, give it this record: - baseline SHA, pooled p50, pooled p95, pass rate -- tip SHA, pooled p50, pooled p95, pass rate -- p50 and p95 delta (tip minus baseline), in milliseconds +- tip SHA; pooled p50, pooled p95, and pass rate when full C1 ran +- full-C1 p50 and p95 delta (tip minus baseline), in milliseconds, when + full C1 ran - clock basis, `device`, and `compute capability` for both sides - RSI phase, alternatives considered, and which change is primary +- fast-screen decision (`drop`, `full_c1`, or `inconclusive`) and, when the + screen ran, its p50/p95 deltas labeled as screen-only - verdict: improved, regressed, or INCONCLUSIVE -- quality/contract result +- quality/contract result when the path reached full C1 or a KEEP decision -The agent uses that record as the next baseline only when the verdict is -improved or regressed and the evidence is complete. An INCONCLUSIVE tip stays -the head until it is fixed and remeasured, or until the tip is reverted. +Improved is only a full-C1 KEEP. A `drop` verdict is regressed, and the +screen numbers stay out of the next baseline. The agent uses a full-C1 +record as the next baseline only when the verdict is improved or regressed +and the evidence is complete. An INCONCLUSIVE tip stays the head until it +is fixed and remeasured, or until the tip is reverted. A dropped tip is +reverted or left unstacked; the previous primary stays the baseline. ## RSI phase, or the tip is not merge-clean @@ -127,8 +173,9 @@ fill in a missing phase. ## Draft, and a separate yes to merge Leave the pull request in draft through the loop. Do not mark it ready for -review and do not merge it because a remeasure improved p50 or p95. Merge -and ready-for-review each need a separate explicit yes from the owner. +review and do not merge it because a remeasure improved p50 or p95, or +because a fast-screen was not a drop. Merge and ready-for-review each need +a separate explicit yes from the owner, and only after a full-C1 KEEP. Passing CI is not that yes. A tip that is not merge-clean on the RSI phase stays unmerged even when the owner has not yet been asked. Room posts of a delta follow the metrics-only rule in [public231-c1](../public231-c1/SKILL.md) diff --git a/.agents/skills/public231-c1/SKILL.md b/.agents/skills/public231-c1/SKILL.md index 75aa1f8e..2b71b38d 100644 --- a/.agents/skills/public231-c1/SKILL.md +++ b/.agents/skills/public231-c1/SKILL.md @@ -1,6 +1,6 @@ --- name: public231-c1 -description: Measure Public231 C1 request latency for a model or pull request. Use when a campaign needs warmup 5, concurrency 1, three measured rounds, SHA, pass rate, and p50/p95 on the bench.py clock. Every paste records measured device and compute capability. The documented production target is H200 / SM90; a different measured identity stays a valid run and pairs only with that same recorded identity. Includes how to treat a thinking field and what a room post may contain. +description: Measure Public231 C1 request latency for a model or pull request. Use when a campaign needs the contract smoke, a 60-request fast-screen that can drop a clear regression, then warmup 5, concurrency 1, three measured rounds, SHA, pass rate, and p50/p95 on the bench.py clock. A fast-screen is not a KEEP and is not a gain. Every paste records measured device and compute capability. The documented production target is H200 / SM90; a different measured identity stays a valid run and pairs only with that same recorded identity. Includes how to treat a thinking field and what a room post may contain. --- # Public231 C1 @@ -13,6 +13,111 @@ concurrency 1. Read [the benchmark protocol](../../../benchmarks/README.md) and This skill records a measurement. It does not authorize a merge, a hardware relabel, or a room post that contains operator metadata. +## Order of operations + +1. **Smoke.** Unchanged. `benchmarks/smoke.jsonl` is the contract smoke, not + this campaign. `python benchmarks/bench.py validate benchmarks/smoke.jsonl` + must succeed before any latency screen. A model recipe that already runs a + server contract smoke still runs that smoke. A failed smoke is not a + latency result and is not a DROP. +2. **Fast-screen.** One 60-request round after smoke. DROP the tip when both + p50 and p95 are clearly worse than that model's current full-C1 primary. + Otherwise run full C1. The screen never KEEPs a tip. +3. **Full Public231 C1.** Unchanged measurement: warmup 5, concurrency 1, + three measured rounds. KEEP only on the full dual gate below. + +## Fast-screen + +Use this on the frozen Public231 manifest after smoke, before spending a +full C1. The manifest cases carry a nonnegative integer `prompt_tokens` next +to `id` / `request` / `expected`. That field is the model's prompt-token +count. `wire_request` does not send it. A file without it cannot be +screened. Do not invent lengths. + +Sample with `benchmarks/bench.py run --fast-screen`. The draw is fixed: + +- seed **231** +- rank is SHA-256 of the UTF-8 string `{seed}\n{bucket}\n{id}` +- each bucket keeps its quota of lowest ranks +- selected requests are replayed in original manifest order +- one invocation, one fresh output directory, one measured round +- concurrency 1, warmup 5 (warmup stays out of the percentiles) +- clock `post_to_body_json_and_schema_validation` +- same recorded `device`, `compute capability`, precision, cache policy, + and thinking treatment as the primary + +Bucket quotas (inclusive lower bound, exclusive upper bound): + +| Bucket | Quota | +|---|---:| +| 0–127 | 10 | +| 128–255 | 10 | +| 256–511 | 8 | +| 512–1023 | 8 | +| 1024–2047 | 8 | +| 2048+ | 16 | + +That is 60 requests. 2048+ is the overweight. A bucket with fewer cases than +its quota is not a screen; fix the length index or stop. Do not fill the +quota from another bucket. Do not run a second screen round and pool it. + +Compare the screen's successful p50 and p95 to the current full-C1 primary +for **this** model. Pass that primary with `--baseline-p50-ms` and +`--baseline-p95-ms`. The StartLux Instant primary named for this gate, until +a full-C1 KEEP replaces it, is runtime revision +`51f80ad681c8f7b1be9b22b5e700bef34e33534c` (`51f80ad`), pooled p50 7.338 ms, +pooled p95 51.816 ms. Clef and pplx-decider keep the primaries recorded in +their recipes and in [the latency leaderboard](../../../benchmarks/latency-leaderboard.md). +Do not paste the StartLux Instant pair onto Clef or pplx-decider. + +```sh +python benchmarks/bench.py run \ + --endpoint --model \ + --metadata --phase measured \ + --concurrency 1 --warmup 5 \ + --fast-screen --baseline-p50-ms --baseline-p95-ms \ + --output +``` + +Delta is tip minus baseline, in milliseconds. A percentile is **worse** when +its delta is ≥ +2% of that baseline value or ≥ +0.5 ms. The runner evaluates +that boundary as `tip >= baseline * 1.02` or `tip >= baseline + 0.5`, so an +exact threshold stays worse. Early **DROP** only when **both** p50 and p95 +are worse. Mixed results and results inside that band proceed to full C1. +The saved summary field `fast_screen_decision` is `drop`, `full_c1`, or +`inconclusive`. It is never `keep`. + +`drop` means do not run full C1. `inconclusive` means any measured request +failed, was incomplete, or was unattempted, warmup failed, or +`evidence_complete` is not true. An inconclusive screen is not a DROP and +not permission to continue. A negative screen delta is not a gain, not a +new primary, and not a KEEP. Do not claim a speedup from fast-screen +numbers. Percentiles use the same median and nearest-rank p95 as full C1, +on this one round of 60, not an average of rounds. + +## Full C1 KEEP + +The measurement commands in [Measure](#measure) stay the full gate: warmup +5, concurrency 1, three rounds, then recompute pooled p50 and p95 from the +concatenated successful samples. KEEP requires all of: + +- pooled p50 strictly lower than the primary +- pooled p95 strictly lower than the primary +- pooled **2048+** p95 strictly lower than the primary's pooled 2048+ p95 +- no quality collapse: serving pass rate not lower, and the quality and + contract checks in + [optimize-remeasure-gate](../optimize-remeasure-gate/SKILL.md) stay green + +When every case has `prompt_tokens`, each round's summary includes +`latency_by_prompt_bucket`. Pool 2048+ the same way as the campaign p95: +concatenate successful `latency_ms` whose `prompt_tokens` is ≥ 2048 across +the three rounds, then take nearest-rank p95. Do not average the three +bucket percentiles. The fast-screen's 16 long requests are not this figure. +A manifest without `prompt_tokens` omits the bucket report; do not KEEP +until the same lengths are joined offline and that pooled p95 is shown. +Equal latency is not "down" and is not a KEEP. The +2% / +0.5 ms band is +only the early DROP rule. It is not a KEEP tolerance. + ## Measured hardware The production GPU, when a document is naming that intended target, is @@ -99,6 +204,11 @@ text campaign. ## Measure +This is the full Public231 C1, run only after smoke and only when the +fast-screen decision is `full_c1` (or when establishing a model's first +primary, which has no screen baseline yet). A fast-screen `drop` stops +here. + `bench.py` defaults `--warmup` to 3. This protocol passes **5** on every measured round. The historical H800 board warmed up once for all three traversals and used a different clock; do not copy that shape. Invoke `run` @@ -147,6 +257,12 @@ Publish: incomplete, and unattempted counts - `evidence_complete` from each `summary.json` +A fast-screen paste is a separate record. Label it fast-screen, and include +the decision (`drop`, `full_c1`, or `inconclusive`), seed 231, both deltas, +the primary SHA those deltas use, and the same `device` and +`compute capability`. Do not publish that paste as a C1 baseline or as a +speedup. + Labelled choice/noul accuracy and score MAE in `quality_on_successful_requests` are conditional on successes. Report them separately. They are not the serving pass rate, and a fast success subset is @@ -166,4 +282,6 @@ p50/p95, plus INCONCLUSIVE when that is the result. A production-target example is `device: H200` and `compute capability: SM90`; if the host string differs, post the recorded pair instead. Omit hostnames, SSH targets, usernames, filesystem paths, and reservation-tool -names. Local output directories stay local. +names. Local output directories stay local. A fast-screen room post uses the +same omission rule and the fast-screen label above. It does not announce a +KEEP. diff --git a/benchmarks/README.md b/benchmarks/README.md index 807cc777..dd539383 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -121,6 +121,27 @@ traversals at concurrency 1, with five warmup requests per traversal and the sam running server. Invoke `run` separately for each output directory. Repeating the manifest three times still represents 231 independent tasks, not 693 tasks. +A Public231 tip that already has a full-C1 primary can be dropped before those +three traversals. Validate `benchmarks/smoke.jsonl` first. `--fast-screen` then +samples 60 requests from a manifest whose cases include integer `prompt_tokens` +(not sent on the wire): quotas 10, 10, 8, 8, 8, and 16 for buckets 0–127, +128–255, 256–511, 512–1023, 1024–2047, and 2048+. Seed 231, one measured round, +concurrency 1, warmup 5. Pass the current primary with `--baseline-p50-ms` and +`--baseline-p95-ms`. The summary field `fast_screen_decision` is `drop` only +when both p50 and p95 are worse by ≥ +2% or ≥ +0.5 ms, `full_c1` when they are +not, and `inconclusive` when the round is incomplete. The field is never a +KEEP. The gate, including the StartLux Instant primary and the Clef and +pplx-decider pairing rules, is `.agents/skills/public231-c1/SKILL.md`. + +```sh +python benchmarks/bench.py run requests.jsonl \ + --endpoint http://127.0.0.1:8000/v1/systemone --model english \ + --metadata run-metadata.json --phase measured \ + --concurrency 1 --warmup 5 \ + --fast-screen --baseline-p50-ms --baseline-p95-ms \ + --output /tmp/public231-fast-screen +``` + By default warmup cycles through the measured manifest. Use `--warmup-manifest warmup-requests.jsonl` to supply independent labelled requests in the same format. They are saved with their checksum and never enter measured diff --git a/benchmarks/bench.py b/benchmarks/bench.py index 4bd60447..fa8a68f8 100755 --- a/benchmarks/bench.py +++ b/benchmarks/bench.py @@ -37,6 +37,22 @@ def response_digest(text): RESPONSE_FORMAT = "httpx-content-base64-utf8-v1" LEGACY_RESPONSE_FORMAT = "httpx-text-v1" +# Public231 fast-screen. One measured round samples these prompt-token buckets +# (inclusive lower bound, exclusive upper bound; 2048+ is unbounded). Selection +# rank is SHA-256 of "{seed}\\n{bucket}\\n{id}"; the quota of lowest ranks is +# replayed in manifest order. Seed 231 is the fixed draw. +FAST_SCREEN_SEED = 231 +FAST_SCREEN_RELATIVE = 0.02 +FAST_SCREEN_ABSOLUTE_MS = 0.5 +FAST_SCREEN_BUCKETS = ( + ("0-127", 0, 128, 10), + ("128-255", 128, 256, 10), + ("256-511", 256, 512, 8), + ("512-1023", 512, 1024, 8), + ("1024-2047", 1024, 2048, 8), + ("2048+", 2048, None, 16), +) + def saved_response_format(config): value = config.get("response_format", LEGACY_RESPONSE_FORMAT) @@ -246,6 +262,179 @@ def read_answers( return normalized +def nearest_rank_p95(samples): + if not samples: + return None + ordered = sorted(samples) + return ordered[math.ceil(0.95 * len(ordered)) - 1] + + +def prompt_token_count(case): + if "prompt_tokens" not in case: + raise ValueError( + f"case {case['id']!r} needs integer prompt_tokens for fast-screen" + ) + value = case["prompt_tokens"] + if type(value) is not int or value < 0: + raise ValueError( + f"case {case['id']!r} prompt_tokens must be a nonnegative integer" + ) + return value + + +def fast_screen_bucket(tokens): + for name, lo, hi, _quota in FAST_SCREEN_BUCKETS: + if tokens >= lo and (hi is None or tokens < hi): + return name + raise ValueError(f"prompt_tokens {tokens} is outside fast-screen buckets") + + +def _fast_screen_rank(seed, bucket, case_id): + return hashlib.sha256(f"{seed}\n{bucket}\n{case_id}".encode()).hexdigest() + + +def fast_screen_sample(cases, seed=FAST_SCREEN_SEED): + """Pick the fixed 60-request screen. Returns cases in manifest order.""" + grouped = {name: [] for name, *_rest in FAST_SCREEN_BUCKETS} + for case in cases: + grouped[fast_screen_bucket(prompt_token_count(case))].append(case) + chosen = set() + counts = {} + for name, _lo, _hi, quota in FAST_SCREEN_BUCKETS: + pool = grouped[name] + if len(pool) < quota: + raise ValueError( + f"fast-screen bucket {name} needs {quota} requests; found {len(pool)}" + ) + ranked = sorted( + pool, + key=lambda case: (_fast_screen_rank(seed, name, case["id"]), case["id"]), + ) + picked = ranked[:quota] + counts[name] = len(picked) + chosen.update(case["id"] for case in picked) + return [case for case in cases if case["id"] in chosen], counts + + +def latency_clearly_worse(baseline_ms, tip_ms): + """True when tip - baseline is at least +2% or +0.5 ms.""" + if ( + type(baseline_ms) is bool + or type(tip_ms) is bool + or not isinstance(baseline_ms, (int, float)) + or not isinstance(tip_ms, (int, float)) + or not math.isfinite(baseline_ms) + or not math.isfinite(tip_ms) + or baseline_ms <= 0 + ): + raise ValueError( + "fast-screen latencies must be finite and the baseline must be positive" + ) + # Compare on the tip side so an exact +2% or +0.5 ms boundary stays worse. + # Subtracting first can land just below the threshold for the same decimal. + return tip_ms >= baseline_ms * ( + 1 + FAST_SCREEN_RELATIVE + ) or tip_ms >= baseline_ms + FAST_SCREEN_ABSOLUTE_MS + + +def fast_screen_decision(baseline_p50, baseline_p95, tip_p50, tip_p95): + """DROP only when both percentiles are clearly worse. Never returns keep.""" + p50_worse = latency_clearly_worse(baseline_p50, tip_p50) + p95_worse = latency_clearly_worse(baseline_p95, tip_p95) + if p50_worse and p95_worse: + return "drop" + return "full_c1" + + +def latency_by_prompt_bucket(cases, records): + """Per-bucket p50/p95 when every case carries prompt_tokens. Else None.""" + if any("prompt_tokens" not in case for case in cases): + return None + by_id = {case["id"]: prompt_token_count(case) for case in cases} + grouped = {name: [] for name, *_rest in FAST_SCREEN_BUCKETS} + for record in records: + if record.get("latency_ms") is None: + continue + grouped[fast_screen_bucket(by_id[record["id"]])].append(record["latency_ms"]) + report = {} + for name, _lo, _hi, _quota in FAST_SCREEN_BUCKETS: + samples = grouped[name] + report[name] = { + "samples": len(samples), + "p50_ms": statistics.median(samples) if samples else None, + "p95_ms": nearest_rank_p95(samples), + } + return report + + +def _require_fast_screen_run(args): + if args.phase != "measured": + raise ValueError("fast-screen requires --phase measured") + if args.concurrency != 1: + raise ValueError("fast-screen requires --concurrency 1") + if args.warmup != 5: + raise ValueError("fast-screen requires --warmup 5") + baseline_p50 = getattr(args, "baseline_p50_ms", None) + baseline_p95 = getattr(args, "baseline_p95_ms", None) + if baseline_p50 is None or baseline_p95 is None: + raise ValueError( + "fast-screen requires --baseline-p50-ms and --baseline-p95-ms" + ) + latency_clearly_worse(baseline_p50, baseline_p50) + latency_clearly_worse(baseline_p95, baseline_p95) + + +def manifest_subset_bytes(path, selected_ids): + lines = [line for line in path.read_text().splitlines() if line.strip()] + by_id = {} + for line in lines: + item = json.loads(line) + by_id[item["id"]] = line + missing = [case_id for case_id in selected_ids if case_id not in by_id] + if missing: + raise ValueError("fast-screen selection is missing manifest lines") + return "".join(by_id[case_id] + "\n" for case_id in selected_ids).encode() + + +def _check_saved_fast_screen(screen, cases): + if screen.get("seed") != FAST_SCREEN_SEED: + raise ValueError("unsupported fast-screen seed") + if ( + screen.get("relative_worse") != FAST_SCREEN_RELATIVE + or screen.get("absolute_worse_ms") != FAST_SCREEN_ABSOLUTE_MS + ): + raise ValueError("fast-screen thresholds do not match this runner") + expected = {name: quota for name, _lo, _hi, quota in FAST_SCREEN_BUCKETS} + if screen.get("bucket_counts") != expected: + raise ValueError("fast-screen bucket counts do not match the protocol") + if "baseline_p50_ms" not in screen or "baseline_p95_ms" not in screen: + raise ValueError("saved fast-screen is missing its baseline") + if screen.get("selected_ids") != [case["id"] for case in cases]: + raise ValueError("fast-screen selection does not match the saved manifest") + if len(cases) != sum(expected.values()): + raise ValueError("fast-screen manifest is not 60 requests") + + +def _saved_fast_screen_decision(screen, summary): + complete = ( + summary["evidence_complete"] + and summary["failed_requests"] == 0 + and summary["incomplete_requests"] == 0 + and summary["unattempted_requests"] == 0 + and summary["successful_requests"] == summary["planned_requests"] + and summary["successful_latency_p50_ms"] is not None + and summary["successful_latency_p95_ms"] is not None + ) + if not complete: + return "inconclusive" + return fast_screen_decision( + screen["baseline_p50_ms"], + screen["baseline_p95_ms"], + summary["successful_latency_p50_ms"], + summary["successful_latency_p95_ms"], + ) + + def quality(cases, records): values = {"choice_accuracy": [], "noul_accuracy": [], "score_mae": []} by_id = {r["id"]: r for r in records} @@ -299,7 +488,7 @@ def summarize(cases, records, elapsed, attempted_ids=None): for record in failed: key = record["error_kind"] errors[key] = errors.get(key, 0) + 1 - return { + result = { "requests": len(records), "planned_requests": len(cases), "attempted_requests": len(attempted), @@ -320,19 +509,17 @@ def summarize(cases, records, elapsed, attempted_ids=None): "successful_latency_p50_ms": statistics.median(latencies) if latencies else None, - "successful_latency_p95_ms": latencies[math.ceil(0.95 * len(latencies)) - 1] - if latencies - else None, + "successful_latency_p95_ms": nearest_rank_p95(latencies), "failed_latency_p50_ms": statistics.median(failed_latencies) if failed_latencies else None, - "failed_latency_p95_ms": failed_latencies[ - math.ceil(0.95 * len(failed_latencies)) - 1 - ] - if failed_latencies - else None, + "failed_latency_p95_ms": nearest_rank_p95(failed_latencies), "quality_on_successful_requests": quality(cases, records), } + buckets = latency_by_prompt_bucket(cases, successful) + if buckets is not None: + result["latency_by_prompt_bucket"] = buckets + return result def wire_request(case, model, protocol): @@ -582,8 +769,6 @@ async def run(args, cases): "validation_protocol": VALIDATION_PROTOCOL, } ) - warmup_manifest = getattr(args, "warmup_manifest", None) - warmup_cases = load_cases(warmup_manifest) if warmup_manifest else cases metadata = json.loads(args.metadata.read_text()) for name in ( "gpu", @@ -599,6 +784,37 @@ async def run(args, cases): ): if not metadata.get(name): raise ValueError(f"metadata requires {name}") + fast_screen = getattr(args, "fast_screen", False) + baseline_p50 = getattr(args, "baseline_p50_ms", None) + baseline_p95 = getattr(args, "baseline_p95_ms", None) + screen_record = None + if fast_screen: + _require_fast_screen_run(args) + cases, bucket_counts = fast_screen_sample(cases) + selected_ids = [case["id"] for case in cases] + screen_record = { + "seed": FAST_SCREEN_SEED, + "relative_worse": FAST_SCREEN_RELATIVE, + "absolute_worse_ms": FAST_SCREEN_ABSOLUTE_MS, + "buckets": [ + { + "name": name, + "min_tokens": lo, + "max_tokens_exclusive": hi, + "quota": quota, + } + for name, lo, hi, quota in FAST_SCREEN_BUCKETS + ], + "selected_ids": selected_ids, + "bucket_counts": bucket_counts, + "parent_manifest_sha256": digest(args.manifest), + "baseline_p50_ms": args.baseline_p50_ms, + "baseline_p95_ms": args.baseline_p95_ms, + } + elif baseline_p50 is not None or baseline_p95 is not None: + raise ValueError("baseline percentiles require --fast-screen") + warmup_manifest = getattr(args, "warmup_manifest", None) + warmup_cases = load_cases(warmup_manifest) if warmup_manifest else cases # A case the adapter cannot wire fails before any HTTP exchange; reject it # before creating the output directory so a saved archive stays summarizable. for case in cases if warmup_cases is cases else [*cases, *warmup_cases]: @@ -609,32 +825,34 @@ async def run(args, cases): f"case {case['id']!r} is incompatible with the {protocol} " f"wire protocol: {exc}" ) from exc + if fast_screen: + request_bytes = manifest_subset_bytes(args.manifest, selected_ids) + else: + request_bytes = args.manifest.read_bytes() args.output.mkdir(parents=True, exist_ok=False) - write_json( - args.output / "config.json", - { - "manifest_sha256": digest(args.manifest), - "runner_sha256": digest(Path(__file__)), - "metadata": metadata, - "validation_protocol": version, - "score_decimals": score_decimals, - "protocol": protocol, - "response_format": RESPONSE_FORMAT, - "endpoint": args.endpoint, - "model": args.model, - "phase": args.phase, - "concurrency": args.concurrency, - "warmup": args.warmup, - "warmup_manifest_sha256": digest(warmup_manifest) - if warmup_manifest - else None, - "timeout_seconds": args.timeout, - "latency_basis": "post_to_body_json_and_schema_validation", - "python": platform.python_version(), - "httpx": httpx.__version__, - }, - ) - (args.output / "requests.jsonl").write_bytes(args.manifest.read_bytes()) + config = { + "manifest_sha256": hashlib.sha256(request_bytes).hexdigest(), + "runner_sha256": digest(Path(__file__)), + "metadata": metadata, + "validation_protocol": version, + "score_decimals": score_decimals, + "protocol": protocol, + "response_format": RESPONSE_FORMAT, + "endpoint": args.endpoint, + "model": args.model, + "phase": args.phase, + "concurrency": args.concurrency, + "warmup": args.warmup, + "warmup_manifest_sha256": digest(warmup_manifest) if warmup_manifest else None, + "timeout_seconds": args.timeout, + "latency_basis": "post_to_body_json_and_schema_validation", + "python": platform.python_version(), + "httpx": httpx.__version__, + } + if screen_record is not None: + config["fast_screen"] = screen_record + write_json(args.output / "config.json", config) + (args.output / "requests.jsonl").write_bytes(request_bytes) if warmup_manifest: (args.output / "warmup-requests.jsonl").write_bytes( warmup_manifest.read_bytes() @@ -737,6 +955,9 @@ def summarize_saved(path): else: records = [json.loads(line) for line in responses_path.read_text().splitlines()] cases = load_cases(path / "requests.jsonl") + screen = config.get("fast_screen") + if screen is not None: + _check_saved_fast_screen(screen, cases) by_id = {c["id"]: c for c in cases} ids = {} for name in ("attempted_ids", "completed_ids", "active_ids"): @@ -846,6 +1067,16 @@ def summarize_saved(path): summary["evidence_complete"] = ( summary["evidence_complete"] and state["stop_reason"] == "completed" ) + if screen is not None: + summary["fast_screen_decision"] = _saved_fast_screen_decision(screen, summary) + p50 = summary["successful_latency_p50_ms"] + p95 = summary["successful_latency_p95_ms"] + summary["fast_screen_delta_p50_ms"] = ( + None if p50 is None else p50 - screen["baseline_p50_ms"] + ) + summary["fast_screen_delta_p95_ms"] = ( + None if p95 is None else p95 - screen["baseline_p95_ms"] + ) return summary @@ -946,6 +1177,24 @@ def main(): runner.add_argument("--phase", choices=("feasibility", "measured"), required=True) runner.add_argument("--concurrency", type=positive, default=1) runner.add_argument("--warmup", type=positive, default=3) + runner.add_argument( + "--fast-screen", + action="store_true", + help=( + "sample 60 Public231 requests (seed 231, length buckets) for one " + "measured round; requires --baseline-p50-ms and --baseline-p95-ms" + ), + ) + runner.add_argument( + "--baseline-p50-ms", + type=float, + help="current full-C1 primary p50, milliseconds; fast-screen only", + ) + runner.add_argument( + "--baseline-p95-ms", + type=float, + help="current full-C1 primary p95, milliseconds; fast-screen only", + ) runner.add_argument( "--warmup-manifest", type=Path, help="independent labelled warmup requests" ) diff --git a/tests/benchmarks/test_bench.py b/tests/benchmarks/test_bench.py index 7d0ed4ae..d929bdd2 100644 --- a/tests/benchmarks/test_bench.py +++ b/tests/benchmarks/test_bench.py @@ -3,6 +3,7 @@ import base64 import io import json +import math import tempfile import unittest from contextlib import redirect_stdout @@ -913,3 +914,237 @@ def test_parity_and_missing_records(self): ) with self.assertRaises(ValueError): bench.compare(args) + + +def screen_case(index, tokens): + return { + "id": f"s{index:03d}", + "prompt_tokens": tokens, + "request": { + "state": f"state {index}", + "questions": {"q": {"type": "noul", "instructions": "Is this a test?"}}, + }, + "expected": {"q": True}, + } + + +def screen_population(): + cases = [] + index = 0 + for base, count in ((0, 12), (128, 12), (256, 10), (512, 10), (1024, 10), (2048, 20)): + for offset in range(count): + cases.append(screen_case(index, base + offset)) + index += 1 + return cases + + +def noul_ok(): + return {"answers": {"q": {"type": "noul", "noul": 1.0}}} + + +class FastScreenTests(unittest.IsolatedAsyncioTestCase): + def test_protocol_constants_match_the_confirmed_quotas(self): + self.assertEqual( + [(name, quota) for name, _lo, _hi, quota in bench.FAST_SCREEN_BUCKETS], + [ + ("0-127", 10), + ("128-255", 10), + ("256-511", 8), + ("512-1023", 8), + ("1024-2047", 8), + ("2048+", 16), + ], + ) + self.assertEqual(sum(quota for *_rest, quota in bench.FAST_SCREEN_BUCKETS), 60) + self.assertEqual(bench.FAST_SCREEN_SEED, 231) + self.assertEqual(bench.FAST_SCREEN_RELATIVE, 0.02) + self.assertEqual(bench.FAST_SCREEN_ABSOLUTE_MS, 0.5) + edges = { + 0: "0-127", + 127: "0-127", + 128: "128-255", + 255: "128-255", + 256: "256-511", + 511: "256-511", + 512: "512-1023", + 1023: "512-1023", + 1024: "1024-2047", + 2047: "1024-2047", + 2048: "2048+", + 100000: "2048+", + } + for tokens, name in edges.items(): + self.assertEqual(bench.fast_screen_bucket(tokens), name) + + def test_sample_is_stable_quota_and_manifest_order(self): + cases = screen_population() + selected, counts = bench.fast_screen_sample(cases) + again, _counts = bench.fast_screen_sample(cases) + self.assertEqual([case["id"] for case in selected], [case["id"] for case in again]) + self.assertEqual(sum(counts.values()), 60) + self.assertEqual( + counts, + { + "0-127": 10, + "128-255": 10, + "256-511": 8, + "512-1023": 8, + "1024-2047": 8, + "2048+": 16, + }, + ) + ids = [case["id"] for case in cases] + chosen = [case["id"] for case in selected] + self.assertEqual(chosen, [case_id for case_id in ids if case_id in set(chosen)]) + for case in selected: + self.assertNotIn("prompt_tokens", case["request"]) + + def test_short_bucket_and_bad_lengths_fail_closed(self): + cases = screen_population() + long = [case for case in cases if case["prompt_tokens"] >= 2048][:15] + short = [case for case in cases if case["prompt_tokens"] < 2048] + long + with self.assertRaisesRegex(ValueError, "2048\\+"): + bench.fast_screen_sample(short) + missing = dict(cases[0]) + missing.pop("prompt_tokens") + with self.assertRaisesRegex(ValueError, "prompt_tokens"): + bench.fast_screen_sample([missing, *cases[1:]]) + bad = dict(cases[0]) + bad["prompt_tokens"] = True + with self.assertRaisesRegex(ValueError, "nonnegative integer"): + bench.fast_screen_sample([bad, *cases[1:]]) + + def test_drop_only_when_both_percentiles_are_clearly_worse(self): + relative_tip = 10 * (1 + bench.FAST_SCREEN_RELATIVE) + self.assertTrue(bench.latency_clearly_worse(10, relative_tip)) + self.assertFalse( + bench.latency_clearly_worse(10, math.nextafter(relative_tip, 10)) + ) + absolute_tip = 1000 + bench.FAST_SCREEN_ABSOLUTE_MS + self.assertTrue(bench.latency_clearly_worse(1000, absolute_tip)) + self.assertFalse( + bench.latency_clearly_worse(1000, math.nextafter(absolute_tip, 1000)) + ) + self.assertEqual( + bench.fast_screen_decision(10, 1000, relative_tip, absolute_tip), "drop" + ) + self.assertEqual( + bench.fast_screen_decision(1000, 1000, absolute_tip, absolute_tip), + "drop", + ) + self.assertEqual(bench.fast_screen_decision(10, 1000, relative_tip, 999), "full_c1") + self.assertEqual(bench.fast_screen_decision(1000, 1000, 1000.5, 1000), "full_c1") + self.assertEqual(bench.fast_screen_decision(10, 1000, 9, 900), "full_c1") + with self.assertRaisesRegex(ValueError, "positive"): + bench.latency_clearly_worse(0, 1) + + async def test_run_fast_screen_keeps_sixty_and_does_not_keep(self): + cases = screen_population() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest = root / "public231.jsonl" + manifest.write_text("".join(json.dumps(case) + "\n" for case in cases)) + args = run_args(root) + args.manifest = manifest + args.fast_screen = True + args.warmup = 5 + args.baseline_p50_ms = 1_000_000.0 + args.baseline_p95_ms = 1_000_000.0 + seen = [] + + def handler(request): + body = json.loads(request.content) + seen.append(body) + self.assertNotIn("prompt_tokens", body) + self.assertEqual(body["model"], "english") + return httpx.Response(200, json=noul_ok()) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + with ( + patch.object(bench.httpx, "AsyncClient", return_value=client), + redirect_stdout(io.StringIO()), + ): + self.assertEqual(await bench.run(args, bench.load_cases(manifest)), 0) + saved = json.loads((args.output / "summary.json").read_text()) + self.assertEqual(saved, bench.summarize_saved(args.output)) + self.assertEqual(saved["planned_requests"], 60) + self.assertEqual(saved["successful_requests"], 60) + self.assertEqual(saved["fast_screen_decision"], "full_c1") + self.assertNotIn("keep", saved["fast_screen_decision"]) + self.assertLess(saved["fast_screen_delta_p50_ms"], 0) + config = json.loads((args.output / "config.json").read_text()) + screen = config["fast_screen"] + self.assertEqual(screen["seed"], 231) + self.assertEqual(screen["parent_manifest_sha256"], bench.digest(manifest)) + self.assertEqual(len(screen["selected_ids"]), 60) + self.assertEqual( + screen["bucket_counts"]["2048+"], + 16, + ) + recorded = [ + json.loads(line) + for line in (args.output / "requests.jsonl").read_text().splitlines() + ] + self.assertEqual([case["id"] for case in recorded], screen["selected_ids"]) + buckets = saved["latency_by_prompt_bucket"] + self.assertEqual(buckets["2048+"]["samples"], 16) + self.assertEqual(buckets["0-127"]["samples"], 10) + self.assertEqual(len(seen), 65) + + async def test_failed_fast_screen_is_inconclusive_and_short_bucket_writes_nothing(self): + cases = screen_population() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest = root / "public231.jsonl" + manifest.write_text("".join(json.dumps(case) + "\n" for case in cases)) + args = run_args(root) + args.manifest = manifest + args.fast_screen = True + args.warmup = 5 + args.baseline_p50_ms = 1_000_000.0 + args.baseline_p95_ms = 1_000_000.0 + calls = {"n": 0} + + def handler(_request): + calls["n"] += 1 + if calls["n"] == 6: + return httpx.Response(503, text="busy") + return httpx.Response(200, json=noul_ok()) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + with ( + patch.object(bench.httpx, "AsyncClient", return_value=client), + redirect_stdout(io.StringIO()), + ): + self.assertEqual(await bench.run(args, bench.load_cases(manifest)), 1) + saved = bench.summarize_saved(args.output) + self.assertEqual(saved["fast_screen_decision"], "inconclusive") + self.assertGreater(saved["failed_requests"], 0) + + refused = run_args(root) + refused.output = root / "unused" + refused.manifest = manifest + refused.fast_screen = True + refused.warmup = 3 + refused.baseline_p50_ms = 7.338 + refused.baseline_p95_ms = 51.816 + with self.assertRaisesRegex(ValueError, "warmup 5"): + await bench.run(refused, bench.load_cases(manifest)) + self.assertFalse(refused.output.exists()) + + long = [case for case in cases if case["prompt_tokens"] >= 2048][:15] + short_cases = [case for case in cases if case["prompt_tokens"] < 2048] + long + short_manifest = root / "short.jsonl" + short_manifest.write_text( + "".join(json.dumps(case) + "\n" for case in short_cases) + ) + short = run_args(root) + short.output = root / "short-out" + short.manifest = short_manifest + short.fast_screen = True + short.warmup = 5 + short.baseline_p50_ms = 7.338 + short.baseline_p95_ms = 51.816 + with self.assertRaisesRegex(ValueError, "2048\\+"): + await bench.run(short, bench.load_cases(short_manifest)) + self.assertFalse(short.output.exists()) From 923c8c739bcaf3cc2b098f3cf373c2b547d2666c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 26 Oct 2026 08:02:18 +0000 Subject: [PATCH 2/5] Compare fast-screen to the same 60 IDs, not pooled C1 A tip screen baseline is the primary SHA's own fast-screen of those request IDs. Typed percentiles and the StartLux 7.338/51.816 pooled pair are refused. A DROP records dropped_at. The screen-primary pair stays TBD until it is measured on those IDs. Co-authored-by: Hongsheng Liu --- .agents/skills/new-model-integration/SKILL.md | 12 +- .../skills/optimize-remeasure-gate/SKILL.md | 68 +++-- .agents/skills/public231-c1/SKILL.md | 91 +++++-- .agents/skills/system1-omni-review/SKILL.md | 29 ++- benchmarks/README.md | 17 +- benchmarks/bench.py | 238 +++++++++++++++--- tests/benchmarks/test_bench.py | 191 ++++++++++---- 7 files changed, 500 insertions(+), 146 deletions(-) diff --git a/.agents/skills/new-model-integration/SKILL.md b/.agents/skills/new-model-integration/SKILL.md index d95a8938..f1e63b56 100644 --- a/.agents/skills/new-model-integration/SKILL.md +++ b/.agents/skills/new-model-integration/SKILL.md @@ -88,10 +88,14 @@ that: do not invent a C1 result. CPU contract tests are not a latency baseline. Further latency work stays on this model's draft branch. Run it under [optimize-remeasure-gate](../optimize-remeasure-gate/SKILL.md): paste the C1 p50/p95, `device`, and `compute capability` first. After that primary -exists, smoke and the fast-screen run before another full C1; a clear -regression is dropped there and is not a KEEP. Do not stack the next tip -commit until that tip has been dropped by the screen or remeasured with -full C1 on the same named hardware. Incomplete evidence is INCONCLUSIVE. +exists, smoke and the fast-screen run before another full C1. The screen +compares the same 60 request IDs against the primary SHA's own screen, not +against the full-C1 pooled pair. That screen-primary pair stays +`TBD — measure on primary SHA same 60 IDs` until it is measured. A clear +regression is dropped there, with `dropped_at`, and is not a KEEP. Do not +stack the next tip commit until that tip has been dropped by the screen or +remeasured with full C1 on the same named hardware. Incomplete evidence is +INCONCLUSIVE. Quality and contract checks stay green. Each tip states its RSI phase (bring-up/scheduling, parallelism/comms, kernel, or launch/post) and which alternatives were diff --git a/.agents/skills/optimize-remeasure-gate/SKILL.md b/.agents/skills/optimize-remeasure-gate/SKILL.md index 92d75c98..6283feca 100644 --- a/.agents/skills/optimize-remeasure-gate/SKILL.md +++ b/.agents/skills/optimize-remeasure-gate/SKILL.md @@ -60,32 +60,46 @@ method, or the named hardware. Follow [public231-c1](../public231-c1/SKILL.md). Smoke must pass before the screen. Run one fast-screen: 60 requests, seed 231, concurrency 1, warmup 5, bucket quotas 0–127:10, 128–255:10, 256–511:8, 512–1023:8, 1024–2047:8, -2048+:16. Compare its p50 and p95 to this model's current full-C1 primary, -on the same device, compute capability, and clock. - -The StartLux Instant primary named for this gate, until a full-C1 KEEP -replaces it, is runtime revision -`51f80ad681c8f7b1be9b22b5e700bef34e33534c` (`51f80ad`), pooled p50 7.338 ms, -pooled p95 51.816 ms. Clef and pplx-decider keep the primaries recorded in -their recipes and in the latency leaderboard. Do not reuse the StartLux -Instant pair for those models. A primary moves only when a later full C1 -KEEPs. - -Early DROP only when both Δp50 and Δp95 are worse. A metric is worse when -its delta (tip minus baseline) is ≥ +2% of that baseline or ≥ +0.5 ms. -Otherwise, including a mixed result, run full C1. -`python benchmarks/bench.py run --fast-screen` records `drop`, `full_c1`, -or `inconclusive`. It does not record a KEEP. - -A `drop` is a regression. Do not run full C1. Do not stack the next commit -on that tip. Do not adopt the screen p50/p95 as the next baseline. A -negative screen delta is not a gain and is not a KEEP. An `inconclusive` -screen stops the loop. +2048+:16. Compare the tip's screen p50 and p95 to the primary SHA's own +screen of those same 60 IDs on the same device, compute capability, and +clock. Pass that directory with `--baseline-screen`. Do not pass full-C1 +pooled percentiles. `--baseline-p50-ms` and `--baseline-p95-ms` are refused. + +StartLux Instant's full-C1 pooled primary remains +`51f80ad681c8f7b1be9b22b5e700bef34e33534c` (`51f80ad`), p50 7.338 ms, p95 +51.816 ms. That pair is not the screen baseline. Clef and pplx-decider keep +the full-C1 primaries in their recipes and the latency leaderboard. For +every model the screen-primary slot is: + +```text +screen_primary_p50_ms: TBD — measure on primary SHA same 60 IDs +screen_primary_p95_ms: TBD — measure on primary SHA same 60 IDs +``` + +Fill it only from an hk2 `--fast-screen` run whose decision is +`screen_primary`. A full-C1 KEEP moves the full-C1 primary; it does not by +itself fill this slot. Measure the new primary on the same 60 IDs before +the next tip screen. + +Early DROP only when both screen Δp50 and Δp95 are worse than that same-60 +baseline. A metric is worse when its delta (tip minus baseline) is ≥ +2% +of that baseline or ≥ +0.5 ms. Otherwise, including a mixed result, run +full C1. The summary records `screen_primary`, `drop`, `full_c1`, or +`inconclusive`. It does not record a KEEP. + +A `drop` is a regression. The artifact includes `dropped_at` in +`fast_screen_drop.json` and on the summary. Do not run full C1. Do not +stack the next commit on that tip. Do not adopt the screen p50/p95 as the +next baseline. A negative screen delta is not a gain and is not a KEEP. An +`inconclusive` screen stops the loop. KEEP, merge, and ready-for-review require the full C1 dual gate: pooled p50 and pooled p95 both strictly lower, pooled 2048+ p95 strictly lower, and no quality collapse under the rules in the next section. The +2% / +0.5 ms -band does not make a full-C1 tie a KEEP. +band does not make a full-C1 tie a KEEP. Ready-for-review also requires hk2 +evidence of `--fast-screen` (screen primary, and the tip screen when a tip +is claimed). Offline MockTransport tests are not that evidence and are not +a review PASS. ## INCONCLUSIVE stops the loop @@ -95,8 +109,13 @@ these are true: - the baseline p50/p95 were not recorded before the edit - smoke did not pass before the latency screen - the fast-screen was skipped on a tip that already has a full-C1 primary +- the screen baseline is a full-C1 pooled p50/p95, a typed percentile, or a + different 60 IDs than `--fast-screen` just sampled - the fast-screen decision is `inconclusive` +- a DROP has no `dropped_at` - a KEEP, a gain, or a new primary is claimed from fast-screen numbers +- ready or review PASS is claimed from offline MockTransport output without + hk2 `--fast-screen` evidence - the fast-screen decision was `full_c1` but fewer than three measured rounds finished - a completed `drop` is treated as a KEEP or as the next baseline @@ -143,8 +162,9 @@ Before the agent proposes the next change, give it this record: full C1 ran - clock basis, `device`, and `compute capability` for both sides - RSI phase, alternatives considered, and which change is primary -- fast-screen decision (`drop`, `full_c1`, or `inconclusive`) and, when the - screen ran, its p50/p95 deltas labeled as screen-only +- fast-screen decision (`screen_primary`, `drop`, `full_c1`, or + `inconclusive`), the same 60 IDs, `dropped_at` when the decision is + `drop`, and the screen p50/p95 deltas labeled as screen-only - verdict: improved, regressed, or INCONCLUSIVE - quality/contract result when the path reached full C1 or a KEEP decision diff --git a/.agents/skills/public231-c1/SKILL.md b/.agents/skills/public231-c1/SKILL.md index 2b71b38d..e7f15d13 100644 --- a/.agents/skills/public231-c1/SKILL.md +++ b/.agents/skills/public231-c1/SKILL.md @@ -20,9 +20,11 @@ relabel, or a room post that contains operator metadata. must succeed before any latency screen. A model recipe that already runs a server contract smoke still runs that smoke. A failed smoke is not a latency result and is not a DROP. -2. **Fast-screen.** One 60-request round after smoke. DROP the tip when both - p50 and p95 are clearly worse than that model's current full-C1 primary. - Otherwise run full C1. The screen never KEEPs a tip. +2. **Fast-screen.** One 60-request round after smoke. The screen baseline is + the primary SHA served on the same machine and measured on these exact 60 + request IDs. It is not the full-C1 pooled p50/p95. DROP the tip when both + screen percentiles are clearly worse than that same-60 baseline. Otherwise + run full C1. The screen never KEEPs a tip. 3. **Full Public231 C1.** Unchanged measurement: warmup 5, concurrency 1, three measured rounds. KEEP only on the full dual gate below. @@ -61,22 +63,46 @@ That is 60 requests. 2048+ is the overweight. A bucket with fewer cases than its quota is not a screen; fix the length index or stop. Do not fill the quota from another bucket. Do not run a second screen round and pool it. -Compare the screen's successful p50 and p95 to the current full-C1 primary -for **this** model. Pass that primary with `--baseline-p50-ms` and -`--baseline-p95-ms`. The StartLux Instant primary named for this gate, until -a full-C1 KEEP replaces it, is runtime revision +Compare the tip screen's successful p50 and p95 only to a screen primary: +the same primary SHA, same `device` / `gpu`, same manifest, same seed 231, +same bucket quotas, and the same 60 request IDs. Pass that saved run as +`--baseline-screen`. Do not pass `--baseline-p50-ms` or `--baseline-p95-ms`. +The runner rejects those flags. A full-C1 output directory is also rejected. + +Full-C1 pooled numbers are not this baseline. StartLux Instant's full-C1 +pooled primary remains runtime revision `51f80ad681c8f7b1be9b22b5e700bef34e33534c` (`51f80ad`), pooled p50 7.338 ms, -pooled p95 51.816 ms. Clef and pplx-decider keep the primaries recorded in -their recipes and in [the latency leaderboard](../../../benchmarks/latency-leaderboard.md). -Do not paste the StartLux Instant pair onto Clef or pplx-decider. +pooled p95 51.816 ms, and the runner refuses that pair as a screen baseline. +Clef and pplx-decider keep their full-C1 primaries in their recipes and in +[the latency leaderboard](../../../benchmarks/latency-leaderboard.md). Do not +paste any model's pooled C1 pair into the screen. + +Screen primary, same 60 IDs (fill only after that screen is measured; do not +invent the pair): + +```text +screen_primary_p50_ms: TBD — measure on primary SHA same 60 IDs +screen_primary_p95_ms: TBD — measure on primary SHA same 60 IDs +``` + +Record the screen primary with `--fast-screen` and no `--baseline-screen`. +That run's decision is `screen_primary`. It is not a tip verdict and not a +KEEP. Later tips pass that directory. ```sh python benchmarks/bench.py run \ --endpoint --model \ --metadata --phase measured \ --concurrency 1 --warmup 5 \ - --fast-screen --baseline-p50-ms --baseline-p95-ms \ - --output + --fast-screen \ + --output + +python benchmarks/bench.py run \ + --endpoint --model \ + --metadata --phase measured \ + --concurrency 1 --warmup 5 \ + --fast-screen --baseline-screen \ + --output ``` Delta is tip minus baseline, in milliseconds. A percentile is **worse** when @@ -84,16 +110,27 @@ its delta is ≥ +2% of that baseline value or ≥ +0.5 ms. The runner evaluates that boundary as `tip >= baseline * 1.02` or `tip >= baseline + 0.5`, so an exact threshold stays worse. Early **DROP** only when **both** p50 and p95 are worse. Mixed results and results inside that band proceed to full C1. -The saved summary field `fast_screen_decision` is `drop`, `full_c1`, or -`inconclusive`. It is never `keep`. - -`drop` means do not run full C1. `inconclusive` means any measured request -failed, was incomplete, or was unattempted, warmup failed, or -`evidence_complete` is not true. An inconclusive screen is not a DROP and -not permission to continue. A negative screen delta is not a gain, not a -new primary, and not a KEEP. Do not claim a speedup from fast-screen -numbers. Percentiles use the same median and nearest-rank p95 as full C1, -on this one round of 60, not an average of rounds. +The saved summary field `fast_screen_decision` is `screen_primary`, `drop`, +`full_c1`, or `inconclusive`. It is never `keep`. + +`drop` means do not run full C1. The run writes `fast_screen_drop.json` and +copies `dropped_at` onto the summary. `dropped_at` is UTC +`YYYY-MM-DDTHH:MM:SSZ`, written once. A DROP without `dropped_at` is not an +auditable DROP. `inconclusive` means any measured request failed, was +incomplete, or was unattempted, warmup failed, or `evidence_complete` is +not true. An inconclusive screen is not a DROP and not permission to +continue. A negative screen delta is not a gain, not a new primary, and +not a KEEP. Do not claim a speedup from fast-screen numbers. Percentiles +use the same median and nearest-rank p95 as full C1, on this one round of +60, not an average of rounds. + +Before this protocol change is marked ready for review, the PR must include +hk2 evidence of `python benchmarks/bench.py run --fast-screen` for the +screen primary (and the tip comparison, when a tip is claimed). That +evidence records the SHA, `device`, `compute capability`, the 60 IDs, and +`summary.json`. An offline `httpx.MockTransport` unit test is not that +evidence and is not a review PASS. Room posts still omit hostnames; the +hk2 requirement is the ready/review evidence, not a room post. ## Full C1 KEEP @@ -258,10 +295,12 @@ Publish: - `evidence_complete` from each `summary.json` A fast-screen paste is a separate record. Label it fast-screen, and include -the decision (`drop`, `full_c1`, or `inconclusive`), seed 231, both deltas, -the primary SHA those deltas use, and the same `device` and -`compute capability`. Do not publish that paste as a C1 baseline or as a -speedup. +the decision (`screen_primary`, `drop`, `full_c1`, or `inconclusive`), seed +231, the same 60 IDs, the primary SHA of the screen baseline when the run +is a tip, both deltas when they exist, `dropped_at` when the decision is +`drop`, and the same `device` and `compute capability`. Do not publish that +paste as a C1 baseline or as a speedup. Do not fill screen-primary p50/p95 +from the full-C1 pooled pair. Labelled choice/noul accuracy and score MAE in `quality_on_successful_requests` are conditional on successes. Report them diff --git a/.agents/skills/system1-omni-review/SKILL.md b/.agents/skills/system1-omni-review/SKILL.md index 392df4c8..d2f70883 100644 --- a/.agents/skills/system1-omni-review/SKILL.md +++ b/.agents/skills/system1-omni-review/SKILL.md @@ -1,6 +1,6 @@ --- name: system1-omni-review -description: Review ThinkFlowLab/system1-omni pull requests for Rust serving, model-contract, CUDA/Metal compatibility and numerical or performance evidence, with pinned review provenance. +description: Review ThinkFlowLab/system1-omni pull requests for Rust serving, model-contract, CUDA/Metal compatibility and numerical or performance evidence, with pinned review provenance. For a Public231 fast-screen, check the same-60-ID screen baseline, hk2 --fast-screen evidence, dropped_at on DROP, and that KEEP is still full C1 only. --- # System1-Omni PR review @@ -29,6 +29,33 @@ baseline; target code and configuration determine current paths and commands. - For native/FFI/CUDA changes, inspect Rust and C ABI in lockstep, buffer and graph lifetimes, stream/device ownership, sizes/overflow, BF16 rounding locations, and error propagation/recovery. Compile success and ignored GPU tests do not validate kernels. Test changed inputs, reuse, growth, eviction and failure recovery when graph/cache behavior changes. - For performance or numerical claims, freeze baseline/candidate, hardware/toolchain, checkpoint, input manifest, precision, variables, tolerances and budget before comparing. Separate quality/parity from speed, native inference from frontend overhead, and cold/capture costs from warmed latency. Preserve errors and denominators; never relax tolerances or discard failures to produce a winner. +## Public231 fast-screen + +When the change touches `benchmarks/bench.py`, the Public231 skills, or a +claim that a tip was screened or dropped, check all four of these. A miss +is a blocking review finding, not a note. + +1. **Same-60 screen baseline.** The tip screen p50/p95 are compared to the + primary SHA's own `--fast-screen` of those exact request IDs (same seed, + buckets, quotas, and device). Full-C1 pooled numbers are not that + baseline. StartLux Instant 7.338/51.816 from `51f80ad` is a pooled C1 + pair. Typed `--baseline-p50-ms` / `--baseline-p95-ms` and a full-C1 + output directory fail closed. The screen-primary slot stays + `TBD — measure on primary SHA same 60 IDs` until that screen exists. + Do not accept a paste that fills the slot from the pooled pair. +2. **hk2 smoke.** Ready and review PASS require hk2 evidence of + `python benchmarks/bench.py run --fast-screen` (the screen-primary run, + and the tip comparison when a tip is claimed), including SHA, device, + compute capability, the 60 IDs, and `summary.json`. An offline + `httpx.MockTransport` unit test does not satisfy this check. +3. **`dropped_at`.** A screen DROP writes `fast_screen_drop.json` and puts + `dropped_at` (`YYYY-MM-DDTHH:MM:SSZ`) on the summary. A DROP missing that + field is not auditable and is not a valid DROP. +4. **KEEP is full C1 only.** KEEP still requires the full Public231 C1 dual + gate: pooled p50 and pooled p95 both strictly lower, pooled 2048+ p95 + strictly lower, and no quality collapse. `screen_primary`, `full_c1` on + the screen, or a negative screen delta is not a KEEP. + ## Validate and report Select checks with the current [change-to-validation matrix](../../../CONTRIBUTING.md#choose-validation-by-changed-area) and [Rust CI commands](../../../CONTRIBUTING.md#rust-ci-checks), then verify them against the target revision's [CI workflow](../../../.github/workflows/ci.yml) and [Docs workflow](../../../.github/workflows/docs.yml). Do not substitute commands from a historical audit for current feature flags, test targets or prerequisites. Benchmark tooling has weight-free Python tests; docs have a strict MkDocs build. Select GPU/model validation only for the changed risk and within separate execution authorization, using the host's reservation rules. Record skipped/ignored/checkpoint-dependent tests explicitly. Missing GPU, model or tooling is an evidence gap, not PASS. diff --git a/benchmarks/README.md b/benchmarks/README.md index dd539383..ef201e8e 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -126,19 +126,22 @@ three traversals. Validate `benchmarks/smoke.jsonl` first. `--fast-screen` then samples 60 requests from a manifest whose cases include integer `prompt_tokens` (not sent on the wire): quotas 10, 10, 8, 8, 8, and 16 for buckets 0–127, 128–255, 256–511, 512–1023, 1024–2047, and 2048+. Seed 231, one measured round, -concurrency 1, warmup 5. Pass the current primary with `--baseline-p50-ms` and -`--baseline-p95-ms`. The summary field `fast_screen_decision` is `drop` only -when both p50 and p95 are worse by ≥ +2% or ≥ +0.5 ms, `full_c1` when they are -not, and `inconclusive` when the round is incomplete. The field is never a -KEEP. The gate, including the StartLux Instant primary and the Clef and -pplx-decider pairing rules, is `.agents/skills/public231-c1/SKILL.md`. +concurrency 1, warmup 5. The screen baseline is an earlier `--fast-screen` of +the primary SHA on those same 60 IDs (`--baseline-screen`). Do not pass +`--baseline-p50-ms` or `--baseline-p95-ms`; full-C1 pooled numbers, including +StartLux Instant 7.338/51.816, are refused. Until that same-60 screen exists, +the screen primary is `TBD — measure on primary SHA same 60 IDs`. +`fast_screen_decision` is `screen_primary`, `drop` (only when both p50 and p95 +are worse by ≥ +2% or ≥ +0.5 ms), `full_c1`, or `inconclusive`. A `drop` writes +`dropped_at` in `fast_screen_drop.json`. The field is never a KEEP. The gate is +`.agents/skills/public231-c1/SKILL.md`. ```sh python benchmarks/bench.py run requests.jsonl \ --endpoint http://127.0.0.1:8000/v1/systemone --model english \ --metadata run-metadata.json --phase measured \ --concurrency 1 --warmup 5 \ - --fast-screen --baseline-p50-ms --baseline-p95-ms \ + --fast-screen --baseline-screen /tmp/public231-screen-primary \ --output /tmp/public231-fast-screen ``` diff --git a/benchmarks/bench.py b/benchmarks/bench.py index fa8a68f8..856a6e68 100755 --- a/benchmarks/bench.py +++ b/benchmarks/bench.py @@ -44,6 +44,12 @@ def response_digest(text): FAST_SCREEN_SEED = 231 FAST_SCREEN_RELATIVE = 0.02 FAST_SCREEN_ABSOLUTE_MS = 0.5 +FAST_SCREEN_DROP_NAME = "fast_screen_drop.json" +# Placeholder until the primary SHA is screened on these same 60 IDs. +SCREEN_PRIMARY_TBD = "screen primary TBD — measure on primary SHA same 60 IDs" +# Full-C1 pooled pairs. These are not screen baselines. StartLux Instant's +# pooled primary is the first entry; do not copy it onto a fast-screen. +FULL_C1_POOLED_NOT_SCREEN = ((7.338, 51.816),) FAST_SCREEN_BUCKETS = ( ("0-127", 0, 128, 10), ("128-255", 128, 256, 10), @@ -346,6 +352,27 @@ def fast_screen_decision(baseline_p50, baseline_p95, tip_p50, tip_p95): return "full_c1" +def is_full_c1_pooled_baseline(p50_ms, p95_ms): + """True for published full-C1 pooled pairs that must not be a screen baseline.""" + if type(p50_ms) is bool or type(p95_ms) is bool: + return False + if not isinstance(p50_ms, (int, float)) or not isinstance(p95_ms, (int, float)): + return False + return any( + math.isclose(p50_ms, pooled_p50, rel_tol=0, abs_tol=5e-4) + and math.isclose(p95_ms, pooled_p95, rel_tol=0, abs_tol=5e-4) + for pooled_p50, pooled_p95 in FULL_C1_POOLED_NOT_SCREEN + ) + + +def refuse_full_c1_pooled_baseline(p50_ms, p95_ms): + if is_full_c1_pooled_baseline(p50_ms, p95_ms): + raise ValueError( + "refusing full-C1 pooled numbers as a fast-screen baseline " + f"({p50_ms} / {p95_ms} ms). {SCREEN_PRIMARY_TBD}" + ) + + def latency_by_prompt_bucket(cases, records): """Per-bucket p50/p95 when every case carries prompt_tokens. Else None.""" if any("prompt_tokens" not in case for case in cases): @@ -367,21 +394,91 @@ def latency_by_prompt_bucket(cases, records): return report -def _require_fast_screen_run(args): +def _refuse_numeric_screen_baseline(args): + """Typed percentiles are how a full-C1 pooled pair gets reused. Refuse them.""" + if ( + getattr(args, "baseline_p50_ms", None) is not None + or getattr(args, "baseline_p95_ms", None) is not None + ): + raise ValueError( + "refusing --baseline-p50-ms and --baseline-p95-ms. " + "Full-C1 pooled percentiles, including StartLux Instant " + "7.338/51.816, are not a screen baseline. " + SCREEN_PRIMARY_TBD + ) + + +def _require_fast_screen_shape(args): if args.phase != "measured": raise ValueError("fast-screen requires --phase measured") if args.concurrency != 1: raise ValueError("fast-screen requires --concurrency 1") if args.warmup != 5: raise ValueError("fast-screen requires --warmup 5") - baseline_p50 = getattr(args, "baseline_p50_ms", None) - baseline_p95 = getattr(args, "baseline_p95_ms", None) - if baseline_p50 is None or baseline_p95 is None: + + +def _valid_dropped_at(value): + if not isinstance(value, str): + return False + try: + parsed = time.strptime(value, "%Y-%m-%dT%H:%M:%SZ") + except ValueError: + return False + return time.strftime("%Y-%m-%dT%H:%M:%SZ", parsed) == value + + +def load_same_60_screen_baseline(path, selected_ids, parent_manifest_sha256, tip_gpu): + """Baseline is the primary SHA's own screen of these exact request IDs.""" + if not isinstance(path, Path) or not path.is_dir(): + raise ValueError("baseline screen directory is missing") + if not (path / "config.json").is_file(): + raise ValueError("baseline screen is missing config.json") + config = json.loads((path / "config.json").read_text()) + screen = config.get("fast_screen") + if not screen: + raise ValueError( + "a full-C1 directory is not a fast-screen baseline. " + SCREEN_PRIMARY_TBD + ) + if screen.get("role") != "screen_primary": raise ValueError( - "fast-screen requires --baseline-p50-ms and --baseline-p95-ms" + "baseline screen must be the primary SHA's own fast-screen " + "(role screen_primary) of these same 60 IDs, not a tip comparison " + "and not full-C1 pooled percentiles. " + SCREEN_PRIMARY_TBD ) - latency_clearly_worse(baseline_p50, baseline_p50) - latency_clearly_worse(baseline_p95, baseline_p95) + if screen.get("selected_ids") != list(selected_ids): + raise ValueError( + "baseline screen request IDs do not match this fast-screen" + ) + if screen.get("seed") != FAST_SCREEN_SEED: + raise ValueError("baseline screen seed does not match") + if screen.get("parent_manifest_sha256") != parent_manifest_sha256: + raise ValueError("baseline screen manifest does not match") + expected = {name: quota for name, _lo, _hi, quota in FAST_SCREEN_BUCKETS} + if screen.get("bucket_counts") != expected: + raise ValueError("baseline screen quotas do not match") + metadata = config.get("metadata") or {} + gpu = metadata.get("gpu") + if gpu != tip_gpu: + raise ValueError("baseline screen gpu does not match this run") + revision = metadata.get("runtime_revision") + if not revision: + raise ValueError("baseline screen is missing runtime_revision") + summary = summarize_saved(path) + if summary.get("fast_screen_decision") != "screen_primary": + raise ValueError( + "baseline screen is not a completed screen-primary measurement. " + + SCREEN_PRIMARY_TBD + ) + p50 = summary["successful_latency_p50_ms"] + p95 = summary["successful_latency_p95_ms"] + refuse_full_c1_pooled_baseline(p50, p95) + return { + "baseline_kind": "same_60_id_screen", + "baseline_runtime_revision": revision, + "baseline_gpu": gpu, + "baseline_p50_ms": p50, + "baseline_p95_ms": p95, + "baseline_selected_ids": list(selected_ids), + } def manifest_subset_bytes(path, selected_ids): @@ -396,7 +493,7 @@ def manifest_subset_bytes(path, selected_ids): return "".join(by_id[case_id] + "\n" for case_id in selected_ids).encode() -def _check_saved_fast_screen(screen, cases): +def _check_saved_fast_screen(screen, cases, config): if screen.get("seed") != FAST_SCREEN_SEED: raise ValueError("unsupported fast-screen seed") if ( @@ -407,12 +504,33 @@ def _check_saved_fast_screen(screen, cases): expected = {name: quota for name, _lo, _hi, quota in FAST_SCREEN_BUCKETS} if screen.get("bucket_counts") != expected: raise ValueError("fast-screen bucket counts do not match the protocol") - if "baseline_p50_ms" not in screen or "baseline_p95_ms" not in screen: - raise ValueError("saved fast-screen is missing its baseline") if screen.get("selected_ids") != [case["id"] for case in cases]: raise ValueError("fast-screen selection does not match the saved manifest") if len(cases) != sum(expected.values()): raise ValueError("fast-screen manifest is not 60 requests") + role = screen.get("role") + if role == "screen_primary": + if "baseline_p50_ms" in screen or "baseline_p95_ms" in screen: + raise ValueError( + "a screen-primary measurement must not carry typed baseline percentiles. " + + SCREEN_PRIMARY_TBD + ) + return + if role != "tip": + raise ValueError("unsupported fast-screen role") + if screen.get("baseline_kind") != "same_60_id_screen": + raise ValueError( + "tip fast-screen baseline must be the same 60 IDs. " + SCREEN_PRIMARY_TBD + ) + if "baseline_p50_ms" not in screen or "baseline_p95_ms" not in screen: + raise ValueError("saved tip fast-screen is missing its same-60 baseline") + refuse_full_c1_pooled_baseline(screen["baseline_p50_ms"], screen["baseline_p95_ms"]) + if screen.get("baseline_selected_ids") != screen["selected_ids"]: + raise ValueError("saved screen baseline is not the same 60 request IDs") + if screen.get("baseline_gpu") != config.get("metadata", {}).get("gpu"): + raise ValueError("saved screen baseline gpu does not match this run") + if not screen.get("baseline_runtime_revision"): + raise ValueError("saved screen baseline is missing the primary SHA") def _saved_fast_screen_decision(screen, summary): @@ -427,6 +545,8 @@ def _saved_fast_screen_decision(screen, summary): ) if not complete: return "inconclusive" + if screen.get("role") == "screen_primary": + return "screen_primary" return fast_screen_decision( screen["baseline_p50_ms"], screen["baseline_p95_ms"], @@ -435,6 +555,22 @@ def _saved_fast_screen_decision(screen, summary): ) +def _attach_dropped_at(path, screen, summary): + drop_path = path / FAST_SCREEN_DROP_NAME + if summary.get("fast_screen_decision") != "drop": + if drop_path.exists(): + raise ValueError("fast_screen_drop.json is only written for a DROP") + return + if not drop_path.is_file(): + return + record = json.loads(drop_path.read_text()) + if record.get("decision") != "drop" or not _valid_dropped_at(record.get("dropped_at")): + raise ValueError("fast-screen DROP record is missing dropped_at") + if record.get("selected_ids") != screen["selected_ids"]: + raise ValueError("fast-screen DROP record does not match the 60 IDs") + summary["dropped_at"] = record["dropped_at"] + + def quality(cases, records): values = {"choice_accuracy": [], "noul_accuracy": [], "score_mae": []} by_id = {r["id"]: r for r in records} @@ -784,14 +920,15 @@ async def run(args, cases): ): if not metadata.get(name): raise ValueError(f"metadata requires {name}") + _refuse_numeric_screen_baseline(args) fast_screen = getattr(args, "fast_screen", False) - baseline_p50 = getattr(args, "baseline_p50_ms", None) - baseline_p95 = getattr(args, "baseline_p95_ms", None) + baseline_screen = getattr(args, "baseline_screen", None) screen_record = None if fast_screen: - _require_fast_screen_run(args) + _require_fast_screen_shape(args) cases, bucket_counts = fast_screen_sample(cases) selected_ids = [case["id"] for case in cases] + parent_sha = digest(args.manifest) screen_record = { "seed": FAST_SCREEN_SEED, "relative_worse": FAST_SCREEN_RELATIVE, @@ -807,12 +944,22 @@ async def run(args, cases): ], "selected_ids": selected_ids, "bucket_counts": bucket_counts, - "parent_manifest_sha256": digest(args.manifest), - "baseline_p50_ms": args.baseline_p50_ms, - "baseline_p95_ms": args.baseline_p95_ms, + "parent_manifest_sha256": parent_sha, } - elif baseline_p50 is not None or baseline_p95 is not None: - raise ValueError("baseline percentiles require --fast-screen") + if baseline_screen is not None: + screen_record["role"] = "tip" + screen_record.update( + load_same_60_screen_baseline( + baseline_screen, + selected_ids, + parent_sha, + metadata.get("gpu"), + ) + ) + else: + screen_record["role"] = "screen_primary" + elif baseline_screen is not None: + raise ValueError("--baseline-screen requires --fast-screen") warmup_manifest = getattr(args, "warmup_manifest", None) warmup_cases = load_cases(warmup_manifest) if warmup_manifest else cases # A case the adapter cannot wire fails before any HTTP exchange; reject it @@ -912,6 +1059,24 @@ async def run(args, cases): # Replay has saved the completed samples and the measured stop boundary. pass summary = summarize_saved(args.output) + if summary.get("fast_screen_decision") == "drop" and "dropped_at" not in summary: + saved_screen = json.loads((args.output / "config.json").read_text())["fast_screen"] + write_json( + args.output / FAST_SCREEN_DROP_NAME, + { + "decision": "drop", + "dropped_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "selected_ids": saved_screen["selected_ids"], + "baseline_runtime_revision": saved_screen["baseline_runtime_revision"], + "baseline_p50_ms": saved_screen["baseline_p50_ms"], + "baseline_p95_ms": saved_screen["baseline_p95_ms"], + "tip_p50_ms": summary["successful_latency_p50_ms"], + "tip_p95_ms": summary["successful_latency_p95_ms"], + }, + ) + summary = summarize_saved(args.output) + if "dropped_at" not in summary: + raise ValueError("fast-screen DROP did not record dropped_at") write_json(args.output / "summary.json", summary) print(json.dumps(summary, indent=2)) return int(summary["failed_requests"] > 0 or not summary["evidence_complete"]) @@ -957,7 +1122,7 @@ def summarize_saved(path): cases = load_cases(path / "requests.jsonl") screen = config.get("fast_screen") if screen is not None: - _check_saved_fast_screen(screen, cases) + _check_saved_fast_screen(screen, cases, config) by_id = {c["id"]: c for c in cases} ids = {} for name in ("attempted_ids", "completed_ids", "active_ids"): @@ -1069,14 +1234,18 @@ def summarize_saved(path): ) if screen is not None: summary["fast_screen_decision"] = _saved_fast_screen_decision(screen, summary) - p50 = summary["successful_latency_p50_ms"] - p95 = summary["successful_latency_p95_ms"] - summary["fast_screen_delta_p50_ms"] = ( - None if p50 is None else p50 - screen["baseline_p50_ms"] - ) - summary["fast_screen_delta_p95_ms"] = ( - None if p95 is None else p95 - screen["baseline_p95_ms"] - ) + if screen.get("role") == "tip": + p50 = summary["successful_latency_p50_ms"] + p95 = summary["successful_latency_p95_ms"] + summary["fast_screen_delta_p50_ms"] = ( + None if p50 is None else p50 - screen["baseline_p50_ms"] + ) + summary["fast_screen_delta_p95_ms"] = ( + None if p95 is None else p95 - screen["baseline_p95_ms"] + ) + _attach_dropped_at(path, screen, summary) + elif (path / FAST_SCREEN_DROP_NAME).exists(): + raise ValueError("fast_screen_drop.json is only written for a DROP") return summary @@ -1182,18 +1351,27 @@ def main(): action="store_true", help=( "sample 60 Public231 requests (seed 231, length buckets) for one " - "measured round; requires --baseline-p50-ms and --baseline-p95-ms" + "measured round. Without --baseline-screen this records the " + "screen primary. A tip passes that directory, not pooled C1 numbers." + ), + ) + runner.add_argument( + "--baseline-screen", + type=Path, + help=( + "primary SHA's own fast-screen directory of these same 60 IDs. " + "Not a full-C1 pooled p50/p95." ), ) runner.add_argument( "--baseline-p50-ms", type=float, - help="current full-C1 primary p50, milliseconds; fast-screen only", + help=argparse.SUPPRESS, ) runner.add_argument( "--baseline-p95-ms", type=float, - help="current full-C1 primary p95, milliseconds; fast-screen only", + help=argparse.SUPPRESS, ) runner.add_argument( "--warmup-manifest", type=Path, help="independent labelled warmup requests" diff --git a/tests/benchmarks/test_bench.py b/tests/benchmarks/test_bench.py index d929bdd2..36b8c9c7 100644 --- a/tests/benchmarks/test_bench.py +++ b/tests/benchmarks/test_bench.py @@ -1037,59 +1037,143 @@ def test_drop_only_when_both_percentiles_are_clearly_worse(self): self.assertEqual(bench.fast_screen_decision(10, 1000, 9, 900), "full_c1") with self.assertRaisesRegex(ValueError, "positive"): bench.latency_clearly_worse(0, 1) + self.assertTrue(bench.is_full_c1_pooled_baseline(7.338, 51.816)) + self.assertFalse(bench.is_full_c1_pooled_baseline(7.338, 40.0)) + with self.assertRaisesRegex(ValueError, "screen primary TBD"): + bench.refuse_full_c1_pooled_baseline(7.338, 51.816) + + async def _run_screen(self, root, manifest, name, *, baseline=None, step=None, handler=None): + args = run_args(root) + args.output = root / name + args.manifest = manifest + args.fast_screen = True + args.warmup = 5 + args.baseline_screen = baseline + + def default_handler(request): + body = json.loads(request.content) + self.assertNotIn("prompt_tokens", body) + self.assertEqual(body["model"], "english") + return httpx.Response(200, json=noul_ok()) + + client = httpx.AsyncClient( + transport=httpx.MockTransport(handler or default_handler) + ) + with patch.object(bench.httpx, "AsyncClient", return_value=client): + if step is None: + with redirect_stdout(io.StringIO()): + code = await bench.run(args, bench.load_cases(manifest)) + else: + with ( + patch.object(bench.time, "perf_counter", _stepping_clock(step)), + redirect_stdout(io.StringIO()), + ): + code = await bench.run(args, bench.load_cases(manifest)) + return args, code - async def test_run_fast_screen_keeps_sixty_and_does_not_keep(self): + async def test_screen_primary_is_sixty_ids_and_not_a_keep(self): cases = screen_population() with tempfile.TemporaryDirectory() as directory: root = Path(directory) manifest = root / "public231.jsonl" manifest.write_text("".join(json.dumps(case) + "\n" for case in cases)) - args = run_args(root) - args.manifest = manifest - args.fast_screen = True - args.warmup = 5 - args.baseline_p50_ms = 1_000_000.0 - args.baseline_p95_ms = 1_000_000.0 - seen = [] - - def handler(request): - body = json.loads(request.content) - seen.append(body) - self.assertNotIn("prompt_tokens", body) - self.assertEqual(body["model"], "english") - return httpx.Response(200, json=noul_ok()) - - client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) - with ( - patch.object(bench.httpx, "AsyncClient", return_value=client), - redirect_stdout(io.StringIO()), - ): - self.assertEqual(await bench.run(args, bench.load_cases(manifest)), 0) + args, code = await self._run_screen(root, manifest, "primary", step=0.002) + self.assertEqual(code, 0) saved = json.loads((args.output / "summary.json").read_text()) self.assertEqual(saved, bench.summarize_saved(args.output)) self.assertEqual(saved["planned_requests"], 60) self.assertEqual(saved["successful_requests"], 60) - self.assertEqual(saved["fast_screen_decision"], "full_c1") - self.assertNotIn("keep", saved["fast_screen_decision"]) - self.assertLess(saved["fast_screen_delta_p50_ms"], 0) + self.assertEqual(saved["fast_screen_decision"], "screen_primary") + self.assertNotIn("dropped_at", saved) + self.assertFalse((args.output / bench.FAST_SCREEN_DROP_NAME).exists()) config = json.loads((args.output / "config.json").read_text()) screen = config["fast_screen"] + self.assertEqual(screen["role"], "screen_primary") + self.assertNotIn("baseline_p50_ms", screen) self.assertEqual(screen["seed"], 231) - self.assertEqual(screen["parent_manifest_sha256"], bench.digest(manifest)) self.assertEqual(len(screen["selected_ids"]), 60) - self.assertEqual( - screen["bucket_counts"]["2048+"], - 16, + self.assertEqual(saved["latency_by_prompt_bucket"]["2048+"]["samples"], 16) + + async def test_tip_compares_same_60_ids_and_drop_records_dropped_at(self): + cases = screen_population() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest = root / "public231.jsonl" + manifest.write_text("".join(json.dumps(case) + "\n" for case in cases)) + primary, _code = await self._run_screen( + root, manifest, "primary", step=0.002 ) - recorded = [ - json.loads(line) - for line in (args.output / "requests.jsonl").read_text().splitlines() - ] - self.assertEqual([case["id"] for case in recorded], screen["selected_ids"]) - buckets = saved["latency_by_prompt_bucket"] - self.assertEqual(buckets["2048+"]["samples"], 16) - self.assertEqual(buckets["0-127"]["samples"], 10) - self.assertEqual(len(seen), 65) + tip, code = await self._run_screen( + root, manifest, "tip", baseline=primary.output, step=0.020 + ) + self.assertEqual(code, 0) + saved = json.loads((tip.output / "summary.json").read_text()) + self.assertEqual(saved, bench.summarize_saved(tip.output)) + self.assertEqual(saved["fast_screen_decision"], "drop") + self.assertRegex(saved["dropped_at"], r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + record = json.loads((tip.output / bench.FAST_SCREEN_DROP_NAME).read_text()) + self.assertEqual(record["dropped_at"], saved["dropped_at"]) + self.assertEqual(record["decision"], "drop") + tip_ids = json.loads((tip.output / "config.json").read_text())["fast_screen"]["selected_ids"] + primary_ids = json.loads((primary.output / "config.json").read_text())["fast_screen"]["selected_ids"] + self.assertEqual(record["selected_ids"], tip_ids) + self.assertEqual(tip_ids, primary_ids) + again = bench.summarize_saved(tip.output) + self.assertEqual(again["dropped_at"], saved["dropped_at"]) + pooled = run_args(root) + pooled.output = root / "pooled" + pooled.manifest = manifest + pooled.fast_screen = True + pooled.warmup = 5 + pooled.baseline_screen = primary.output + pooled.baseline_p50_ms = 7.338 + pooled.baseline_p95_ms = 51.816 + with self.assertRaisesRegex(ValueError, "screen primary TBD"): + await bench.run(pooled, bench.load_cases(manifest)) + self.assertFalse(pooled.output.exists()) + + kept, kept_code = await self._run_screen( + root, manifest, "faster", baseline=primary.output, step=0.001 + ) + self.assertEqual(kept_code, 0) + kept_summary = bench.summarize_saved(kept.output) + self.assertEqual(kept_summary["fast_screen_decision"], "full_c1") + self.assertNotIn("dropped_at", kept_summary) + self.assertFalse((kept.output / bench.FAST_SCREEN_DROP_NAME).exists()) + + async def test_full_c1_directory_and_other_ids_are_refused(self): + cases = screen_population() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest = root / "public231.jsonl" + manifest.write_text("".join(json.dumps(case) + "\n" for case in cases)) + primary, _code = await self._run_screen(root, manifest, "primary", step=0.002) + full_c1 = root / "full-c1" + full_c1.mkdir() + bench.write_json(full_c1 / "config.json", {"metadata": {"gpu": "mock-only"}}) + refused = run_args(root) + refused.output = root / "tip" + refused.manifest = manifest + refused.fast_screen = True + refused.warmup = 5 + refused.baseline_screen = full_c1 + with self.assertRaisesRegex(ValueError, "full-C1"): + await bench.run(refused, bench.load_cases(manifest)) + self.assertFalse(refused.output.exists()) + + config_path = primary.output / "config.json" + config = json.loads(config_path.read_text()) + config["metadata"]["gpu"] = "other-gpu" + bench.write_json(config_path, config) + mismatched = run_args(root) + mismatched.output = root / "mismatch" + mismatched.manifest = manifest + mismatched.fast_screen = True + mismatched.warmup = 5 + mismatched.baseline_screen = primary.output + with self.assertRaisesRegex(ValueError, "gpu does not match"): + await bench.run(mismatched, bench.load_cases(manifest)) + self.assertFalse(mismatched.output.exists()) async def test_failed_fast_screen_is_inconclusive_and_short_bucket_writes_nothing(self): cases = screen_population() @@ -1097,12 +1181,6 @@ async def test_failed_fast_screen_is_inconclusive_and_short_bucket_writes_nothin root = Path(directory) manifest = root / "public231.jsonl" manifest.write_text("".join(json.dumps(case) + "\n" for case in cases)) - args = run_args(root) - args.manifest = manifest - args.fast_screen = True - args.warmup = 5 - args.baseline_p50_ms = 1_000_000.0 - args.baseline_p95_ms = 1_000_000.0 calls = {"n": 0} def handler(_request): @@ -1111,23 +1189,20 @@ def handler(_request): return httpx.Response(503, text="busy") return httpx.Response(200, json=noul_ok()) - client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) - with ( - patch.object(bench.httpx, "AsyncClient", return_value=client), - redirect_stdout(io.StringIO()), - ): - self.assertEqual(await bench.run(args, bench.load_cases(manifest)), 1) + args, code = await self._run_screen( + root, manifest, "broken", handler=handler + ) + self.assertEqual(code, 1) saved = bench.summarize_saved(args.output) self.assertEqual(saved["fast_screen_decision"], "inconclusive") self.assertGreater(saved["failed_requests"], 0) + self.assertNotIn("dropped_at", saved) refused = run_args(root) refused.output = root / "unused" refused.manifest = manifest refused.fast_screen = True refused.warmup = 3 - refused.baseline_p50_ms = 7.338 - refused.baseline_p95_ms = 51.816 with self.assertRaisesRegex(ValueError, "warmup 5"): await bench.run(refused, bench.load_cases(manifest)) self.assertFalse(refused.output.exists()) @@ -1143,8 +1218,16 @@ def handler(_request): short.manifest = short_manifest short.fast_screen = True short.warmup = 5 - short.baseline_p50_ms = 7.338 - short.baseline_p95_ms = 51.816 with self.assertRaisesRegex(ValueError, "2048\\+"): await bench.run(short, bench.load_cases(short_manifest)) self.assertFalse(short.output.exists()) + + +def _stepping_clock(step): + state = {"t": 0.0} + + def tick(): + state["t"] += step + return state["t"] + + return tick From 2b4257f38f51597e1b9c82e08db1e12c58ce4f1f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 26 Oct 2026 08:05:01 +0000 Subject: [PATCH 3/5] Record hk2 smoke PASS and the blocked Instant screen Contract smoke of d2fab1e against Instant at 51f80ad passed 4/4. The 60-request screen primary did not: bucket 1024-2047 has 6 prompts and needs 8. Leave that pair TBD and do not substitute the full-C1 pooled numbers. The quota stays fail-closed. Co-authored-by: Hongsheng Liu --- .../skills/optimize-remeasure-gate/SKILL.md | 28 +++++++++++----- .agents/skills/public231-c1/SKILL.md | 33 +++++++++++++------ .agents/skills/system1-omni-review/SKILL.md | 14 +++++--- benchmarks/README.md | 7 ++-- tests/benchmarks/test_bench.py | 11 +++++++ 5 files changed, 67 insertions(+), 26 deletions(-) diff --git a/.agents/skills/optimize-remeasure-gate/SKILL.md b/.agents/skills/optimize-remeasure-gate/SKILL.md index 6283feca..3798cc62 100644 --- a/.agents/skills/optimize-remeasure-gate/SKILL.md +++ b/.agents/skills/optimize-remeasure-gate/SKILL.md @@ -76,10 +76,15 @@ screen_primary_p50_ms: TBD — measure on primary SHA same 60 IDs screen_primary_p95_ms: TBD — measure on primary SHA same 60 IDs ``` -Fill it only from an hk2 `--fast-screen` run whose decision is -`screen_primary`. A full-C1 KEEP moves the full-C1 primary; it does not by -itself fill this slot. Measure the new primary on the same 60 IDs before -the next tip screen. +Fill it only from a completed `--fast-screen` whose decision is +`screen_primary` on the primary SHA, same machine, same 60 IDs. A full-C1 +KEEP moves the full-C1 primary; it does not by itself fill this slot. +StartLux Instant is still TBD. hk2 refused the Instant +`usage.input_tokens` index before any screened request: bucket 1024–2047 +has 6 prompts, quota 8 (`fast-screen bucket 1024-2047 needs 8 requests; +found 6`). Do not write 7.338/51.816 as that screen primary. Unblock only +with an Instant-valid length index of at least 8 prompts in 1024–2047, or +an InferRSI-approved quota change. This gate does not choose between those. Early DROP only when both screen Δp50 and Δp95 are worse than that same-60 baseline. A metric is worse when its delta (tip minus baseline) is ≥ +2% @@ -96,10 +101,13 @@ next baseline. A negative screen delta is not a gain and is not a KEEP. An KEEP, merge, and ready-for-review require the full C1 dual gate: pooled p50 and pooled p95 both strictly lower, pooled 2048+ p95 strictly lower, and no quality collapse under the rules in the next section. The +2% / +0.5 ms -band does not make a full-C1 tie a KEEP. Ready-for-review also requires hk2 -evidence of `--fast-screen` (screen primary, and the tip screen when a tip -is claimed). Offline MockTransport tests are not that evidence and are not -a review PASS. +band does not make a full-C1 tie a KEEP. hk2 contract smoke of client +`d2fab1e08f231fe2e23540f3b25b2650cd416d09` against Instant at `51f80ad` +passed 4/4 with `evidence_complete` true; that client has no StartLux or +CUDA diff versus `51f80ad`, and offline `tests/benchmarks` there were 109 +OK. That smoke is not the screen primary. Offline MockTransport tests are +not a review PASS. The Instant screen primary is still blocked on the short +1024–2047 bucket. ## INCONCLUSIVE stops the loop @@ -115,7 +123,9 @@ these are true: - a DROP has no `dropped_at` - a KEEP, a gain, or a new primary is claimed from fast-screen numbers - ready or review PASS is claimed from offline MockTransport output without - hk2 `--fast-screen` evidence + the hk2 contract smoke, or a screen-primary p50/p95 is filled from the + full-C1 pooled pair or from the refused Instant index (1024–2047 found 6, + needs 8) - the fast-screen decision was `full_c1` but fewer than three measured rounds finished - a completed `drop` is treated as a KEEP or as the next baseline diff --git a/.agents/skills/public231-c1/SKILL.md b/.agents/skills/public231-c1/SKILL.md index e7f15d13..70c3cbbb 100644 --- a/.agents/skills/public231-c1/SKILL.md +++ b/.agents/skills/public231-c1/SKILL.md @@ -77,15 +77,25 @@ Clef and pplx-decider keep their full-C1 primaries in their recipes and in [the latency leaderboard](../../../benchmarks/latency-leaderboard.md). Do not paste any model's pooled C1 pair into the screen. -Screen primary, same 60 IDs (fill only after that screen is measured; do not -invent the pair): +Screen primary, same 60 IDs. Fill this only from a completed +`--fast-screen` whose decision is `screen_primary`. Do not invent the pair, +and do not copy the full-C1 pooled pair into it: ```text screen_primary_p50_ms: TBD — measure on primary SHA same 60 IDs screen_primary_p95_ms: TBD — measure on primary SHA same 60 IDs ``` -Record the screen primary with `--fast-screen` and no `--baseline-screen`. +StartLux Instant is still that TBD. An hk2 `--fast-screen` of Instant, +indexed by `usage.input_tokens`, was refused before any screened request: +bucket 1024–2047 has 6 prompts and the quota is 8 +(`fast-screen bucket 1024-2047 needs 8 requests; found 6`). Fail-closed is +correct. Unblock only with an Instant-valid length index that has at least +8 prompts in 1024–2047, or with an InferRSI-approved quota change. This +protocol does not choose between those. Do not write 7.338/51.816, or any +other stand-in, as the screen primary. + +Record a screen primary with `--fast-screen` and no `--baseline-screen`. That run's decision is `screen_primary`. It is not a tip verdict and not a KEEP. Later tips pass that directory. @@ -124,13 +134,16 @@ not a KEEP. Do not claim a speedup from fast-screen numbers. Percentiles use the same median and nearest-rank p95 as full C1, on this one round of 60, not an average of rounds. -Before this protocol change is marked ready for review, the PR must include -hk2 evidence of `python benchmarks/bench.py run --fast-screen` for the -screen primary (and the tip comparison, when a tip is claimed). That -evidence records the SHA, `device`, `compute capability`, the 60 IDs, and -`summary.json`. An offline `httpx.MockTransport` unit test is not that -evidence and is not a review PASS. Room posts still omit hostnames; the -hk2 requirement is the ready/review evidence, not a room post. +hk2 contract smoke of client +`d2fab1e08f231fe2e23540f3b25b2650cd416d09` against StartLux Instant served +at `51f80ad681c8f7b1be9b22b5e700bef34e33534c` passed: 4/4, +`evidence_complete` true. That client has no StartLux or CUDA diff versus +`51f80ad`. Offline `tests/benchmarks` on that client: 109 OK. An offline +`httpx.MockTransport` unit test is not a substitute for that smoke and is +not a review PASS. The 4/4 smoke is not a screen primary, not a 60-request +latency pair, and not a KEEP. The Instant screen primary remains the TBD +above because the 1024–2047 bucket was short. Room posts still omit +hostnames; the hk2 note is PR evidence. ## Full C1 KEEP diff --git a/.agents/skills/system1-omni-review/SKILL.md b/.agents/skills/system1-omni-review/SKILL.md index d2f70883..393b9725 100644 --- a/.agents/skills/system1-omni-review/SKILL.md +++ b/.agents/skills/system1-omni-review/SKILL.md @@ -43,11 +43,15 @@ is a blocking review finding, not a note. output directory fail closed. The screen-primary slot stays `TBD — measure on primary SHA same 60 IDs` until that screen exists. Do not accept a paste that fills the slot from the pooled pair. -2. **hk2 smoke.** Ready and review PASS require hk2 evidence of - `python benchmarks/bench.py run --fast-screen` (the screen-primary run, - and the tip comparison when a tip is claimed), including SHA, device, - compute capability, the 60 IDs, and `summary.json`. An offline - `httpx.MockTransport` unit test does not satisfy this check. +2. **hk2 smoke.** Contract smoke on hk2 is required before ready. + Recorded PASS: client `d2fab1e08f231fe2e23540f3b25b2650cd416d09` against + StartLux Instant served at `51f80ad681c8f7b1be9b22b5e700bef34e33534c`, + 4/4, `evidence_complete` true, no StartLux or CUDA diff versus + `51f80ad`, offline `tests/benchmarks` 109 OK. MockTransport output is + not that smoke and is not a PASS. The 4/4 smoke is not a screen primary. + The Instant `--fast-screen` attempt was refused: bucket 1024–2047 has 6 + prompts, quota 8. Do not PASS a review that writes 7.338/51.816, or any + other fabricated pair, into the screen-primary slot. 3. **`dropped_at`.** A screen DROP writes `fast_screen_drop.json` and puts `dropped_at` (`YYYY-MM-DDTHH:MM:SSZ`) on the summary. A DROP missing that field is not auditable and is not a valid DROP. diff --git a/benchmarks/README.md b/benchmarks/README.md index ef201e8e..8374d234 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -129,8 +129,11 @@ samples 60 requests from a manifest whose cases include integer `prompt_tokens` concurrency 1, warmup 5. The screen baseline is an earlier `--fast-screen` of the primary SHA on those same 60 IDs (`--baseline-screen`). Do not pass `--baseline-p50-ms` or `--baseline-p95-ms`; full-C1 pooled numbers, including -StartLux Instant 7.338/51.816, are refused. Until that same-60 screen exists, -the screen primary is `TBD — measure on primary SHA same 60 IDs`. +StartLux Instant 7.338/51.816, are refused. The StartLux Instant screen +primary is `TBD — measure on primary SHA same 60 IDs`. An Instant +`usage.input_tokens` index was refused: bucket 1024–2047 has 6 prompts and +the quota is 8. Do not fill the slot until that bucket has at least 8, or +InferRSI approves a quota change. `fast_screen_decision` is `screen_primary`, `drop` (only when both p50 and p95 are worse by ≥ +2% or ≥ +0.5 ms), `full_c1`, or `inconclusive`. A `drop` writes `dropped_at` in `fast_screen_drop.json`. The field is never a KEEP. The gate is diff --git a/tests/benchmarks/test_bench.py b/tests/benchmarks/test_bench.py index 36b8c9c7..0973f647 100644 --- a/tests/benchmarks/test_bench.py +++ b/tests/benchmarks/test_bench.py @@ -1013,6 +1013,17 @@ def test_short_bucket_and_bad_lengths_fail_closed(self): bad["prompt_tokens"] = True with self.assertRaisesRegex(ValueError, "nonnegative integer"): bench.fast_screen_sample([bad, *cases[1:]]) + mid = [case for case in cases if 1024 <= case["prompt_tokens"] < 2048] + self.assertGreaterEqual(len(mid), 8) + under = [ + case for case in cases if not (1024 <= case["prompt_tokens"] < 2048) + ] + under.extend(mid[:6]) + with self.assertRaisesRegex( + ValueError, + r"fast-screen bucket 1024-2047 needs 8 requests; found 6", + ): + bench.fast_screen_sample(under) def test_drop_only_when_both_percentiles_are_clearly_worse(self): relative_tip = 10 * (1 + bench.FAST_SCREEN_RELATIVE) From 76a920e3d6705e8cdb60290e9ab63e93e21283b6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 26 Oct 2026 08:12:21 +0000 Subject: [PATCH 4/5] Cap the Public231 fast-screen at quotas 10/10/8/8/6/16. A short bucket keeps every available prompt and records quota_shortfall. 2048+ below 16 is inconclusive and cannot early-DROP. Screen primary stays TBD. Co-authored-by: Hongsheng Liu --- .agents/skills/new-model-integration/SKILL.md | 8 +- .../skills/optimize-remeasure-gate/SKILL.md | 59 ++++---- .agents/skills/public231-c1/SKILL.md | 67 +++++---- .agents/skills/system1-omni-review/SKILL.md | 31 ++-- benchmarks/README.md | 24 +-- benchmarks/bench.py | 138 ++++++++++++++---- tests/benchmarks/test_bench.py | 115 +++++++++++---- 7 files changed, 302 insertions(+), 140 deletions(-) diff --git a/.agents/skills/new-model-integration/SKILL.md b/.agents/skills/new-model-integration/SKILL.md index f1e63b56..77b54e57 100644 --- a/.agents/skills/new-model-integration/SKILL.md +++ b/.agents/skills/new-model-integration/SKILL.md @@ -89,9 +89,11 @@ Further latency work stays on this model's draft branch. Run it under [optimize-remeasure-gate](../optimize-remeasure-gate/SKILL.md): paste the C1 p50/p95, `device`, and `compute capability` first. After that primary exists, smoke and the fast-screen run before another full C1. The screen -compares the same 60 request IDs against the primary SHA's own screen, not -against the full-C1 pooled pair. That screen-primary pair stays -`TBD — measure on primary SHA same 60 IDs` until it is measured. A clear +compares the same request IDs against the primary SHA's own screen, not +against the full-C1 pooled pair. Quotas are 10/10/8/8/6/16. A short bucket +records `quota_shortfall` and is not filled from another bucket. 2048+ below +16 is inconclusive and cannot early-DROP. That screen-primary pair stays +`TBD — measure on primary SHA same request IDs` until it is measured. A clear regression is dropped there, with `dropped_at`, and is not a KEEP. Do not stack the next tip commit until that tip has been dropped by the screen or remeasured with full C1 on the same named hardware. Incomplete evidence is diff --git a/.agents/skills/optimize-remeasure-gate/SKILL.md b/.agents/skills/optimize-remeasure-gate/SKILL.md index 3798cc62..e09e06e2 100644 --- a/.agents/skills/optimize-remeasure-gate/SKILL.md +++ b/.agents/skills/optimize-remeasure-gate/SKILL.md @@ -58,10 +58,14 @@ method, or the named hardware. ## Fast-screen before full C1 Follow [public231-c1](../public231-c1/SKILL.md). Smoke must pass before the -screen. Run one fast-screen: 60 requests, seed 231, concurrency 1, warmup 5, -bucket quotas 0–127:10, 128–255:10, 256–511:8, 512–1023:8, 1024–2047:8, -2048+:16. Compare the tip's screen p50 and p95 to the primary SHA's own -screen of those same 60 IDs on the same device, compute capability, and +screen. Run one fast-screen: seed 231, concurrency 1, warmup 5, bucket +quotas 0–127:10, 128–255:10, 256–511:8, 512–1023:8, 1024–2047:6, 2048+:16 +(58 requests when every bucket is full). If a bucket has fewer prompts than +its quota, take every prompt in that bucket and record `quota_shortfall`. +Do not fill from another bucket and do not substitute another model's +lengths. If 2048+ has fewer than 16, the screen is `inconclusive` and cannot +early-DROP. Compare the tip's screen p50 and p95 to the primary SHA's own +screen of those same request IDs on the same device, compute capability, and clock. Pass that directory with `--baseline-screen`. Do not pass full-C1 pooled percentiles. `--baseline-p50-ms` and `--baseline-p95-ms` are refused. @@ -72,21 +76,20 @@ the full-C1 primaries in their recipes and the latency leaderboard. For every model the screen-primary slot is: ```text -screen_primary_p50_ms: TBD — measure on primary SHA same 60 IDs -screen_primary_p95_ms: TBD — measure on primary SHA same 60 IDs +screen_primary_p50_ms: TBD — measure on primary SHA same request IDs +screen_primary_p95_ms: TBD — measure on primary SHA same request IDs ``` Fill it only from a completed `--fast-screen` whose decision is -`screen_primary` on the primary SHA, same machine, same 60 IDs. A full-C1 +`screen_primary` on the primary SHA, same machine, same request IDs. A full-C1 KEEP moves the full-C1 primary; it does not by itself fill this slot. -StartLux Instant is still TBD. hk2 refused the Instant -`usage.input_tokens` index before any screened request: bucket 1024–2047 -has 6 prompts, quota 8 (`fast-screen bucket 1024-2047 needs 8 requests; -found 6`). Do not write 7.338/51.816 as that screen primary. Unblock only -with an Instant-valid length index of at least 8 prompts in 1024–2047, or -an InferRSI-approved quota change. This gate does not choose between those. - -Early DROP only when both screen Δp50 and Δp95 are worse than that same-60 +StartLux Instant is still TBD until measured with quotas 10/10/8/8/6/16. +An earlier hk2 screen of the Instant `usage.input_tokens` index was refused +under the previous 1024–2047 quota of 8 (that bucket has 6 prompts). InferRSI +set the quota to 6 on 2026-10-26. Do not write 7.338/51.816 as that screen +primary. Remeasure with the new quotas before filling the slot. + +Early DROP only when both screen Δp50 and Δp95 are worse than that same-ID baseline. A metric is worse when its delta (tip minus baseline) is ≥ +2% of that baseline or ≥ +0.5 ms. Otherwise, including a mixed result, run full C1. The summary records `screen_primary`, `drop`, `full_c1`, or @@ -102,12 +105,14 @@ KEEP, merge, and ready-for-review require the full C1 dual gate: pooled p50 and pooled p95 both strictly lower, pooled 2048+ p95 strictly lower, and no quality collapse under the rules in the next section. The +2% / +0.5 ms band does not make a full-C1 tie a KEEP. hk2 contract smoke of client -`d2fab1e08f231fe2e23540f3b25b2650cd416d09` against Instant at `51f80ad` -passed 4/4 with `evidence_complete` true; that client has no StartLux or -CUDA diff versus `51f80ad`, and offline `tests/benchmarks` there were 109 -OK. That smoke is not the screen primary. Offline MockTransport tests are -not a review PASS. The Instant screen primary is still blocked on the short -1024–2047 bucket. +`d2fab1e08f231fe2e23540f3b25b2650cd416d09` against Instant served at +`51f80ad681c8f7b1be9b22b5e700bef34e33534c` passed 4/4 with +`evidence_complete` true. Artifacts are under +`~/experiments/2026-10-26-pr181-screen-primary/primary/smoke/` on hk2. +That client has no StartLux or CUDA diff versus `51f80ad`, and offline +`tests/benchmarks` there were 109 OK. That smoke is not the screen primary. +Offline MockTransport tests are not a review PASS. The Instant screen +primary stays TBD until measured with quotas 10/10/8/8/6/16. ## INCONCLUSIVE stops the loop @@ -117,15 +122,17 @@ these are true: - the baseline p50/p95 were not recorded before the edit - smoke did not pass before the latency screen - the fast-screen was skipped on a tip that already has a full-C1 primary -- the screen baseline is a full-C1 pooled p50/p95, a typed percentile, or a - different 60 IDs than `--fast-screen` just sampled +- the screen baseline is a full-C1 pooled p50/p95, a typed percentile, or + different request IDs than `--fast-screen` just sampled +- a short bucket was filled from another bucket, or 2048+ has fewer than 16 + and the screen was treated as a DROP or as a screen primary - the fast-screen decision is `inconclusive` - a DROP has no `dropped_at` - a KEEP, a gain, or a new primary is claimed from fast-screen numbers - ready or review PASS is claimed from offline MockTransport output without the hk2 contract smoke, or a screen-primary p50/p95 is filled from the - full-C1 pooled pair or from the refused Instant index (1024–2047 found 6, - needs 8) + full-C1 pooled pair, or from a screen that has not been measured with + quotas 10/10/8/8/6/16 - the fast-screen decision was `full_c1` but fewer than three measured rounds finished - a completed `drop` is treated as a KEEP or as the next baseline @@ -173,7 +180,7 @@ Before the agent proposes the next change, give it this record: - clock basis, `device`, and `compute capability` for both sides - RSI phase, alternatives considered, and which change is primary - fast-screen decision (`screen_primary`, `drop`, `full_c1`, or - `inconclusive`), the same 60 IDs, `dropped_at` when the decision is + `inconclusive`), the request IDs and any `quota_shortfall`, `dropped_at` when the decision is `drop`, and the screen p50/p95 deltas labeled as screen-only - verdict: improved, regressed, or INCONCLUSIVE - quality/contract result when the path reached full C1 or a KEEP decision diff --git a/.agents/skills/public231-c1/SKILL.md b/.agents/skills/public231-c1/SKILL.md index 70c3cbbb..71c937b7 100644 --- a/.agents/skills/public231-c1/SKILL.md +++ b/.agents/skills/public231-c1/SKILL.md @@ -1,6 +1,6 @@ --- name: public231-c1 -description: Measure Public231 C1 request latency for a model or pull request. Use when a campaign needs the contract smoke, a 60-request fast-screen that can drop a clear regression, then warmup 5, concurrency 1, three measured rounds, SHA, pass rate, and p50/p95 on the bench.py clock. A fast-screen is not a KEEP and is not a gain. Every paste records measured device and compute capability. The documented production target is H200 / SM90; a different measured identity stays a valid run and pairs only with that same recorded identity. Includes how to treat a thinking field and what a room post may contain. +description: Measure Public231 C1 request latency for a model or pull request. Use when a campaign needs the contract smoke, a fast-screen (quotas 10/10/8/8/6/16, 58 requests when every bucket is full) that can drop a clear regression, then warmup 5, concurrency 1, three measured rounds, SHA, pass rate, and p50/p95 on the bench.py clock. A fast-screen is not a KEEP and is not a gain. A short 2048+ bucket is inconclusive and cannot early-DROP. Every paste records measured device and compute capability. The documented production target is H200 / SM90; a different measured identity stays a valid run and pairs only with that same recorded identity. Includes how to treat a thinking field and what a room post may contain. --- # Public231 C1 @@ -20,11 +20,13 @@ relabel, or a room post that contains operator metadata. must succeed before any latency screen. A model recipe that already runs a server contract smoke still runs that smoke. A failed smoke is not a latency result and is not a DROP. -2. **Fast-screen.** One 60-request round after smoke. The screen baseline is - the primary SHA served on the same machine and measured on these exact 60 - request IDs. It is not the full-C1 pooled p50/p95. DROP the tip when both - screen percentiles are clearly worse than that same-60 baseline. Otherwise - run full C1. The screen never KEEPs a tip. +2. **Fast-screen.** One round after smoke. Quotas are 10/10/8/8/6/16 (58 + requests when every bucket is full). The screen baseline is the primary + SHA served on the same machine and measured on these exact request IDs. + It is not the full-C1 pooled p50/p95. DROP the tip when both screen + percentiles are clearly worse than that same-ID baseline. Otherwise run + full C1. The screen never KEEPs a tip. If 2048+ has fewer than 16 + prompts, the screen is inconclusive and cannot early-DROP. 3. **Full Public231 C1.** Unchanged measurement: warmup 5, concurrency 1, three measured rounds. KEEP only on the full dual gate below. @@ -56,16 +58,21 @@ Bucket quotas (inclusive lower bound, exclusive upper bound): | 128–255 | 10 | | 256–511 | 8 | | 512–1023 | 8 | -| 1024–2047 | 8 | +| 1024–2047 | 6 | | 2048+ | 16 | -That is 60 requests. 2048+ is the overweight. A bucket with fewer cases than -its quota is not a screen; fix the length index or stop. Do not fill the -quota from another bucket. Do not run a second screen round and pool it. +That is 58 requests when every bucket is full. 2048+ is the overweight. + +Cap-to-available: if a bucket has fewer prompts than its quota, take every +prompt in that bucket and record `quota_shortfall` as `{name, quota, found}`. +Do not fill the gap from another bucket. Do not substitute another model's +lengths. Do not run a second screen round and pool it. If **2048+ has fewer +than 16**, the whole screen is `inconclusive`. It cannot early-DROP, and it +cannot be a screen primary. Compare the tip screen's successful p50 and p95 only to a screen primary: the same primary SHA, same `device` / `gpu`, same manifest, same seed 231, -same bucket quotas, and the same 60 request IDs. Pass that saved run as +same bucket quotas, and the same request IDs. Pass that saved run as `--baseline-screen`. Do not pass `--baseline-p50-ms` or `--baseline-p95-ms`. The runner rejects those flags. A full-C1 output directory is also rejected. @@ -77,23 +84,22 @@ Clef and pplx-decider keep their full-C1 primaries in their recipes and in [the latency leaderboard](../../../benchmarks/latency-leaderboard.md). Do not paste any model's pooled C1 pair into the screen. -Screen primary, same 60 IDs. Fill this only from a completed +Screen primary, same request IDs. Fill this only from a completed `--fast-screen` whose decision is `screen_primary`. Do not invent the pair, and do not copy the full-C1 pooled pair into it: ```text -screen_primary_p50_ms: TBD — measure on primary SHA same 60 IDs -screen_primary_p95_ms: TBD — measure on primary SHA same 60 IDs +screen_primary_p50_ms: TBD — measure on primary SHA same request IDs +screen_primary_p95_ms: TBD — measure on primary SHA same request IDs ``` -StartLux Instant is still that TBD. An hk2 `--fast-screen` of Instant, -indexed by `usage.input_tokens`, was refused before any screened request: -bucket 1024–2047 has 6 prompts and the quota is 8 -(`fast-screen bucket 1024-2047 needs 8 requests; found 6`). Fail-closed is -correct. Unblock only with an Instant-valid length index that has at least -8 prompts in 1024–2047, or with an InferRSI-approved quota change. This -protocol does not choose between those. Do not write 7.338/51.816, or any -other stand-in, as the screen primary. +StartLux Instant is still that TBD until a screen is measured with these +quotas (10/10/8/8/6/16). An earlier hk2 `--fast-screen` of Instant, indexed +by `usage.input_tokens`, was refused under the previous quota of 8 for +1024–2047 because that bucket has 6 prompts. InferRSI set the quota to 6 +on 2026-10-26. That refusal is not a screen-primary pair. Do not write +7.338/51.816, or any other stand-in, as the screen primary. Remeasure the +primary SHA with the new quotas before filling the slot. Record a screen primary with `--fast-screen` and no `--baseline-screen`. That run's decision is `screen_primary`. It is not a tip verdict and not a @@ -127,12 +133,13 @@ The saved summary field `fast_screen_decision` is `screen_primary`, `drop`, copies `dropped_at` onto the summary. `dropped_at` is UTC `YYYY-MM-DDTHH:MM:SSZ`, written once. A DROP without `dropped_at` is not an auditable DROP. `inconclusive` means any measured request failed, was -incomplete, or was unattempted, warmup failed, or `evidence_complete` is -not true. An inconclusive screen is not a DROP and not permission to +incomplete, or was unattempted, warmup failed, `evidence_complete` is not +true, or bucket 2048+ has fewer than 16 prompts. A short 2048+ tail cannot +early-DROP. An inconclusive screen is not a DROP and not permission to continue. A negative screen delta is not a gain, not a new primary, and not a KEEP. Do not claim a speedup from fast-screen numbers. Percentiles -use the same median and nearest-rank p95 as full C1, on this one round of -60, not an average of rounds. +use the same median and nearest-rank p95 as full C1, on this one capped +round (58 requests when every bucket is full), not an average of rounds. hk2 contract smoke of client `d2fab1e08f231fe2e23540f3b25b2650cd416d09` against StartLux Instant served @@ -140,9 +147,11 @@ at `51f80ad681c8f7b1be9b22b5e700bef34e33534c` passed: 4/4, `evidence_complete` true. That client has no StartLux or CUDA diff versus `51f80ad`. Offline `tests/benchmarks` on that client: 109 OK. An offline `httpx.MockTransport` unit test is not a substitute for that smoke and is -not a review PASS. The 4/4 smoke is not a screen primary, not a 60-request +not a review PASS. Smoke artifacts are under +`~/experiments/2026-10-26-pr181-screen-primary/primary/smoke/` on hk2. +The 4/4 smoke is not a screen primary, not a fast-screen latency pair, and not a KEEP. The Instant screen primary remains the TBD -above because the 1024–2047 bucket was short. Room posts still omit +above until it is measured with quotas 10/10/8/8/6/16. Room posts still omit hostnames; the hk2 note is PR evidence. ## Full C1 KEEP @@ -309,7 +318,7 @@ Publish: A fast-screen paste is a separate record. Label it fast-screen, and include the decision (`screen_primary`, `drop`, `full_c1`, or `inconclusive`), seed -231, the same 60 IDs, the primary SHA of the screen baseline when the run +231, the request IDs, any `quota_shortfall`, the primary SHA of the screen baseline when the run is a tip, both deltas when they exist, `dropped_at` when the decision is `drop`, and the same `device` and `compute capability`. Do not publish that paste as a C1 baseline or as a speedup. Do not fill screen-primary p50/p95 diff --git a/.agents/skills/system1-omni-review/SKILL.md b/.agents/skills/system1-omni-review/SKILL.md index 393b9725..09b8c04f 100644 --- a/.agents/skills/system1-omni-review/SKILL.md +++ b/.agents/skills/system1-omni-review/SKILL.md @@ -1,6 +1,6 @@ --- name: system1-omni-review -description: Review ThinkFlowLab/system1-omni pull requests for Rust serving, model-contract, CUDA/Metal compatibility and numerical or performance evidence, with pinned review provenance. For a Public231 fast-screen, check the same-60-ID screen baseline, hk2 --fast-screen evidence, dropped_at on DROP, and that KEEP is still full C1 only. +description: Review ThinkFlowLab/system1-omni pull requests for Rust serving, model-contract, CUDA/Metal compatibility and numerical or performance evidence, with pinned review provenance. For a Public231 fast-screen, check the same-request-ID screen baseline (quotas 10/10/8/8/6/16, cap-to-available, 2048+ below 16 is inconclusive), hk2 smoke evidence, dropped_at on DROP, and that KEEP is still full C1 only. --- # System1-Omni PR review @@ -35,23 +35,28 @@ When the change touches `benchmarks/bench.py`, the Public231 skills, or a claim that a tip was screened or dropped, check all four of these. A miss is a blocking review finding, not a note. -1. **Same-60 screen baseline.** The tip screen p50/p95 are compared to the +1. **Same-ID screen baseline.** The tip screen p50/p95 are compared to the primary SHA's own `--fast-screen` of those exact request IDs (same seed, - buckets, quotas, and device). Full-C1 pooled numbers are not that - baseline. StartLux Instant 7.338/51.816 from `51f80ad` is a pooled C1 - pair. Typed `--baseline-p50-ms` / `--baseline-p95-ms` and a full-C1 - output directory fail closed. The screen-primary slot stays - `TBD — measure on primary SHA same 60 IDs` until that screen exists. - Do not accept a paste that fills the slot from the pooled pair. + buckets, quotas 10/10/8/8/6/16, and device). Full-C1 pooled numbers are + not that baseline. StartLux Instant 7.338/51.816 from `51f80ad` is a + pooled C1 pair. Typed `--baseline-p50-ms` / `--baseline-p95-ms` and a + full-C1 output directory fail closed. A bucket shorter than its quota + records `quota_shortfall` and is not filled from another bucket. If + 2048+ has fewer than 16, the screen is `inconclusive` and cannot + early-DROP or stand as a screen primary. The screen-primary slot stays + `TBD — measure on primary SHA same request IDs` until that screen exists + under these quotas. Do not accept a paste that fills the slot from the + pooled pair. 2. **hk2 smoke.** Contract smoke on hk2 is required before ready. Recorded PASS: client `d2fab1e08f231fe2e23540f3b25b2650cd416d09` against StartLux Instant served at `51f80ad681c8f7b1be9b22b5e700bef34e33534c`, 4/4, `evidence_complete` true, no StartLux or CUDA diff versus - `51f80ad`, offline `tests/benchmarks` 109 OK. MockTransport output is - not that smoke and is not a PASS. The 4/4 smoke is not a screen primary. - The Instant `--fast-screen` attempt was refused: bucket 1024–2047 has 6 - prompts, quota 8. Do not PASS a review that writes 7.338/51.816, or any - other fabricated pair, into the screen-primary slot. + `51f80ad`, offline `tests/benchmarks` 109 OK. Artifacts are under + `~/experiments/2026-10-26-pr181-screen-primary/primary/smoke/` on hk2. + MockTransport output is not that smoke and is not a PASS. The 4/4 smoke + is not a screen primary. The Instant screen primary stays TBD until + measured with quotas 10/10/8/8/6/16. Do not PASS a review that writes + 7.338/51.816, or any other fabricated pair, into the screen-primary slot. 3. **`dropped_at`.** A screen DROP writes `fast_screen_drop.json` and puts `dropped_at` (`YYYY-MM-DDTHH:MM:SSZ`) on the summary. A DROP missing that field is not auditable and is not a valid DROP. diff --git a/benchmarks/README.md b/benchmarks/README.md index 8374d234..144a3128 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -123,17 +123,19 @@ manifest three times still represents 231 independent tasks, not 693 tasks. A Public231 tip that already has a full-C1 primary can be dropped before those three traversals. Validate `benchmarks/smoke.jsonl` first. `--fast-screen` then -samples 60 requests from a manifest whose cases include integer `prompt_tokens` -(not sent on the wire): quotas 10, 10, 8, 8, 8, and 16 for buckets 0–127, -128–255, 256–511, 512–1023, 1024–2047, and 2048+. Seed 231, one measured round, -concurrency 1, warmup 5. The screen baseline is an earlier `--fast-screen` of -the primary SHA on those same 60 IDs (`--baseline-screen`). Do not pass -`--baseline-p50-ms` or `--baseline-p95-ms`; full-C1 pooled numbers, including -StartLux Instant 7.338/51.816, are refused. The StartLux Instant screen -primary is `TBD — measure on primary SHA same 60 IDs`. An Instant -`usage.input_tokens` index was refused: bucket 1024–2047 has 6 prompts and -the quota is 8. Do not fill the slot until that bucket has at least 8, or -InferRSI approves a quota change. +samples a manifest whose cases include integer `prompt_tokens` (not sent on +the wire): quotas 10, 10, 8, 8, 6, and 16 for buckets 0–127, 128–255, +256–511, 512–1023, 1024–2047, and 2048+ (58 requests when every bucket is +full). A shorter bucket keeps every available prompt and records +`quota_shortfall`; nothing is borrowed from another bucket. If 2048+ has +fewer than 16, the decision is `inconclusive` and cannot early-DROP. Seed +231, one measured round, concurrency 1, warmup 5. The screen baseline is an +earlier `--fast-screen` of the primary SHA on those same request IDs +(`--baseline-screen`). Do not pass `--baseline-p50-ms` or +`--baseline-p95-ms`; full-C1 pooled numbers, including StartLux Instant +7.338/51.816, are refused. The StartLux Instant screen primary is +`TBD — measure on primary SHA same request IDs` until measured with these +quotas. `fast_screen_decision` is `screen_primary`, `drop` (only when both p50 and p95 are worse by ≥ +2% or ≥ +0.5 ms), `full_c1`, or `inconclusive`. A `drop` writes `dropped_at` in `fast_screen_drop.json`. The field is never a KEEP. The gate is diff --git a/benchmarks/bench.py b/benchmarks/bench.py index 856a6e68..21d9da86 100755 --- a/benchmarks/bench.py +++ b/benchmarks/bench.py @@ -45,19 +45,22 @@ def response_digest(text): FAST_SCREEN_RELATIVE = 0.02 FAST_SCREEN_ABSOLUTE_MS = 0.5 FAST_SCREEN_DROP_NAME = "fast_screen_drop.json" -# Placeholder until the primary SHA is screened on these same 60 IDs. -SCREEN_PRIMARY_TBD = "screen primary TBD — measure on primary SHA same 60 IDs" +# Placeholder until the primary SHA is screened on these same request IDs. +SCREEN_PRIMARY_TBD = "screen primary TBD — measure on primary SHA same request IDs" # Full-C1 pooled pairs. These are not screen baselines. StartLux Instant's # pooled primary is the first entry; do not copy it onto a fast-screen. FULL_C1_POOLED_NOT_SCREEN = ((7.338, 51.816),) +# Quotas sum to 58 when every bucket is full. 1024-2047 is 6 (InferRSI +# 2026-10-26). A shorter bucket is capped, not filled from another bucket. FAST_SCREEN_BUCKETS = ( ("0-127", 0, 128, 10), ("128-255", 128, 256, 10), ("256-511", 256, 512, 8), ("512-1023", 512, 1024, 8), - ("1024-2047", 1024, 2048, 8), + ("1024-2047", 1024, 2048, 6), ("2048+", 2048, None, 16), ) +FAST_SCREEN_TAIL = "2048+" def saved_response_format(config): @@ -299,27 +302,37 @@ def _fast_screen_rank(seed, bucket, case_id): return hashlib.sha256(f"{seed}\n{bucket}\n{case_id}".encode()).hexdigest() +def fast_screen_quotas(): + return {name: quota for name, _lo, _hi, quota in FAST_SCREEN_BUCKETS} + + def fast_screen_sample(cases, seed=FAST_SCREEN_SEED): - """Pick the fixed 60-request screen. Returns cases in manifest order.""" + """Cap each bucket to its quota. A short bucket keeps every case. + + Nothing is borrowed from another bucket. `quota_shortfall` lists each + bucket whose pool was smaller than its quota. 2048+ below 16 does not + raise here; the saved decision is inconclusive and cannot early-DROP. + """ grouped = {name: [] for name, *_rest in FAST_SCREEN_BUCKETS} for case in cases: grouped[fast_screen_bucket(prompt_token_count(case))].append(case) chosen = set() counts = {} + shortfall = [] for name, _lo, _hi, quota in FAST_SCREEN_BUCKETS: pool = grouped[name] - if len(pool) < quota: - raise ValueError( - f"fast-screen bucket {name} needs {quota} requests; found {len(pool)}" - ) ranked = sorted( pool, key=lambda case: (_fast_screen_rank(seed, name, case["id"]), case["id"]), ) picked = ranked[:quota] counts[name] = len(picked) + if len(pool) < quota: + shortfall.append({"name": name, "quota": quota, "found": len(pool)}) chosen.update(case["id"] for case in picked) - return [case for case in cases if case["id"] in chosen], counts + if not chosen: + raise ValueError("fast-screen selected no requests") + return [case for case in cases if case["id"] in chosen], counts, shortfall def latency_clearly_worse(baseline_ms, tip_ms): @@ -426,7 +439,7 @@ def _valid_dropped_at(value): return time.strftime("%Y-%m-%dT%H:%M:%SZ", parsed) == value -def load_same_60_screen_baseline(path, selected_ids, parent_manifest_sha256, tip_gpu): +def load_same_request_screen_baseline(path, selected_ids, parent_manifest_sha256, tip_gpu): """Baseline is the primary SHA's own screen of these exact request IDs.""" if not isinstance(path, Path) or not path.is_dir(): raise ValueError("baseline screen directory is missing") @@ -441,7 +454,7 @@ def load_same_60_screen_baseline(path, selected_ids, parent_manifest_sha256, tip if screen.get("role") != "screen_primary": raise ValueError( "baseline screen must be the primary SHA's own fast-screen " - "(role screen_primary) of these same 60 IDs, not a tip comparison " + "(role screen_primary) of these same request IDs, not a tip comparison " "and not full-C1 pooled percentiles. " + SCREEN_PRIMARY_TBD ) if screen.get("selected_ids") != list(selected_ids): @@ -452,8 +465,7 @@ def load_same_60_screen_baseline(path, selected_ids, parent_manifest_sha256, tip raise ValueError("baseline screen seed does not match") if screen.get("parent_manifest_sha256") != parent_manifest_sha256: raise ValueError("baseline screen manifest does not match") - expected = {name: quota for name, _lo, _hi, quota in FAST_SCREEN_BUCKETS} - if screen.get("bucket_counts") != expected: + if screen.get("quotas") != fast_screen_quotas(): raise ValueError("baseline screen quotas do not match") metadata = config.get("metadata") or {} gpu = metadata.get("gpu") @@ -472,7 +484,7 @@ def load_same_60_screen_baseline(path, selected_ids, parent_manifest_sha256, tip p95 = summary["successful_latency_p95_ms"] refuse_full_c1_pooled_baseline(p50, p95) return { - "baseline_kind": "same_60_id_screen", + "baseline_kind": "same_request_id_screen", "baseline_runtime_revision": revision, "baseline_gpu": gpu, "baseline_p50_ms": p50, @@ -501,13 +513,9 @@ def _check_saved_fast_screen(screen, cases, config): or screen.get("absolute_worse_ms") != FAST_SCREEN_ABSOLUTE_MS ): raise ValueError("fast-screen thresholds do not match this runner") - expected = {name: quota for name, _lo, _hi, quota in FAST_SCREEN_BUCKETS} - if screen.get("bucket_counts") != expected: - raise ValueError("fast-screen bucket counts do not match the protocol") + _validate_screen_counts(screen, cases) if screen.get("selected_ids") != [case["id"] for case in cases]: raise ValueError("fast-screen selection does not match the saved manifest") - if len(cases) != sum(expected.values()): - raise ValueError("fast-screen manifest is not 60 requests") role = screen.get("role") if role == "screen_primary": if "baseline_p50_ms" in screen or "baseline_p95_ms" in screen: @@ -518,21 +526,85 @@ def _check_saved_fast_screen(screen, cases, config): return if role != "tip": raise ValueError("unsupported fast-screen role") - if screen.get("baseline_kind") != "same_60_id_screen": + if screen.get("baseline_kind") != "same_request_id_screen": raise ValueError( - "tip fast-screen baseline must be the same 60 IDs. " + SCREEN_PRIMARY_TBD + "tip fast-screen baseline must be the same request IDs. " + + SCREEN_PRIMARY_TBD ) if "baseline_p50_ms" not in screen or "baseline_p95_ms" not in screen: - raise ValueError("saved tip fast-screen is missing its same-60 baseline") + raise ValueError("saved tip fast-screen is missing its same-ID baseline") refuse_full_c1_pooled_baseline(screen["baseline_p50_ms"], screen["baseline_p95_ms"]) if screen.get("baseline_selected_ids") != screen["selected_ids"]: - raise ValueError("saved screen baseline is not the same 60 request IDs") + raise ValueError("saved screen baseline is not the same request IDs") if screen.get("baseline_gpu") != config.get("metadata", {}).get("gpu"): raise ValueError("saved screen baseline gpu does not match this run") if not screen.get("baseline_runtime_revision"): raise ValueError("saved screen baseline is missing the primary SHA") +def _validate_screen_counts(screen, cases): + """Taken counts stay inside each quota. A gap is a recorded shortfall.""" + quotas = fast_screen_quotas() + if screen.get("quotas") != quotas: + raise ValueError("fast-screen quotas do not match this runner") + counts = screen.get("bucket_counts") + if not isinstance(counts, dict) or set(counts) != set(quotas): + raise ValueError("fast-screen bucket counts do not match the protocol") + shortfall = screen.get("quota_shortfall") + if not isinstance(shortfall, list): + raise ValueError("fast-screen quota_shortfall is missing") + short_by_name = {} + for item in shortfall: + if not isinstance(item, dict): + raise ValueError("fast-screen quota_shortfall has a bad bucket") + name = item.get("name") + if name not in quotas or name in short_by_name: + raise ValueError("fast-screen quota_shortfall has a bad bucket") + if item.get("quota") != quotas[name]: + raise ValueError("fast-screen quota_shortfall quota does not match") + found = item.get("found") + if type(found) is not int or found < 0 or found >= quotas[name]: + raise ValueError("fast-screen quota_shortfall found is not below quota") + short_by_name[name] = found + order = [name for name, *_rest in FAST_SCREEN_BUCKETS if name in short_by_name] + if [item["name"] for item in shortfall] != order: + raise ValueError("fast-screen quota_shortfall is out of bucket order") + for name, quota in quotas.items(): + taken = counts[name] + if type(taken) is not int or taken < 0 or taken > quota: + raise ValueError("fast-screen bucket count is outside the quota") + if name in short_by_name: + if taken != short_by_name[name]: + raise ValueError( + "fast-screen shortfall does not match the taken count" + ) + elif taken != quota: + raise ValueError( + "fast-screen bucket is below quota without quota_shortfall" + ) + if sum(counts.values()) != len(cases): + raise ValueError("fast-screen manifest does not match the taken bucket counts") + actual = {name: 0 for name in quotas} + for case in cases: + actual[fast_screen_bucket(prompt_token_count(case))] += 1 + if actual != counts: + raise ValueError("fast-screen cases do not match bucket_counts") + + +def _tail_short(screen): + """2048+ below 16 makes the whole screen inconclusive. No early DROP.""" + counts = screen.get("bucket_counts") or {} + taken = counts.get(FAST_SCREEN_TAIL) + if type(taken) is int and taken < fast_screen_quotas()[FAST_SCREEN_TAIL]: + return True + for item in screen.get("quota_shortfall") or []: + if isinstance(item, dict) and item.get("name") == FAST_SCREEN_TAIL: + found = item.get("found") + if type(found) is not int or found < fast_screen_quotas()[FAST_SCREEN_TAIL]: + return True + return False + + def _saved_fast_screen_decision(screen, summary): complete = ( summary["evidence_complete"] @@ -545,6 +617,8 @@ def _saved_fast_screen_decision(screen, summary): ) if not complete: return "inconclusive" + if _tail_short(screen): + return "inconclusive" if screen.get("role") == "screen_primary": return "screen_primary" return fast_screen_decision( @@ -567,7 +641,7 @@ def _attach_dropped_at(path, screen, summary): if record.get("decision") != "drop" or not _valid_dropped_at(record.get("dropped_at")): raise ValueError("fast-screen DROP record is missing dropped_at") if record.get("selected_ids") != screen["selected_ids"]: - raise ValueError("fast-screen DROP record does not match the 60 IDs") + raise ValueError("fast-screen DROP record does not match the screen request IDs") summary["dropped_at"] = record["dropped_at"] @@ -926,7 +1000,7 @@ async def run(args, cases): screen_record = None if fast_screen: _require_fast_screen_shape(args) - cases, bucket_counts = fast_screen_sample(cases) + cases, bucket_counts, quota_shortfall = fast_screen_sample(cases) selected_ids = [case["id"] for case in cases] parent_sha = digest(args.manifest) screen_record = { @@ -942,14 +1016,16 @@ async def run(args, cases): } for name, lo, hi, quota in FAST_SCREEN_BUCKETS ], + "quotas": fast_screen_quotas(), "selected_ids": selected_ids, "bucket_counts": bucket_counts, + "quota_shortfall": quota_shortfall, "parent_manifest_sha256": parent_sha, } if baseline_screen is not None: screen_record["role"] = "tip" screen_record.update( - load_same_60_screen_baseline( + load_same_request_screen_baseline( baseline_screen, selected_ids, parent_sha, @@ -1234,6 +1310,7 @@ def summarize_saved(path): ) if screen is not None: summary["fast_screen_decision"] = _saved_fast_screen_decision(screen, summary) + summary["quota_shortfall"] = screen.get("quota_shortfall") or [] if screen.get("role") == "tip": p50 = summary["successful_latency_p50_ms"] p95 = summary["successful_latency_p95_ms"] @@ -1350,16 +1427,17 @@ def main(): "--fast-screen", action="store_true", help=( - "sample 60 Public231 requests (seed 231, length buckets) for one " - "measured round. Without --baseline-screen this records the " - "screen primary. A tip passes that directory, not pooled C1 numbers." + "sample Public231 requests (seed 231, quotas 10/10/8/8/6/16) for " + "one measured round. A short bucket is capped and recorded. " + "Without --baseline-screen this records the screen primary. " + "A tip passes that directory, not pooled C1 numbers." ), ) runner.add_argument( "--baseline-screen", type=Path, help=( - "primary SHA's own fast-screen directory of these same 60 IDs. " + "primary SHA's own fast-screen directory of these same request IDs. " "Not a full-C1 pooled p50/p95." ), ) diff --git a/tests/benchmarks/test_bench.py b/tests/benchmarks/test_bench.py index 0973f647..48e4b691 100644 --- a/tests/benchmarks/test_bench.py +++ b/tests/benchmarks/test_bench.py @@ -951,11 +951,12 @@ def test_protocol_constants_match_the_confirmed_quotas(self): ("128-255", 10), ("256-511", 8), ("512-1023", 8), - ("1024-2047", 8), + ("1024-2047", 6), ("2048+", 16), ], ) - self.assertEqual(sum(quota for *_rest, quota in bench.FAST_SCREEN_BUCKETS), 60) + self.assertEqual(sum(quota for *_rest, quota in bench.FAST_SCREEN_BUCKETS), 58) + self.assertEqual(bench.FAST_SCREEN_BUCKETS[4][3], 6) self.assertEqual(bench.FAST_SCREEN_SEED, 231) self.assertEqual(bench.FAST_SCREEN_RELATIVE, 0.02) self.assertEqual(bench.FAST_SCREEN_ABSOLUTE_MS, 0.5) @@ -978,10 +979,11 @@ def test_protocol_constants_match_the_confirmed_quotas(self): def test_sample_is_stable_quota_and_manifest_order(self): cases = screen_population() - selected, counts = bench.fast_screen_sample(cases) - again, _counts = bench.fast_screen_sample(cases) + selected, counts, shortfall = bench.fast_screen_sample(cases) + again, _counts, _short = bench.fast_screen_sample(cases) self.assertEqual([case["id"] for case in selected], [case["id"] for case in again]) - self.assertEqual(sum(counts.values()), 60) + self.assertEqual(sum(counts.values()), 58) + self.assertEqual(shortfall, []) self.assertEqual( counts, { @@ -989,7 +991,7 @@ def test_sample_is_stable_quota_and_manifest_order(self): "128-255": 10, "256-511": 8, "512-1023": 8, - "1024-2047": 8, + "1024-2047": 6, "2048+": 16, }, ) @@ -999,12 +1001,15 @@ def test_sample_is_stable_quota_and_manifest_order(self): for case in selected: self.assertNotIn("prompt_tokens", case["request"]) - def test_short_bucket_and_bad_lengths_fail_closed(self): + def test_short_bucket_caps_without_borrowing(self): cases = screen_population() long = [case for case in cases if case["prompt_tokens"] >= 2048][:15] - short = [case for case in cases if case["prompt_tokens"] < 2048] + long - with self.assertRaisesRegex(ValueError, "2048\\+"): - bench.fast_screen_sample(short) + short_tail = [case for case in cases if case["prompt_tokens"] < 2048] + long + _selected, counts, shortfall = bench.fast_screen_sample(short_tail) + self.assertEqual(counts["2048+"], 15) + self.assertEqual(counts["1024-2047"], 6) + self.assertEqual(shortfall, [{"name": "2048+", "quota": 16, "found": 15}]) + self.assertEqual(sum(counts.values()), 57) missing = dict(cases[0]) missing.pop("prompt_tokens") with self.assertRaisesRegex(ValueError, "prompt_tokens"): @@ -1014,16 +1019,65 @@ def test_short_bucket_and_bad_lengths_fail_closed(self): with self.assertRaisesRegex(ValueError, "nonnegative integer"): bench.fast_screen_sample([bad, *cases[1:]]) mid = [case for case in cases if 1024 <= case["prompt_tokens"] < 2048] - self.assertGreaterEqual(len(mid), 8) + self.assertGreaterEqual(len(mid), 6) under = [ case for case in cases if not (1024 <= case["prompt_tokens"] < 2048) ] - under.extend(mid[:6]) - with self.assertRaisesRegex( - ValueError, - r"fast-screen bucket 1024-2047 needs 8 requests; found 6", - ): - bench.fast_screen_sample(under) + under.extend(mid[:5]) + selected, counts, shortfall = bench.fast_screen_sample(under) + self.assertEqual(counts["1024-2047"], 5) + self.assertEqual(counts["2048+"], 16) + self.assertEqual(counts["512-1023"], 8) + self.assertEqual(shortfall, [{"name": "1024-2047", "quota": 6, "found": 5}]) + self.assertEqual( + sum(1 for case in selected if 1024 <= case["prompt_tokens"] < 2048), + 5, + ) + exact = [ + case for case in cases if not (1024 <= case["prompt_tokens"] < 2048) + ] + exact.extend(mid[:6]) + _selected, counts, shortfall = bench.fast_screen_sample(exact) + self.assertEqual(counts["1024-2047"], 6) + self.assertEqual(shortfall, []) + with self.assertRaisesRegex(ValueError, "selected no requests"): + bench.fast_screen_sample([]) + + def test_short_tail_cannot_early_drop(self): + summary = { + "evidence_complete": True, + "failed_requests": 0, + "incomplete_requests": 0, + "unattempted_requests": 0, + "successful_requests": 57, + "planned_requests": 57, + "successful_latency_p50_ms": 100.0, + "successful_latency_p95_ms": 100.0, + } + screen = { + "role": "tip", + "bucket_counts": {"2048+": 15}, + "quota_shortfall": [{"name": "2048+", "quota": 16, "found": 15}], + "baseline_p50_ms": 1.0, + "baseline_p95_ms": 1.0, + } + self.assertEqual( + bench._saved_fast_screen_decision(screen, summary), "inconclusive" + ) + screen["role"] = "screen_primary" + self.assertEqual( + bench._saved_fast_screen_decision(screen, summary), "inconclusive" + ) + shorter_mid = { + "role": "tip", + "bucket_counts": {"2048+": 16, "1024-2047": 5}, + "quota_shortfall": [{"name": "1024-2047", "quota": 6, "found": 5}], + "baseline_p50_ms": 1.0, + "baseline_p95_ms": 1.0, + } + self.assertEqual( + bench._saved_fast_screen_decision(shorter_mid, summary), "drop" + ) def test_drop_only_when_both_percentiles_are_clearly_worse(self): relative_tip = 10 * (1 + bench.FAST_SCREEN_RELATIVE) @@ -1092,8 +1146,9 @@ async def test_screen_primary_is_sixty_ids_and_not_a_keep(self): self.assertEqual(code, 0) saved = json.loads((args.output / "summary.json").read_text()) self.assertEqual(saved, bench.summarize_saved(args.output)) - self.assertEqual(saved["planned_requests"], 60) - self.assertEqual(saved["successful_requests"], 60) + self.assertEqual(saved["planned_requests"], 58) + self.assertEqual(saved["successful_requests"], 58) + self.assertEqual(saved["quota_shortfall"], []) self.assertEqual(saved["fast_screen_decision"], "screen_primary") self.assertNotIn("dropped_at", saved) self.assertFalse((args.output / bench.FAST_SCREEN_DROP_NAME).exists()) @@ -1102,7 +1157,9 @@ async def test_screen_primary_is_sixty_ids_and_not_a_keep(self): self.assertEqual(screen["role"], "screen_primary") self.assertNotIn("baseline_p50_ms", screen) self.assertEqual(screen["seed"], 231) - self.assertEqual(len(screen["selected_ids"]), 60) + self.assertEqual(len(screen["selected_ids"]), 58) + self.assertEqual(screen["quotas"]["1024-2047"], 6) + self.assertEqual(screen["quota_shortfall"], []) self.assertEqual(saved["latency_by_prompt_bucket"]["2048+"]["samples"], 16) async def test_tip_compares_same_60_ids_and_drop_records_dropped_at(self): @@ -1224,14 +1281,16 @@ def handler(_request): short_manifest.write_text( "".join(json.dumps(case) + "\n" for case in short_cases) ) - short = run_args(root) - short.output = root / "short-out" - short.manifest = short_manifest - short.fast_screen = True - short.warmup = 5 - with self.assertRaisesRegex(ValueError, "2048\\+"): - await bench.run(short, bench.load_cases(short_manifest)) - self.assertFalse(short.output.exists()) + short_args, code = await self._run_screen(root, short_manifest, "short-out") + self.assertEqual(code, 0) + saved = bench.summarize_saved(short_args.output) + self.assertEqual(saved["fast_screen_decision"], "inconclusive") + self.assertEqual( + saved["quota_shortfall"], + [{"name": "2048+", "quota": 16, "found": 15}], + ) + self.assertNotIn("dropped_at", saved) + self.assertFalse((short_args.output / "fast_screen_drop.json").exists()) def _stepping_clock(step): From f675976db36167d0ec6911805990c184d21701c9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 26 Oct 2026 08:15:22 +0000 Subject: [PATCH 5/5] Record the measured StartLux Instant n=58 screen primary. hk2 screen p50 is 19.112976267933846 ms and p95 is 73.46072793006897 ms on 51f80ad. Full-C1 pooled 7.338/51.816 stays refused as a screen baseline. Co-authored-by: Hongsheng Liu --- .agents/skills/new-model-integration/SKILL.md | 7 +++- .../skills/optimize-remeasure-gate/SKILL.md | 38 +++++++++++------ .agents/skills/public231-c1/SKILL.md | 41 +++++++++++++------ .agents/skills/system1-omni-review/SKILL.md | 24 +++++++---- benchmarks/README.md | 11 +++-- benchmarks/bench.py | 8 +++- tests/benchmarks/test_bench.py | 4 +- 7 files changed, 91 insertions(+), 42 deletions(-) diff --git a/.agents/skills/new-model-integration/SKILL.md b/.agents/skills/new-model-integration/SKILL.md index 77b54e57..6a5a12bb 100644 --- a/.agents/skills/new-model-integration/SKILL.md +++ b/.agents/skills/new-model-integration/SKILL.md @@ -92,8 +92,11 @@ exists, smoke and the fast-screen run before another full C1. The screen compares the same request IDs against the primary SHA's own screen, not against the full-C1 pooled pair. Quotas are 10/10/8/8/6/16. A short bucket records `quota_shortfall` and is not filled from another bucket. 2048+ below -16 is inconclusive and cannot early-DROP. That screen-primary pair stays -`TBD — measure on primary SHA same request IDs` until it is measured. A clear +16 is inconclusive and cannot early-DROP. StartLux Instant's measured +screen primary is SHA `51f80ad681c8f7b1be9b22b5e700bef34e33534c`, p50 +19.112976267933846 ms, p95 73.46072793006897 ms, n=58 on L20X / sm_89. +A model that has not been screened has no screen-primary pair. Do not copy +Instant's screen pair or a full-C1 pooled pair into it. A clear regression is dropped there, with `dropped_at`, and is not a KEEP. Do not stack the next tip commit until that tip has been dropped by the screen or remeasured with full C1 on the same named hardware. Incomplete evidence is diff --git a/.agents/skills/optimize-remeasure-gate/SKILL.md b/.agents/skills/optimize-remeasure-gate/SKILL.md index e09e06e2..ce0207b2 100644 --- a/.agents/skills/optimize-remeasure-gate/SKILL.md +++ b/.agents/skills/optimize-remeasure-gate/SKILL.md @@ -72,22 +72,34 @@ pooled percentiles. `--baseline-p50-ms` and `--baseline-p95-ms` are refused. StartLux Instant's full-C1 pooled primary remains `51f80ad681c8f7b1be9b22b5e700bef34e33534c` (`51f80ad`), p50 7.338 ms, p95 51.816 ms. That pair is not the screen baseline. Clef and pplx-decider keep -the full-C1 primaries in their recipes and the latency leaderboard. For -every model the screen-primary slot is: +the full-C1 primaries in their recipes and the latency leaderboard. Fill a +screen-primary slot only from a completed `--fast-screen` whose decision is +`screen_primary` on the primary SHA, same machine, same request IDs. A full-C1 +KEEP moves the full-C1 primary; it does not by itself fill this slot. + +StartLux Instant screen primary, hk2, quotas 10/10/8/8/6/16, 58/58 success, +`evidence_complete` true, decision `screen_primary`, `quota_shortfall` `{}`: ```text -screen_primary_p50_ms: TBD — measure on primary SHA same request IDs -screen_primary_p95_ms: TBD — measure on primary SHA same request IDs +screen_primary_sha: 51f80ad681c8f7b1be9b22b5e700bef34e33534c +screen_primary_p50_ms: 19.112976267933846 +screen_primary_p95_ms: 73.46072793006897 +n: 58 +bucket_counts: 0-127:10, 128-255:10, 256-511:8, 512-1023:8, 1024-2047:6, 2048+:16 +device: L20X +compute capability: sm_89 ``` -Fill it only from a completed `--fast-screen` whose decision is -`screen_primary` on the primary SHA, same machine, same request IDs. A full-C1 -KEEP moves the full-C1 primary; it does not by itself fill this slot. -StartLux Instant is still TBD until measured with quotas 10/10/8/8/6/16. -An earlier hk2 screen of the Instant `usage.input_tokens` index was refused -under the previous 1024–2047 quota of 8 (that bucket has 6 prompts). InferRSI -set the quota to 6 on 2026-10-26. Do not write 7.338/51.816 as that screen -primary. Remeasure with the new quotas before filling the slot. +The measuring client was `923c8c739bcaf3cc2b098f3cf373c2b547d2666c` plus a +local-only 1024–2047 quota patch from 8 to 6. The remote tip may still have +been on quota 8. Later screens use the committed n=58 quotas and do not +need that patch. Artifacts: +`~/experiments/2026-10-26-pr181-screen-primary/screen-primary-n58/` +(`SCREEN_PRIMARY.json`, `fast-screen/`, `screen_ids.json`, +`local-quota-patch.diff`). Do not write 7.338/51.816 as this screen +primary. Do not type the screen percentiles; pass `--baseline-screen`. +Clef and pplx-decider have no screen primary yet. Do not copy Instant's +screen pair onto them. Early DROP only when both screen Δp50 and Δp95 are worse than that same-ID baseline. A metric is worse when its delta (tip minus baseline) is ≥ +2% @@ -112,7 +124,7 @@ band does not make a full-C1 tie a KEEP. hk2 contract smoke of client That client has no StartLux or CUDA diff versus `51f80ad`, and offline `tests/benchmarks` there were 109 OK. That smoke is not the screen primary. Offline MockTransport tests are not a review PASS. The Instant screen -primary stays TBD until measured with quotas 10/10/8/8/6/16. +primary is the n=58 block above. ## INCONCLUSIVE stops the loop diff --git a/.agents/skills/public231-c1/SKILL.md b/.agents/skills/public231-c1/SKILL.md index 71c937b7..d08be8b4 100644 --- a/.agents/skills/public231-c1/SKILL.md +++ b/.agents/skills/public231-c1/SKILL.md @@ -86,20 +86,36 @@ paste any model's pooled C1 pair into the screen. Screen primary, same request IDs. Fill this only from a completed `--fast-screen` whose decision is `screen_primary`. Do not invent the pair, -and do not copy the full-C1 pooled pair into it: +and do not copy the full-C1 pooled pair into it. + +StartLux Instant screen primary, measured on hk2 with quotas 10/10/8/8/6/16. +58/58 success, `evidence_complete` true, decision `screen_primary`, +`quota_shortfall` `{}`: ```text -screen_primary_p50_ms: TBD — measure on primary SHA same request IDs -screen_primary_p95_ms: TBD — measure on primary SHA same request IDs +screen_primary_sha: 51f80ad681c8f7b1be9b22b5e700bef34e33534c +screen_primary_p50_ms: 19.112976267933846 +screen_primary_p95_ms: 73.46072793006897 +n: 58 +bucket_counts: 0-127:10, 128-255:10, 256-511:8, 512-1023:8, 1024-2047:6, 2048+:16 +device: L20X +compute capability: sm_89 ``` -StartLux Instant is still that TBD until a screen is measured with these -quotas (10/10/8/8/6/16). An earlier hk2 `--fast-screen` of Instant, indexed -by `usage.input_tokens`, was refused under the previous quota of 8 for -1024–2047 because that bucket has 6 prompts. InferRSI set the quota to 6 -on 2026-10-26. That refusal is not a screen-primary pair. Do not write -7.338/51.816, or any other stand-in, as the screen primary. Remeasure the -primary SHA with the new quotas before filling the slot. +The measuring client was `923c8c739bcaf3cc2b098f3cf373c2b547d2666c` plus a +local-only patch that changed the 1024–2047 quota from 8 to 6. The remote +tip may still have been on quota 8 when this was measured. The committed +runner samples these quotas (n=58), so the next screen does not need that +local patch. Artifacts on hk2: +`~/experiments/2026-10-26-pr181-screen-primary/screen-primary-n58/` +(`SCREEN_PRIMARY.json`, `fast-screen/`, `screen_ids.json`, +`local-quota-patch.diff`). + +Pass that saved run as `--baseline-screen`. Do not type +19.112976267933846 / 73.46072793006897, and do not type the full-C1 pooled +7.338/51.816. Other models do not inherit this pair. An earlier hk2 screen +of Instant was refused under the previous 1024–2047 quota of 8; that +refusal is not this screen primary. Record a screen primary with `--fast-screen` and no `--baseline-screen`. That run's decision is `screen_primary`. It is not a tip verdict and not a @@ -150,9 +166,8 @@ at `51f80ad681c8f7b1be9b22b5e700bef34e33534c` passed: 4/4, not a review PASS. Smoke artifacts are under `~/experiments/2026-10-26-pr181-screen-primary/primary/smoke/` on hk2. The 4/4 smoke is not a screen primary, not a fast-screen -latency pair, and not a KEEP. The Instant screen primary remains the TBD -above until it is measured with quotas 10/10/8/8/6/16. Room posts still omit -hostnames; the hk2 note is PR evidence. +latency pair, and not a KEEP. The Instant screen primary is the n=58 block +above. Room posts still omit hostnames; the hk2 note is PR evidence. ## Full C1 KEEP diff --git a/.agents/skills/system1-omni-review/SKILL.md b/.agents/skills/system1-omni-review/SKILL.md index 09b8c04f..f3633eed 100644 --- a/.agents/skills/system1-omni-review/SKILL.md +++ b/.agents/skills/system1-omni-review/SKILL.md @@ -43,10 +43,20 @@ is a blocking review finding, not a note. full-C1 output directory fail closed. A bucket shorter than its quota records `quota_shortfall` and is not filled from another bucket. If 2048+ has fewer than 16, the screen is `inconclusive` and cannot - early-DROP or stand as a screen primary. The screen-primary slot stays - `TBD — measure on primary SHA same request IDs` until that screen exists - under these quotas. Do not accept a paste that fills the slot from the - pooled pair. + early-DROP or stand as a screen primary. StartLux Instant's measured + screen primary is SHA `51f80ad681c8f7b1be9b22b5e700bef34e33534c`, p50 + 19.112976267933846 ms, p95 73.46072793006897 ms, n=58, 58/58, + `evidence_complete` true, decision `screen_primary`, `quota_shortfall` + `{}`, buckets 10/10/8/8/6/16, device L20X, compute capability sm_89. + The measuring client was `923c8c739bcaf3cc2b098f3cf373c2b547d2666c` with + a local-only 1024–2047 quota patch from 8 to 6; the remote tip may still + have been on quota 8. Later screens use the committed n=58 quotas. + Artifacts: + `~/experiments/2026-10-26-pr181-screen-primary/screen-primary-n58/` + (`SCREEN_PRIMARY.json`, `fast-screen/`, `screen_ids.json`, + `local-quota-patch.diff`). + Do not accept a paste that fills the slot from the pooled pair, from + typed screen percentiles, or from another model's lengths. 2. **hk2 smoke.** Contract smoke on hk2 is required before ready. Recorded PASS: client `d2fab1e08f231fe2e23540f3b25b2650cd416d09` against StartLux Instant served at `51f80ad681c8f7b1be9b22b5e700bef34e33534c`, @@ -54,9 +64,9 @@ is a blocking review finding, not a note. `51f80ad`, offline `tests/benchmarks` 109 OK. Artifacts are under `~/experiments/2026-10-26-pr181-screen-primary/primary/smoke/` on hk2. MockTransport output is not that smoke and is not a PASS. The 4/4 smoke - is not a screen primary. The Instant screen primary stays TBD until - measured with quotas 10/10/8/8/6/16. Do not PASS a review that writes - 7.338/51.816, or any other fabricated pair, into the screen-primary slot. + is not a screen primary. The Instant screen primary is the n=58 pair in + check 1, not 7.338/51.816. Do not PASS a review that writes 7.338/51.816, + or any other fabricated pair, into the screen-primary slot. 3. **`dropped_at`.** A screen DROP writes `fast_screen_drop.json` and puts `dropped_at` (`YYYY-MM-DDTHH:MM:SSZ`) on the summary. A DROP missing that field is not auditable and is not a valid DROP. diff --git a/benchmarks/README.md b/benchmarks/README.md index 144a3128..ab3a3e98 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -133,9 +133,14 @@ fewer than 16, the decision is `inconclusive` and cannot early-DROP. Seed earlier `--fast-screen` of the primary SHA on those same request IDs (`--baseline-screen`). Do not pass `--baseline-p50-ms` or `--baseline-p95-ms`; full-C1 pooled numbers, including StartLux Instant -7.338/51.816, are refused. The StartLux Instant screen primary is -`TBD — measure on primary SHA same request IDs` until measured with these -quotas. +7.338/51.816, are refused. The StartLux Instant screen primary, measured +on hk2 (L20X / sm_89) with these quotas, is SHA +`51f80ad681c8f7b1be9b22b5e700bef34e33534c`, p50 19.112976267933846 ms, p95 +73.46072793006897 ms, n=58, 58/58, `evidence_complete` true, decision +`screen_primary`, `quota_shortfall` `{}`, buckets 10/10/8/8/6/16. The +measuring client was `923c8c739bcaf3cc2b098f3cf373c2b547d2666c` plus a +local-only 1024–2047 quota patch from 8 to 6. Pass that saved run as +`--baseline-screen`; do not type the screen percentiles. `fast_screen_decision` is `screen_primary`, `drop` (only when both p50 and p95 are worse by ≥ +2% or ≥ +0.5 ms), `full_c1`, or `inconclusive`. A `drop` writes `dropped_at` in `fast_screen_drop.json`. The field is never a KEEP. The gate is diff --git a/benchmarks/bench.py b/benchmarks/bench.py index 21d9da86..c27f691b 100755 --- a/benchmarks/bench.py +++ b/benchmarks/bench.py @@ -45,8 +45,12 @@ def response_digest(text): FAST_SCREEN_RELATIVE = 0.02 FAST_SCREEN_ABSOLUTE_MS = 0.5 FAST_SCREEN_DROP_NAME = "fast_screen_drop.json" -# Placeholder until the primary SHA is screened on these same request IDs. -SCREEN_PRIMARY_TBD = "screen primary TBD — measure on primary SHA same request IDs" +# Typed percentiles are not a screen baseline. StartLux Instant's recorded +# screen primary is 51f80ad, n=58; pass that saved run, do not type it. +SCREEN_PRIMARY_TBD = ( + "pass the primary SHA's saved --fast-screen of the same request IDs " + "(StartLux Instant 51f80ad, n=58). Do not type percentiles" +) # Full-C1 pooled pairs. These are not screen baselines. StartLux Instant's # pooled primary is the first entry; do not copy it onto a fast-screen. FULL_C1_POOLED_NOT_SCREEN = ((7.338, 51.816),) diff --git a/tests/benchmarks/test_bench.py b/tests/benchmarks/test_bench.py index 48e4b691..7d3a92e7 100644 --- a/tests/benchmarks/test_bench.py +++ b/tests/benchmarks/test_bench.py @@ -1104,7 +1104,7 @@ def test_drop_only_when_both_percentiles_are_clearly_worse(self): bench.latency_clearly_worse(0, 1) self.assertTrue(bench.is_full_c1_pooled_baseline(7.338, 51.816)) self.assertFalse(bench.is_full_c1_pooled_baseline(7.338, 40.0)) - with self.assertRaisesRegex(ValueError, "screen primary TBD"): + with self.assertRaisesRegex(ValueError, "Do not type percentiles"): bench.refuse_full_c1_pooled_baseline(7.338, 51.816) async def _run_screen(self, root, manifest, name, *, baseline=None, step=None, handler=None): @@ -1196,7 +1196,7 @@ async def test_tip_compares_same_60_ids_and_drop_records_dropped_at(self): pooled.baseline_screen = primary.output pooled.baseline_p50_ms = 7.338 pooled.baseline_p95_ms = 51.816 - with self.assertRaisesRegex(ValueError, "screen primary TBD"): + with self.assertRaisesRegex(ValueError, "Do not type percentiles"): await bench.run(pooled, bench.load_cases(manifest)) self.assertFalse(pooled.output.exists())