Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions loopx/capabilities/benchmark_toolkit/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1105,6 +1105,19 @@ loopx benchmark experiment-board-upsert \
--format json
```

Upsert now returns the complete normalized affected `row` and `write` receipt
by default, for both preview and execution. Check `write.dry_run` and
`write.write_performed` before treating a preview as a saved update. The receipt
retains metrics, countability, protocol identity and total ledger `row_count`;
it does not imply a matched comparison or replace whole-study inspection.
Consumers of the previous board-shaped response can add `--include-board`.
`--benchmark-id` and `--study-id` filter that optional board projection; they do
not constrain the write or hide the affected row in the default receipt.
For a later comparison, call `experiment-board-show` with the same Goal/project
route and relevant benchmark/study filters. Reuse a current board already read
when it answers the decision; avoid rebuilding and returning it after every
single-row update. Validation and invalid-transition rejection still apply.

The default ledger is locked, atomically updated, and keyed by benchmark, study,
case, and run identity. A compact row carries arm role, exact and comparison
protocol ids, model, score metrics, countability, treatment fidelity, bounded
Expand Down
40 changes: 39 additions & 1 deletion loopx/cli_commands/benchmark_experiment_board.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from typing import Any

from ..capabilities.benchmark_toolkit import (
BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION,
benchmark_experiment_board_row_key,
build_benchmark_experiment_board,
default_benchmark_experiment_board_path,
Expand Down Expand Up @@ -92,6 +93,14 @@ def register_benchmark_experiment_board_commands(
required=True,
help="Compact benchmark_experiment_board_row_v0 JSON path, or - for stdin.",
)
upsert_parser.add_argument(
"--include-board",
action="store_true",
help=(
"Return the full board projection (legacy response) instead of the "
"default affected row and write receipt. Board filters apply to this projection."
),
)
upsert_parser.add_argument(
"--execute",
action="store_true",
Expand Down Expand Up @@ -135,6 +144,16 @@ def _board_payload(
)


def _render_upsert_receipt(payload: dict[str, object]) -> str:
return "\n".join([
"# Benchmark experiment row",
"",
"```json",
json.dumps(payload, ensure_ascii=False, indent=2),
"```",
])


def _reconcile_experiment_board(
args: argparse.Namespace,
*,
Expand Down Expand Up @@ -200,7 +219,9 @@ def handle_benchmark_experiment_board_command(
row = _read_json_object(args.row_json)
if args.execute:
receipt = upsert_benchmark_experiment_board_row(ledger_path, row)
rows = read_benchmark_experiment_board_rows(ledger_path)
affected_row = receipt["row"]
if args.include_board:
rows = read_benchmark_experiment_board_rows(ledger_path)
write = {
**receipt["write"],
"dry_run": False,
Expand All @@ -209,13 +230,30 @@ def handle_benchmark_experiment_board_command(
rows, preview_status = preview_benchmark_experiment_board_upsert(
rows, row
)
affected_row = next(item for item in rows if _row_key(item) == _row_key(row))
write = {
"status": f"preview_{preview_status}",
"write_performed": False,
"row_count": len(rows),
"path_recorded": False,
"dry_run": True,
}
if not args.include_board:
payload = {
"ok": True,
"schema_version": BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION,
"goal_id": args.goal_id,
"path_recorded": False,
"row": affected_row,
"write": write,
"read_hint": (
"Use this row and receipt for the update decision. For comparisons, "
"use experiment-board-show with the same route and relevant "
"--benchmark-id/--study-id; --include-board restores the full upsert response."
),
}
print_payload(payload, output_format(args), _render_upsert_receipt)
return 0
elif args.benchmark_command == "experiment-board-reconcile":
rows, write = _reconcile_experiment_board(
args,
Expand Down
6 changes: 6 additions & 0 deletions skills/loopx-benchmark/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,12 @@ separate: fewer bytes, calls or tokens are not proof of a better task outcome.
The running row uses `status=running`, empty `metrics`, and
`countability={integrity_qualified:false, official_result_present:false,
score_countable:false}`. Keep the same stable `run_id` for every transition.
Upsert returns the normalized affected `row` and `write` receipt by default.
Check its identity and preview/write outcome directly. Read the board again
when the next comparison needs it, using the same route and relevant
`--benchmark-id`/`--study-id`; do not re-fetch a whole study merely to confirm
one update. `--include-board` restores the previous full-board response for
consumers that need it. A row receipt does not establish comparison eligibility.

4. **Preview and upsert terminal score, countability, effort, and insight.**
First run integrity qualification. An automated restricted-access match is a
Expand Down
59 changes: 58 additions & 1 deletion tests/capabilities/test_benchmark_experiment_board.py
Original file line number Diff line number Diff line change
Expand Up @@ -882,11 +882,14 @@ def test_cli_previews_then_writes_and_reads_board(tmp_path: Path) -> None:
)
assert preview_payload["write"]["status"] == "preview_inserted"
assert preview_payload["write"]["write_performed"] is False
assert preview_payload["row"] == normalize_benchmark_experiment_board_row(_baseline())
assert "runs" not in preview_payload
assert not ledger.exists()

executed = run_source_cli([*command, "--execute"], check=True)
executed_payload = json.loads(executed.stdout)
assert executed_payload["write"]["status"] == "inserted"
assert executed_payload["row"] == preview_payload["row"]
assert executed_payload["path_recorded"] is False
assert str(tmp_path) not in executed.stdout
assert ledger.exists()
Expand All @@ -911,6 +914,60 @@ def test_cli_previews_then_writes_and_reads_board(tmp_path: Path) -> None:
)


@pytest.mark.parametrize("execute", [False, True])
def test_cli_upsert_returns_exact_row_without_board_growth(tmp_path: Path, execute: bool) -> None:
ledger = default_benchmark_experiment_board_path(project=tmp_path, goal_id="fixture-goal")
ledger.parent.mkdir(parents=True, exist_ok=True)
# Run ids may repeat across cases. Preserve an uncountable zero exactly.
candidate = _baseline()
candidate["countability"] = {
"integrity_qualified": False, "official_result_present": True, "score_countable": False,
}
rows = [_baseline(case_id=f"case-{index}") for index in range(1000)]
ledger.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
original = ledger.read_bytes()
row_path = tmp_path / "candidate.json"
row_path.write_text(json.dumps(candidate), encoding="utf-8")
command = ["benchmark", "experiment-board-upsert", "--goal-id", "fixture-goal",
"--project", str(tmp_path), "--row-json", str(row_path), "--format", "json"]
if execute:
command.append("--execute")
result = run_source_cli(command)
payload = json.loads(result.stdout)
assert payload["row"] == normalize_benchmark_experiment_board_row(candidate)
assert payload["write"]["row_count"] == 1000
assert payload["write"]["write_performed"] is execute
assert "experiment-board-show" in payload["read_hint"]
assert "runs" not in payload and "summary" not in payload
assert len(result.stdout.encode()) < 6000 # One complete row, not 1000 rows.
if execute:
observed = read_benchmark_experiment_board_rows(ledger)
assert len(observed) == 1000
assert all(row["countability"]["score_countable"] is (row["case_id"] != "case-12")
for row in observed)
else:
assert ledger.read_bytes() == original
full = json.loads(run_source_cli([*command, "--include-board"]).stdout)
assert full["summary"]["run_count"] == 1000
assert len(full["runs"]) == 1000
assert full["write"]["write_performed"] is execute


def test_cli_compact_upsert_rejects_invalid_transition_without_writing(tmp_path: Path) -> None:
ledger = default_benchmark_experiment_board_path(project=tmp_path, goal_id="fixture-goal")
upsert_benchmark_experiment_board_row(ledger, _baseline())
before = ledger.read_bytes()
row_path = tmp_path / "row.json"
row_path.write_text(json.dumps(_running_baseline()), encoding="utf-8")
result = run_source_cli([
"benchmark", "experiment-board-upsert", "--goal-id", "fixture-goal",
"--project", str(tmp_path), "--row-json", str(row_path), "--execute", "--format", "json",
], check=False)
assert result.returncode != 0
assert "invalid benchmark experiment board input" in result.stderr
assert ledger.read_bytes() == before


def test_cli_show_projects_factorial_contrast_from_explicit_contract(
tmp_path: Path,
) -> None:
Expand Down Expand Up @@ -1031,7 +1088,7 @@ def test_cli_omitted_project_routes_board_to_registered_goal_repo(
goal_id="fixture-goal",
)

assert payload["summary"]["run_count"] == 1
assert payload["write"]["row_count"] == 1
assert ledger.is_file()
assert not (unrelated / ".loopx").exists()

Expand Down
Loading