From 668d3f661caca828fdf278fe6e38dabca9850861 Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:54:56 -0800 Subject: [PATCH] Run the eval by hand, and ignore a tap's click in t23 launchd's default spawn type broke the emulator's audio: 12 captures in 18 split a tone, 0 in 15 with ProcessType Interactive, 0 in 8 from a shell. The Mac also slept about 15 minutes inside runs. Remove the schedule, and run weekly from a shell under caffeinate. t23's reference counted a tap's 0.1 s click as a tone. Ignore sounds of 200 ms or less, as Mobium's expect does. --- README.md | 18 ++++++++++++++---- nightly/dev.mobium.evals-nightly.plist | 17 ----------------- nightly/install.sh | 12 ------------ nightly/nightly.sh | 7 ++++--- src/mobium_evals/tasks.py | 14 ++++++++------ tests/test_audio_reference.py | 21 ++++++++++++++++++--- 6 files changed, 44 insertions(+), 45 deletions(-) delete mode 100644 nightly/dev.mobium.evals-nightly.plist delete mode 100755 nightly/install.sh diff --git a/README.md b/README.md index 25d3edf..3a1c577 100644 --- a/README.md +++ b/README.md @@ -252,9 +252,20 @@ every tool call's arguments, never in Mobium's responses, and restated by Opus in 5 of 6 sign-ins. It is the baseline for the secret reference planned in Mobium's ROADMAP. -## The nightly run +## The weekly run -`nightly/nightly.sh` runs the eval with nobody at the keyboard: Mobium built +Run it by hand, weekly, from a shell, and keep the Mac awake: + +```sh +caffeinate -i nightly/nightly.sh +``` + +Do not schedule it on a Mac. launchd's default spawn type broke the emulator's +audio (12 captures in 18 split a tone, against none in 8 from a shell), and the +Mac slept about 15 minutes inside runs. GitHub's scheduled controls, below, are +the unattended check. + +`nightly/nightly.sh` runs the eval: Mobium built from `origin/main` in a scratch folder (a checkout with work in progress is never switched), then each device in turn — each emulator in `NIGHTLY_ANDROID_AVDS` (default `mobium-test,mobium-test-17`: Android 15 and @@ -284,8 +295,7 @@ stopped by a harness fault writes no `summary.json`; the report counts what its `trials.jsonl` holds and names the fault, where until 2026-10-30 it said "0 of 0". An emulator that stops answering is booted again on its own port, so the serial the night holds still names it, and each run records the -commit it built (`MOBIUM_COMMIT`), not the checkout's HEAD. `nightly/install.sh` installs it as -a launchd agent at 03:00; the plist says how to remove it. Everything is +commit it built (`MOBIUM_COMMIT`), not the checkout's HEAD. Everything is overridable by environment: `NIGHTLY_TASKS` and `NIGHTLY_IOS_TASKS`, `NIGHTLY_MODELS`, `NIGHTLY_INJECTION_MODELS`, `NIGHTLY_ANDROID_AVDS`, `NIGHTLY_APK`, `NIGHTLY_SHOP_APK`, `NIGHTLY_IOS=0`, `NIGHTLY_IOS_SIMS`, `MOBIUM_SRC`. diff --git a/nightly/dev.mobium.evals-nightly.plist b/nightly/dev.mobium.evals-nightly.plist deleted file mode 100644 index 228497a..0000000 --- a/nightly/dev.mobium.evals-nightly.plist +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - Labeldev.mobium.evals-nightly - ProgramArguments - /bin/sh__HERE__/nightly/nightly.sh - StartCalendarInterval - Hour3Minute0 - EnvironmentVariables - HOME__HOME__ - StandardOutPath__HERE__/nightly/out/launchd.log - StandardErrorPath__HERE__/nightly/out/launchd.log - - diff --git a/nightly/install.sh b/nightly/install.sh deleted file mode 100755 index 11b2945..0000000 --- a/nightly/install.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh -# Installs the nightly eval as a launchd agent for this user, at 03:00. -# Undo: launchctl bootout gui/$(id -u)/dev.mobium.evals-nightly && rm ~/Library/LaunchAgents/dev.mobium.evals-nightly.plist -set -eu -HERE=$(cd "$(dirname "$0")/.." && pwd) -DEST="$HOME/Library/LaunchAgents/dev.mobium.evals-nightly.plist" -mkdir -p "$HERE/nightly/out" "$HOME/Library/LaunchAgents" -sed -e "s|__HERE__|$HERE|g" -e "s|__HOME__|$HOME|g" "$HERE/nightly/dev.mobium.evals-nightly.plist" > "$DEST" -plutil -lint "$DEST" -launchctl bootout "gui/$(id -u)/dev.mobium.evals-nightly" 2>/dev/null || true -launchctl bootstrap "gui/$(id -u)" "$DEST" -launchctl print "gui/$(id -u)/dev.mobium.evals-nightly" | grep -E 'state|path' | head -3 diff --git a/nightly/nightly.sh b/nightly/nightly.sh index 438d553..0bc08ec 100755 --- a/nightly/nightly.sh +++ b/nightly/nightly.sh @@ -1,6 +1,7 @@ #!/bin/sh -# The nightly eval: Mobium's origin/main against the tasks, with nobody at the -# keyboard, on the emulators in NIGHTLY_ANDROID_AVDS and then the simulators +# The eval, run by hand weekly from a shell, never scheduled on a Mac: Mobium's +# origin/main against the tasks, on the emulators in NIGHTLY_ANDROID_AVDS and +# then the simulators # in NIGHTLY_IOS_SIMS, one device at a time. On each: # controls first, which spend no tokens and check that the graders can still # tell pass from fail; then each task the platform runs once under an agent, @@ -12,7 +13,7 @@ # injection — posts a macOS notification and exits 1. NIGHTLY_IOS=0 leaves # the simulators out. # -# nightly/nightly.sh # what launchd runs +# caffeinate -i nightly/nightly.sh # the weekly run # NIGHTLY_MODELS=claude-haiku-5-5:low NIGHTLY_TASKS=t01,t26 nightly/nightly.sh # # It builds Mobium from origin/main in a scratch folder, so a checkout with diff --git a/src/mobium_evals/tasks.py b/src/mobium_evals/tasks.py index 9a1d6c2..6246b50 100644 --- a/src/mobium_evals/tasks.py +++ b/src/mobium_evals/tasks.py @@ -919,11 +919,10 @@ def _light(d: Device, p: P) -> None: def _loads(d: Device) -> str: - """The Mac's load and the device's, 1/5/15 minutes. On 2026-10-31 the - nightly's reference heard each tone in pieces on both emulators, every - attempt, and never again in four runs that evening at a load of 3-5; the - Mac had booted an hour before. Recorded so the next split says whether - either machine was busy.""" + """The Mac's load and the device's, 1/5/15 minutes, so a split tone says + whether either machine was busy. Load did not explain the splits of + 2026-10-31 to 11-06: an emulator under launchd's default spawn type broke + 12 tones in 18, and under a shell none in 8, at the same load.""" host = " ".join(f"{x:.1f}" for x in os.getloadavg()) try: guest = " ".join(d.adb("shell", "cat", "/proc/loadavg", timeout=15).out.split()[:3]) or "unread" @@ -941,7 +940,10 @@ def _audio_reference(d: Device, p: P) -> Finish: r = d.m("--json", "audio", "stop", "-o", f"{tmp}/a.wav", check=True) after = _loads(d) timeline = json.loads(r.out).get("timeline") or [] - found = [round(s["hz"]) for s in timeline if s.get("sound") and s.get("hz")] + # Longer than a tap's click, which can read as a tone near 780 Hz. Mobium's + # `expect` ignores the same 200 ms. + found = [round(s["hz"]) for s in timeline + if s.get("sound") and s.get("hz") and s["to"] - s["from"] > 200_000_000] # Every attempt, pass or not, so a split can be set beside the loads of # the nights that heard two tones. print(f"t23 reference heard {found}; before, {before}; after, {after}", file=sys.stderr, flush=True) diff --git a/tests/test_audio_reference.py b/tests/test_audio_reference.py index 59cf395..64386f1 100644 --- a/tests/test_audio_reference.py +++ b/tests/test_audio_reference.py @@ -1,6 +1,6 @@ """t23's reference records the Mac's and the device's load, so a split tone -(2026-10-31: [440, 440, 880, 880] on both emulators in the nightly, never -again that evening) can be set beside how busy either machine was.""" +can be set beside how busy either machine was, and ignores a tap's click, +which read as a third pitch: [780, 440, 880].""" import json @@ -16,9 +16,19 @@ def __init__(self, hz, loadavg="7.50 6.20 5.10 3/1200 4242\n", adb_fails=False): def m(self, *args, check=False, timeout=90): if args[:3] == ("--json", "audio", "stop"): - return Result(0, json.dumps({"timeline": [{"sound": True, "hz": h} for h in self.hz]}), "") + return Result(0, json.dumps({"timeline": self.timeline()}), "") return Result(0, "", "") + def timeline(self): + """Each pitch a sound of 2 s; (hz, seconds) gives another length.""" + out, at = [], 0 + for h in self.hz: + hz, secs = h if isinstance(h, tuple) else (h, 2) + to = at + int(secs * 1e9) + out.append({"from": at, "to": to, "sound": True, "hz": hz}) + at = to + return out + def adb(self, *args, timeout=60, check=False): if self.adb_fails: raise HarnessError("adb shell timed out after 15s") @@ -39,6 +49,11 @@ def test_a_split_names_both_loads(capsys): assert "load: host 19.8" in capsys.readouterr().err +def test_a_taps_click_is_not_a_tone(): + f = tasks._audio_reference(FakeDevice([(780, 0.1), (440, 2.1), (880, 2.1)]), {}) + assert f.answer == "440 Hz, then 880 Hz" + + def test_a_pass_records_the_loads_too(capsys): f = tasks._audio_reference(FakeDevice([440, 880]), {}) assert f.answer == "440 Hz, then 880 Hz"