diff --git a/.gitignore b/.gitignore index 0c981383..99ca47e2 100644 --- a/.gitignore +++ b/.gitignore @@ -76,4 +76,7 @@ e2e/node_modules/ /package-lock.json # Planning docs β€” keep locally, never publish to the repo -/PLAN-* \ No newline at end of file +/PLAN-* +# Python bytecode (interpreter-generated) +__pycache__/ +*.pyc diff --git a/benchmarks/slopcodebench/bin/scb-live b/benchmarks/slopcodebench/bin/scb-live new file mode 100755 index 00000000..09c5b2d4 --- /dev/null +++ b/benchmarks/slopcodebench/bin/scb-live @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""scb-live β€” dependency-free LIVE view of a running slop-code-bench eval. + +Serves an auto-refreshing HTML page (stdlib only, no deps, pinned port) showing +what's happening right now + a running tally (per-problem checkpoint / steps / +cost / state, and the cumulative cost). Reads the latest /workspace/.scb-run*.log +that scb-run writes β€” so it works for inference-only runs (no scoring needed), +unlike the Dash dashboard which needs evaluated runs. + +Point an Orcabot browser block at http://localhost: (default 8051; +deliberately NOT 8050/8080 to avoid the Dash / sandbox-server ports). + +Run: bin/scb-live (foreground) + SCB_LIVE_PORT=8055 bin/scb-live +""" +import glob +import html +import os +import re +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +WS = os.environ.get("SCB_WORKSPACE", "/workspace") +PORT = int(os.environ.get("SCB_LIVE_PORT", "8051")) + +# slop-code's rich console wraps each progress update across several padded +# lines, so we flatten whitespace before matching. +PROG = re.compile( + r"'(?P[^']+)': progress update " + r"checkpoint='(?P[^']+)' " + r"cost=(?P[\d.]+) " + r"elapsed=(?P[\d.]+) " + r"state='(?P[^']+)' " + r"steps=(?P\d+) " + r"total_steps=(?P\d+)" +) + + +ARM_RE = re.compile(r"\[scb-matrix\] arm (\S+) x (\S+)(?: x (\S+))?") +TS_RE = re.compile(r"\.(\d{8}T\d{6})\.log$") + + + +def subscription_mode(): + """Codex on a ChatGPT/Codex plan reports NO token_count/total_cost events, so cost + is legitimately always $0. Showing $0.0000 next to a healthy run reads as failure, + so label it instead.""" + try: + txt = open(os.path.join(WS, ".scb-config.yaml"), encoding="utf-8").read() + except OSError: + return False + for line in txt.splitlines(): + line = line.split("#", 1)[0].strip() + if line.startswith("codex_auth:"): + return line.split(":", 1)[1].strip().lower() == "subscription" + return False + + +def cost_cell(value, sub): + """$0.0000 on a subscription is correct (no per-token billing) but reads as a + failed run, so show the plan label instead.""" + return "plan" if sub else f"${float(value):.4f}" + + +def batch_logs(): + """All logs from the LATEST run batch. scb-matrix stamps every arm's logfile + with one shared timestamp, so grouping by that timestamp shows the whole matrix + (and a single scb-run is just a 1-log batch).""" + logs = glob.glob(f"{WS}/.scb-run.*.log") + by_ts = {} + for p in logs: + m = TS_RE.search(p) + by_ts.setdefault(m.group(1) if m else "00000000T000000", []).append(p) + if not by_ts: + return None, [] + latest = max(by_ts) + return latest, sorted(by_ts[latest], key=os.path.getmtime) + + +def arm_label(logpath, raw): + m = ARM_RE.search(raw) + if m: + parts = [m.group(1), m.group(2)] + ([m.group(3)] if m.group(3) else []) + return " Β· ".join(parts) + base = re.sub(r"^\.scb-run\.|\.\d{8}T\d{6}\.log$|\.log$", "", os.path.basename(logpath)) + return base or "run" + + +def parse_log(logpath): + """Return (arm label, {problem: latest progress}, tail lines) for one arm log.""" + with open(logpath, errors="replace") as fh: + raw = fh.read() + flat = re.sub(r"\s+", " ", raw) + problems = {} + for m in PROG.finditer(flat): + d = m.groupdict() + problems[d["problem"]] = d + return arm_label(logpath, raw), problems, raw.splitlines()[-12:] + + +def render(): + now = time.strftime("%H:%M:%S") + ts, logs = batch_logs() + if not logs: + body = "

No run yet. Launch one with scb-matrix or scb-run.

" + return page(body, "idle", now) + + sub = subscription_mode() + total_cost, max_elapsed, any_running, any_prog = 0.0, 0.0, False, False + rows, tail = [], [] + multi = len(logs) > 1 + for logpath in logs: + arm, problems, t = parse_log(logpath) + tail = t # newest-updated arm's tail (logs sorted by mtime) + for name, p in sorted(problems.items()): + any_prog = True + total_cost += float(p["cost"]) + max_elapsed = max(max_elapsed, float(p["elapsed"])) + if p["state"] == "running": + any_running = True + dot = "🟒" if p["state"] == "running" else "βšͺ" + arm_cell = f"{html.escape(arm)}" if multi else "" + rows.append( + f"{arm_cell}{dot} {html.escape(name)}" + f"{html.escape(p['cp'])}" + f"{p['steps']}/{p['total']}" + f"{cost_cell(p['cost'], sub)}" + f"{html.escape(p['state'])}" + ) + overall = "running" if any_running else ("done" if any_prog else "starting") + ncols = 6 if multi else 5 + arm_th = "arm" if multi else "" + table = ( + f"{arm_th}" + "" + + ("".join(rows) or f"") + + "
problemcheckpointstepscoststate
waiting for first progress…
" + ) + tally = ( + f"
{len(logs)} arm(s)" + f"total {'plan' if sub else f'${total_cost:.4f}'}" + f"elapsed {max_elapsed:.0f}s" + f"{overall.upper()}
" + ) + log = "
" + html.escape("\n".join(tail)) + "
" + body = ( + f"

batch {html.escape(ts)} Β· {len(logs)} arm(s)

" + + tally + table + "

activity (latest arm)

" + log + ) + return page(body, overall, now) + + +def page(body, overall, now): + return f""" + +slop-code-bench β€” live + +

slop-code-bench Β· live

+

updated {now} Β· auto-refresh 2s

+{body} +""" + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + out = render().encode() + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(out))) + self.end_headers() + self.wfile.write(out) + + def log_message(self, *_): # quiet + pass + + +if __name__ == "__main__": + print(f"scb-live serving http://127.0.0.1:{PORT} (workspace={WS})", flush=True) + ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever() diff --git a/benchmarks/slopcodebench/bin/scb-matrix b/benchmarks/slopcodebench/bin/scb-matrix new file mode 100644 index 00000000..e1e9c4e4 --- /dev/null +++ b/benchmarks/slopcodebench/bin/scb-matrix @@ -0,0 +1,549 @@ +#!/usr/bin/env python3 +"""scb-matrix β€” expand a benchmark config into arms and run them, W at a time. + +An ARM is one (harness Γ— model) pair; each arm runs all `problems` sequentially +via slop-code. Arms run with `workers`-way parallelism. Each arm's output is tee'd +to /workspace/.scb-run...log (scb-live aggregates these into the live +per-arm tally) and mirrored to a tmux window (scb-visualize surfaces a per-arm +read-only viewer). + +Config: /workspace/.scb-config.yaml β€” created with defaults if missing. CLI flags +override the file for a one-off run (the panel/chat write the file to persist). + +Usage: + scb-matrix [--config PATH] [--print-config] + [--harness H]... [--model M]... [--problem P]... + [--workers N] [--thinking low|medium|high] [--prompt NAME] [--evaluate] + +Dependency-free (stdlib only), like scb-live / scb-visualize. +""" +import os +import re +import subprocess +import sys +import threading +import time + +WS = os.environ.get("SCB_WORKSPACE", "/workspace") +REPO = os.environ.get("SCB_REPO_DIR", f"{WS}/slop-code-bench") +VENV = os.environ.get("SCB_VENV", f"{WS}/scb-venv") +CONFIG = os.environ.get("SCB_CONFIG", f"{WS}/.scb-config.yaml") + +# slop-code's local runtime spawns codex with HOME=/tmp/agent_home (agents/utils.py +# HOME_PATH). Codex CLI reads its ChatGPT/Codex subscription from $HOME/.codex/auth.json. +AGENT_HOME = os.environ.get("SCB_AGENT_HOME", "/tmp/agent_home") +CODEX_AUTH_DEST = f"{AGENT_HOME}/.codex/auth.json" + +DEFAULT_CONFIG = """\ +# SlopCodeBench run config β€” edit here, ask Orcabot chat ("add claude", "2 workers"), +# or use the config panel. An ARM is one (harness x model x skill); each arm runs all +# `problems` sequentially. `workers` arms run in parallel. +harnesses: [opencode] # opencode claude_code codex gemini kimi_cli cursor_cli openhands miniswe pi +models: [openrouter/kimi-k2.6] # OpenRouter ("openrouter/") or native model ids +skills: [baseline] # baseline gsd omc superpowers karpathy addyosmani (agent-skill packs) +problems: [] # empty = the FULL benchmark set (every problem, in sequence) +workers: 1 # how many arms run in parallel +prompt: just-solve # baseline prompt (skills override this); just-solve plan_first ... +thinking: low # low medium high +evaluate: false # true = score the run, false = inference-only +codex_auth: broker # broker = brokered API key; subscription = codex login (ChatGPT/Codex plan) +""" + +SCALARS = {"workers", "prompt", "thinking", "evaluate", "codex_auth"} +LISTS = {"harnesses", "models", "problems", "skills"} + +# A "skill" is a public agent-skill pack; running one swaps the slop-code +# environment + prompt (see the fork's REPRODUCE_PUBLIC_SKILLS.md). Every skill +# now has a LOCAL (in-VM) env + `-local-trigger` prompt: scb-matrix clones the +# skill's public plugin repo, points SCB_SKILL__DIR at it, and the env's +# setup command copies it into the run's `.claude/skills/` (the Docker +# variants that bind-mount from a host path are still in the fork for reference). +SKILL_ENV = { + "baseline": "configs/environments/local-tmux-py.yaml", + "gsd": "configs/environments/local-tmux-py-with-gsd.yaml", + "omc": "configs/environments/local-tmux-py-with-omc.yaml", + "superpowers": "configs/environments/local-tmux-py-with-superpowers.yaml", + "karpathy": "configs/environments/local-tmux-py-with-karpathy.yaml", + "addyosmani": "configs/environments/local-tmux-py-with-addyosmani.yaml", +} +SKILL_PROMPT = { + "baseline": None, # use config `prompt` + "gsd": "just-solve-with-gsd-local-trigger", + "omc": "just-solve-with-omc-local-trigger", + "superpowers": "just-solve-with-superpowers-local-trigger", + "karpathy": "just-solve-with-karpathy-local-trigger", + "addyosmani": "just-solve-with-addyosmani-local-trigger", +} +# skill -> (git repo, subpath within the repo that IS the plugin dir, env var the +# env yaml reads to stage the plugin locally). subpath "" = repo root. +SKILL_PLUGIN = { + "gsd": ("https://github.com/open-gsd/get-shit-done-redux", "", "SCB_SKILL_GSD_DIR"), + "omc": ("https://github.com/Yeachan-Heo/oh-my-claudecode", "", "SCB_SKILL_OMC_DIR"), + "superpowers": ("https://github.com/obra/superpowers", "", "SCB_SKILL_SUPERPOWERS_DIR"), + "karpathy": ("https://github.com/multica-ai/andrej-karpathy-skills", + "skills/karpathy-guidelines", "SCB_SKILL_KARPATHY_DIR"), + "addyosmani": ("https://github.com/addyosmani/agent-skills", "", "SCB_SKILL_ADDYOSMANI_DIR"), +} + + +def _log(msg): + sys.stderr.write(f"[scb-matrix] {msg}\n") + sys.stderr.flush() + + +def parse_config(text): + """Minimal YAML subset for the flat config: `key: scalar`, `key: [a, b]`, and + block lists (`key:` then ` - item`). Comments (#) and blanks ignored.""" + cfg = {} + cur_list = None + for raw in text.splitlines(): + line = raw.split("#", 1)[0].rstrip() + if not line.strip(): + continue + if line.startswith(" ") and line.lstrip().startswith("- ") and cur_list: + cfg[cur_list].append(line.lstrip()[2:].strip()) + continue + m = re.match(r"^([A-Za-z_]+):\s*(.*)$", line) + if not m: + continue + key, val = m.group(1), m.group(2).strip() + cur_list = None + if val.startswith("[") and val.endswith("]"): + cfg[key] = [v.strip() for v in val[1:-1].split(",") if v.strip()] + elif val == "": + cfg[key] = [] + cur_list = key + else: + cfg[key] = val + return cfg + + +def load_config(): + if not os.path.isfile(CONFIG): + try: + with open(CONFIG, "w", encoding="utf-8") as fh: + fh.write(DEFAULT_CONFIG) + _log(f"created default config {CONFIG}") + except OSError as exc: + _log(f"could not create {CONFIG}: {exc}") + return parse_config(DEFAULT_CONFIG) + return parse_config(open(CONFIG, encoding="utf-8").read()) + + +def apply_overrides(cfg, args): + """CLI flags override the config file (for a one-off run).""" + i, harn, mods, probs, skls = 0, [], [], [], [] + while i < len(args): + a = args[i] + if a == "--harness": + harn.append(args[i + 1]); i += 2 + elif a == "--model": + mods.append(args[i + 1]); i += 2 + elif a == "--problem": + probs.append(args[i + 1]); i += 2 + elif a == "--skill": + skls.append(args[i + 1]); i += 2 + elif a in ("--workers", "--thinking", "--prompt", "--codex-auth"): + cfg[a[2:].replace("-", "_")] = args[i + 1]; i += 2 + elif a == "--evaluate": + cfg["evaluate"] = "true"; i += 1 + elif a == "--subscription": # shorthand for --codex-auth subscription + cfg["codex_auth"] = "subscription"; i += 1 + else: + _log(f"ignoring unknown arg: {a}"); i += 1 + if harn: + cfg["harnesses"] = harn + if mods: + cfg["models"] = mods + if probs: + cfg["problems"] = probs + if skls: + cfg["skills"] = skls + return cfg + + +def patch_broker(): + """Restore a pristine providers.yaml and route openrouter through this session's + broker (mirrors scb-run; injects the real key server-side).""" + broker = os.environ.get("OPENROUTER_BASE_URL", "").strip().replace("localhost", "127.0.0.1") + orig = os.path.join(REPO, "configs", "providers.yaml.orig") + prov = os.path.join(REPO, "configs", "providers.yaml") + if os.path.isfile(orig): + try: + open(prov, "w", encoding="utf-8").write(open(orig, encoding="utf-8").read()) + except OSError: + pass + if not broker: + _log("OPENROUTER_BASE_URL not set β€” is OPENROUTER_API_KEY a broker secret?") + return + try: + txt = open(prov, encoding="utf-8").read() + new = txt.replace("api_base: https://openrouter.ai/api/v1", "api_base: " + broker) + # Also rewrite a STALE broker URL (a prior session's, e.g. captured into a + # non-pristine providers.yaml.orig) so opencode targets THIS session's + # broker path β€” otherwise the request hits a dead session and 401s. + new = re.sub(r"http://127\.0\.0\.1:\d+/broker/[0-9a-f]+/openrouter(?![-\w])", broker, new) + if new != txt: + open(prov, "w", encoding="utf-8").write(new) + _log(f"routed OpenRouter via broker -> {broker}") + except OSError as exc: + _log(f"providers.yaml patch failed: {exc}") + + +def find_codex_auth(): + """Locate a Codex subscription credential (auth.json written by `codex login`). + Search order: $SCB_CODEX_AUTH_FILE, $WS/.scb-codex-auth.json (staged by the + panel/user), then the usual CODEX_HOME / HOME locations. Returns a path or None.""" + candidates = [ + os.environ.get("SCB_CODEX_AUTH_FILE"), + os.path.join(WS, ".scb-codex-auth.json"), + os.path.join(os.environ.get("CODEX_HOME", ""), "auth.json") if os.environ.get("CODEX_HOME") else None, + os.path.join(os.path.expanduser("~"), ".codex", "auth.json"), + ] + for c in candidates: + if c and os.path.isfile(c): + return c + return None + + +def stage_codex_auth(): + """Copy the located Codex subscription auth.json into the agent's HOME so the + in-VM (local runtime) codex process finds it at $HOME/.codex/auth.json. Returns + True if staged. Does NOT overwrite a newer dest.""" + src = find_codex_auth() + if not src: + _log("codex_auth=subscription but no auth.json found " + f"(looked at $SCB_CODEX_AUTH_FILE, {WS}/.scb-codex-auth.json, ~/.codex/auth.json). " + "Run `codex login` or stage the file; the arm will fail auth without it.") + return False + try: + os.makedirs(os.path.dirname(CODEX_AUTH_DEST), exist_ok=True) + with open(src, "rb") as fh: + data = fh.read() + with open(CODEX_AUTH_DEST, "wb") as fh: + fh.write(data) + os.chmod(CODEX_AUTH_DEST, 0o600) + _log(f"staged codex subscription auth {src} -> {CODEX_AUTH_DEST}") + return True + except OSError as exc: + _log(f"codex auth staging failed: {exc}") + return False + + +def patch_codex_provider(): + """For subscription runs, flip the `openai` provider from an API-key env_var to + a file credential (the staged auth.json). slop-code then resolves a FILE + credential for codex (no OPENAI_API_KEY injected) and codex uses the ChatGPT/ + Codex subscription. Idempotent; runs after patch_broker restores .orig.""" + prov = os.path.join(REPO, "configs", "providers.yaml") + try: + txt = open(prov, encoding="utf-8").read() + except OSError: + return + new = re.sub( + r"(\n openai:\n )type: env_var\n env_var: OPENAI_API_KEY", + r"\1type: file\n file_path: " + CODEX_AUTH_DEST, + txt, + count=1, + ) + if new != txt: + try: + open(prov, "w", encoding="utf-8").write(new) + _log("openai provider -> file credential (codex subscription)") + except OSError as exc: + _log(f"codex provider patch failed: {exc}") + + +STOP_FLAG = f"{WS}/.scb-stop" + + +def stop_requested(): + return os.path.exists(STOP_FLAG) + + +def preflight_codex_subscription(cfg, models): + """Fail BEFORE launching anything if a codex-subscription run can't possibly work. + Without this, every problem fails identically and the matrix happily grinds through + the whole suite spawning a viewer per run (observed: dozens of dead terminals).""" + src = find_codex_auth() + if not src: + _log("codex_auth=subscription but no auth.json found. Run `codex login` in a " + "terminal, or stage the file at " + os.path.join(WS, ".scb-codex-auth.json") + + ". Not starting the run.") + return False + # Deliberately NOT blocking model ids here: which models a given ChatGPT/Codex plan + # can drive is account-dependent and not something we can check up front. A wrong + # model is caught at runtime by the fatal-error watchdog in run_arm() instead, which + # also covers auth/quota failures generally. + return True + + +def is_codex_subscription(cfg): + return str(cfg.get("codex_auth", "broker")).strip().lower() == "subscription" + + +def slug(s): + return re.sub(r"[^A-Za-z0-9]+", "-", s).strip("-") + + +STAGE_MARKER = ".scb-staged" + + +def stage_skill(skill): + """Clone a skill's public plugin once and return the env var its env yaml reads. + Clones to a temp dir and renames into place, so a concurrent reader can never see + a half-populated checkout. Returns {} if the clone failed β€” callers MUST treat that + as fatal rather than silently running the arm with no skill.""" + info = SKILL_PLUGIN.get(skill) + if not info: + return {} + repo, subpath, envvar = info + dest = os.path.join(WS, ".scb-skills", skill) + plugin_dir = os.path.join(dest, subpath) if subpath else dest + # "non-empty dir" is NOT proof of a good checkout β€” an interrupted clone leaves a + # dir containing only .git, which the old check happily accepted (and legacy dirs + # from the earlier clone race look exactly like that). Require a marker that is + # only written after a validated, completed clone. + marker = os.path.join(dest, STAGE_MARKER) + if os.path.isfile(marker) and os.path.isdir(plugin_dir) and os.listdir(plugin_dir): + return {envvar: plugin_dir} + if os.path.exists(dest): + _log(f"skill '{skill}': existing checkout is unverified/partial β€” re-cloning") + + tmp = dest + ".tmp" + subprocess.call(["rm", "-rf", tmp]) + try: + os.makedirs(os.path.dirname(dest), exist_ok=True) + rc = subprocess.call(["git", "clone", "--depth", "1", repo, tmp], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if rc != 0: + _log(f"skill plugin clone FAILED ({skill} <- {repo}, rc={rc})") + subprocess.call(["rm", "-rf", tmp]) + return {} + tmp_plugin = os.path.join(tmp, subpath) if subpath else tmp + if not (os.path.isdir(tmp_plugin) and os.listdir(tmp_plugin)): + _log(f"skill plugin {skill} incomplete after clone (missing {subpath or 'repo root'})") + subprocess.call(["rm", "-rf", tmp]) + return {} + with open(os.path.join(tmp, STAGE_MARKER), "w", encoding="utf-8") as fh: + fh.write(f"{repo}\n") # written ONLY after a validated clone + subprocess.call(["rm", "-rf", dest]) + os.rename(tmp, dest) # atomic: readers see complete+marked, or nothing + except Exception as exc: # noqa: BLE001 + _log(f"skill plugin staging failed ({skill}): {exc}") + subprocess.call(["rm", "-rf", tmp]) + return {} + + _log(f"staged skill plugin {skill} <- {repo}") + return {envvar: plugin_dir} + + +def stage_skills(skills): + """Stage every skill up front, serially, BEFORE any worker starts. Previously each + worker raced to clone the same destination, so with several models sharing one skill + a worker could observe a partial checkout β€” or get no skill env at all.""" + staged = {} + for skill in dict.fromkeys(skills): + if skill not in SKILL_PLUGIN: + staged[skill] = {} + continue + env = stage_skill(skill) + if not env: + _log(f"skill '{skill}' could not be staged β€” its arms would silently run " + "WITHOUT the skill, so they will be failed instead.") + staged[skill] = env + return staged + + +# Failures that will repeat identically for EVERY problem (bad model for this account, +# bad/missing auth, exhausted quota). Without aborting on these, slop-code marches +# through the whole suite failing the same way and scb-visualize spawns a dead viewer +# per run β€” the "it kept spawning more tasks" failure mode. +FATAL_PATTERNS = ( + "not supported when using Codex with a ChatGPT account", + "401 Unauthorized", + "Incorrect API key", + "insufficient_quota", + "AuthenticationError", + "No auth credentials found", +) + + +def _watch_for_fatal(proc, logpath, label, fatal): + """Abort an arm whose every problem is failing for the same fundamental reason.""" + while proc.poll() is None: + time.sleep(5) + if stop_requested(): + _log(f"stop requested β€” terminating {label}") + try: + proc.terminate() + except Exception: # noqa: BLE001 + pass + return + try: + txt = open(logpath, errors="replace").read() + except OSError: + continue + for pat in FATAL_PATTERNS: + if pat in txt: + # Abort THIS arm only. Writing the global stop flag here (as an earlier + # version did) cancelled every other arm too β€” so one bad model killed + # the valid comparison arms it was supposed to be compared against. + _log(f"FATAL for {label}: {pat!r} β€” aborting this arm; it would repeat " + "for every problem. Other arms continue.") + fatal["reason"] = pat + try: + proc.terminate() + except Exception: # noqa: BLE001 + pass + return + + +def run_arm(harness, model, skill, cfg, ts, skill_envs): + problems = cfg.get("problems") or [] + thinking = cfg.get("thinking", "low") + evaluate = str(cfg.get("evaluate", "false")).lower() in ("1", "true", "yes", "on") + env_cfg = SKILL_ENV.get(skill, SKILL_ENV["baseline"]) + prompt = SKILL_PROMPT.get(skill) or cfg.get("prompt", "just-solve") + skill_env = skill_envs.get(skill, {}) + if skill in SKILL_PLUGIN and not skill_env: + _log(f"arm FAILED before start: {harness} x {model} x {skill} β€” skill not staged") + return {"arm": f"{harness} x {model} x {skill}", "ok": False, "reason": "skill not staged"} + env = {**os.environ, **skill_env} + # Codex subscription arm: the auth is a file (auth.json) + providers.yaml already + # points openai at it. Strip the brokered OPENAI_* / OPENROUTER_* so the codex CLI + # doesn't see a placeholder API key and falls back to the ChatGPT/Codex subscription. + if harness == "codex" and is_codex_subscription(cfg): + for k in ("OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_MODEL", + "OPENROUTER_API_KEY", "OPENROUTER_BASE_URL"): + env.pop(k, None) + log = f"{WS}/.scb-run.{slug(harness)}_{slug(model)}_{slug(skill)}.{ts}.log" + cmd = [f"{VENV}/bin/slop-code", "run", "--agent", harness, "--model", model, + "--environment", env_cfg, "--prompt", f"configs/prompts/{prompt}.jinja"] + for p in problems: + cmd += ["--problem", p] + if not evaluate: + cmd.append("--no-evaluate") + cmd.append(f"thinking={thinking}") + _log(f"arm start: {harness} x {model} x {skill} -> {log}") + try: + with open(log, "w", encoding="utf-8") as fh: + fh.write(f"[scb-matrix] arm {harness} x {model} x {skill} problems={problems or 'ALL'} " + f"env={env_cfg} prompt={prompt} thinking={thinking} evaluate={evaluate}\n") + fh.flush() + proc = subprocess.Popen(cmd, cwd=REPO, env=env, stdout=fh, + stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL) + label = f"{harness} x {model} x {skill}" + fatal = {} + watcher = threading.Thread(target=_watch_for_fatal, args=(proc, log, label, fatal), + daemon=True) + watcher.start() + rc = proc.wait() + reason = fatal.get("reason") or (None if rc == 0 else f"rc={rc}") + _log(f"arm done: {label} rc={rc}" + (f" ({reason})" if reason else "")) + return {"arm": label, "ok": rc == 0 and not fatal, "reason": reason} + except Exception as exc: # noqa: BLE001 β€” one arm failing must not kill the matrix + _log(f"arm error: {harness} x {model} x {skill}: {exc}") + return {"arm": f"{harness} x {model} x {skill}", "ok": False, "reason": str(exc)} + + +def main(): + args = sys.argv[1:] + cfg_path_override = None + if "--config" in args: + i = args.index("--config"); cfg_path_override = args[i + 1] + args = args[:i] + args[i + 2:] + global CONFIG + if cfg_path_override: + CONFIG = cfg_path_override + + cfg = load_config() + if "--print-config" in args: + print(open(CONFIG, encoding="utf-8").read() if os.path.isfile(CONFIG) else DEFAULT_CONFIG) + return + cfg = apply_overrides(cfg, args) + + harnesses = cfg.get("harnesses") or ["opencode"] + models = cfg.get("models") or ["openrouter/kimi-k2.6"] + skills = cfg.get("skills") or ["baseline"] + # Empty `problems` = run the FULL benchmark set. slop-code auto-discovers every + # valid problem when no --problem is passed (_resolve_problem_names -> discovery), + # so we simply omit the flags rather than hardcoding a default problem. + problems = cfg.get("problems") or [] + try: + workers = max(1, int(str(cfg.get("workers", "1")).strip())) + except ValueError: + workers = 1 + arms = [(h, m, s) for h in harnesses for m in models for s in skills] + _log(f"{len(arms)} arm(s) [{len(harnesses)}h x {len(models)}m x {len(skills)}s], " + f"{len(problems) if problems else 'ALL'} problem(s), workers={workers}") + + # Clear any stop flag from a previous run before starting a new one. + try: + os.remove(STOP_FLAG) + except OSError: + pass + + if is_codex_subscription(cfg): + non_codex = [h for h in harnesses if h != "codex"] + if non_codex: + _log("codex_auth=subscription needs the 'codex' harness: the ChatGPT/Codex " + f"login (~/.codex/auth.json) is read only by the codex CLI, not {non_codex}. " + "Set harness=codex (and model openai/gpt-5.5), or switch the provider to " + "OpenRouter. Not starting.") + sys.exit(2) + if not preflight_codex_subscription(cfg, models): + sys.exit(2) + + patch_broker() + + # Codex subscription mode: stage the ChatGPT/Codex auth.json + point the openai + # provider at it (must run AFTER patch_broker, which restores a pristine + # providers.yaml). Only relevant when a codex arm is present. + if is_codex_subscription(cfg): # harnesses already validated to be codex + _log("codex_auth=subscription β€” using ChatGPT/Codex login (no brokered key)") + stage_codex_auth() + patch_codex_provider() + + ts = time.strftime("%Y%m%dT%H%M%S") + # Stage every skill BEFORE any worker starts (no concurrent clones of one dest). + skill_envs = stage_skills(skills) + + sem = threading.Semaphore(workers) + threads = [] + results = [] + results_lock = threading.Lock() + + def worker(h, m, s): + with sem: + if stop_requested(): + _log(f"stop requested β€” skipping arm {h} x {m} x {s}") + with results_lock: + results.append({"arm": f"{h} x {m} x {s}", "ok": False, "reason": "stopped"}) + return + res = run_arm(h, m, s, cfg, ts, skill_envs) + with results_lock: + results.append(res) + + for h, m, s in arms: + t = threading.Thread(target=worker, args=(h, m, s), daemon=False) + t.start() + threads.append(t) + time.sleep(0.2) # stagger so tmux windows / logfiles don't collide + for t in threads: + t.join() + + # Exit code must reflect reality: an orchestrator can't tell "ran fine" from + # "nothing ran" if we always exit 0 and print "matrix complete". + failed = [r for r in results if r and not r.get("ok")] + ok = len(results) - len(failed) + if failed: + for r in failed: + _log(f" FAILED: {r['arm']} β€” {r.get('reason')}") + _log(f"matrix finished with FAILURES: {ok} ok, {len(failed)} failed") + sys.exit(1) + _log(f"matrix complete: {ok} arm(s) ok") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/slopcodebench/bin/scb-visualize b/benchmarks/slopcodebench/bin/scb-visualize index 60fa18eb..1f61b8e0 100755 --- a/benchmarks/slopcodebench/bin/scb-visualize +++ b/benchmarks/slopcodebench/bin/scb-visualize @@ -21,10 +21,15 @@ MCP server using ORCABOT_MCP_SECRET; the dashboard is resolved server-side from the session, so no dashboard id is needed. Usage: - scb-visualize watch [--workdir DIR] # watch an existing run's manifest - scb-visualize run [--workdir DIR] -- CMD… # run the benchmark AND visualize + scb-visualize watch [--workdir DIR] [--no-viewers] [--skip-existing] # watch a run + scb-visualize run [--workdir DIR] [--no-viewers] -- CMD… # run + visualize + +--no-viewers (or SCB_VIEWERS=0): don't spawn a read-only PTY per run β€” rely on the +scb-live browser dashboard instead. The run and dashboard are unaffected. """ +import glob import json +import re import os import shlex import subprocess @@ -34,13 +39,14 @@ import time import urllib.error import urllib.request +WS_ROOT = os.environ.get("SCB_WORKSPACE", "/workspace") MCP_BASE = os.environ.get("ORCABOT_MCP_BASE", "http://127.0.0.1:8081") SID = os.environ.get("ORCABOT_SESSION_ID", "").strip() PTY = os.environ.get("ORCABOT_PTY_ID", "").strip() SECRET = os.environ.get("ORCABOT_MCP_SECRET", "").strip() -# Canvas layout: a grid of columns; each run = a note stacked over its terminal. -COL_W, NOTE_H, TERM_H, GAP, COLS = 380, 96, 300, 24, 3 +# Canvas layout: a grid of columns; each run = one terminal (title names the run). +COL_W, TERM_H, GAP, COLS = 380, 300, 24, 3 BASE_X, BASE_Y, COL_GAP, ROW_GAP = 40, 40, 40, 48 @@ -53,6 +59,59 @@ def mcp_ready(): return bool(SID and PTY and SECRET) +def prepare_configs(workdir): + """Run-time config fixes applied HERE (in the runner PTY) so they can't be + skipped by the orchestrator and use THIS session's broker URL. + + 1. Route OpenRouter through the session-local secrets broker. The sandbox sets + OPENROUTER_BASE_URL=http://127.0.0.1:8082/broker//openrouter when + OPENROUTER_API_KEY is a broker secret; the broker injects the real key + server-side. Without this, opencode hits openrouter.ai directly with the + broker *placeholder* key and fails auth (total_cost=0, "OpenCode error"). + 2. Pin the tmux log dir to a STABLE path under the checkout. It defaults to the + run's temp working dir (which is cleaned up), so runs.jsonl + the per-run + logfiles vanish and the live agent viewer never surfaces. Point it at + /.scb_tmux β€” exactly where watch() looks.""" + broker = os.environ.get("OPENROUTER_BASE_URL", "").strip() + providers = os.path.join(workdir, "configs", "providers.yaml") + if broker and os.path.isfile(providers): + try: + txt = open(providers, encoding="utf-8").read() + # Point the openrouter openai endpoint at this session's broker. Handle + # both the pristine openrouter.ai URL AND a stale broker URL left by a + # previous run in the shared /workspace (idempotent, always current sid). + new = txt.replace("https://openrouter.ai/api/v1", broker) + new = re.sub(r"http://127\.0\.0\.1:\d+/broker/[0-9a-f]+/openrouter(?![-\w])", + broker, new) + if new != txt: + with open(providers, "w", encoding="utf-8") as fh: + fh.write(new) + _log(f"routed OpenRouter via broker -> {broker}") + except OSError as exc: + _log(f"providers.yaml patch failed: {exc}") + elif not broker: + _log("OPENROUTER_BASE_URL not set β€” is OPENROUTER_API_KEY a broker secret?") + + log_dir = os.path.join(workdir, ".scb_tmux") + # EVERY local env, not just the baseline one. The skill variants + # (local-tmux-py-with-gsd/omc/superpowers/karpathy/addyosmani.yaml) also run on + # the tmux executor; without this patch their manifests land in the run's temp + # dir, watch() never sees runs.jsonl, and those arms surface no viewer at all. + env_dir = os.path.join(workdir, "configs", "environments") + for env_yaml in sorted(glob.glob(os.path.join(env_dir, "local-tmux-py*.yaml"))): + try: + os.makedirs(log_dir, exist_ok=True) + txt = open(env_yaml, encoding="utf-8").read() + if "tmux_log_dir" not in txt and " tmux_session: scb" in txt: + txt = txt.replace(" tmux_session: scb", + f" tmux_session: scb\n tmux_log_dir: {log_dir}") + with open(env_yaml, "w", encoding="utf-8") as fh: + fh.write(txt) + _log(f"pinned tmux_log_dir -> {log_dir} in {os.path.basename(env_yaml)}") + except OSError as exc: + _log(f"env yaml patch failed: {exc}") + + def call_tool(name, arguments): """Invoke an Orcabot MCP UI tool via the local sandbox MCP server.""" url = f"{MCP_BASE}/sessions/{SID}/mcp/tools/call?pty_id={PTY}" @@ -74,34 +133,69 @@ def call_tool(name, arguments): def slot(idx): col, row = idx % COLS, idx // COLS x = BASE_X + col * (COL_W + COL_GAP) - y = BASE_Y + row * (NOTE_H + TERM_H + GAP + ROW_GAP) + y = BASE_Y + row * (TERM_H + ROW_GAP) return x, y def surface_run(idx, problem, logfile): - x, y = slot(idx) - call_tool("create_note", { - "content": f"### Solving: {problem}\n\nAgent-under-test β€” live, read-only.", - "position": {"x": x, "y": y}, - "size": {"width": COL_W, "height": NOTE_H}, - "color": "blue", - }) + # The terminal's own title ("agent: ") already names the run, so there + # is no separate note block β€” it was redundant canvas clutter (and rendered empty). + # + # No explicit position: our old fixed grid started at a hardcoded origin and was + # blind to whatever else is already on the canvas (the benchmark panel, the runner + # terminal), so viewers landed on top of them. Omitting it lets Orcabot's canvas + # auto-placement find genuinely free space. call_tool("create_terminal", { "name": f"agent: {problem}", "boot_command": f"tail -n +1 -F {shlex.quote(logfile)}", "agentic": False, - "position": {"x": x, "y": y + NOTE_H + GAP}, "size": {"width": COL_W, "height": TERM_H}, }) _log(f"surfaced run '{problem}' -> {logfile}") -def watch(workdir, stop=None, skip_existing=False): + +def surface_results_browser(workdir): + """Open the live results browser ONCE scb-live actually answers on its port. Done + server-side here (not from the panel's React state, which resets when the run adds + canvas items) via the same MCP path that surfaces viewers β€” so it's robust. A + marker file makes it at-most-once so re-launches don't stack browsers; the block + persists + auto-refreshes across runs anyway.""" + if not mcp_ready(): + return + port = os.environ.get("SCB_LIVE_PORT", "8051") + url = f"http://127.0.0.1:{port}" + marker = os.path.join(WS_ROOT, ".scb-browser-open") + if os.path.exists(marker): + return + # Wait for scb-live to actually serve (up to ~2min) before creating the block β€” + # a browser navigates once, so opening it early parks it on connection-refused. + for _ in range(120): + try: + urllib.request.urlopen(url + "/", timeout=2) + break + except Exception: # noqa: BLE001 + time.sleep(1) + else: + _log(f"results browser: {url} never answered β€” not surfacing") + return + call_tool("create_browser", {"url": url, "size": {"width": 1200, "height": 750}}) + try: + open(marker, "w").close() + except OSError: + pass + _log(f"surfaced results browser -> {url}") + + +def watch(workdir, stop=None, skip_existing=False, spawn_viewers=True): manifest = os.path.join(workdir, ".scb_tmux", "runs.jsonl") _log(f"watching {manifest}") if not mcp_ready(): _log("not in an Orcabot PTY (no ORCABOT_MCP_SECRET) β€” visualization disabled; still tailing manifest for logs") seen, idx = set(), 0 + # Open the results browser as soon as scb-live is live (independent of the + # per-run viewer loop below). + threading.Thread(target=surface_results_browser, args=(workdir,), daemon=True).start() # runs.jsonl is append-only across ALL runs. In `run` mode we launch a fresh # watcher per benchmark run, so without this it would re-surface a note + a # permanent `tail -F` terminal for every historical run on each launch. @@ -148,7 +242,9 @@ def watch(workdir, stop=None, skip_existing=False): continue if not os.path.isabs(logfile): logfile = os.path.join(workdir, logfile) - if mcp_ready(): + if not spawn_viewers: + _log(f"viewers off β€” not surfacing: {problem} (use the scb-live browser dashboard)") + elif mcp_ready(): surface_run(idx, problem, logfile) else: _log(f"(disabled) would surface: {problem} -> {logfile}") @@ -170,14 +266,38 @@ def main(): workdir = rest[i + 1] rest = rest[:i] + rest[i + 2:] + # Per-run PTY viewers are optional: a big multi-problem run would otherwise + # spawn one read-only terminal per problem, cluttering the canvas β€” the + # scb-live browser dashboard is the low-noise overview. Default on; + # `--no-viewers` or SCB_VIEWERS=0 turns them off (the run + dashboard are + # unaffected). + spawn_viewers = os.environ.get("SCB_VIEWERS", "1").strip().lower() not in ("0", "false", "no", "off") + if "--no-viewers" in rest: + spawn_viewers = False + rest = [a for a in rest if a != "--no-viewers"] + if "--viewers" in rest: + spawn_viewers = True + rest = [a for a in rest if a != "--viewers"] + + # --skip-existing: for standalone `watch` in the pipeline β€” snapshot the runs + # already in the manifest and only surface runs that appear AFTER we start, so + # a shared/append-only runs.jsonl doesn't re-surface historical runs. (`run` + # mode always does this.) + skip_existing = "--skip-existing" in rest + if skip_existing: + rest = [a for a in rest if a != "--skip-existing"] + if mode == "watch": - watch(workdir) + watch(workdir, skip_existing=skip_existing, spawn_viewers=spawn_viewers) elif mode == "run": cmd = rest[1:] if rest[:1] == ["--"] else rest if not cmd: - _log("usage: scb-visualize run [--workdir DIR] -- ") + _log("usage: scb-visualize run [--workdir DIR] [--no-viewers] -- ") sys.exit(2) - t = threading.Thread(target=watch, args=(workdir,), kwargs={"skip_existing": True}, daemon=True) + prepare_configs(workdir) # broker routing + stable log dir, before the run + t = threading.Thread(target=watch, args=(workdir,), + kwargs={"skip_existing": True, "spawn_viewers": spawn_viewers}, + daemon=True) t.start() rc = subprocess.call(cmd, cwd=workdir) time.sleep(2) # let the watcher surface the final run diff --git a/benchmarks/slopcodebench/template.json b/benchmarks/slopcodebench/template.json index 872d27d5..725ce767 100644 --- a/benchmarks/slopcodebench/template.json +++ b/benchmarks/slopcodebench/template.json @@ -1,33 +1,20 @@ { - "_comment": "Importable Orcabot dashboard template. Orcabot chat is the orchestrator (no auto-opened agent terminal); it drives runs via its tools (create_terminal/read_file/secrets_create) and surfaces each agent-under-test as a live read-only component via bin/scb-visualize. One-time VM setup (fork + venv) still required. Seed with bin/seed-benchmark-templates.", + "_comment": "Importable Orcabot dashboard template. Orcabot chat is the orchestrator: it runs setup itself via run_command (no setup PTY, no note block) and drives runs via create_terminal/read_file, surfacing each agent-under-test as a live read-only component via bin/scb-visualize. Seed with bin/seed-benchmark-templates.", "name": "SlopCodeBench Runner", - "description": "Run slop-code-bench arms driven by Orcabot chat in plain English; each agent-under-test opens as a live read-only terminal on the canvas. One-time VM setup required (fork + venv).", + "description": "Compare coding agents on slop-code-bench", "category": "coding", "items": [ { - "placeholderId": "runbook", - "type": "note", - "content": "# SlopCodeBench Runner\n\nRun slop-code-bench arms driven by **Orcabot chat** β€” just tell it what to run in\nplain English. Each agent-under-test opens as a **live, read-only terminal** on\nthe canvas, labelled with the problem it's solving.\n\n## One-time setup (per VM)\n1. Fork at `/workspace/slop-code-bench` with venv `/workspace/scb-venv`.\n2. Add your provider key in the **secrets panel**, broker-protected (e.g.\n `OPENROUTER_API_KEY`).\n\n## Drive it from chat\n- \"Run `file_backup` with opencode on kimi-k2.6\"\n- \"How's the run going?\" (chat reads the progress log for you)\n- \"Score the finished run\"\n\nValidated arm: opencode + Kimi K2.6 via OpenRouter. Broker URL must be\n`127.0.0.1`, not `localhost`.", + "placeholderId": "config", + "type": "benchmark", + "content": "{\"harnesses\": [\"opencode\"], \"skills\": [\"baseline\"], \"models\": [\"openrouter/kimi-k2.6\"], \"problems\": [], \"workers\": 1, \"prompt\": \"just-solve\", \"thinking\": \"low\", \"evaluate\": false}", "position": { "x": 0, "y": 0 }, "size": { - "width": 420, - "height": 340 - } - }, - { - "placeholderId": "results", - "type": "browser", - "content": "{\"name\": \"SlopCodeBench Results\", \"url\": \"http://localhost:8050\"}", - "position": { - "x": 0, - "y": 360 - }, - "size": { - "width": 720, - "height": 500 + "width": 340, + "height": 470 } } ], @@ -37,5 +24,5 @@ "y": 0, "zoom": 1 }, - "setupGuide": "# SlopCodeBench orchestrator (you are Orcabot chat)\n\nYou run slop-code-bench for the user in this dashboard's VM using your tools β€” you\nare the orchestrator; do NOT open a separate agent terminal to drive it. Translate\nplain English into slop-code commands, launch them, and report progress by reading\nfiles. You do NOT solve the benchmark problems β€” the agent-under-test does.\n\nBe brief. Do each step with a tool and confirm before the next.\n\n## Your tools\n- create_terminal(boot_command=…) β€” run setup/benchmark commands in the VM.\n- terminal_send_input β€” type follow-up commands into a terminal you created.\n- read_file β€” check progress/results; this is how you answer \"how's it going?\".\n- secrets_create β€” help the user add a broker-protected provider key.\n\n## 1. Set up + VERIFY the harness β€” in a VISIBLE \"setup\" terminal\nUse this create_terminal call VERBATIM. Its boot_command is ONE line (do not split\nit) that prints to the terminal so the user watches uv sync scroll live, and writes a\ndone-marker only on success. It is idempotent; uv is baked into the image (the curl\nis just a fallback for older images):\n\n create_terminal(name=\"setup\", boot_command=\"sleep 2; echo '== Orcabot setup =='; command -v uv >/dev/null || { curl -LsSf https://astral.sh/uv/install.sh | sh; export PATH=$HOME/.local/bin:$PATH; }; [ -e /workspace/slop-code-bench/bin/scb-visualize ] || git clone --branch feat/host-tmux-executor https://github.com/robdmac/slop-code-bench /workspace/slop-code-bench; cd /workspace/slop-code-bench && { [ -e /workspace/scb-venv/bin/slop-code ] || UV_PYTHON_INSTALL_DIR=/workspace/.uv-python UV_PROJECT_ENVIRONMENT=/workspace/scb-venv uv sync --python 3.12; } && ls /workspace/scb-venv/bin/slop-code && echo SETUP_OK > /workspace/.scb-setup.done && echo SETUP_OK; exec bash\")\n\nThen check ONCE with read_file(\".scb-setup.done\"): if it contains SETUP_OK, setup was\nalready done β€” proceed to step 2. If it ERRORS (marker not there), setup is still\nrunning: tell the user it's running in the 'setup' terminal, to watch it, and that\nwhen it prints SETUP_OK they should say \"go\" (or ask you to \"check setup\") and you'll\ncontinue. Do NOT claim you are monitoring in the background β€” you cannot; you only act\nwhen the user messages you. uv sync is slow the first time β€” that's expected. If the\nterminal shows an error (e.g. github.com or astral.sh held by the egress proxy), ask\nthe user to Allow it, then re-run this same call.\n\n## 2. Provider key\nEnsure an OPENROUTER_API_KEY broker secret exists. If not, ask the user to add it\n(secrets_create, broker-protected). Never display or echo the value.\n\n## 3. Broker URL -> 127.0.0.1 (not localhost)\nPatch configs/providers.yaml so the openrouter api_base uses 127.0.0.1 (opencode /\nNode resolves localhost to IPv6 ::1, which the IPv4-only broker never answers).\n\n## 4. Run + visualize\nLaunch the run THROUGH the visualizer so each agent opens as a live read-only\ncomponent with a note naming its problem (scb-visualize watches\n.scb_tmux/runs.jsonl and surfaces each run):\n create_terminal(name=\"benchmark runner\", boot_command=\"cd /workspace/slop-code-bench && bin/scb-visualize run -- /workspace/scb-venv/bin/slop-code run --agent opencode --model openrouter/kimi-k2.6 --environment configs/environments/local-tmux-py.yaml --prompt configs/prompts/just-solve.jinja --problem file_backup --no-evaluate thinking=low ; echo \\\"[runner exited $?]\\\" ; exec bash\")\nscb-visualize lives at bin/scb-visualize in the checkout (not on PATH). Always\nappend `; echo \"[runner exited $?]\" ; exec bash` so a boot failure stays visible\ninstead of killing the PTY (a dead PTY shows as \"failed to connect\").\n\n## 5. Report progress\nAnswer \"how's it going?\" with read_file(\".scb-run.log\") or\nread_file(\".scb_tmux/runs.jsonl\"). Success = advances past checkpoint_1 with\ncost>0 and steps>0. To score: drop --no-evaluate or run\n`slop-code eval outputs//`." -} \ No newline at end of file + "setupGuide": "# SlopCodeBench orchestrator (you are Orcabot chat)\n\nYou run slop-code-bench for the user in this dashboard's VM using your tools β€” you\nare the orchestrator; do NOT open a separate agent terminal to drive it. Translate\nplain English into slop-code commands, launch them, and report progress by reading\nfiles. You do NOT solve the benchmark problems β€” the agent-under-test does.\n\nBe brief. Do each step with a tool and confirm before the next.\n\n## Your tools\n- run_command β€” run a command and WAIT for its exit code. Use this for SETUP and any\n other finite step, so you can sequence steps automatically without the user.\n- create_terminal(boot_command=…) β€” ONLY for long-running things (the benchmark run),\n which would time out under run_command.\n- terminal_send_input β€” type follow-up commands into a terminal you created.\n- read_file β€” check progress/results; this is how you answer \"how's it going?\".\n- list_secret_names β€” check which keys are already set (names only) BEFORE asking.\n- secrets_create β€” help the user add a broker-protected provider key.\n\n## 1. Setup β€” nothing to do, it runs itself\nDo NOT run setup separately and do NOT create a 'setup' terminal. Setup is folded\ninto the benchmark run: the runner terminal's boot command fetches/clones the fork,\nbuilds the venv, starts the live results server, then runs the matrix. It is\nidempotent, so it's fast on every run after the first.\n\nIMPORTANT: run_command / read_file need a sandbox, and a sandbox only exists once a\nterminal item exists. On a brand-new dashboard there is none yet, so run_command will\nfail with 'No sandbox is running for this dashboard yet'. That is EXPECTED β€” do not\nwork around it by creating an empty terminal. Just launch the run (step 4); the runner\nterminal creates the sandbox, and from then on run_command/read_file work normally.\n\n## 2. Provider first β€” do NOT check or ask for keys before this\nThe Benchmark panel has a Provider control with two choices, and they need different\ncredentials. ASK THE USER WHICH ONE before touching secrets:\n- **OpenRouter (brokered key)** -> needs OPENROUTER_API_KEY. Only now use\n list_secret_names to check; if present say so and move on, if absent ask them to add\n it (secrets_create, broker-protected). Never display the value.\n- **Codex subscription** -> needs NO API key. It uses `codex login` (~/.codex/auth.json).\n You MUST set harness=codex AND model=openai/gpt-5.5 for this mode β€” the subscription\n login is read ONLY by the codex CLI, so opencode/other harnesses cannot use it (a run\n with harness=opencode + codex_auth=subscription is refused). Write the config as\n harnesses:[codex], models:[openai/gpt-5.5], codex_auth:subscription. Do NOT check for\n or ask about OPENROUTER_API_KEY in this mode. If the run reports no auth.json, tell the\n user to run `codex login` in a terminal.\n Model: use an id the user's Codex plan actually exposes. openai/gpt-5.5 is the known-\n good default (it's what the SWE-bench Pro harness drives). If the run aborts with\n \"not supported when using Codex with a ChatGPT account\", the id isn't available on\n that plan β€” ask the user which model their plan exposes rather than guessing.\n\n## 3. Broker routing β€” automatic (nothing to do)\nscb-matrix (step 4) restores a pristine providers.yaml (from providers.yaml.orig that\nsetup saved) and rewrites the openrouter api_base to this session's local broker\n($OPENROUTER_BASE_URL β€” 127.0.0.1, injects the real key server-side). Do NOT patch\nproviders.yaml yourself.\n\n## 4. Configure + run\nThe run is configured by /workspace/.scb-config.yaml β€” harnesses, models, skills,\nproblems, workers, prompt, thinking, evaluate, codex_auth. For a Codex arm on a\nChatGPT/Codex subscription (no API key), set codex_auth: subscription and model\nopenai/gpt-5.5-codex, and make sure `codex login` has been run (auth.json). Note the\nsubscription credential then lives in the VM, readable by the agent-under-test. read_file(\".scb-config.yaml\") to show\nit. Each (harness x model x skill) is an ARM; each arm runs all `problems`; `workers`\narms run in parallel. `problems` defaults to EMPTY, which means run the FULL benchmark\nset β€” every problem in sequence (slop-code auto-discovers them). Only set `problems`\nwhen the user asks for specific ones (e.g. file_backup, etl_pipeline,\nlayered_config_synthesizer). Don't invent a single default. Skills are public agent-skill packs (baseline, gsd, omc,\nsuperpowers, karpathy, addyosmani) β€” all run in the VM; scb-matrix clones each skill's\npublic plugin repo and stages it into the run. When the\nuser says what to run:\n- One-off: pass flags to scb-matrix β€” --harness H / --model M / --skill S / --problem P\n (each repeatable), --workers N, --thinking low|medium|high, --prompt NAME, --evaluate.\n- Persist: rewrite /workspace/.scb-config.yaml (run_command) so it sticks / the panel\n reflects it.\nLaunch with ONE create_terminal (VERBATIM). scb-matrix reads the config + patches the\nbroker; scb-live serves the live per-ARM tally (cost/steps/state) to the results\nbrowser; scb-visualize surfaces a viewer per arm:\n create_terminal(name=\"benchmark runner\", boot_command=\"K=/workspace/.scb-running.lock; L=/workspace/.scb-running; exec 9>\"$K\"; if ! flock -n 9; then echo \"[scb] a benchmark run is already active β€” not starting a second one\"; exec 9>&-; exec bash; fi; printf \"pid=%s beat=%s\\n\" \"$$\" \"$(date +%s)\" > \"$L\"; OWNER=$$; ( while kill -0 \"$OWNER\" 2>/dev/null; do printf \"pid=%s beat=%s\\n\" \"$OWNER\" \"$(date +%s)\" > \"$L\"; sleep 20; done ) & HB=$!; cleanup_lock() { trap - EXIT INT TERM; kill \"$HB\" 2>/dev/null; wait \"$HB\" 2>/dev/null; rm -f \"$L\"; flock -u 9 2>/dev/null; exec 9>&-; }; trap cleanup_lock EXIT; trap 'exit 130' INT; trap 'exit 143' TERM; echo '== Orcabot: preparing harness (first run installs deps) =='; rm -f /workspace/.scb-live.ready; export PATH=$HOME/.local/bin:$PATH; command -v uv >/dev/null || { curl -LsSf https://astral.sh/uv/install.sh | sh; }; D=/workspace/slop-code-bench; B=feat/host-tmux-executor; R=https://github.com/robdmac/slop-code-bench; if [ -d $D/.git ]; then git -C $D fetch --depth 1 origin $B && git -C $D reset --hard FETCH_HEAD; else if [ -e $D ]; then mv $D $D.stale.$(date +%s); fi; git clone --depth 1 --branch $B $R $D; fi; cd $D; curl -sf -o /dev/null --max-time 2 http://127.0.0.1:8051/ || ( exec 9>&-; SCB_LIVE_PORT=8051 nohup bin/scb-live >/workspace/.scb-live.log 2>&1 & ); ( exec 9>&-; for i in $(seq 1 120); do curl -sf -o /dev/null --max-time 2 http://127.0.0.1:8051/ && { : > /workspace/.scb-live.ready; break; }; sleep 1; done ) >/dev/null 2>&1 & { [ -e /workspace/scb-venv/bin/slop-code ] || UV_PYTHON_INSTALL_DIR=/workspace/.uv-python UV_PROJECT_ENVIRONMENT=/workspace/scb-venv uv sync --python 3.12; } && cp -f configs/providers.yaml configs/providers.yaml.orig 2>/dev/null; ls /workspace/scb-venv/bin/slop-code && echo SETUP_OK > /workspace/.scb-setup.done && echo SETUP_OK; echo '== running benchmark =='; cd /workspace/slop-code-bench; ( exec 9>&-; nohup bin/scb-visualize watch --skip-existing >/workspace/.scb-viz.log 2>&1 ) & bin/scb-matrix; rc=$?; cleanup_lock; echo \"[matrix done rc=$rc]\"; exec bash\")\nFor a one-off matrix, append flags to `bin/scb-matrix` (e.g. `bin/scb-matrix --model\nopenrouter/kimi-k2.6 --model anthropic/claude-3.5 --problem file_backup --workers 2`).\nMany arms? add `--no-viewers` to the scb-visualize watch β€” the browser tally covers all.\n\n## 5. Results browser β€” opens itself, do NOT create one\nThe runner opens the live results browser automatically once scb-live is serving\n(scb-visualize surfaces it, the same way the per-run agent viewers appear). Do NOT\ncall create_browser yourself β€” that would add a duplicate. If the user says they\ndon't see it, tell them it appears a minute or so into the run once the live server\nis up, and to check the canvas.\n\n## 6. Report progress\nThe runner prints `[matrix done rc=N]`. rc=0 means every arm completed; rc=1 means at\nleast one arm FAILED β€” read the runner terminal for the per-arm `FAILED:` lines and\ntell the user which arm failed and why. Never report success on a non-zero rc.\n\n\nAnswer \"how's it going?\" by pointing the user at the live results browser (per-arm\ncost/steps/state), or read_file(\".scb-run.log\") for the latest arm's log. Success per arm = advances past checkpoint_1 and the agent viewer shows real tool\ncalls. Do NOT use cost>0 as the signal on a Codex SUBSCRIPTION: codex emits no\ntoken_count/total_cost events on a plan, so cost is legitimately $0.0000 forever\n(the live view shows \"plan\" instead). Only broker/API-key runs bill per token. To score, set `evaluate:\ntrue` in /workspace/.scb-config.yaml (or pass --evaluate), or run\n`slop-code eval outputs//`." +} diff --git a/controlplane/src/chat/handler.ts b/controlplane/src/chat/handler.ts index 56dd852e..7b9f6ca6 100644 --- a/controlplane/src/chat/handler.ts +++ b/controlplane/src/chat/handler.ts @@ -1,6 +1,6 @@ // Copyright 2026 Rob Macrae. All rights reserved. // SPDX-License-Identifier: LicenseRef-Proprietary -// REVISION: chat-v22-viewer-rbac-timeout-report +// REVISION: chat-v23-list-secret-names /** * Orcabot Chat Handler @@ -14,7 +14,7 @@ * - DELETE /chat/history - Clear conversation history */ -console.log(`[chat] REVISION: chat-v22-viewer-rbac-timeout-report loaded at ${new Date().toISOString()}`); +console.log(`[chat] REVISION: chat-v23-list-secret-names loaded at ${new Date().toISOString()}`); import type { Env, ChatMessage, ChatToolCall, ChatToolResult, ChatStreamEvent, AnyUIGuidanceCommand } from '../types'; import { type GeminiTool } from '../gemini/client'; @@ -65,6 +65,7 @@ When working with dashboards, use dashboard_list first to find existing ones, or AI PROVIDER SETUP (when user has NO stored keys and wants to set one up): - The three supported coding agents are: Claude Code (needs ANTHROPIC_API_KEY), Gemini CLI (needs GEMINI_API_KEY), and Codex (needs OPENAI_API_KEY). +- NEVER ask the user to add a key without checking first: call list_secret_names (names only, no values) and see if it is already set. If it IS listed, say so and proceed β€” do not ask them to add it again. - If no keys are stored, ask which provider they want and use secrets_create to store it with dashboard_id="_global". - Keys are stored encrypted and are never visible to AI agents (secrets broker protection). - After storing the key, offer to create a terminal with the chosen agent. @@ -311,6 +312,23 @@ const SECRETS_TOOLS: GeminiTool[] = [ required: ['dashboard_id'], }, }, + { + name: 'list_secret_names', + description: + 'List the NAMES of secrets/env vars already configured for a dashboard (plus global ones). ' + + 'Returns names only β€” never values. Use this to check whether a required key (e.g. ' + + 'OPENROUTER_API_KEY) is already set BEFORE asking the user to add it.', + inputSchema: { + type: 'object', + properties: { + dashboard_id: { + type: 'string', + description: 'The dashboard ID whose secrets to list (global "_global" secrets are always included).', + }, + }, + required: ['dashboard_id'], + }, + }, { name: 'secrets_create', description: 'Create a new secret or environment variable', @@ -1212,6 +1230,58 @@ async function executeTool( return { result: data as Record, isError: !response.ok }; } + if (toolName === 'list_secret_names') { + const dashboardId = args.dashboard_id as string; + if (!dashboardId) { + return { result: { error: 'dashboard_id is required' }, isError: true }; + } + + // Enumerating configured key names is metadata, not credentials β€” but it still + // reveals which providers/services a dashboard is wired to, so require write + // access (owner/editor) like the other non-read-only tools. A viewer must not + // be able to enumerate. + const access = await env.DB.prepare(` + SELECT role FROM dashboard_members WHERE dashboard_id = ? AND user_id = ? + `).bind(dashboardId, userId).first<{ role: string }>(); + if (!access || (access.role !== 'owner' && access.role !== 'editor')) { + console.log(`[list_secret_names] DENIED role=${access?.role ?? 'none'} dashboard=${dashboardId} user=${userId}`); + return { + result: { error: 'Access denied. list_secret_names requires owner or editor access to this dashboard.' }, + isError: true, + }; + } + + // NAMES ONLY. Never select `value` β€” not even a prefix/suffix: a "helpful" + // preview like sk-ant-api03-abc… is a real leak. The result is strictly + // set/not-set metadata, so the broker's guarantee (LLM never sees values) + // is preserved. + const rows = await env.DB.prepare(` + SELECT name, type, broker_protected, dashboard_id + FROM user_secrets + WHERE user_id = ? AND dashboard_id IN (?, '_global') + ORDER BY name + `).bind(userId, dashboardId).all<{ + name: string; type: string; broker_protected: number; dashboard_id: string; + }>(); + + const secrets_ = (rows.results ?? []).map((r) => ({ + name: r.name, + type: r.type, + broker_protected: !!r.broker_protected, + scope: r.dashboard_id === '_global' ? 'global' : 'dashboard', + })); + + return { + result: { + names: secrets_.map((s) => s.name), + secrets: secrets_, + count: secrets_.length, + note: 'Names only β€” values are never exposed. A listed name means the key IS set.', + }, + isError: false, + }; + } + if (toolName === 'secrets_create') { const response = await secrets.createSecret(env, userId, { dashboardId: args.dashboard_id as string, diff --git a/controlplane/src/index.ts b/controlplane/src/index.ts index 15a1608c..fb58f08c 100644 --- a/controlplane/src/index.ts +++ b/controlplane/src/index.ts @@ -1500,7 +1500,7 @@ async function handleRequest(request: Request, env: EnvWithBindings, ctx: Pick { if (!env.SURFACE_TOKEN) return; await ensureTemplateColumns(env); - const marker = 'seed_desktop_starter_templates_v2'; - const already = await env.DB.prepare( + // Two markers on purpose. The v2 marker still guards the ORIGINAL catalog, so a + // user who deliberately deleted one of those starters keeps it deleted. Bumping a + // single marker to v3 would have re-inserted the whole catalog and resurrected + // them. Newly-added starters get their own marker and are seeded on their own. + const baseMarker = 'seed_desktop_starter_templates_v2'; + const seededBase = await env.DB.prepare( `SELECT 1 FROM schema_migrations WHERE name = ?` - ).bind(marker).first(); - if (already) return; + ).bind(baseMarker).first(); - // Drop the earlier hand-made starters, superseded by these prod copies. - await env.DB.prepare( - `DELETE FROM dashboard_templates WHERE id IN - ('starter-agentic-coding', 'starter-automation', 'starter-documentation')` - ).run(); + // Starters introduced after the v2 catalog β€” seeded independently so they reach + // existing DBs without touching what the user has already curated. + const LATER_STARTERS = new Set(['starter-slopcodebench']); + const laterMarker = 'seed_desktop_starter_slopcodebench_v1'; + const seededLater = await env.DB.prepare( + `SELECT 1 FROM schema_migrations WHERE name = ?` + ).bind(laterMarker).first(); + + if (seededBase && seededLater) return; + + if (!seededBase) { + // Drop the earlier hand-made starters, superseded by these prod copies. + await env.DB.prepare( + `DELETE FROM dashboard_templates WHERE id IN + ('starter-agentic-coding', 'starter-automation', 'starter-documentation')` + ).run(); + } const now = new Date().toISOString(); for (const t of starterTemplates) { + const isLater = LATER_STARTERS.has(t.id); + if (isLater ? seededLater : seededBase) continue; // already handled for this group await env.DB.prepare( `INSERT INTO dashboard_templates (id, name, description, category, author_id, author_name, - items_json, edges_json, viewport_json, item_count, is_featured, status, created_at, updated_at) - VALUES (?, ?, ?, ?, 'orcabot', 'Orcabot', ?, ?, ?, ?, 1, 'approved', ?, ?) + items_json, edges_json, viewport_json, setup_guide, item_count, is_featured, status, created_at, updated_at) + VALUES (?, ?, ?, ?, 'orcabot', 'Orcabot', ?, ?, ?, ?, ?, 1, 'approved', ?, ?) ON CONFLICT(id) DO NOTHING` ).bind( t.id, t.name, t.description, t.category, JSON.stringify(t.items), JSON.stringify(t.edges), t.viewport ? JSON.stringify(t.viewport) : null, + (t as { setupGuide?: string }).setupGuide ?? null, t.itemCount, now, now ).run(); } - await env.DB.prepare( - `INSERT INTO schema_migrations (name) VALUES (?)` - ).bind(marker).run(); + if (!seededBase) { + await env.DB.prepare(`INSERT INTO schema_migrations (name) VALUES (?)`).bind(baseMarker).run(); + } + if (!seededLater) { + await env.DB.prepare(`INSERT INTO schema_migrations (name) VALUES (?)`).bind(laterMarker).run(); + } } /** diff --git a/controlplane/src/templates/scrubber.ts b/controlplane/src/templates/scrubber.ts index ed91761b..e25ceab0 100644 --- a/controlplane/src/templates/scrubber.ts +++ b/controlplane/src/templates/scrubber.ts @@ -27,7 +27,8 @@ export type DashboardItemType = | 'workspace' | 'recipe' | 'prompt' - | 'schedule'; + | 'schedule' + | 'benchmark'; /** * Safely parse JSON, returning null on failure @@ -170,6 +171,25 @@ export function scrubItemContent( return JSON.stringify({ name: '', cron: '', eventTrigger: '', enabled: true }); } + case 'benchmark': { + // Benchmark config is run configuration, not user data β€” no names, URLs or + // credentials β€” so it survives export intact. Falling through to `default` + // blanked it, silently resetting exported dashboards to generic defaults. + const parsed = safeParseJson(content) as Record | null; + if (!parsed || typeof parsed !== 'object') return ''; + return JSON.stringify({ + harnesses: parsed.harnesses ?? [], + skills: parsed.skills ?? [], + models: parsed.models ?? [], + problems: parsed.problems ?? [], + workers: parsed.workers ?? 1, + prompt: parsed.prompt ?? 'just-solve', + thinking: parsed.thinking ?? 'low', + evaluate: parsed.evaluate ?? false, + codexAuth: parsed.codexAuth ?? 'broker', + }); + } + default: // Unknown type - return empty for safety return ''; diff --git a/controlplane/src/templates/starter-templates.json b/controlplane/src/templates/starter-templates.json index 8261e752..e8378f4f 100644 --- a/controlplane/src/templates/starter-templates.json +++ b/controlplane/src/templates/starter-templates.json @@ -349,5 +349,34 @@ "zoom": 0.6876693766937669 }, "itemCount": 5 + }, + { + "id": "starter-slopcodebench", + "name": "SlopCodeBench Runner", + "description": "Compare coding agents on slop-code-bench", + "category": "coding", + "items": [ + { + "placeholderId": "config", + "type": "benchmark", + "content": "{\"harnesses\": [\"opencode\"], \"skills\": [\"baseline\"], \"models\": [\"openrouter/kimi-k2.6\"], \"problems\": [], \"workers\": 1, \"prompt\": \"just-solve\", \"thinking\": \"low\", \"evaluate\": false}", + "position": { + "x": 0, + "y": 0 + }, + "size": { + "width": 340, + "height": 470 + } + } + ], + "edges": [], + "viewport": { + "x": 0, + "y": 0, + "zoom": 1 + }, + "itemCount": 1, + "setupGuide": "# SlopCodeBench orchestrator (you are Orcabot chat)\n\nYou run slop-code-bench for the user in this dashboard's VM using your tools β€” you\nare the orchestrator; do NOT open a separate agent terminal to drive it. Translate\nplain English into slop-code commands, launch them, and report progress by reading\nfiles. You do NOT solve the benchmark problems β€” the agent-under-test does.\n\nBe brief. Do each step with a tool and confirm before the next.\n\n## Your tools\n- run_command β€” run a command and WAIT for its exit code. Use this for SETUP and any\n other finite step, so you can sequence steps automatically without the user.\n- create_terminal(boot_command=…) β€” ONLY for long-running things (the benchmark run),\n which would time out under run_command.\n- terminal_send_input β€” type follow-up commands into a terminal you created.\n- read_file β€” check progress/results; this is how you answer \"how's it going?\".\n- list_secret_names β€” check which keys are already set (names only) BEFORE asking.\n- secrets_create β€” help the user add a broker-protected provider key.\n\n## 1. Setup β€” nothing to do, it runs itself\nDo NOT run setup separately and do NOT create a 'setup' terminal. Setup is folded\ninto the benchmark run: the runner terminal's boot command fetches/clones the fork,\nbuilds the venv, starts the live results server, then runs the matrix. It is\nidempotent, so it's fast on every run after the first.\n\nIMPORTANT: run_command / read_file need a sandbox, and a sandbox only exists once a\nterminal item exists. On a brand-new dashboard there is none yet, so run_command will\nfail with 'No sandbox is running for this dashboard yet'. That is EXPECTED β€” do not\nwork around it by creating an empty terminal. Just launch the run (step 4); the runner\nterminal creates the sandbox, and from then on run_command/read_file work normally.\n\n## 2. Provider first β€” do NOT check or ask for keys before this\nThe Benchmark panel has a Provider control with two choices, and they need different\ncredentials. ASK THE USER WHICH ONE before touching secrets:\n- **OpenRouter (brokered key)** -> needs OPENROUTER_API_KEY. Only now use\n list_secret_names to check; if present say so and move on, if absent ask them to add\n it (secrets_create, broker-protected). Never display the value.\n- **Codex subscription** -> needs NO API key. It uses `codex login` (~/.codex/auth.json).\n You MUST set harness=codex AND model=openai/gpt-5.5 for this mode β€” the subscription\n login is read ONLY by the codex CLI, so opencode/other harnesses cannot use it (a run\n with harness=opencode + codex_auth=subscription is refused). Write the config as\n harnesses:[codex], models:[openai/gpt-5.5], codex_auth:subscription. Do NOT check for\n or ask about OPENROUTER_API_KEY in this mode. If the run reports no auth.json, tell the\n user to run `codex login` in a terminal.\n Model: use an id the user's Codex plan actually exposes. openai/gpt-5.5 is the known-\n good default (it's what the SWE-bench Pro harness drives). If the run aborts with\n \"not supported when using Codex with a ChatGPT account\", the id isn't available on\n that plan β€” ask the user which model their plan exposes rather than guessing.\n\n## 3. Broker routing β€” automatic (nothing to do)\nscb-matrix (step 4) restores a pristine providers.yaml (from providers.yaml.orig that\nsetup saved) and rewrites the openrouter api_base to this session's local broker\n($OPENROUTER_BASE_URL β€” 127.0.0.1, injects the real key server-side). Do NOT patch\nproviders.yaml yourself.\n\n## 4. Configure + run\nThe run is configured by /workspace/.scb-config.yaml β€” harnesses, models, skills,\nproblems, workers, prompt, thinking, evaluate, codex_auth. For a Codex arm on a\nChatGPT/Codex subscription (no API key), set codex_auth: subscription and model\nopenai/gpt-5.5-codex, and make sure `codex login` has been run (auth.json). Note the\nsubscription credential then lives in the VM, readable by the agent-under-test. read_file(\".scb-config.yaml\") to show\nit. Each (harness x model x skill) is an ARM; each arm runs all `problems`; `workers`\narms run in parallel. `problems` defaults to EMPTY, which means run the FULL benchmark\nset β€” every problem in sequence (slop-code auto-discovers them). Only set `problems`\nwhen the user asks for specific ones (e.g. file_backup, etl_pipeline,\nlayered_config_synthesizer). Don't invent a single default. Skills are public agent-skill packs (baseline, gsd, omc,\nsuperpowers, karpathy, addyosmani) β€” all run in the VM; scb-matrix clones each skill's\npublic plugin repo and stages it into the run. When the\nuser says what to run:\n- One-off: pass flags to scb-matrix β€” --harness H / --model M / --skill S / --problem P\n (each repeatable), --workers N, --thinking low|medium|high, --prompt NAME, --evaluate.\n- Persist: rewrite /workspace/.scb-config.yaml (run_command) so it sticks / the panel\n reflects it.\nLaunch with ONE create_terminal (VERBATIM). scb-matrix reads the config + patches the\nbroker; scb-live serves the live per-ARM tally (cost/steps/state) to the results\nbrowser; scb-visualize surfaces a viewer per arm:\n create_terminal(name=\"benchmark runner\", boot_command=\"K=/workspace/.scb-running.lock; L=/workspace/.scb-running; exec 9>\"$K\"; if ! flock -n 9; then echo \"[scb] a benchmark run is already active β€” not starting a second one\"; exec 9>&-; exec bash; fi; printf \"pid=%s beat=%s\\n\" \"$$\" \"$(date +%s)\" > \"$L\"; OWNER=$$; ( while kill -0 \"$OWNER\" 2>/dev/null; do printf \"pid=%s beat=%s\\n\" \"$OWNER\" \"$(date +%s)\" > \"$L\"; sleep 20; done ) & HB=$!; cleanup_lock() { trap - EXIT INT TERM; kill \"$HB\" 2>/dev/null; wait \"$HB\" 2>/dev/null; rm -f \"$L\"; flock -u 9 2>/dev/null; exec 9>&-; }; trap cleanup_lock EXIT; trap 'exit 130' INT; trap 'exit 143' TERM; echo '== Orcabot: preparing harness (first run installs deps) =='; rm -f /workspace/.scb-live.ready; export PATH=$HOME/.local/bin:$PATH; command -v uv >/dev/null || { curl -LsSf https://astral.sh/uv/install.sh | sh; }; D=/workspace/slop-code-bench; B=feat/host-tmux-executor; R=https://github.com/robdmac/slop-code-bench; if [ -d $D/.git ]; then git -C $D fetch --depth 1 origin $B && git -C $D reset --hard FETCH_HEAD; else if [ -e $D ]; then mv $D $D.stale.$(date +%s); fi; git clone --depth 1 --branch $B $R $D; fi; cd $D; curl -sf -o /dev/null --max-time 2 http://127.0.0.1:8051/ || ( exec 9>&-; SCB_LIVE_PORT=8051 nohup bin/scb-live >/workspace/.scb-live.log 2>&1 & ); ( exec 9>&-; for i in $(seq 1 120); do curl -sf -o /dev/null --max-time 2 http://127.0.0.1:8051/ && { : > /workspace/.scb-live.ready; break; }; sleep 1; done ) >/dev/null 2>&1 & { [ -e /workspace/scb-venv/bin/slop-code ] || UV_PYTHON_INSTALL_DIR=/workspace/.uv-python UV_PROJECT_ENVIRONMENT=/workspace/scb-venv uv sync --python 3.12; } && cp -f configs/providers.yaml configs/providers.yaml.orig 2>/dev/null; ls /workspace/scb-venv/bin/slop-code && echo SETUP_OK > /workspace/.scb-setup.done && echo SETUP_OK; echo '== running benchmark =='; cd /workspace/slop-code-bench; ( exec 9>&-; nohup bin/scb-visualize watch --skip-existing >/workspace/.scb-viz.log 2>&1 ) & bin/scb-matrix; rc=$?; cleanup_lock; echo \"[matrix done rc=$rc]\"; exec bash\")\nFor a one-off matrix, append flags to `bin/scb-matrix` (e.g. `bin/scb-matrix --model\nopenrouter/kimi-k2.6 --model anthropic/claude-3.5 --problem file_backup --workers 2`).\nMany arms? add `--no-viewers` to the scb-visualize watch β€” the browser tally covers all.\n\n## 5. Results browser β€” opens itself, do NOT create one\nThe runner opens the live results browser automatically once scb-live is serving\n(scb-visualize surfaces it, the same way the per-run agent viewers appear). Do NOT\ncall create_browser yourself β€” that would add a duplicate. If the user says they\ndon't see it, tell them it appears a minute or so into the run once the live server\nis up, and to check the canvas.\n\n## 6. Report progress\nThe runner prints `[matrix done rc=N]`. rc=0 means every arm completed; rc=1 means at\nleast one arm FAILED β€” read the runner terminal for the per-arm `FAILED:` lines and\ntell the user which arm failed and why. Never report success on a non-zero rc.\n\n\nAnswer \"how's it going?\" by pointing the user at the live results browser (per-arm\ncost/steps/state), or read_file(\".scb-run.log\") for the latest arm's log. Success per arm = advances past checkpoint_1 and the agent viewer shows real tool\ncalls. Do NOT use cost>0 as the signal on a Codex SUBSCRIPTION: codex emits no\ntoken_count/total_cost events on a plan, so cost is legitimately $0.0000 forever\n(the live view shows \"plan\" instead). Only broker/API-key runs bill per token. To score, set `evaluate:\ntrue` in /workspace/.scb-config.yaml (or pass --evaluate), or run\n`slop-code eval outputs//`." } -] \ No newline at end of file +] diff --git a/controlplane/src/types.ts b/controlplane/src/types.ts index 056793a9..e292a81d 100644 --- a/controlplane/src/types.ts +++ b/controlplane/src/types.ts @@ -174,7 +174,7 @@ export interface Dashboard { export interface DashboardItem { id: string; dashboardId: string; - type: 'note' | 'todo' | 'terminal' | 'link' | 'browser' | 'workspace' | 'prompt' | 'schedule' | 'gmail' | 'calendar' | 'contacts' | 'sheets' | 'forms' | 'slack' | 'discord' | 'telegram' | 'whatsapp' | 'teams' | 'matrix' | 'google_chat' | 'twitter' | 'outlook'; + type: 'note' | 'todo' | 'terminal' | 'link' | 'browser' | 'workspace' | 'prompt' | 'schedule' | 'gmail' | 'calendar' | 'contacts' | 'sheets' | 'forms' | 'slack' | 'discord' | 'telegram' | 'whatsapp' | 'teams' | 'matrix' | 'google_chat' | 'twitter' | 'outlook' | 'benchmark'; content: string; position: { x: number; y: number }; size: { width: number; height: number }; diff --git a/desktop/app/src-tauri/Cargo.lock b/desktop/app/src-tauri/Cargo.lock index a7544f6d..c355f689 100644 --- a/desktop/app/src-tauri/Cargo.lock +++ b/desktop/app/src-tauri/Cargo.lock @@ -2555,7 +2555,7 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "orcabot-desktop" -version = "0.9.2" +version = "0.9.3" dependencies = [ "ctrlc", "filetime", diff --git a/desktop/app/src-tauri/Cargo.toml b/desktop/app/src-tauri/Cargo.toml index 2e71cbb2..0e1682c7 100644 --- a/desktop/app/src-tauri/Cargo.toml +++ b/desktop/app/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "orcabot-desktop" -version = "0.9.2" +version = "0.9.3" edition = "2021" description = "Orcabot Desktop shell" license = "UNLICENSED" diff --git a/desktop/app/src-tauri/tauri.conf.json b/desktop/app/src-tauri/tauri.conf.json index 957c6814..10b8b3b3 100644 --- a/desktop/app/src-tauri/tauri.conf.json +++ b/desktop/app/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Orcabot", - "version": "0.9.2", + "version": "0.9.3", "identifier": "com.orcabot.desktop", "build": { "beforeDevCommand": "sh -c \"cd ../../frontend && NEXT_PUBLIC_API_URL=http://localhost:8787 NEXT_PUBLIC_SITE_URL=http://localhost:8788 NEXT_PUBLIC_DEV_MODE_ENABLED=true NEXT_PUBLIC_DESKTOP_MODE=true npx wrangler dev -c wrangler.toml --port 8788\"", diff --git a/frontend/public/blog/free_orcabot_desktop.png b/frontend/public/blog/free_orcabot_desktop.png index 4676a7d4..14824749 100644 Binary files a/frontend/public/blog/free_orcabot_desktop.png and b/frontend/public/blog/free_orcabot_desktop.png differ diff --git a/frontend/src/app/(app)/dashboards/[id]/page.tsx b/frontend/src/app/(app)/dashboards/[id]/page.tsx index 0b6a7a48..2045f450 100644 --- a/frontend/src/app/(app)/dashboards/[id]/page.tsx +++ b/frontend/src/app/(app)/dashboards/[id]/page.tsx @@ -15,6 +15,7 @@ import { StickyNote, CheckSquare, Globe, + FlaskConical, SquareTerminal, Users, Settings, @@ -76,6 +77,7 @@ import { ShareDashboardDialog } from "@/components/dialogs/ShareDashboardDialog" import { BugReportDialog } from "@/components/dialogs/BugReportDialog"; import { OnboardingDialog } from "@/components/dialogs/OnboardingDialog"; import { Canvas } from "@/components/canvas"; +import { findAvailableSpace } from "@/lib/canvas/placement"; import { DesktopVersionBadge } from "@/components/DesktopVersionBadge"; import { CursorOverlay, PresenceList } from "@/components/multiplayer"; import { useAuthStore } from "@/stores/auth-store"; @@ -169,6 +171,7 @@ const blockTools: BlockTool[] = [ { type: "decision", icon: , label: "Decision" }, { type: "schedule", icon: , label: "Schedule" }, { type: "browser", icon: , label: "Browser" }, + { type: "benchmark", icon: , label: "Benchmark" }, // Recipe is not in DB schema yet - uncomment when added: // { type: "recipe", icon: , label: "Recipe" }, ]; @@ -262,6 +265,7 @@ const defaultSizes: Record = { link: { width: 260, height: 140 }, terminal: { width: 480, height: 500 }, browser: { width: 800, height: 500 }, + benchmark: { width: 340, height: 470 }, workspace: { width: 620, height: 130 }, recipe: { width: 320, height: 200 }, gmail: { width: 280, height: 280 }, @@ -281,7 +285,6 @@ const defaultSizes: Record = { outlook_calendar: { width: 280, height: 280 }, }; -const PLACEMENT_GAP = 32; // gap between items when finding space const COLLAPSED_SIDEBAR_WIDTH = 36; // w-9 = 2.25rem β‰ˆ 36px const SIDEBAR_WIDTH_KEY = "orcabot:sidebar-width"; const DEFAULT_SIDEBAR_WIDTH = 200; @@ -297,83 +300,6 @@ function getSidebarWidth(collapsed: boolean): number { } } -/** - * Find available space for a new component that doesn't overlap existing items. - * Strategy: - * 1. Try to place within the visible viewport area, scanning positions on a grid. - * 2. If no space in viewport, place to the right of all existing items. - * Returns a snapped position (16px grid). - * - * @param sidebarInset Pixels of the left edge occluded by the workspace sidebar. - */ -function findAvailableSpace( - existingItems: Array<{ position: { x: number; y: number }; size: { width: number; height: number } }>, - newSize: { width: number; height: number }, - viewport: { x: number; y: number; zoom: number }, - containerWidth: number, - containerHeight: number, - sidebarInset: number, -): { x: number; y: number } { - // Convert viewport to flow coordinates (visible area) - const zoom = viewport.zoom || 1; - const viewLeft = (-viewport.x + sidebarInset) / zoom; // shift right past sidebar - const viewTop = -viewport.y / zoom; - const viewWidth = (containerWidth - sidebarInset) / zoom; - const viewHeight = containerHeight / zoom; - - // Check if a candidate position overlaps any existing item - function overlaps(cx: number, cy: number): boolean { - for (const item of existingItems) { - const ax = item.position.x; - const ay = item.position.y; - const aw = item.size.width; - const ah = item.size.height; - - if ( - cx < ax + aw + PLACEMENT_GAP && - cx + newSize.width + PLACEMENT_GAP > ax && - cy < ay + ah + PLACEMENT_GAP && - cy + newSize.height + PLACEMENT_GAP > ay - ) { - return true; - } - } - return false; - } - - // Snap to 16px grid - const snap = (v: number) => Math.round(v / 16) * 16; - - // 1. Try to place within the visible viewport with some margin - const margin = 48; - const stepX = Math.max(64, snap(newSize.width / 2)); - const stepY = Math.max(64, snap(newSize.height / 2)); - - for (let y = snap(viewTop + margin); y + newSize.height < viewTop + viewHeight - margin; y += stepY) { - for (let x = snap(viewLeft + margin); x + newSize.width < viewLeft + viewWidth - margin; x += stepX) { - if (!overlaps(x, y)) { - return { x, y }; - } - } - } - - // 2. No space in viewport β€” place to the right of all existing items - if (existingItems.length === 0) { - return { x: snap(viewLeft + margin), y: snap(viewTop + margin) }; - } - - let maxRight = -Infinity; - let topAtMaxRight = 0; - for (const item of existingItems) { - const right = item.position.x + item.size.width; - if (right > maxRight) { - maxRight = right; - topAtMaxRight = item.position.y; - } - } - - return { x: snap(maxRight + PLACEMENT_GAP * 2), y: snap(topAtMaxRight) }; -} /** * Bridge component that watches for inbound messaging events (WhatsApp/Slack/Discord) @@ -903,6 +829,15 @@ export default function DashboardPage() { }, [dashboardId, isAuthenticated, isAuthResolved]); // Compute a non-overlapping position for a new block and ensure it's visible + // Placements handed out in the last few seconds. `items` comes from React state, so + // blocks created in quick succession (scb-visualize surfacing several agent viewers, + // or chat creating a terminal then a browser) would all be placed against the SAME + // stale list and land on top of each other. Reserving each returned rect until the + // real item shows up in `items` makes back-to-back placement collision-free. + const recentPlacementsRef = React.useRef< + Array<{ position: { x: number; y: number }; size: { width: number; height: number }; at: number }> + >([]); + const computePlacement = React.useCallback( (newSize: { width: number; height: number }) => { const container = canvasContainerRef.current; @@ -910,14 +845,20 @@ export default function DashboardPage() { const containerHeight = container?.clientHeight ?? 800; const sidebarInset = getSidebarWidth(sidebarCollapsed); - return findAvailableSpace( - items, + const now = Date.now(); + recentPlacementsRef.current = recentPlacementsRef.current.filter((p) => now - p.at < 5000); + + const position = findAvailableSpace( + [...items, ...recentPlacementsRef.current], newSize, viewportRef.current, containerWidth, containerHeight, sidebarInset, ); + + recentPlacementsRef.current.push({ position, size: newSize, at: now }); + return position; }, [items, sidebarCollapsed], ); @@ -2599,7 +2540,11 @@ export default function DashboardPage() { return; } - const defaultContent = tool.type === "todo" ? "[]" : ""; + const defaultContent = + tool.type === "todo" ? "[]" + : tool.type === "benchmark" + ? JSON.stringify({ harnesses: ["opencode"], skills: ["baseline"], models: ["openrouter/kimi-k2.6"], problems: [], workers: 1, prompt: "just-solve", thinking: "low", evaluate: false }) + : ""; // workspaceCwd is a file-tree-relative path like "/test" or "/src/lib". // The PTY starts in the workspace root (~), so use a relative cd. const relCwd = workspaceCwd !== "/" ? workspaceCwd.replace(/^\//, "") : ""; @@ -2635,9 +2580,22 @@ export default function DashboardPage() { }; const handleCreateBrowserBlock = React.useCallback( - (url: string, anchor?: { x: number; y: number }, sourceId?: string) => { + ( + url: string, + anchor?: { x: number; y: number }, + sourceId?: string, + sizeOverride?: { width: number; height: number }, + ) => { if (!url) return; - const size = defaultSizes.browser; + // Reuse an existing browser on the same URL rather than stacking duplicates + // (e.g. a run launched twice, or chat and the panel both opening the results + // view). A browser block navigates once, so a second one is pure clutter. + const existing = items.find((i) => i.type === "browser" && i.content === url); + if (existing) { + ensureVisible(existing.position, existing.size); + return; + } + const size = sizeOverride ?? defaultSizes.browser; const position = anchor ? { x: Math.round(anchor.x), y: Math.round(anchor.y) } : computePlacement(size); @@ -2652,6 +2610,23 @@ export default function DashboardPage() { }); ensureVisible(position, size); }, + [createItemMutation, computePlacement, ensureVisible, items] + ); + + // Benchmark block "Run": create a terminal that runs the pipeline boot command + // (same shape as the chat's create_terminal β€” the backend only reads bootCommand). + const handleCreateTerminalBlock = React.useCallback( + (name: string, bootCommand: string) => { + const size = defaultSizes.terminal; + const position = computePlacement(size); + createItemMutation.mutate({ + type: "terminal", + content: JSON.stringify({ name, subagentIds: [], skillIds: [], agentic: false, bootCommand }), + position, + size, + }); + ensureVisible(position, size); + }, [createItemMutation, computePlacement, ensureVisible] ); @@ -4052,6 +4027,7 @@ export default function DashboardPage() { edges={edgesToRender} onEdgesChange={onEdgesChange} onCreateBrowserBlock={role === "viewer" ? undefined : handleCreateBrowserBlock} + onCreateTerminalBlock={role === "viewer" ? undefined : handleCreateTerminalBlock} onViewportChange={(next) => { viewportRef.current = next; }} diff --git a/frontend/src/components/blocks/BenchmarkBlock.tsx b/frontend/src/components/blocks/BenchmarkBlock.tsx new file mode 100644 index 00000000..977949af --- /dev/null +++ b/frontend/src/components/blocks/BenchmarkBlock.tsx @@ -0,0 +1,583 @@ +// Copyright 2026 Rob Macrae. All rights reserved. +// SPDX-License-Identifier: LicenseRef-Proprietary +// REVISION: benchmark-block-v2-flock-run-lock + +"use client"; + +const BENCHMARK_BLOCK_REVISION = "benchmark-block-v2-flock-run-lock"; +if (typeof window !== "undefined" && !(window as unknown as { __benchmarkBlockLogged?: boolean }).__benchmarkBlockLogged) { + (window as unknown as { __benchmarkBlockLogged?: boolean }).__benchmarkBlockLogged = true; + console.log(`[BenchmarkBlock] REVISION: ${BENCHMARK_BLOCK_REVISION} loaded at ${new Date().toISOString()}`); +} + +import * as React from "react"; +import { type NodeProps, type Node } from "@xyflow/react"; +import { FlaskConical, Play, Square, X, Minimize2, Settings, Copy, Loader2 } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { BlockWrapper } from "./BlockWrapper"; +import { ConnectionHandles } from "./ConnectionHandles"; +import { MinimizedBlockView, MINIMIZED_SIZE } from "./MinimizedBlockView"; +import { + Button, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui"; +import { useDebouncedCallback } from "@/hooks/useDebounce"; +import { BlockSettingsFooter } from "./BlockSettingsFooter"; +import { writeSessionFile, readSessionFileText } from "@/lib/api/cloudflare/files"; +import catalog from "@/data/openrouter-models.json"; +import type { DashboardItem, BenchmarkContent } from "@/types/dashboard"; + +// Slop-code harnesses (from the fork's agent_runner/agents dir). +const HARNESSES = ["opencode", "claude_code", "codex", "gemini", "kimi_cli", "cursor_cli", "openhands", "miniswe", "pi"]; +const PROBLEMS_KNOWN = ["file_backup", "etl_pipeline", "layered_config_synthesizer"]; +const PROMPTS = ["just-solve", "plan_first", "anti_slop", "plan-and-test"]; +const THINKING = ["low", "medium", "high"] as const; +// One-time (idempotent) harness setup, folded into the run so no separate setup +// terminal is needed. A sandbox session requires a canvas item to exist, so the +// runner terminal is the ONE component that does everything: fetch/clone the fork, +// build the venv, start scb-live, then run the matrix. Re-runs are fast (fetch + +// reset, venv already present). +const SETUP_PRELUDE = + "rm -f /workspace/.scb-live.ready; export PATH=$HOME/.local/bin:$PATH; command -v uv >/dev/null || { curl -LsSf https://astral.sh/uv/install.sh | sh; }; D=/workspace/slop-code-bench; B=feat/host-tmux-executor; R=https://github.com/robdmac/slop-code-bench; if [ -d $D/.git ]; then git -C $D fetch --depth 1 origin $B && git -C $D reset --hard FETCH_HEAD; else if [ -e $D ]; then mv $D $D.stale.$(date +%s); fi; git clone --depth 1 --branch $B $R $D; fi; cd $D; curl -sf -o /dev/null --max-time 2 http://127.0.0.1:8051/ || ( exec 9>&-; SCB_LIVE_PORT=8051 nohup bin/scb-live >/workspace/.scb-live.log 2>&1 & ); ( exec 9>&-; for i in $(seq 1 120); do curl -sf -o /dev/null --max-time 2 http://127.0.0.1:8051/ && { : > /workspace/.scb-live.ready; break; }; sleep 1; done ) >/dev/null 2>&1 & { [ -e /workspace/scb-venv/bin/slop-code ] || UV_PYTHON_INSTALL_DIR=/workspace/.uv-python UV_PROJECT_ENVIRONMENT=/workspace/scb-venv uv sync --python 3.12; } && cp -f configs/providers.yaml configs/providers.yaml.orig 2>/dev/null; ls /workspace/scb-venv/bin/slop-code && echo SETUP_OK > /workspace/.scb-setup.done && echo SETUP_OK"; + +// Live results view served by scb-live inside the VM (started during setup). +const LIVE_URL = "http://127.0.0.1:8051"; +// Codex CLI native models β€” used with a ChatGPT/Codex subscription instead of a +// brokered OpenRouter key. slop-code needs {provider}/{model}; provider "openai" +// resolves the auth.json file credential. +// Suggested codex model for a subscription β€” gpt-5.5 is what the SWE-bench Pro +// harness drives. Only a SUGGESTION: which ids a given Codex plan exposes is +// account-dependent, and the field accepts any free-typed value. +const NATIVE_CODEX_MODELS = ["openai/gpt-5.5"]; +const DEFAULT_OPENROUTER_MODEL = "openrouter/kimi-k2.6"; +// Public agent-skill packs. Every skill now runs in the Orcabot VM (local envs); +// scb-matrix clones each skill's public plugin repo and stages it into the run. +const SKILLS = ["baseline", "gsd", "omc", "superpowers", "karpathy", "addyosmani"]; + +// Model suggestions: the validated arms + the OpenRouter catalog (routed via the +// broker, so prefixed openrouter/). The field also accepts free-typed ids. +const MODEL_SUGGESTIONS = Array.from(new Set([ + "openrouter/kimi-k2.6", + "openrouter/z-ai/glm-4.6", + ...((catalog as { models: { id: string }[] }).models || []).map((m) => `openrouter/${m.id}`), +])); + +const DEFAULT_CONFIG: BenchmarkContent = { + harnesses: ["opencode"], + skills: ["baseline"], + models: ["openrouter/kimi-k2.6"], + problems: [], + workers: 1, + prompt: "just-solve", + thinking: "low", + evaluate: false, + codexAuth: "broker", +}; + +function parseConfig(content: string): BenchmarkContent { + try { + const p = JSON.parse(content || "{}"); + return { + harnesses: Array.isArray(p.harnesses) && p.harnesses.length ? p.harnesses : DEFAULT_CONFIG.harnesses, + skills: Array.isArray(p.skills) && p.skills.length ? p.skills : DEFAULT_CONFIG.skills, + models: Array.isArray(p.models) && p.models.length ? p.models : DEFAULT_CONFIG.models, + problems: Array.isArray(p.problems) && p.problems.length ? p.problems : DEFAULT_CONFIG.problems, + workers: Number.isFinite(p.workers) && p.workers > 0 ? Math.min(8, Math.floor(p.workers)) : 1, + prompt: typeof p.prompt === "string" ? p.prompt : DEFAULT_CONFIG.prompt, + thinking: THINKING.includes(p.thinking) ? p.thinking : DEFAULT_CONFIG.thinking, + evaluate: p.evaluate === true, + codexAuth: p.codexAuth === "subscription" ? "subscription" : "broker", + }; + } catch { + return { ...DEFAULT_CONFIG }; + } +} + +// Minimal YAML subset for /workspace/.scb-config.yaml (matches scb-matrix's format). +function toYaml(c: BenchmarkContent): string { + return [ + "# SlopCodeBench run config β€” written by the config panel (also edited by chat).", + `harnesses: [${c.harnesses.join(", ")}]`, + `models: [${c.models.join(", ")}]`, + `skills: [${c.skills.join(", ")}]`, + `problems: [${c.problems.join(", ")}]`, + `workers: ${c.workers}`, + `prompt: ${c.prompt}`, + `thinking: ${c.thinking}`, + `evaluate: ${c.evaluate}`, + `codex_auth: ${c.codexAuth ?? "broker"}`, + "", + ].join("\n"); +} + +function fromYaml(text: string): Partial { + const out: Record = {}; + for (const raw of text.split("\n")) { + const line = raw.split("#", 1)[0].trim(); + const m = line.match(/^([a-z_]+):\s*(.*)$/); + if (!m) continue; + const [, key, val] = m; + if (val.startsWith("[") && val.endsWith("]")) { + out[key] = val.slice(1, -1).split(",").map((s) => s.trim()).filter(Boolean); + } else if (val !== "") { + out[key] = val; + } + } + return { + harnesses: out.harnesses as string[] | undefined, + skills: out.skills as string[] | undefined, + models: out.models as string[] | undefined, + problems: out.problems as string[] | undefined, + workers: out.workers != null ? Number(out.workers) : undefined, + prompt: out.prompt as string | undefined, + thinking: out.thinking as BenchmarkContent["thinking"] | undefined, + evaluate: out.evaluate != null ? String(out.evaluate) === "true" : undefined, + codexAuth: out.codex_auth === "subscription" ? "subscription" : out.codex_auth === "broker" ? "broker" : undefined, + }; +} + + +// Write /workspace/.scb-config.yaml from the shell so the launched configuration is +// ALWAYS persisted. Writing it via the file API only worked once a session existed, +// so the very first run on a new dashboard left the file at defaults while the run +// used panel flags β€” chat and later reloads then saw a config that never ran. +function configWriteCommand(c: BenchmarkContent): string { + const lit = (v: string) => `'${String(v).replace(/'/g, "")}'`; + const lines = toYaml(c).split("\n").filter((l) => l.length > 0); + return `printf '%s\\n' ${lines.map(lit).join(" ")} > /workspace/.scb-config.yaml`; +} + +function buildBootCommand(c: BenchmarkContent): string { + // No CLI flags: configWriteCommand persists the panel's config to + // /workspace/.scb-config.yaml and scb-matrix reads that, so the file stays the + // single source of truth shared by the panel, chat and the run itself. + const matrix = `bin/scb-matrix`; + return ( + // Acquire an advisory kernel lock FIRST β€” before clone/venv/setup, which can + // take minutes. The persistent lock file is never unlinked: flock makes both + // initial acquisition and crash recovery atomic across the shared PTY UID pool. + // The heartbeat file is only a UI status marker, not the synchronization primitive. + "K=/workspace/.scb-running.lock; L=/workspace/.scb-running; exec 9>\"$K\"; if ! flock -n 9; then echo \"[scb] a benchmark run is already active \u2014 not starting a second one\"; exec 9>&-; exec bash; fi; printf \"pid=%s beat=%s\\n\" \"$$\" \"$(date +%s)\" > \"$L\"; OWNER=$$; ( while kill -0 \"$OWNER\" 2>/dev/null; do printf \"pid=%s beat=%s\\n\" \"$OWNER\" \"$(date +%s)\" > \"$L\"; sleep 20; done ) & HB=$!; cleanup_lock() { trap - EXIT INT TERM; kill \"$HB\" 2>/dev/null; wait \"$HB\" 2>/dev/null; rm -f \"$L\"; flock -u 9 2>/dev/null; exec 9>&-; }; trap cleanup_lock EXIT; trap 'exit 130' INT; trap 'exit 143' TERM; " + + "echo '== Orcabot: preparing harness (first run installs deps) =='; " + + SETUP_PRELUDE + "; " + + "echo '== running benchmark =='; cd /workspace/slop-code-bench; " + + configWriteCommand(c) + "; " + + "( exec 9>&-; nohup bin/scb-visualize watch --skip-existing >/workspace/.scb-viz.log 2>&1 ) & " + + // Release explicitly BEFORE exec: exec replaces the shell so the EXIT trap never fires. + matrix + "; rc=$?; cleanup_lock; echo \"[matrix done rc=$rc]\"; exec bash" + ); +} + +interface BenchmarkData extends Record { + content: string; + size: { width: number; height: number }; + metadata?: { minimized?: boolean; expandedSize?: { width: number; height: number }; [key: string]: unknown }; + /** Sandbox session id (for reading/writing /workspace/.scb-config.yaml). */ + sessionId?: string; + /** Create a terminal that runs bootCommand (same path the chat's create_terminal uses). */ + onCreateTerminal?: (name: string, bootCommand: string) => void; + onCreateBrowserBlock?: (url: string, anchor?: { x: number; y: number }, sourceId?: string, size?: { width: number; height: number }) => void; + onContentChange?: (content: string) => void; + onItemChange?: (changes: Partial) => void; + onDuplicate?: () => void; + connectorMode?: boolean; + onConnectorClick?: (nodeId: string, handleId: string, kind: "source" | "target") => void; +} + +type BenchmarkNode = Node; + +// A removable-chip list with an add input + datalist suggestions. +function TokenList({ label, values, onChange, suggestions, placeholder }: { + label: string; values: string[]; onChange: (v: string[]) => void; suggestions: string[]; placeholder: string; +}) { + const [draft, setDraft] = React.useState(""); + const listId = React.useId(); + const add = (v: string) => { + const t = v.trim(); + if (t && !values.includes(t)) onChange([...values, t]); + setDraft(""); + }; + return ( +
+
{label}
+
+ {values.map((v) => ( + + {v} + + + ))} +
+ setDraft(e.target.value)} + onKeyDown={(e) => { if (e.key === "Enter") { e.preventDefault(); add(draft); } }} + onBlur={() => draft && add(draft)} + placeholder={placeholder} + className="nodrag w-full text-[11px] bg-[var(--background)] border border-[var(--border)] rounded px-1.5 py-1 focus:outline-none focus:border-[var(--border-focus)]" + /> + + {suggestions.filter((s) => !values.includes(s)).map((s) => +
+ ); +} + +export function BenchmarkBlock({ id, data, selected }: NodeProps) { + const initial = React.useMemo(() => parseConfig(data.content), [data.content]); + const [cfg, setCfg] = React.useState(initial); + const [running, setRunning] = React.useState(false); + const [status, setStatus] = React.useState(null); + const connectorsVisible = selected || Boolean(data.connectorMode); + const isMinimized = data.metadata?.minimized === true; + const [expandAnimation, setExpandAnimation] = React.useState(null); + const [isAnimatingMinimize, setIsAnimatingMinimize] = React.useState(false); + const syncedRef = React.useRef(false); + + const persist = useDebouncedCallback((c: BenchmarkContent) => { + data.onContentChange?.(JSON.stringify(c)); + }, 500); + + const update = (patch: Partial) => { + setCfg((prev) => { + const next = { ...prev, ...patch }; + persist(next); + return next; + }); + }; + + // Sync from server-persisted content. + React.useEffect(() => { setCfg(parseConfig(data.content)); }, [data.content]); + + // On first mount with a live session, sync from /workspace/.scb-config.yaml so the + // panel reflects whatever the chat last set (shared source of truth). + React.useEffect(() => { + if (syncedRef.current || !data.sessionId) return; + syncedRef.current = true; + readSessionFileText(data.sessionId, ".scb-config.yaml").then((text) => { + if (!text) return; + const y = fromYaml(text); + setCfg((prev) => { + const merged: BenchmarkContent = { + harnesses: y.harnesses?.length ? y.harnesses : prev.harnesses, + skills: y.skills?.length ? y.skills : prev.skills, + models: y.models?.length ? y.models : prev.models, + problems: y.problems?.length ? y.problems : prev.problems, + workers: y.workers && y.workers > 0 ? y.workers : prev.workers, + prompt: y.prompt || prev.prompt, + thinking: y.thinking || prev.thinking, + evaluate: y.evaluate ?? prev.evaluate, + codexAuth: y.codexAuth ?? prev.codexAuth, + }; + data.onContentChange?.(JSON.stringify(merged)); + return merged; + }); + }); + }, [data.sessionId]); // eslint-disable-line react-hooks/exhaustive-deps + + const arms = cfg.harnesses.length * cfg.models.length * cfg.skills.length; + // No problems selected = run the FULL benchmark set (slop-code auto-discovers every + // problem when no --problem is passed), so it must NOT block Run. + const canRun = cfg.harnesses.length > 0 && cfg.models.length > 0 && cfg.skills.length > 0; + + + // Provider/auth is the one setting that constrains harness AND model, so make it + // explicit rather than implied by a model prefix. Subscription mode only exists for + // the codex CLI (it reads ~/.codex/auth.json), so selecting it pins the harness. + const setProvider = (mode: "broker" | "subscription") => { + if (mode === "subscription") { + update({ + codexAuth: "subscription", + harnesses: ["codex"], + models: cfg.models.every((m) => m.startsWith("openrouter/")) + ? [NATIVE_CODEX_MODELS[0]] + : cfg.models, + }); + } else { + update({ + codexAuth: "broker", + models: cfg.models.some((m) => m.startsWith("openai/")) + ? [DEFAULT_OPENROUTER_MODEL] + : cfg.models, + }); + } + }; + + // A run is in flight while /workspace/.scb-running exists with a fresh heartbeat. + // Polling the file (not React state) keeps Run disabled for the run's real duration + // and survives this component re-rendering β€” the fragility that broke the old + // browser-open poll. A crash leaves a stale marker, so require a recent heartbeat. + const [runActive, setRunActive] = React.useState(false); + React.useEffect(() => { + if (!data.sessionId) return; + let cancelled = false; + const poll = async () => { + if (cancelled) return; + const marker = await readSessionFileText(data.sessionId!, ".scb-running").catch(() => null); + const beat = marker ? Number(/beat=(\d+)/.exec(marker)?.[1] ?? 0) : 0; + const fresh = beat > 0 && Date.now() / 1000 - beat < 90; + if (!cancelled) setRunActive(marker !== null && fresh); + if (!cancelled) timer = setTimeout(poll, 4000); + }; + let timer = setTimeout(poll, 1000); + return () => { cancelled = true; clearTimeout(timer); }; + }, [data.sessionId]); + + const handleStop = async () => { + if (!data.sessionId) { setStatus("No sandbox session yet β€” nothing to stop."); return; } + // scb-matrix polls /workspace/.scb-stop and stops launching further arms. It can't + // interrupt an agent mid-problem, so this is "stop after the current one". + const ok = await writeSessionFile(data.sessionId, ".scb-stop", "stop\n").catch(() => false); + setStatus(ok ? "Stop requested β€” finishing the current problem, then halting." : "Couldn't write the stop flag."); + }; + + const handleRun = async () => { + if (!canRun) { setStatus("Pick β‰₯1 harness, model, and skill."); return; } + setRunning(true); + setStatus(null); + try { + if (data.sessionId) { + await writeSessionFile(data.sessionId, ".scb-config.yaml", toYaml(cfg)).catch(() => false); + } + data.onCreateTerminal?.("benchmark runner", buildBootCommand(cfg)); + // Open the live results view only now that a run exists β€” a browser block + // navigates once and never retries, so opening it earlier just parks it on + // ERR_CONNECTION_REFUSED. Deduped upstream, so re-running won't stack blocks. + // The runner opens the results browser itself once scb-live is serving + // (scb-visualize -> MCP create_browser), which is robust to this panel + // re-rendering. So we don't poll/open it from React state here anymore. + setStatus(`Launched ${arms} arm${arms === 1 ? "" : "s"} β€” results browser opens when live.`); + } finally { + setRunning(false); + } + }; + + const handleMinimize = () => { + setIsAnimatingMinimize(true); + data.onItemChange?.({ metadata: { ...data.metadata, expandedSize: data.size }, size: MINIMIZED_SIZE }); + setTimeout(() => { + setIsAnimatingMinimize(false); + data.onItemChange?.({ metadata: { ...data.metadata, minimized: true, expandedSize: data.size } }); + }, 350); + }; + const handleExpand = () => { + const saved = data.metadata?.expandedSize; + setExpandAnimation("animate-expand-bounce"); + setTimeout(() => setExpandAnimation(null), 300); + data.onItemChange?.({ metadata: { ...data.metadata, minimized: false }, size: saved || { width: 320, height: 460 } }); + }; + + if (isMinimized && !isAnimatingMinimize) { + return ( + } + label={`Benchmark (${arms} arm${arms === 1 ? "" : "s"})`} + onExpand={handleExpand} + connectorsVisible={connectorsVisible} + onConnectorClick={data.onConnectorClick} + /> + ); + } + + return ( + +
+ {/* Header */} +
+
+ + Benchmark +
+
+ {arms} arm{arms === 1 ? "" : "s"} + + + + + + data.onDuplicate?.()} className="gap-2"> + Duplicate + + + + + +
+
+ + {/* Body / form */} +
+ {/* Provider / auth β€” drives which models and harness make sense */} +
+
Provider
+
+ {([ + ["broker", "OpenRouter (brokered key)", "Routes through the session secrets broker using your OPENROUTER_API_KEY. The LLM never sees the key."], + ["subscription", "Codex subscription", "Uses your ChatGPT/Codex login (~/.codex/auth.json) β€” no API key, no OpenRouter. Forces the codex harness."], + ] as const).map(([mode, label, tip]) => { + const on = (cfg.codexAuth ?? "broker") === mode; + return ( + + ); + })} +
+ {cfg.codexAuth === "subscription" && ( +
+ Needs codex login β€” stage its auth.json at{" "} + /workspace/.scb-codex-auth.json. ⚠ That credential lives in the + VM and is readable by the agent-under-test. +
+ )} +
+ + {/* Harnesses */} +
+
Harnesses
+
+ {HARNESSES.map((h) => { + const on = cfg.harnesses.includes(h); + return ( + + ); + })} +
+
+ + {/* Skills (public agent-skill packs; all run in the sandbox VM) */} +
+
Skills
+
+ {SKILLS.map((s) => { + const on = cfg.skills.includes(s); + return ( + + ); + })} +
+
+ + update({ models: v })} /> + update({ problems: v })} /> + {cfg.problems.length === 0 && ( +
+ No problems selected β†’ runs the full benchmark set, one after another. +
+ )} + +
+ + + +
+ + + + +
+ {arms} arm{arms === 1 ? "" : "s"} ({cfg.harnesses.length}h Γ— {cfg.models.length}m Γ— {cfg.skills.length}s) + {" Γ— "}{cfg.problems.length || "all"} problem{cfg.problems.length === 1 ? "" : "s"} +
+
+ + {/* Footer / Run */} +
+ + +
+ {status &&
{status}
} +
+ + + ); +} diff --git a/frontend/src/components/canvas/Canvas.tsx b/frontend/src/components/canvas/Canvas.tsx index 1e86f91b..d57bddb7 100644 --- a/frontend/src/components/canvas/Canvas.tsx +++ b/frontend/src/components/canvas/Canvas.tsx @@ -62,6 +62,7 @@ import { GoogleChatBlock } from "@/components/blocks/GoogleChatBlock"; import { TwitterBlock } from "@/components/blocks/TwitterBlock"; import OutlookBlock from "@/components/blocks/OutlookBlock"; import { OutlookCalendarBlock } from "@/components/blocks/OutlookCalendarBlock"; +import { BenchmarkBlock } from "@/components/blocks/BenchmarkBlock"; import { CursorNode } from "@/components/canvas/CursorNode"; import type { DashboardItem, Session } from "@/types/dashboard"; import type { TerminalHandle } from "@/components/terminal"; @@ -95,6 +96,7 @@ const nodeTypes: NodeTypes = { twitter: TwitterBlock, outlook: OutlookBlock, outlook_calendar: OutlookCalendarBlock, + benchmark: BenchmarkBlock, cursor: CursorNode, }; @@ -112,7 +114,8 @@ function itemsToNodes( onCreateBrowserBlock?: ( url: string, anchor?: { x: number; y: number }, - sourceId?: string + sourceId?: string, + size?: { width: number; height: number } ) => void, onConnectorClick?: (nodeId: string, handleId: string, kind: "source" | "target") => void, connectorMode?: boolean, @@ -122,7 +125,8 @@ function itemsToNodes( onStorageLinked?: (workspaceItemId: string, provider: "google_drive" | "onedrive" | "box" | "github") => void, onStorageDisconnected?: (provider: "google_drive" | "onedrive" | "box" | "github") => void, onDuplicate?: (itemId: string) => void, - onTerminalCwdChange?: (itemId: string, cwd: string) => void + onTerminalCwdChange?: (itemId: string, cwd: string) => void, + onCreateTerminalBlock?: (name: string, bootCommand: string) => void ): Node[] { const workspaceSession = sessions.find((s) => s.status === "active") @@ -160,7 +164,9 @@ function itemsToNodes( dashboardId: item.dashboardId, itemId: item.id, // Pass actual item ID for API calls session, // Pass session to terminal blocks - sessionId: item.type === "workspace" ? workspaceSession?.id : undefined, + sessionId: item.type === "workspace" || item.type === "benchmark" ? workspaceSession?.id : undefined, + // For benchmark blocks: launch a run by creating a terminal (same path as chat create_terminal) + onCreateTerminal: item.type === "benchmark" ? onCreateTerminalBlock : undefined, color, // Note color from metadata metadata: item.metadata, // Pass full metadata for type-specific use onRegisterTerminal, @@ -214,7 +220,9 @@ interface CanvasProps { onItemDelete?: (itemId: string) => void; /** Called when multiple items are deleted at once (e.g. multi-select delete) */ onItemsDelete?: (itemIds: string[]) => void; - onCreateBrowserBlock?: (url: string, anchor?: { x: number; y: number }, sourceId?: string) => void; + onCreateBrowserBlock?: (url: string, anchor?: { x: number; y: number }, sourceId?: string, size?: { width: number; height: number }) => void; + /** Benchmark blocks call this to launch a run (creates a terminal with a boot command). */ + onCreateTerminalBlock?: (name: string, bootCommand: string) => void; onViewportChange?: (viewport: { x: number; y: number; zoom: number }) => void; onCursorMove?: (point: { x: number; y: number }) => void; onCanvasClick?: () => void; @@ -263,6 +271,7 @@ export function Canvas({ onItemDelete, onItemsDelete, onCreateBrowserBlock, + onCreateTerminalBlock, onViewportChange, onCursorMove, onCanvasClick, @@ -340,7 +349,8 @@ export function Canvas({ onStorageLinked, onStorageDisconnected, readOnly ? undefined : onDuplicate, - onTerminalCwdChange + onTerminalCwdChange, + onCreateTerminalBlock ) ); return [...baseNodes, ...extraNodes]; @@ -432,7 +442,8 @@ export function Canvas({ onStorageLinked, onStorageDisconnected, readOnly ? undefined : onDuplicate, - onTerminalCwdChange + onTerminalCwdChange, + onCreateTerminalBlock ) ); // Preserve selection state from current nodes when rebuilding @@ -457,7 +468,7 @@ export function Canvas({ if (selectedIds.size === 0) return newNodes; return newNodes.map(n => selectedIds.has(n.id) ? { ...n, selected: true } : n); }); - }, [items, sessions, setNodes, onItemChange, readOnly, onCreateBrowserBlock, onConnectorClick, connectorMode, applyZIndex, extraNodes, bringToFront, onPolicyUpdate, onIntegrationAttached, onIntegrationDetached, onStorageLinked, onStorageDisconnected, onDuplicate, onTerminalCwdChange]); + }, [items, sessions, setNodes, onItemChange, readOnly, onCreateBrowserBlock, onCreateTerminalBlock, onConnectorClick, connectorMode, applyZIndex, extraNodes, bringToFront, onPolicyUpdate, onIntegrationAttached, onIntegrationDetached, onStorageLinked, onStorageDisconnected, onDuplicate, onTerminalCwdChange]); // Update nodes when items or sessions change from server // Deferred during active drag to prevent mid-drag position jumps @@ -677,6 +688,33 @@ export function Canvas({ onViewportChange?.(nextViewport); }, [onViewportChange, reactFlowRef]); + // Cmd/Ctrl +/- zooms the CANVAS rather than the whole webview, which is what you + // actually want on a dashboard. Cmd+0 resets to 100%. We only intercept when the + // user isn't typing, so the shortcut can't hijack text fields. + React.useEffect(() => { + const onKeyDown = (e: KeyboardEvent) => { + if (!(e.metaKey || e.ctrlKey) || e.altKey) return; + const el = document.activeElement as HTMLElement | null; + if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable)) return; + const inst = instanceRef.current; + if (!inst) return; + // "=" is the unshifted "+" key; also accept the numpad variants. + if (e.key === "+" || e.key === "=" || e.code === "NumpadAdd") { + e.preventDefault(); + inst.zoomIn({ duration: 120 }); + } else if (e.key === "-" || e.key === "_" || e.code === "NumpadSubtract") { + e.preventDefault(); + inst.zoomOut({ duration: 120 }); + } else if (e.key === "0") { + e.preventDefault(); + const vp = inst.getViewport(); + inst.setViewport({ ...vp, zoom: 1 }, { duration: 120 }); + } + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, []); + const handlePaneMouseMove = React.useCallback( (event: React.MouseEvent) => { if (!onCursorMove || !instanceRef.current) return; diff --git a/frontend/src/hooks/useUICommands.ts b/frontend/src/hooks/useUICommands.ts index c4846f54..ff5a480e 100644 --- a/frontend/src/hooks/useUICommands.ts +++ b/frontend/src/hooks/useUICommands.ts @@ -48,6 +48,7 @@ const defaultSizes: Record = { todo: { width: 280, height: 160 }, terminal: { width: 480, height: 500 }, browser: { width: 800, height: 500 }, + benchmark: { width: 340, height: 470 }, integration: { width: 320, height: 400 }, }; diff --git a/frontend/src/lib/api/cloudflare/files.ts b/frontend/src/lib/api/cloudflare/files.ts index b2e5dcdc..c3880b29 100644 --- a/frontend/src/lib/api/cloudflare/files.ts +++ b/frontend/src/lib/api/cloudflare/files.ts @@ -33,6 +33,42 @@ export async function deleteSessionFile(sessionId: string, path: string): Promis await apiDelete(url); } +/** + * Read a workspace file's text content. `path` is relative to the workspace root + * (e.g. ".scb-config.yaml"). Returns null on any error / missing file (best-effort). + */ +export async function readSessionFileText(sessionId: string, path: string): Promise { + if (!sessionId) return null; + const url = `${API.cloudflare.base}/sessions/${sessionId}/file?path=${encodeURIComponent(path)}`; + try { + const res = await fetch(url, { headers: { ...getAuthHeaders() }, credentials: "include" }); + if (!res.ok) return null; + return await res.text(); + } catch { + return null; + } +} + +/** + * Write raw text to a workspace file (owner-only; PUT /sessions/:id/file). `path` + * is workspace-relative. Returns true on success. + */ +export async function writeSessionFile(sessionId: string, path: string, content: string): Promise { + if (!sessionId) return false; + const url = `${API.cloudflare.base}/sessions/${sessionId}/file?path=${encodeURIComponent(path)}`; + try { + const res = await fetch(url, { + method: "PUT", + headers: { ...getAuthHeaders(), "Content-Type": "text/plain" }, + credentials: "include", + body: content, + }); + return res.ok; + } catch { + return false; + } +} + export interface WorkspaceSnapshot { version: number; dashboardId: string; diff --git a/frontend/src/lib/canvas/placement.test.ts b/frontend/src/lib/canvas/placement.test.ts new file mode 100644 index 00000000..369cb0e7 --- /dev/null +++ b/frontend/src/lib/canvas/placement.test.ts @@ -0,0 +1,119 @@ +import { describe, it, expect } from "vitest"; +import { findAvailableSpace, PLACEMENT_GAP, type PlacedRect } from "./placement"; + +// A 1200x800 container at zoom 1 with no sidebar => viewport is flow (0,0)-(1200,800). +const VIEW = { x: 0, y: 0, zoom: 1 }; +const W = 1200, H = 800, INSET = 0; + +const rect = (x: number, y: number, width: number, height: number): PlacedRect => ({ + position: { x, y }, size: { width, height }, +}); + +const overlaps = (a: PlacedRect, b: PlacedRect) => + a.position.x < b.position.x + b.size.width && + a.position.x + a.size.width > b.position.x && + a.position.y < b.position.y + b.size.height && + a.position.y + a.size.height > b.position.y; + +const place = (existing: PlacedRect[], size: { width: number; height: number }) => + findAvailableSpace(existing, size, VIEW, W, H, INSET); + +describe("findAvailableSpace", () => { + it("never overlaps an existing block", () => { + const existing = [rect(0, 0, 340, 470), rect(400, 32, 480, 500)]; + const size = { width: 300, height: 200 }; + const pos = place(existing, size); + const placed = { position: pos, size }; + for (const e of existing) expect(overlaps(placed, e)).toBe(false); + }); + + it("keeps at least PLACEMENT_GAP from neighbours", () => { + const existing = [rect(0, 0, 340, 470)]; + const size = { width: 200, height: 150 }; + const pos = place(existing, size); + const gapX = Math.max(existing[0].position.x - (pos.x + size.width), pos.x - 340, 0); + const gapY = Math.max(existing[0].position.y - (pos.y + size.height), pos.y - 470, 0); + expect(Math.max(gapX, gapY)).toBeGreaterThanOrEqual(PLACEMENT_GAP); + }); + + it("picks the roomier side rather than the first top-left fit", () => { + // Narrow sliver on the left, wide open space on the right. + const existing = [rect(180, 0, 60, 800)]; + const size = { width: 120, height: 120 }; + const pos = place(existing, size); + expect(pos.x).toBeGreaterThan(240); // chose the open right side, not the sliver + }); + + it("places N blocks in a row without any mutual overlap (rapid creation)", () => { + const placed: PlacedRect[] = []; + const size = { width: 260, height: 180 }; + for (let i = 0; i < 8; i++) { + const pos = findAvailableSpace(placed, size, VIEW, W, H, INSET); + const p = { position: pos, size }; + for (const q of placed) expect(overlaps(p, q)).toBe(false); + placed.push(p); + } + expect(placed).toHaveLength(8); + }); + + it("falls back to minimal growth when the viewport is full", () => { + // Fill the viewport completely. + const existing: PlacedRect[] = []; + for (let y = 0; y < 800; y += 200) for (let x = 0; x < 1200; x += 300) existing.push(rect(x, y, 280, 180)); + const size = { width: 400, height: 300 }; + const pos = place(existing, size); + const placed = { position: pos, size }; + for (const e of existing) expect(overlaps(placed, e)).toBe(false); + // It must pick whichever direction grows the content bounding box LESS. + const bboxW = 1180, bboxH = 780; // grid spans (0,0)-(1180,780) + const growRight = Math.max(bboxW, 1180 + 32 + 400) * Math.max(bboxH, 300) - bboxW * bboxH; + const growDown = Math.max(bboxW, 400) * Math.max(bboxH, 780 + 32 + 300) - bboxW * bboxH; + const wentRight = pos.x >= bboxW; + expect(wentRight).toBe(growRight <= growDown); + }); + + it("uses the viewport origin on an empty canvas", () => { + const pos = place([], { width: 300, height: 200 }); + expect(pos.x).toBeGreaterThanOrEqual(0); + expect(pos.y).toBeGreaterThanOrEqual(0); + expect(pos.x).toBeLessThan(200); + expect(pos.y).toBeLessThan(200); + }); + + it("respects the sidebar inset (never places under the sidebar)", () => { + const pos = findAvailableSpace([], { width: 300, height: 200 }, VIEW, W, H, 320); + expect(pos.x).toBeGreaterThanOrEqual(320); + }); +}); + +// Regression: the real SlopCodeBench dashboard sequence. Previously the runner +// terminal and an agent viewer ended up overlapping (observed in D1 as +// terminal(400,32,480x500) vs terminal(460,388,380x300)), because blocks created +// back-to-back were all placed against the same stale item list. +describe("SlopCodeBench dashboard sequence", () => { + it("places panel -> runner -> 3 agent viewers -> browser with zero overlaps", () => { + const sizes = [ + { width: 340, height: 470 }, // benchmark panel (from template) + { width: 480, height: 500 }, // benchmark runner terminal + { width: 380, height: 300 }, // agent viewer 1 + { width: 380, height: 300 }, // agent viewer 2 + { width: 380, height: 300 }, // agent viewer 3 + { width: 800, height: 500 }, // results browser + ]; + const placed: PlacedRect[] = []; + for (const size of sizes) { + // Same call the app makes; `placed` stands in for items + reserved placements. + const pos = findAvailableSpace(placed, size, VIEW, W, H, INSET); + placed.push({ position: pos, size }); + } + for (let i = 0; i < placed.length; i++) { + for (let j = i + 1; j < placed.length; j++) { + expect( + overlaps(placed[i], placed[j]), + `block ${i} at (${placed[i].position.x},${placed[i].position.y}) overlaps ` + + `block ${j} at (${placed[j].position.x},${placed[j].position.y})`, + ).toBe(false); + } + } + }); +}); diff --git a/frontend/src/lib/canvas/placement.ts b/frontend/src/lib/canvas/placement.ts new file mode 100644 index 00000000..1460e310 --- /dev/null +++ b/frontend/src/lib/canvas/placement.ts @@ -0,0 +1,136 @@ +// Copyright 2026 Rob Macrae. All rights reserved. +// SPDX-License-Identifier: LicenseRef-Proprietary + +// Canvas auto-placement. Extracted from the dashboard page so it can be unit tested +// independently of React β€” placement bugs (overlapping blocks) are pure geometry. + +export const PLACEMENT_GAP = 32; // gap between items when finding space + +export interface PlacedRect { + position: { x: number; y: number }; + size: { width: number; height: number }; +} + +/** + * Find available space for a new component that doesn't overlap existing items. + * + * Strategy: + * 1. Scan the visible viewport on a fine grid and keep every position where the new + * block fits. Rather than taking the FIRST fit (which hugs the top-left and packs + * blocks against each other), score each candidate by its CLEARANCE β€” the distance + * to the nearest neighbour or viewport edge β€” and take the roomiest. That lands new + * blocks in the largest open pocket. + * 2. If nothing fits in view, extend the canvas in whichever direction costs the least: + * pick the placement that grows the content bounding box by the smallest area, so + * the viewport has to expand as little as possible. + * + * Returns a snapped position (16px grid). + * + * @param sidebarInset Pixels of the left edge occluded by the workspace sidebar. + */ +export function findAvailableSpace( + existingItems: PlacedRect[], + newSize: { width: number; height: number }, + viewport: { x: number; y: number; zoom: number }, + containerWidth: number, + containerHeight: number, + sidebarInset: number, +): { x: number; y: number } { + // Convert viewport to flow coordinates (visible area) + const zoom = viewport.zoom || 1; + const viewLeft = (-viewport.x + sidebarInset) / zoom; // shift right past sidebar + const viewTop = -viewport.y / zoom; + const viewWidth = (containerWidth - sidebarInset) / zoom; + const viewHeight = containerHeight / zoom; + + const snap = (v: number) => Math.round(v / 16) * 16; + const rects = existingItems.map((i) => ({ + x: i.position.x, y: i.position.y, w: i.size.width, h: i.size.height, + })); + + const fits = (x: number, y: number): boolean => { + for (const r of rects) { + if ( + x < r.x + r.w + PLACEMENT_GAP && + x + newSize.width + PLACEMENT_GAP > r.x && + y < r.y + r.h + PLACEMENT_GAP && + y + newSize.height + PLACEMENT_GAP > r.y + ) return false; // overlaps a neighbour (incl. gap) + } + return true; + }; + + // Distance from the candidate rect to the nearest NEIGHBOUR. Deliberately ignores + // viewport edges: counting them scores the middle of the canvas highest and parks + // every block dead centre. Measuring only against other blocks means the roomiest + // open pocket wins, and (once quantised below) the block still packs to that + // pocket's top-left corner rather than floating in its middle. + const ROOMY = 320; // clearance beyond this is "wide open" β€” treat as equally good + const clearance = (x: number, y: number): number => { + let best = ROOMY; + for (const r of rects) { + const dx = Math.max(r.x - (x + newSize.width), x - (r.x + r.w), 0); + const dy = Math.max(r.y - (y + newSize.height), y - (r.y + r.h), 0); + best = Math.min(best, Math.max(dx, dy)); + if (best <= 0) return 0; + } + return best; + }; + + // 1. Roomiest spot inside the viewport. Step is capped so a zoomed-out viewport + // can't blow up the scan (bounded ~60x60 candidates). + const margin = 24; + const usableW = viewWidth - margin * 2; + const usableH = viewHeight - margin * 2; + if (usableW >= newSize.width && usableH >= newSize.height) { + const stepX = Math.max(32, Math.ceil((usableW - newSize.width) / 60 / 16) * 16 || 32); + const stepY = Math.max(32, Math.ceil((usableH - newSize.height) / 60 / 16) * 16 || 32); + let best: { x: number; y: number } | null = null; + let bestScore = -Infinity; + const maxY = viewTop + viewHeight - margin - newSize.height; + const maxX = viewLeft + viewWidth - margin - newSize.width; + for (let y = snap(viewTop + margin); y <= maxY; y += stepY) { + for (let x = snap(viewLeft + margin); x <= maxX; x += stepX) { + if (!fits(x, y)) continue; + // Pack row-major (left-to-right, then down). Packing tightly is what actually + // minimises canvas growth β€” an earlier version scored by clearance, which + // pushed each block away from its neighbours, stranded usable gaps, and made + // the canvas expand far sooner. Clearance is kept only as a tie-breaker so + // that among equally-packed spots we still prefer the less cramped one. + const packing = (y - viewTop) * 4 + (x - viewLeft); + const score = -packing * 1000 + Math.min(clearance(x, y), ROOMY); + if (score > bestScore) { bestScore = score; best = { x: snap(x), y: snap(y) }; } + } + } + if (best) return best; + } + + // 2. Nothing fits in view β€” grow the canvas as little as possible. + if (rects.length === 0) return { x: snap(viewLeft + margin), y: snap(viewTop + margin) }; + + const bbox = rects.reduce( + (a, r) => ({ + left: Math.min(a.left, r.x), top: Math.min(a.top, r.y), + right: Math.max(a.right, r.x + r.w), bottom: Math.max(a.bottom, r.y + r.h), + }), + { left: Infinity, top: Infinity, right: -Infinity, bottom: -Infinity }, + ); + const bboxW = bbox.right - bbox.left; + const bboxH = bbox.bottom - bbox.top; + const candidates = [ + { x: bbox.right + PLACEMENT_GAP, y: Math.max(bbox.top, viewTop + margin) }, // to the right + { x: Math.max(bbox.left, viewLeft + margin), y: bbox.bottom + PLACEMENT_GAP }, // below + ]; + let bestPos = candidates[0]; + let bestCost = Infinity; + for (const c of candidates) { + const w = Math.max(bbox.right, c.x + newSize.width) - Math.min(bbox.left, c.x); + const h = Math.max(bbox.bottom, c.y + newSize.height) - Math.min(bbox.top, c.y); + // Minimise how far the user must zoom OUT to see everything, not raw area. + // Area growth always favours extending the already-long axis, which produces a + // runaway single-row strip; this keeps the canvas roughly viewport-shaped. + const cost = Math.max(w / Math.max(viewWidth, 1), h / Math.max(viewHeight, 1)); + if (cost < bestCost) { bestCost = cost; bestPos = c; } + } + return { x: snap(bestPos.x), y: snap(bestPos.y) }; +} diff --git a/frontend/src/types/dashboard.ts b/frontend/src/types/dashboard.ts index c49a4598..22275e0a 100644 --- a/frontend/src/types/dashboard.ts +++ b/frontend/src/types/dashboard.ts @@ -22,7 +22,7 @@ export interface Dashboard { /** * Dashboard item types */ -export type DashboardItemType = "note" | "todo" | "terminal" | "link" | "browser" | "workspace" | "recipe" | "prompt" | "schedule" | "decision" | "gmail" | "calendar" | "contacts" | "sheets" | "forms" | "slack" | "discord" | "telegram" | "whatsapp" | "teams" | "matrix" | "google_chat" | "twitter" | "outlook" | "outlook_calendar"; +export type DashboardItemType = "note" | "todo" | "terminal" | "link" | "browser" | "workspace" | "recipe" | "prompt" | "schedule" | "decision" | "gmail" | "calendar" | "contacts" | "sheets" | "forms" | "slack" | "discord" | "telegram" | "whatsapp" | "teams" | "matrix" | "google_chat" | "twitter" | "outlook" | "outlook_calendar" | "benchmark"; /** * Position on the canvas @@ -98,6 +98,25 @@ export interface TodoContent { items: TodoItem[]; } +/** + * Benchmark config block content (SlopCodeBench control panel). + * Mirrors /workspace/.scb-config.yaml β€” the single source of truth the chat and + * scb-matrix also read. A run is a matrix of (harness x model) arms. + */ +export interface BenchmarkContent { + harnesses: string[]; + /** Public agent-skill packs (baseline/gsd/omc/superpowers/karpathy). Each = an arm axis. */ + skills: string[]; + models: string[]; + problems: string[]; + workers: number; + prompt: string; + thinking: "low" | "medium" | "high"; + evaluate: boolean; + /** Codex auth: "broker" = brokered API key (default); "subscription" = codex login (ChatGPT/Codex plan). */ + codexAuth?: "broker" | "subscription"; +} + /** * Link block content */ diff --git a/sandbox/docker/Dockerfile b/sandbox/docker/Dockerfile index a4c52b8e..64ad6d95 100644 --- a/sandbox/docker/Dockerfile +++ b/sandbox/docker/Dockerfile @@ -28,7 +28,7 @@ FROM debian:bookworm-slim # Prevent interactive prompts during package installation ENV DEBIAN_FRONTEND=noninteractive -# REVISION: sandbox-version-cli-v1-build-arg +# REVISION: sandbox-version-cli-v2-flock-lock ARG SANDBOX_GIT_COMMIT=unknown ENV ORCABOT_SANDBOX_VERSION=${SANDBOX_GIT_COMMIT} @@ -56,6 +56,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ nano \ less \ jq \ + util-linux \ # Process management procps \ kmod \