diff --git a/CLAUDE.md b/CLAUDE.md index 68e19a1..fa2ff48 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,13 +16,13 @@ cargo test --offline --test e2e push_workflow_succeeds # one e2e test cargo run -- -C tests/fixtures/repo --host -n # try the CLI against the fixture repo ``` -- `tests/e2e.rs` runs the built binary (through `bin()`, which adds `--no-history`) with `--host --no-cache-server` against `tests/fixtures/repo` (needs `bash`, `node` and `git` on PATH, no Docker). Assertions grep stdout, so changing log/summary line formats (`" / : "`, masking `***`) can break them. +- `tests/e2e.rs` runs the built binary (through `bin()`, which adds `--no-history`) with `--host --no-cache-server` against `tests/fixtures/repo` (needs `bash`, `node` and `git` on PATH, no Docker). Assertions grep stdout, so changing log/summary line formats (`" / : "`, the closing `๐Ÿ`/`โŒ N workflows, M jobs: โ€ฆ` line, masking `***`) can break them. - The e2e test binary bakes in absolute paths via `CARGO_BIN_EXE_runthrough` / `CARGO_MANIFEST_DIR`. If it fails with `run runthrough: ... NotFound` after the repo directory was moved or renamed, `touch tests/e2e.rs` (or `cargo clean`) to force a rebuild. - Exit codes: `0` all jobs succeeded/skipped, `1` a job failed, `2` an internal error (`anyhow` error from `run()`). ## Architecture -Pipeline: `main.rs` (CLI + config assembly) โ†’ `workflow.rs` (load YAML) โ†’ `filters.rs` (`on.` branch/tag/path/type filters) โ†’ `planner.rs` (stages, matrix) โ†’ `runner.rs` (execute) using `expr.rs`, `actions.rs`, `backend.rs`, `commands.rs`, `matchers.rs`, `context.rs`, `servers.rs`; `info.rs` serves the exit-early modes (`--graph`, `--man-page`, `--list-options`, `--bug-report`) and `--watch`'s fingerprint; `history.rs` and `dashboard.rs` record runs and serve `--dashboard`; `markers.rs` formats the job and step status lines the runner logs and `history.rs` parses back. +Pipeline: `main.rs` (CLI + config assembly) โ†’ `workflow.rs` (load YAML) โ†’ `filters.rs` (`on.` branch/tag/path/type filters) โ†’ `planner.rs` (stages, matrix) โ†’ `runner.rs` (execute) using `expr.rs`, `actions.rs`, `backend.rs`, `commands.rs`, `matchers.rs`, `context.rs`, `servers.rs`; `info.rs` serves the exit-early modes (`--graph`, `--man-page`, `--list-options`, `--bug-report`, `--install-hook`) and `--watch`'s fingerprint; `history.rs` and `dashboard.rs` record runs and serve `--dashboard`; `markers.rs` formats the job and step status lines the runner logs and `history.rs` parses back. - **`main.rs`** parses CLI (clap derive; flag names follow act, see `docs/act-parity.md`), discovers workflows (`workflow_paths`: `-W`, or `.github/workflows` plus `.runthrough/workflows`; recursively unless `--no-recurse`), filters by event and its `on.` filters (an explicit `-j` or a single workflow file bypasses both), handles `-l`/`-n`/`-g`/`--validate`, merges `.secrets`/`.vars`/`.env`/`.input` files with CLI flags (flags win), and builds `runner::Config`. Arguments from `.actrc` files are spliced in before the real ones (`actrc_args`). `Cli` is `pub` so `info.rs` and `servers.rs` read it directly. - **`workflow.rs`** validates every workflow (`schema_errors`: required keys and value types; `strict_check` adds unknown keys under `--strict`), then deserializes it with `serde_yaml`; many fields stay as raw `serde_yaml::Value` because they may contain `${{ }}` and are evaluated later. `yaml_to_json` / `yaml_scalar_to_string` bridge YAML into the JSON values the expression engine uses. @@ -38,13 +38,13 @@ Pipeline: `main.rs` (CLI + config assembly) โ†’ `workflow.rs` (load YAML) โ†’ `f - `exec_with` is the single point that spawns step processes: it creates fresh `GITHUB_OUTPUT`/`ENV`/`PATH`/`STATE`/`STEP_SUMMARY` files under `files//`, injects env (runner โ†’ job โ†’ step), parses workflow commands from output, and applies masking. `Target::Backend` runs in the job's backend; `Target::Container` runs a Docker action or `docker://` step as a sibling container started from the host (`run_container_step`), with env passed as `-e`. - `Logger` prints through `emit`, which ignores stdout write errors so a closed pipe never skips cleanup. - `resolve_platform` maps `runs-on` labels to an image or the host (`-self-hosted` sentinel; `ubuntu*`/`linux` โ†’ `DEFAULT_IMAGE`; `self-hosted` and `macos*` on macOS โ†’ host). - - `actions/checkout` is special-cased (`checkout`) unless `--no-skip-checkout`: the own repo is copied (no-op with `--bind`), another `repository` is shallow-cloned. + - `actions/checkout` is special-cased (`checkout`) unless `--no-skip-checkout`: the own repo is copied (no-op with `--bind`; only the HEAD commit with `--committed`, which also reads the repository's workflows, called ones included, from that commit exported by `actions::export_head` into `Config.committed`), another `repository` is shallow-cloned. - **`backend.rs`**: `Backend::Host` or `Backend::Container`, driven through a CLI chosen by `Engine` (`docker`, or Apple's `container` via `--engine container`); no container API crate. Engine differences are handled here, as `Engine` methods the runner asks (`quiet_args`, `host_gateway_arg`, `has_network_aliases`, `has_health_checks`, `has_host_network`, `needs_temp_volume`, `is_remote`): Apple `container` gets `--progress none`, has no `--network-alias` (the runner writes service IPs into the job container's `/etc/hosts`) and no health checks (`take_health` strips `--health-*` and `wait_healthy` polls the command with `exec`); `host_address` is `host.docker.internal` for Docker and the network's `ipv4Gateway` for Apple. The job scratch dir is bind-mounted at the **same absolute path** in the container, so host and container paths are interchangeable throughout the runner. Exceptions are the job's volumes (`JobRun.job_volumes`, mounted over those paths): RUNNER_TEMP for Docker jobs, and, where Docker shares host files through a VM (`Engine::shares_files_through_vm`, i.e. Docker Desktop outside Linux) and without `--bind` or the engine socket (whose jobs hand workspace paths to the engine), the workspace; the host does not see writes there, and `sync_in` copies checkouts into it. With a remote Docker (`engine_is_remote`: `DOCKER_HOST=ssh://`/`tcp://`, or `RUNTHROUGH_REMOTE_ENGINE=1` to force it locally) `Config.copy_mode` replaces the mounts with per-job volumes at those same paths; `JobRun::sync_in` and `exec_with` keep them in sync via `docker cp` (temp/ and files/ before each step, files/ back after, checkout results and cached actions once). - **`actions.rs`**: parses `uses:` (`Local`, `Remote`, `Docker`), loads `action.yml`, and `fetch_remote` git-clones into the action cache (`$XDG_CACHE_HOME/runthrough/actions` or `~/.cache/...`), serialized by a global mutex; `--offline` only uses the cache. - **`commands.rs`**: parsing of `GITHUB_ENV`-style files (including heredoc delimiters), `::command::` lines, and secret masking. - **`matchers.rs`**: problem matchers (`::add-matcher::`). `JobRun.matchers` is the job-wide list (shared with forks); `exec_with` feeds each output line to a per-step copy and logs matches as annotations. - **`context.rs`**: `RepoInfo::detect` shells out to `git` to fill the `github` context (repo slug from `--remote-name`, sha, ref, default branch); `host` (`--github-instance`) drives the server/API URLs. `event_ref_and_sha` then overrides ref/sha from the event payload the way GitHub does (`refs/pull/N/merge`, release tags, push `ref`/`after`, โ€ฆ), and `GITHUB_*_URL` from the caller's environment win. -- **`history.rs`**: `Recorder` (in `Config.history`, unless `--no-history` or `--insecure-secrets`) writes each run to `/runs//`: `meta.json` from `main.rs` (masked argv, repo, event, jobs by stage, and `config`: env/vars/inputs masked, secret names, platforms), every `Logger::line` after masking into `log.jsonl`, and `result.json` at the end. `Summary` / `Job::feed` rebuild jobs and steps from the log's status lines (`โญ`, `โœ…`/`โŒ`/`โš ๏ธ`/`๐Ÿ›‘`, `โญ๏ธ`, `๐Ÿš€`/`๐Ÿ`), formatted and parsed by `markers.rs`, whose tests check each line parses back. +- **`history.rs`**: `Recorder` (in `Config.history`, unless `--no-history` or `--insecure-secrets`) writes each run to `/runs//`: `meta.json` from `main.rs` (masked argv, repo, event, jobs by stage, and `config`: env/vars/inputs masked, secret names, platforms), every `Logger::line` after masking into `log.jsonl`, and `result.json` at the end. `Summary` / `Job::feed` rebuild jobs and steps from the log's status lines (`โญ`, `โœ…`/`โŒ`/`โš ๏ธ`/`๐Ÿ›‘`, `โญ๏ธ`, `๐Ÿš€`/`๐Ÿ`, `โ„น๏ธ` notes on what a local run does not reproduce) and keeps step summaries recorded beside them, formatted and parsed by `markers.rs`, whose tests check each line parses back. - **`dashboard.rs`**: `--dashboard` serves `dashboard.html` (one self-contained page, no build step) and a JSON API (`/api/runs`, `/api/runs/`, `/api/runs//job?workflow=&job=`) through `servers::serve`. - **`servers.rs`**: std-only HTTP/1.1 servers jobs reach through `ACTIONS_CACHE_URL` (`@actions/cache` v1 API, on by default, entries persisted in `index.json`) and `ACTIONS_RESULTS_URL` (`@actions/artifact` v2 Twirp API plus Azure-blob-style block uploads, with `--artifact-server-path`; the toolkit sends proto field names such as `workflow_run_backend_id`) plus `ACTIONS_RUNTIME_URL` (the v1 API of the v3 actions, files under `v3//`, gzipped chunks stored with a `.gz__` suffix). `ACTIONS_RUNTIME_TOKEN` is a JWT signed with a per-run key (`RuntimeAuth`, or the caller's own token) whose `scp` claim carries the run/job IDs the toolkit parses; the artifact API refuses calls without it, except the signed block URLs. The cache API lives under a random per-run prefix (`cache_token`), lowercases keys, and garbage-collects entries (`CacheStore::gc`) on start and after each upload. They bind `127.0.0.1`, or `0.0.0.0` for Apple containers (reached via the vmnet gateway); a failure to start the cache server only disables it. diff --git a/README.md b/README.md index 6aaf288..2b66c2e 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,14 @@ runthrough --watch # rerun whenever a file changes runthrough --graph # draw the job graph runthrough --dashboard # browse recorded runs at http://127.0.0.1:7766 runthrough --validate --strict # check the workflows, then exit +runthrough --committed # rehearse the HEAD commit, leaving uncommitted changes out +runthrough --install-hook # add a git pre-push hook that runs the push workflows first ``` +A run ends with one line for all of its workflows: jobs by result, the time it took and, when it was recorded, its dashboard URL. With `--watch`, each rerun also says which jobs changed result since the last one. + +`--install-hook` writes `.git/hooks/pre-push` (or the hook in `core.hooksPath`), never over an existing hook. It runs `runthrough --committed --quiet push`, so the commit being pushed is rehearsed rather than the working tree (its checkout and the workflows themselves come from HEAD), and a failed job stops the push, as does pushing a commit other than HEAD, which the hook does not rehearse; `git push --no-verify` skips it once. Add flags such as `--host` to the hook's `runthrough` line. + Workflows are read from `.github/workflows` and, laid out the same way, `.runthrough/workflows` (for workflows meant only for local runs); `-W` reads a single file or directory instead. Flags follow act's names (`--matrix`, `--reuse`, `--json`, `--input-file`, โ€ฆ); `runthrough --help` lists them and [docs/act-parity.md](docs/act-parity.md) tracks parity with act. Day-to-day operation and troubleshooting are in [RUNBOOK.md](RUNBOOK.md). @@ -47,7 +53,7 @@ Flags win over files. Arguments from `.actrc` files are read before the command ### Dashboard -Every run is recorded (with secrets masked) under `~/.cache/runthrough/runs`, and `runthrough --dashboard` serves a local web UI over them in the style of GitHub's Actions tab: the runs list (optionally grouped by repository or date), a run's job graph by stage with its matrix legs and annotations, and each job's steps with their logs, refreshed while a run is in progress. A failed job's page has **Copy for LLM**, which copies the job's context, its steps and the last 200 lines of each failed step as Markdown to paste into an LLM. **Resources** lists the ports runthrough listens on and the Docker containers, networks and volumes it created, with the run each belongs to; ones whose run has ended are marked left behind, with a command to remove them. `--dashboard-addr` / `--dashboard-port` change where it listens; `--no-history` skips recording a run, and runs with `--insecure-secrets` are never recorded. The newest 100 runs are kept. +Every run is recorded (with secrets masked) under `~/.cache/runthrough/runs`, and `runthrough --dashboard` serves a local web UI over them in the style of GitHub's Actions tab: the runs list (optionally grouped by repository or date), a run's job graph by stage with its matrix legs and annotations, and each job's steps with their logs, refreshed while a run is in progress. A run's summary page shows the jobs' step summaries (`GITHUB_STEP_SUMMARY`, as Markdown) and, as annotations, what the run did not reproduce locally; **Copy rehearsal note** copies the jobs and their results as Markdown for a pull request. A failed job's page has **Copy for LLM**, which copies the job's context, its steps and the last 200 lines of each failed step as Markdown to paste into an LLM. **Resources** lists the ports runthrough listens on and the Docker containers, networks and volumes it created, with the run each belongs to; ones whose run has ended are marked left behind, with a command to remove them. `--dashboard-addr` / `--dashboard-port` change where it listens; `--no-history` skips recording a run, and runs with `--insecure-secrets` are never recorded. The newest 100 runs are kept. ### Exit codes @@ -72,7 +78,7 @@ Steps see `RUNTHROUGH=true` and `ACT=true` (also in the `env` context, so workfl ## Not supported -The cache service v2 API (`ACTIONS_CACHE_SERVICE_V2`). `permissions` and `jobs..snapshot` are validated but have no local effect; `environment` is reported (name and `url`) without environment secrets or protection rules. `concurrency` groups span one `runthrough` invocation, and a job cancelled by `cancel-in-progress` stops before its next step. +A job using one of these logs a note (`โ„น๏ธ`) when it starts. The cache service v2 API (`ACTIONS_CACHE_SERVICE_V2`). `permissions` and `jobs..snapshot` are validated but have no local effect; `environment` is reported (name and `url`) without environment secrets or protection rules. `concurrency` groups span one `runthrough` invocation, and a job cancelled by `cancel-in-progress` stops before its next step. ## Platforms diff --git a/docs/favicon.svg b/docs/favicon.svg new file mode 100644 index 0000000..2f52d9b --- /dev/null +++ b/docs/favicon.svg @@ -0,0 +1 @@ + diff --git a/docs/index.html b/docs/index.html index ae008f2..28065eb 100644 --- a/docs/index.html +++ b/docs/index.html @@ -5,6 +5,15 @@ runthrough + + + + + + + + + + + +
runthrough
+

The full rehearsal before you push.

+ $ runthrough # GitHub Actions, run locally + + diff --git a/docs/og.png b/docs/og.png new file mode 100644 index 0000000..7c2cb8e Binary files /dev/null and b/docs/og.png differ diff --git a/src/actions.rs b/src/actions.rs index f20b8d3..35516dd 100644 --- a/src/actions.rs +++ b/src/actions.rs @@ -368,16 +368,29 @@ pub fn list_repo_files(root: &Path, exclude_ignored: bool) -> Vec { /// Copy the local repository into `dest` (emulates `actions/checkout`). /// Tracked and untracked files are copied, plus `.git`; ignored files only -/// when `use_gitignore` is false. Objects come from `objects`. +/// when `use_gitignore` is false. With `committed`, only `.git` is copied and +/// the HEAD commit's files are checked out of it, so uncommitted and untracked +/// changes stay behind. Objects come from `objects`. pub fn copy_repo( src: &Path, dest: &Path, use_gitignore: bool, + committed: bool, objects: &ObjectSnapshot, ) -> Result<()> { - let files = list_repo_files(src, use_gitignore); - let mut plan: Vec<(PathBuf, Place)> = files.into_iter().map(|f| (f, Place::Copy)).collect(); let git_dir = src.join(".git"); + if committed && !git_dir.is_dir() { + bail!( + "--committed needs the repository's .git directory ({} is not one; worktrees and submodules are not supported)", + git_dir.display() + ); + } + let files = if committed { + Vec::new() + } else { + list_repo_files(src, use_gitignore) + }; + let mut plan: Vec<(PathBuf, Place)> = files.into_iter().map(|f| (f, Place::Copy)).collect(); if git_dir.is_dir() { let (shared, own): (Vec, Vec) = walkdir::WalkDir::new(&git_dir) .into_iter() @@ -415,7 +428,76 @@ pub fn copy_repo( } } } - }) + })?; + if committed { + let out = Command::new("git") + .arg("-C") + .arg(dest) + .args(["reset", "--quiet", "--hard", "HEAD"]) + .output()?; + if !out.status.success() { + bail!( + "--committed: checking out the HEAD commit failed (no commit yet?): {}", + String::from_utf8_lossy(&out.stderr).trim() + ); + } + } + Ok(()) +} + +/// The files of `root`'s HEAD commit, exported into a new temporary directory +/// (`git archive`), for `--committed` to read workflows as the commit has them. +pub fn export_head(root: &Path) -> Result { + let dir = tempfile::Builder::new() + .prefix("runthrough-head") + .tempdir()?; + let mut git = Command::new("git") + .arg("-C") + .arg(root) + .args(["archive", "--format=tar", "HEAD"]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .context("running git archive")?; + let tar = Command::new("tar") + .arg("-xf") + .arg("-") + .arg("-C") + .arg(dir.path()) + .stdin(git.stdout.take().expect("piped stdout")) + .status() + .context("running tar")?; + let out = git.wait_with_output()?; + if !out.status.success() { + bail!( + "--committed needs a HEAD commit in {}: {}", + root.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + if !tar.success() { + bail!("extracting the HEAD commit of {} failed", root.display()); + } + Ok(dir) +} + +/// How many changes `git status` shows in `root` (uncommitted, untracked); +/// none when git cannot tell. +pub fn uncommitted_changes(root: &Path) -> usize { + Command::new("git") + .arg("-C") + .arg(root) + .args(["status", "--porcelain"]) + .output() + .ok() + .filter(|o| o.status.success()) + // One line per change; a path with a newline is quoted by git. + .map_or(0, |o| { + o.stdout + .split(|b| *b == b'\n') + .filter(|l| !l.is_empty()) + .count() + }) } /// `f` for every item, on a few threads: copying file by file is bound by @@ -531,7 +613,7 @@ mod tests { let objects = ObjectSnapshot::default(); let first = tempfile::tempdir().unwrap(); - copy_repo(src.path(), first.path(), true, &objects).unwrap(); + copy_repo(src.path(), first.path(), true, false, &objects).unwrap(); for n in 0..200 { let f = first.path().join(format!("d{}/f{n}.txt", n % 7)); assert_eq!(std::fs::read_to_string(f).unwrap(), format!("{n}")); @@ -545,7 +627,7 @@ mod tests { git(src.path(), &["add", "late.txt"]); git(src.path(), &["commit", "--quiet", "-m", "late"]); let second = tempfile::tempdir().unwrap(); - copy_repo(src.path(), second.path(), true, &objects).unwrap(); + copy_repo(src.path(), second.path(), true, false, &objects).unwrap(); assert_eq!(git(second.path(), &["log", "--format=%s"]), "late\nc"); git(second.path(), &["fsck", "--no-progress"]); @@ -568,6 +650,46 @@ mod tests { } } + #[test] + fn committed_copy_leaves_uncommitted_changes_behind() { + let src = tempfile::tempdir().unwrap(); + let git = |args: &[&str]| { + let out = Command::new("git") + .arg("-C") + .arg(src.path()) + .args(["-c", "user.name=t", "-c", "user.email=t@t"]) + .args(args) + .output() + .unwrap(); + assert!(out.status.success(), "git {args:?}: {out:?}"); + }; + git(&["init", "--quiet"]); + let objects = ObjectSnapshot::default(); + let empty = tempfile::tempdir().unwrap(); + assert!(copy_repo(src.path(), empty.path(), true, true, &objects).is_err()); + + std::fs::write(src.path().join("a.txt"), "committed").unwrap(); + git(&["add", "a.txt"]); + git(&["commit", "--quiet", "-m", "c"]); + std::fs::write(src.path().join("a.txt"), "edited").unwrap(); + std::fs::write(src.path().join("new.txt"), "untracked").unwrap(); + assert_eq!(uncommitted_changes(src.path()), 2); + + let dest = tempfile::tempdir().unwrap(); + copy_repo( + src.path(), + dest.path(), + true, + true, + &ObjectSnapshot::default(), + ) + .unwrap(); + let a = std::fs::read_to_string(dest.path().join("a.txt")).unwrap(); + assert_eq!(a, "committed"); + assert!(!dest.path().join("new.txt").exists()); + assert_eq!(uncommitted_changes(dest.path()), 0); + } + #[test] fn only_content_addressed_git_files_are_shared() { for rel in [ diff --git a/src/dashboard.html b/src/dashboard.html index 40b599a..522eac9 100644 --- a/src/dashboard.html +++ b/src/dashboard.html @@ -3,7 +3,8 @@ -RunThrough +runthrough +