From 9a45fa787fb83e7f8def46d02e21133669399e59 Mon Sep 17 00:00:00 2001 From: TristanKruse Date: Thu, 22 Oct 2026 11:00:25 +0200 Subject: [PATCH] Add pinned static canaries for Rack, Rake, and RSpec --- .github/workflows/canary.yml | 76 +++++++++ README.md | 5 + benchmark/README.md | 3 + benchmark/canary/README.md | 70 ++++++++ benchmark/canary/projects.json | 61 +++++++ benchmark/real_world_canary.rb | 178 ++++++++++++++++++++ spec/benchmark/extraction_spec.rb | 83 +++++++++ spec/benchmark/real_world_canary_spec.rb | 206 +++++++++++++++++++++++ 8 files changed, 682 insertions(+) create mode 100644 .github/workflows/canary.yml create mode 100644 benchmark/canary/README.md create mode 100644 benchmark/canary/projects.json create mode 100644 benchmark/real_world_canary.rb create mode 100644 spec/benchmark/extraction_spec.rb create mode 100644 spec/benchmark/real_world_canary_spec.rb diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 0000000..5d22c97 --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,76 @@ +name: Real-world static canary + +on: + workflow_dispatch: + pull_request: + paths: + - 'lib/**' + - 'Gemfile' + - 'Gemfile.lock' + - 'benchmark/extraction.rb' + - 'benchmark/real_world_canary.rb' + - 'benchmark/canary/**' + - 'spec/benchmark/**' + - '.github/workflows/canary.yml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + canary: + name: Static lib analysis — ${{ matrix.name }} + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - name: rack + repository: rack/rack + revision: e1f22fdbe99afd2126b6fbf05bb12399359574b7 + - name: rake + repository: ruby/rake + revision: 15df62f11e7f3980bb404bf1471e326cb7ad40ec + - name: rspec-core + repository: rspec/rspec-core + revision: aec5f49485d97908183dbe790a7fefb8baaa8091 + defaults: + run: + working-directory: archunit + steps: + - name: Check out ArchUnitRuby + uses: actions/checkout@v7 + with: + path: archunit + persist-credentials: false + - name: Check out pinned corpus as data only + uses: actions/checkout@v7 + with: + repository: ${{ matrix.repository }} + ref: ${{ matrix.revision }} + path: corpus + fetch-depth: 1 + persist-credentials: false + - name: Set up this project's Ruby and dependencies only + uses: ruby/setup-ruby@v1 + with: + ruby-version: '4.0' + bundler-cache: true + working-directory: archunit + - name: Validate baseline, cache reuse, and passing/failing rules + run: >- + bundle exec ruby benchmark/real_world_canary.rb + --project ${{ matrix.name }} --root ../corpus + --output pkg/canary-${{ matrix.name }}.json + - name: Retain performance and correctness evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: canary-${{ matrix.name }}-${{ github.run_id }} + path: archunit/pkg/canary-${{ matrix.name }}.json + if-no-files-found: warn + retention-days: 14 diff --git a/README.md b/README.md index 90046be..e90d73f 100644 --- a/README.md +++ b/README.md @@ -588,6 +588,11 @@ bundle exec ruby benchmark/extraction.rb See the [benchmark guide](https://github.com/LukasNiessen/ArchUnitRuby/blob/main/benchmark/README.md) for corpus controls, JSON output, and CI limits. +The separate [real-world canary](https://github.com/LukasNiessen/ArchUnitRuby/blob/main/benchmark/canary/README.md) +checks pinned Rack, Rake, and RSpec production-library sources in both dependency modes, including +passing and deliberately failing rules, graph baselines, and cold/warm performance. It never runs +those projects or installs their dependencies, and is not scheduled automatically yet. + The implementation conventions and intended dependency directions live in [`AGENTS.md`](AGENTS.md). ## 📅 Plans and Current Limitations diff --git a/benchmark/README.md b/benchmark/README.md index e9e8348..53e429c 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,5 +1,8 @@ # Extraction performance benchmark +For pinned production-library compatibility checks, public passing/failing rule witnesses, and +real-world timing artifacts, see the [static canary guide](canary/README.md). + The benchmark generates a deterministic multi-gem Ruby repository, then measures one cold graph build and one same-process cached lookup. Target application code and generated gemspecs are never executed. diff --git a/benchmark/canary/README.md b/benchmark/canary/README.md new file mode 100644 index 0000000..eab58c5 --- /dev/null +++ b/benchmark/canary/README.md @@ -0,0 +1,70 @@ +# Real-world static canary + +The canary statically analyzes the **production `lib` directory only** of three pinned Ruby +repositories. It does not install their dependencies, evaluate their Gemfiles or gemspecs, execute +their source, or run their tests. It is not an upstream architecture-quality judgment. + +| Corpus | Pinned revision | Ruby files | Internal edges: imports / expanded | +| --- | --- | ---: | ---: | +| [Rack](https://github.com/rack/rack) 3.2.6 | `e1f22fdbe99afd2126b6fbf05bb12399359574b7` | 49 | 133 / 140 | +| [Rake](https://github.com/ruby/rake) snapshot | `15df62f11e7f3980bb404bf1471e326cb7ad40ec` | 44 | 57 / 78 | +| [RSpec Core](https://github.com/rspec/rspec-core) snapshot | `aec5f49485d97908183dbe790a7fefb8baaa8091` | 74 | 18 / 157 | + +These are reviewed baselines recorded on 2026-10-22, not moving branch names. `projects.json` +records the full commits, a known import witness, and per-mode file/edge counts and graph hashes. +The workflow matrix must match that manifest; an offline contract test enforces it. + +## What it proves + +For both `imports` and `imports_and_constants`, the canary: + +- Confirms the exact checkout revision and a clean `lib` source scope. +- Measures cold extraction, warm cache reuse, stage timings, and available heap/RSS counters. +- Compares file counts, internal/external edge counts, and a sorted internal graph fingerprint. +- Runs a passing public rule, then deliberately forbids one known real import and requires exactly + one `FileDependencyViolation` and an assertion failure containing the rule rationale. + +For example, the Rack witness is `rack.rb -> rack/builder.rb`, relative to its `lib` root. Forbidding +this existing edge is an intentionally failing **test policy**, not evidence that Rack's design is wrong. +Unknown dynamic imports, unresolved constants, and ambiguous names remain visible in profile counters; +the canary does not claim complete modeling of Ruby or framework autoloading. + +## Run it + +After cloning a corpus and checking out the exact manifest commit: + +~~~bash +bundle exec ruby benchmark/real_world_canary.rb \ + --project rack --root /path/to/rack \ + --output pkg/canary-rack.json +~~~ + +Corpus dependencies are never needed. The script uses ArchUnitRuby's own locked parser dependencies. +Do not run `bundle install` or build tasks inside the corpus. Dirty or untracked files under `lib` +are rejected; unrelated upstream test fixtures are outside the analysis scope. Some security test +fixtures can trigger host antivirus products. Do not disable those protections or attempt alternate +reads of blocked files: keep the declared source scope consistent across platforms. + +The separate **Real-world static canary** workflow runs on relevant pull requests and supports manual +dispatch once merged. It is intentionally not scheduled yet. It uses read-only repository access, +three isolated ten-minute jobs, and 14-day JSON artifacts, including error evidence on failures. +Keep it advisory rather than adding it to required branch checks until its cost and stability are reviewed. +Failures remain red; they are not hidden with `continue-on-error`. + +There are broad 120-second cold / 5-second warm ceilings per mode, plus the job timeout. Those are +runaway guards, not performance promises. Local cold times for this lib-only corpus were roughly +0.1–0.4 seconds per mode on Ruby 4.0.6 / Windows x64; compare repeated like-for-like samples before +claiming a speedup. These small libraries are not substitutes for a full Rails monorepo benchmark. + +## Updating a baseline + +1. Choose and review a new full upstream commit. Update the workflow matrix and manifest together. +2. Inspect the known witness in that source. Do not replace it with an arbitrary edge simply to pass. +3. Run the script with `--record` to produce **proposed observations**. This option never rewrites the + manifest and marks its report `baseline_verified: false`. +4. Review file/edge/hash differences and unresolved/ambiguous counters. If the extractor changed, + determine whether differences are intended or a regression before accepting new expectations. +5. Edit the reviewed `modes` values in `projects.json`, rerun without `--record`, and verify Linux CI. + +Regular CI never records or automatically accepts a new baseline. The ordinary unit suite uses +small synthetic projects and mocked git inspection; it does not clone these repositories or need a network. diff --git a/benchmark/canary/projects.json b/benchmark/canary/projects.json new file mode 100644 index 0000000..c6765ec --- /dev/null +++ b/benchmark/canary/projects.json @@ -0,0 +1,61 @@ +{ + "projects": { + "rack": { + "repository": "rack/rack", + "revision": "e1f22fdbe99afd2126b6fbf05bb12399359574b7", + "witness": { "source": "rack.rb", "target": "rack/builder.rb" }, + "modes": { + "imports": { + "file_count": 49, + "internal_dependencies": 133, + "external_dependencies": 28, + "internal_graph_sha256": "8749f31622cfb15062f77de0f32eb870b5eb6f782bdc3365604b62236712510c" + }, + "imports_and_constants": { + "file_count": 49, + "internal_dependencies": 140, + "external_dependencies": 28, + "internal_graph_sha256": "a3897a8c67acee0a1d372c5bb97dcfd7a862048e5759d8f3fb1507f3121498d6" + } + } + }, + "rake": { + "repository": "ruby/rake", + "revision": "15df62f11e7f3980bb404bf1471e326cb7ad40ec", + "witness": { "source": "rake.rb", "target": "rake/application.rb" }, + "modes": { + "imports": { + "file_count": 44, + "internal_dependencies": 57, + "external_dependencies": 12, + "internal_graph_sha256": "1b188eb049249b6f12cbb8d3307c6c3d4ad786e29a19d4a4f1b31f31fe3dae04" + }, + "imports_and_constants": { + "file_count": 44, + "internal_dependencies": 78, + "external_dependencies": 12, + "internal_graph_sha256": "a2bcb171c24678a6a1277fe4b3b4b32afef5c23ba3f8b6ca263db7b7ee41e466" + } + } + }, + "rspec-core": { + "repository": "rspec/rspec-core", + "revision": "aec5f49485d97908183dbe790a7fefb8baaa8091", + "witness": { "source": "rspec/autorun.rb", "target": "rspec/core.rb" }, + "modes": { + "imports": { + "file_count": 74, + "internal_dependencies": 18, + "external_dependencies": 34, + "internal_graph_sha256": "81e0621a26487d438391595a7407d651b15025247caa37c20f1bb7fab77f8415" + }, + "imports_and_constants": { + "file_count": 74, + "internal_dependencies": 157, + "external_dependencies": 34, + "internal_graph_sha256": "34f388962ada484dc84d055dd261cfadc16f17d5538dec2ea45de821f1ee0a4c" + } + } + } + } +} diff --git a/benchmark/real_world_canary.rb b/benchmark/real_world_canary.rb new file mode 100644 index 0000000..6f2a4e4 --- /dev/null +++ b/benchmark/real_world_canary.rb @@ -0,0 +1,178 @@ +# frozen_string_literal: true + +require 'digest' +require 'open3' +require 'pathname' +require_relative 'extraction' + +# Static analysis only. Never install a corpus's gems or load its application code. +module RealWorldCanary + MANIFEST = File.expand_path('canary/projects.json', __dir__) + MODES = %i[imports imports_and_constants].freeze + + # Compares pinned graph semantics and public rule behavior while collecting performance evidence. + class Runner + def initialize(root:, project:) + @root = File.realpath(root) + @project = project + end + + def run(record: false) + verify_checkout + @source_root = library_root + modes = MODES.to_h { |mode| [mode, check_mode(mode, record)] } + { repository: @project.fetch('repository'), revision: @project.fetch('revision'), + source_directory: 'lib', + archunit_version: ArchUnit::VERSION, baseline_verified: !record, modes: } + ensure + ArchUnit.clear_graph_cache + end + + private + + def library_root + path = File.realpath(File.join(@root, 'lib')) + relative = Pathname.new(path).relative_path_from(Pathname.new(@root)).to_s.tr('\\', '/') + if relative == '..' || relative.start_with?('../') + raise 'Library source directory must stay inside the corpus' + end + + path + end + + def verify_checkout + expected = @project.fetch('revision') + raise 'Canary requires full immutable commit hashes' unless expected.match?(/\A\h{40}\z/) + raise 'Corpus revision does not match pin' unless git('rev-parse', 'HEAD') == expected + return if git('status', '--porcelain', '--untracked-files=normal', '--', 'lib').empty? + + raise 'Corpus library source must be clean; do not install or generate files inside lib' + end + + def git(*) + stdout, stderr, status = Open3.capture3( + { 'GIT_OPTIONAL_LOCKS' => '0' }, 'git', '-C', @root, * + ) + raise "Could not inspect corpus checkout: #{stderr.strip}" unless status.success? + + stdout.strip + end + + def check_mode(mode, record) + configuration = { dependency_analysis: mode, max_cold_seconds: 120, max_warm_seconds: 5 } + performance = ExtractionBenchmark::Runner.new(@source_root, configuration).run + raise 'Warm analysis did not reuse the same graph' unless performance[:graph_cache_reused] + + options = ArchUnit::CheckOptions.new(dependency_analysis: mode) + graph = ArchUnit::Extraction.extract_graph(@source_root, options:) + observed = RealWorldCanary.snapshot(graph) + verify_baseline(mode, observed) unless record + verify_rules(options) + { snapshot: observed, performance:, passing_rule: true, forbidden_rule_violations: 1 } + end + + def verify_baseline(mode, observed) + expected = @project.fetch('modes').fetch(mode.to_s) + return if expected == observed.transform_keys(&:to_s) + + raise "#{mode} graph changed: expected #{expected.inspect}, observed #{observed.inspect}" + end + + def verify_rules(options) + witness = @project.fetch('witness') + scope = ArchUnit.project_files(@source_root).in_file(witness.fetch('source')) + ArchUnit.assert_passes(scope.should.depend_on_files.in_path(/\.rb\z/), options) + verify_forbidden(forbidden_rule(scope, witness), options) + end + + def forbidden_rule(scope, witness) + scope.should_not.depend_on_files.in_path( + /\A#{Regexp.escape(witness.fetch('target'))}\z/ + ).because('Canary must expose this known cross-file dependency') + end + + def verify_forbidden(rule, options) + violations = rule.check(options) + unless violations.one? && violations.first.is_a?(ArchUnit::FileDependencyViolation) + raise "Expected one known forbidden dependency, got #{violations.length}" + end + + validate_assertion_failure(rule, options) + end + + def validate_assertion_failure(rule, options) + ArchUnit.assert_passes(rule, options) + raise 'The forbidden architecture assertion unexpectedly passed' + rescue ArchUnit::AssertionFailure => e + raise 'Failure omitted rule rationale' unless e.message.include?(rule.rationale) + end + end + + module_function + + def snapshot(graph) + sources = graph.map(&:source).uniq.sort + pairs = internal_pairs(graph) + { file_count: sources.length, internal_dependencies: pairs.length, + external_dependencies: graph.count(&:external), + internal_graph_sha256: Digest::SHA256.hexdigest(JSON.generate([sources, pairs])) } + end + + def internal_pairs(graph) + graph.filter_map do |edge| + next if edge.external || edge.source == edge.target + + [edge.source, edge.target, edge.import_kinds.map(&:to_s).sort] + end.sort + end + + def execute(arguments) + options = parse_options(arguments) + project = load_project(options) + result = Runner.new(root: options.fetch(:root), project:).run(record: options[:record]) + emit_result(options, result) + rescue StandardError => e + emit_error(options, e) + end + + def load_project(options) + projects = JSON.parse(File.read(options[:manifest])).fetch('projects') + projects.fetch(options.fetch(:project)) + end + + def emit_result(options, result) + payload = JSON.pretty_generate(result) + ExtractionBenchmark.write_payload(options[:output], payload) + puts payload + 0 + end + + def emit_error(options, error) + payload = JSON.pretty_generate(error: "#{error.class}: #{error.message}") + ExtractionBenchmark.write_payload(options[:output], payload) if options + warn error.message + 1 + end + + def parse_options(arguments) + options = { manifest: MANIFEST, output: nil, record: false } + option_parser(options).parse!(arguments) + raise ArgumentError, 'Unexpected positional arguments' unless arguments.empty? + + options + end + + def option_parser(options) + OptionParser.new do |parser| + parser.on('--project NAME') { |value| options[:project] = value } + parser.on('--root PATH') { |value| options[:root] = value } + parser.on('--manifest PATH') { |value| options[:manifest] = value } + parser.on('--output PATH') { |value| options[:output] = value } + parser.on('--record', 'Observe proposed baselines; never edits the manifest') do + options[:record] = true + end + end + end +end + +exit RealWorldCanary.execute(ARGV) if $PROGRAM_NAME == __FILE__ diff --git a/spec/benchmark/extraction_spec.rb b/spec/benchmark/extraction_spec.rb new file mode 100644 index 0000000..9630c3d --- /dev/null +++ b/spec/benchmark/extraction_spec.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +require 'tmpdir' +require_relative '../../benchmark/extraction' + +RSpec.describe ExtractionBenchmark do + around do |example| + Dir.mktmpdir('archunit-benchmark-spec') do |root| + @root = File.realpath(root) + example.run + ensure + ArchUnit.clear_graph_cache + end + end + + def corpus_configuration(**overrides) + { file_count: 6, gem_count: 2, imports_per_file: 3 }.merge(overrides) + end + + it 'generates deterministic cross-gem imports without executing gemspecs' do + corpus = described_class::Corpus.new(@root, **corpus_configuration) + expect(corpus.generate).to eq(@root) + paths = Dir.glob(File.join(@root, '**/*.rb')) + expect(paths.length).to eq(7) + graph = ArchUnit::Extraction.extract_graph(@root) + expect(graph.count { |edge| !edge.external && edge.source != edge.target }).to eq(18) + expect(Dir.glob(File.join(@root, '**/*.gemspec')).length).to eq(2) + end + + it 'generates isolated files when no imports are requested' do + described_class::Corpus.new(@root, **corpus_configuration(imports_per_file: 0)).generate + graph = ArchUnit::Extraction.extract_graph(@root) + expect(graph.reject { |edge| edge.source == edge.target }).to be_empty + end + + it 'rejects non-positive corpus sizes and negative import counts' do + invalid = [{ file_count: 0 }, { file_count: '6' }, { gem_count: -1 }, { imports_per_file: -1 }] + invalid.each do |bad| + expect do + described_class::Corpus.new(@root, **corpus_configuration(**bad)) + end.to raise_error(ArgumentError) + end + end + + it 'runs the CLI in a temporary corpus and writes a cache and performance report' do + output = File.join(@root, 'reports/benchmark.json') + arguments = %w[--files 6 --gems 2 --imports-per-file 0 --constants + --max-cold-seconds 120 --max-warm-seconds 5] + ['--output', output] + expect { described_class.execute(arguments) }.to output(/graph_cache_reused/).to_stdout + report = JSON.parse(File.read(output)) + expect(report).to include('graph_cache_reused' => true) + expect(report.fetch('corpus')).to include('dependency_analysis' => 'imports_and_constants') + expect(report.dig('cold', 'memory', 'ruby_heap_bytes')).to be_positive + end + + it 'supports an output-free run without threshold limits' do + configuration = described_class::DEFAULTS.merge(files: 2, gems: 1, imports_per_file: 1) + expect do + described_class.run_in(@root, configuration) + end.to output(/graph_cache_reused/).to_stdout + end + + it 'fails when a configured cold or warm time budget is exceeded' do + described_class::Corpus.new(@root, **corpus_configuration).generate + %i[max_cold_seconds max_warm_seconds].each do |threshold| + configuration = described_class::DEFAULTS.merge(threshold => -1) + runner = described_class::Runner.new(@root, configuration) + expect { runner.run }.to raise_error(/#{threshold} exceeded/) + end + end + + it 'parses optional Linux peak memory without assuming the host has procfs' do + runner = described_class::Runner.new(@root, described_class::DEFAULTS) + allow(File).to receive(:file?).and_call_original + allow(File).to receive(:file?).with('/proc/self/status').and_return(true) + allow(File).to receive(:read).with('/proc/self/status').and_return("VmHWM: 1234 kB\n") + expect(runner.send(:linux_peak_rss_kb)).to eq(1234) + allow(File).to receive(:read).with('/proc/self/status').and_return("Name: ruby\n") + expect(runner.send(:linux_peak_rss_kb)).to be_nil + allow(File).to receive(:file?).with('/proc/self/status').and_return(false) + expect(runner.send(:linux_peak_rss_kb)).to be_nil + end +end diff --git a/spec/benchmark/real_world_canary_spec.rb b/spec/benchmark/real_world_canary_spec.rb new file mode 100644 index 0000000..760d402 --- /dev/null +++ b/spec/benchmark/real_world_canary_spec.rb @@ -0,0 +1,206 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'tmpdir' +require 'yaml' +require_relative '../../benchmark/real_world_canary' + +RSpec.describe RealWorldCanary::Runner do + let(:revision) { 'a' * 40 } + let(:project) do + { 'repository' => 'example/fixture', 'revision' => revision, + 'witness' => { 'source' => 'api.rb', 'target' => 'store.rb' }, + 'modes' => { + 'imports' => expected_snapshot(['require_relative']), + 'imports_and_constants' => expected_snapshot(%w[constant_reference require_relative]) + } } + end + + around do |example| + Dir.mktmpdir('archunit-canary-spec') do |root| + @root = File.realpath(root) + FileUtils.mkdir_p(File.join(@root, 'lib')) + File.write(File.join(@root, 'Gemfile'), "raise 'must never execute'\n") + File.write(File.join(@root, 'lib/api.rb'), <<~RUBY) + require_relative 'store' + class Api + def call = ::Store.new + end + RUBY + File.write(File.join(@root, 'lib/store.rb'), "raise 'must never execute'\nclass Store; end\n") + example.run + ensure + ArchUnit.clear_graph_cache + end + end + + before do + status = instance_double(Process::Status, success?: true) + allow(Open3).to receive(:capture3).and_return([revision, '', status]) + allow(Open3).to receive(:capture3).with( + { 'GIT_OPTIONAL_LOCKS' => '0' }, 'git', '-C', @root, + 'status', '--porcelain', '--untracked-files=normal', '--', 'lib' + ).and_return(['', '', status]) + end + + def expected_snapshot(kinds) + graph_data = [%w[api.rb store.rb], [['api.rb', 'store.rb', kinds]]] + { 'file_count' => 2, 'internal_dependencies' => 1, 'external_dependencies' => 0, + 'internal_graph_sha256' => Digest::SHA256.hexdigest(JSON.generate(graph_data)) } + end + + def run_canary(record: false) + described_class.new(root: @root, project:).run(record:) + end + + it 'checks both modes, cache reuse, and pass/fail rules without executing source' do + result = run_canary + expect(result).to include(baseline_verified: true, source_directory: 'lib') + expected = { passing_rule: true, forbidden_rule_violations: 1, + performance: include(graph_cache_reused: true) } + expect(result.fetch(:modes).values).to all(include(expected)) + end + + it 'fails on changed counts and missing or changed graph fingerprints' do + project['modes']['imports']['file_count'] = 3 + expect { run_canary }.to raise_error(/graph changed/) + project['modes']['imports'] = expected_snapshot(['require_relative']) + project['modes']['imports']['internal_graph_sha256'] = '0' * 64 + expect { run_canary }.to raise_error(/graph changed/) + end + + it 'does not silently bless missing baselines except in explicit observation mode' do + project.delete('modes') + expect { run_canary }.to raise_error(KeyError) + expect(run_canary(record: true).fetch(:baseline_verified)).to be(false) + expect(project).not_to have_key('modes') + end + + it 'rejects the wrong revision and mutable ref names' do + project['revision'] = 'main' + expect { run_canary }.to raise_error(/immutable commit/) + project['revision'] = 'b' * 40 + expect { run_canary }.to raise_error(/revision does not match/) + end + + it 'rejects dirty library sources and failed git inspection' do + status = instance_double(Process::Status, success?: true) + allow(Open3).to receive(:capture3).with( + { 'GIT_OPTIONAL_LOCKS' => '0' }, 'git', '-C', @root, + 'status', '--porcelain', '--untracked-files=normal', '--', 'lib' + ).and_return([' M lib/api.rb', '', status]) + expect { run_canary }.to raise_error(/source must be clean/) + allow(Open3).to receive(:capture3).and_return( + ['', 'not a repository', instance_double(Process::Status, success?: false)] + ) + expect { run_canary }.to raise_error(/Could not inspect corpus/) + end + + it 'rejects a missing witness instead of treating zero violations as success' do + project['witness']['target'] = 'missing.rb' + expect { run_canary }.to raise_error(/Expected one known forbidden/) + end + + it 'detects a regression that disables warm graph reuse' do + runner = instance_double(ExtractionBenchmark::Runner, run: { graph_cache_reused: false }) + allow(ExtractionBenchmark::Runner).to receive(:new).and_return(runner) + expect { run_canary }.to raise_error(/Warm analysis/) + end + + it 'refuses a canonical lib directory outside the corpus' do + source = File.join(@root, 'lib') + allow(File).to receive(:realpath).and_call_original + allow(File).to receive(:realpath).with(source).and_return(File.dirname(@root)) + expect { run_canary }.to raise_error(/source directory must stay inside/) + end + + it 'does not accept a forbidden assertion that unexpectedly passes' do + allow(ArchUnit).to receive(:assert_passes).and_return(nil) + expect { run_canary }.to raise_error(/forbidden architecture assertion unexpectedly passed/) + end + + it 'requires assertion failures to retain the rule rationale' do + allow(ArchUnit).to receive(:assert_passes).and_wrap_original do |original, *arguments| + original.call(*arguments) + rescue ArchUnit::AssertionFailure => e + allow(e).to receive(:message).and_return('missing rationale') + raise + end + expect { run_canary }.to raise_error(/Failure omitted rule rationale/) + end + + def cli_arguments + manifest = File.join(@root, 'manifest.json') + File.write(manifest, JSON.generate('projects' => { 'fixture' => project })) + ['--project', 'fixture', '--root', @root, '--manifest', manifest, + '--output', File.join(@root, 'reports/canary.json')] + end + + it 'writes verified CLI evidence and returns a successful exit status' do + status = nil + expect do + status = RealWorldCanary.execute(cli_arguments) + end.to output(/baseline_verified/).to_stdout + expect(status).to eq(0) + report = JSON.parse(File.read(File.join(@root, 'reports/canary.json'))) + expect(report).to include('baseline_verified' => true, 'revision' => revision) + end + + it 'records observations only when explicitly requested and never changes the manifest' do + project.delete('modes') + arguments = cli_arguments + manifest = File.read(File.join(@root, 'manifest.json')) + status = nil + expect do + status = RealWorldCanary.execute(arguments + ['--record']) + end.to output(/"baseline_verified": false/).to_stdout + expect(status).to eq(0) + expect(File.read(File.join(@root, 'manifest.json'))).to eq(manifest) + end + + it 'persists CLI failure evidence and returns a failing exit status' do + project['revision'] = 'b' * 40 + status = nil + expect do + status = RealWorldCanary.execute(cli_arguments) + end.to output(/revision does not match/).to_stderr + expect(status).to eq(1) + report = JSON.parse(File.read(File.join(@root, 'reports/canary.json'))) + expect(report.fetch('error')).to include('revision does not match') + end + + it 'fails closed on unknown options and unexpected positional arguments' do + status = nil + expect do + status = RealWorldCanary.execute(['--invalid']) + end.to output(/invalid option/).to_stderr + expect(status).to eq(1) + expect do + status = RealWorldCanary.execute(['unexpected']) + end.to output(/Unexpected positional arguments/).to_stderr + expect(status).to eq(1) + end +end + +RSpec.describe 'Real-world canary workflow' do + it 'pins the same immutable corpus revisions in workflow and reviewed manifest' do + root = File.expand_path('../..', __dir__) + manifest = File.read(File.join(root, 'benchmark/canary/projects.json')) + projects = JSON.parse(manifest).fetch('projects') + workflow = YAML.safe_load_file(File.join(root, '.github/workflows/canary.yml')) + job = workflow.fetch('jobs').fetch('canary') + entries = job.fetch('strategy').fetch('matrix').fetch('include') + expect(entries.map { |item| item.fetch('name') }).to match_array(projects.keys) + entries.each do |entry| + expected = projects.fetch(entry.fetch('name')) + expect(entry).to include(expected.slice('repository', 'revision')) + end + expect(workflow.fetch('permissions')).to eq('contents' => 'read') + expect(job.fetch('timeout-minutes')).to eq(10) + commands = job.fetch('steps').filter_map { |step| step['run'] }.join("\n") + expect(commands).not_to include('--record', 'bundle install', 'rspec', 'rake') + triggers = workflow['on'] || workflow[true] + expect(triggers).to include('workflow_dispatch', 'pull_request') + expect(triggers).not_to include('schedule') + end +end