From 7a2351ec0f47b9816d9075c5471ab4bf0f21f6e2 Mon Sep 17 00:00:00 2001 From: Rob Sanheim Date: Sat, 21 Feb 2026 10:17:04 -0600 Subject: [PATCH 1/3] Improve diff output with context lines, field summaries, and truncation Add context lines (3 lines before/after) around changes in diffs with "..." separators between distant hunks. Include a field summary showing which fields (stdout, stderr, status) changed or stayed the same. Truncate large diffs to 50 lines by default, with BACKSPIN_FULL_DIFF=1 env var to show the full output. --- lib/backspin/backspin_result.rb | 16 ++- lib/backspin/command_diff.rb | 94 +++++++++++-- spec/backspin/backspin_result_spec.rb | 4 +- spec/backspin/command_diff_spec.rb | 193 ++++++++++++++++++++++++++ spec/backspin_capture_spec.rb | 6 +- spec/backspin_run_spec.rb | 42 +++++- 6 files changed, 337 insertions(+), 18 deletions(-) diff --git a/lib/backspin/backspin_result.rb b/lib/backspin/backspin_result.rb index a61295a..15e986b 100644 --- a/lib/backspin/backspin_result.rb +++ b/lib/backspin/backspin_result.rb @@ -36,10 +36,18 @@ def error_message return nil unless verified? == false return "Output verification failed" unless @command_diff - msg = "Output verification failed:\n\n" - msg += @command_diff.summary - msg += "\n#{@command_diff.diff}" if @command_diff.diff - msg + parts = ["Summary:"] + @command_diff.field_summary.each do |line| + parts << " #{line}" + end + + diff = @command_diff.diff + if diff && !diff.empty? + parts << "" + parts << diff + end + + parts.join("\n") end def success? diff --git a/lib/backspin/command_diff.rb b/lib/backspin/command_diff.rb index db1c27a..a5f6bbf 100644 --- a/lib/backspin/command_diff.rb +++ b/lib/backspin/command_diff.rb @@ -3,6 +3,9 @@ module Backspin # Represents the difference between expected and actual snapshots. class CommandDiff + CONTEXT_LINES = 3 + MAX_DIFF_LINES = 50 + attr_reader :expected, :actual, :matcher def initialize(expected:, actual:, matcher: nil, filter: nil, filter_on: :both) @@ -26,6 +29,15 @@ def verified? @verified = @matcher.match? end + # @return [Array] Per-field change status lines + def field_summary + lines = [] + lines << stdout_field_summary + lines << stderr_field_summary + lines << status_field_summary + lines + end + # @return [String, nil] Human-readable diff if not verified def diff return nil if verified? @@ -48,7 +60,8 @@ def diff parts << "Exit status: expected #{expected_compare.status}, got #{actual_compare.status}" end - parts.join("\n\n") + result = parts.join("\n\n") + maybe_truncate(result) end # @return [String] Single line summary for error messages @@ -62,6 +75,30 @@ def summary private + def stdout_field_summary + if expected_compare.stdout == actual_compare.stdout + "stdout: unchanged" + else + "stdout: changed" + end + end + + def stderr_field_summary + if expected_compare.stderr == actual_compare.stderr + "stderr: unchanged" + else + "stderr: changed" + end + end + + def status_field_summary + if expected_compare.status == actual_compare.status + "status: unchanged" + else + "status: changed (expected #{expected_compare.status}, actual #{actual_compare.status})" + end + end + def command_types_match? expected.command_type == actual.command_type end @@ -83,25 +120,64 @@ def stderr_diff(expected, actual) end def generate_line_diff(expected, actual) - expected_lines = (expected || "").lines - actual_lines = (actual || "").lines + expected_lines = (expected || "").lines.map(&:chomp) + actual_lines = (actual || "").lines.map(&:chomp) + max_lines = [expected_lines.length, actual_lines.length].max + + changed = Array.new(max_lines, false) + max_lines.times do |i| + changed[i] = (expected_lines[i] != actual_lines[i]) + end + + visible = Array.new(max_lines, false) + max_lines.times do |i| + next unless changed[i] + range_start = [i - CONTEXT_LINES, 0].max + range_end = [i + CONTEXT_LINES, max_lines - 1].min + (range_start..range_end).each { |j| visible[j] = true } + end diff_lines = [] - max_lines = [expected_lines.length, actual_lines.length].max + in_hunk = false max_lines.times do |i| - expected_line = expected_lines[i] - actual_line = actual_lines[i] + unless visible[i] + if in_hunk + diff_lines << "..." + in_hunk = false + end + next + end - if expected_line != actual_line - diff_lines << "-#{expected_line.chomp}" if expected_line - diff_lines << "+#{actual_line.chomp}" if actual_line + in_hunk = true + + if changed[i] + diff_lines << "-#{expected_lines[i]}" if i < expected_lines.length + diff_lines << "+#{actual_lines[i]}" if i < actual_lines.length + else + line = expected_lines[i] || actual_lines[i] + diff_lines << " #{line}" end end diff_lines.join("\n") end + def maybe_truncate(diff_text) + return diff_text if full_diff? + + lines = diff_text.lines + return diff_text if lines.length <= MAX_DIFF_LINES + + truncated = lines.first(MAX_DIFF_LINES).join + truncated.chomp! + truncated + "\n(diff truncated, set BACKSPIN_FULL_DIFF=1 for full output)" + end + + def full_diff? + ENV["BACKSPIN_FULL_DIFF"] == "1" + end + attr_reader :expected_compare attr_reader :actual_compare diff --git a/spec/backspin/backspin_result_spec.rb b/spec/backspin/backspin_result_spec.rb index 2e7f255..ffc1a0d 100644 --- a/spec/backspin/backspin_result_spec.rb +++ b/spec/backspin/backspin_result_spec.rb @@ -52,8 +52,8 @@ command_diff: command_diff ) - expect(result.error_message).to include("Output verification failed:") - expect(result.error_message).to include("Command failed") + expect(result.error_message).to include("Summary:") + expect(result.error_message).to include("stdout: changed") expect(result.error_message).to include("[stdout]") end diff --git a/spec/backspin/command_diff_spec.rb b/spec/backspin/command_diff_spec.rb index e3b76f5..1d68927 100644 --- a/spec/backspin/command_diff_spec.rb +++ b/spec/backspin/command_diff_spec.rb @@ -80,4 +80,197 @@ expect(command_diff.diff).to include("[stderr]") expect(command_diff.diff).to include("Exit status: expected 0, got 1") end + + context "field_summary" do + it "reports all fields as changed when everything differs" do + expected_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["echo", "recorded"], + stdout: "expected output\n", + stderr: "expected err\n", + status: 0 + ) + actual_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["echo", "actual"], + stdout: "actual output\n", + stderr: "actual err\n", + status: 1 + ) + + command_diff = described_class.new( + expected: expected_snapshot, + actual: actual_snapshot + ) + + summary = command_diff.field_summary + expect(summary).to include("stdout: changed") + expect(summary).to include("stderr: changed") + expect(summary).to include("status: changed (expected 0, actual 1)") + end + + it "reports unchanged fields correctly when only stdout differs" do + expected_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["echo", "recorded"], + stdout: "expected\n", + stderr: "same\n", + status: 0 + ) + actual_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["echo", "actual"], + stdout: "actual\n", + stderr: "same\n", + status: 0 + ) + + command_diff = described_class.new( + expected: expected_snapshot, + actual: actual_snapshot + ) + + summary = command_diff.field_summary + expect(summary).to include("stdout: changed") + expect(summary).to include("stderr: unchanged") + expect(summary).to include("status: unchanged") + end + end + + context "context lines in diff" do + it "shows context lines around changes" do + expected_lines = (1..10).map { |i| "line #{i}" }.join("\n") + "\n" + actual_lines = (1..10).map { |i| (i == 5) ? "CHANGED 5" : "line #{i}" }.join("\n") + "\n" + + expected_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["test"], + stdout: expected_lines, + stderr: "", + status: 0 + ) + actual_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["test"], + stdout: actual_lines, + stderr: "", + status: 0 + ) + + diff = described_class.new( + expected: expected_snapshot, + actual: actual_snapshot + ).diff + + expect(diff).to include(" line 2") + expect(diff).to include(" line 3") + expect(diff).to include(" line 4") + expect(diff).to include("-line 5") + expect(diff).to include("+CHANGED 5") + expect(diff).to include(" line 6") + expect(diff).to include(" line 7") + expect(diff).to include(" line 8") + expect(diff).not_to include(" line 1") + expect(diff).not_to include(" line 9") + end + + it "uses ... separator for distant changes" do + expected_lines = (1..20).map { |i| "line #{i}" }.join("\n") + "\n" + actual_lines = (1..20).map { |i| + case i + when 2 then "CHANGED 2" + when 18 then "CHANGED 18" + else "line #{i}" + end + }.join("\n") + "\n" + + expected_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["test"], + stdout: expected_lines, + stderr: "", + status: 0 + ) + actual_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["test"], + stdout: actual_lines, + stderr: "", + status: 0 + ) + + diff = described_class.new( + expected: expected_snapshot, + actual: actual_snapshot + ).diff + + expect(diff).to include("...") + expect(diff).to include("-line 2") + expect(diff).to include("+CHANGED 2") + expect(diff).to include("-line 18") + expect(diff).to include("+CHANGED 18") + end + end + + context "diff truncation" do + it "truncates large diffs with a message" do + expected_lines = (1..100).map { |i| "expected #{i}" }.join("\n") + "\n" + actual_lines = (1..100).map { |i| "actual #{i}" }.join("\n") + "\n" + + expected_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["test"], + stdout: expected_lines, + stderr: "", + status: 0 + ) + actual_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["test"], + stdout: actual_lines, + stderr: "", + status: 0 + ) + + diff = described_class.new( + expected: expected_snapshot, + actual: actual_snapshot + ).diff + + expect(diff).to include("BACKSPIN_FULL_DIFF=1") + expect(diff).to include("truncated") + end + + it "shows full diff when BACKSPIN_FULL_DIFF=1" do + expected_lines = (1..100).map { |i| "expected #{i}" }.join("\n") + "\n" + actual_lines = (1..100).map { |i| "actual #{i}" }.join("\n") + "\n" + + expected_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["test"], + stdout: expected_lines, + stderr: "", + status: 0 + ) + actual_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["test"], + stdout: actual_lines, + stderr: "", + status: 0 + ) + + ENV["BACKSPIN_FULL_DIFF"] = "1" + diff = described_class.new( + expected: expected_snapshot, + actual: actual_snapshot + ).diff + + expect(diff).not_to include("truncated") + expect(diff).to include("-expected 100") + expect(diff).to include("+actual 100") + ensure + ENV.delete("BACKSPIN_FULL_DIFF") + end + end end diff --git a/spec/backspin_capture_spec.rb b/spec/backspin_capture_spec.rb index 76ff0cc..49f0b56 100644 --- a/spec/backspin_capture_spec.rb +++ b/spec/backspin_capture_spec.rb @@ -129,7 +129,8 @@ end expect(result.verified?).to be false - expect(result.error_message).to include("Output verification failed") + expect(result.error_message).to include("Summary:") + expect(result.error_message).to include("stdout: changed") expect(result.actual.stdout).to eq("Different output\n") expect(result.expected.stdout).to eq("Expected output\n") end @@ -161,7 +162,8 @@ end end.to raise_error(Backspin::VerificationError) do |error| expect(error.message).to include("Backspin verification failed!") - expect(error.message).to include("Output verification failed") + expect(error.message).to include("Summary:") + expect(error.message).to include("stdout: changed") end end diff --git a/spec/backspin_run_spec.rb b/spec/backspin_run_spec.rb index 9525a72..d409c25 100644 --- a/spec/backspin_run_spec.rb +++ b/spec/backspin_run_spec.rb @@ -152,7 +152,8 @@ result = Backspin.run(["echo", "actual"], name: "config_no_raise") expect(result.verified?).to be false - expect(result.error_message).to include("Output verification failed") + expect(result.error_message).to include("Summary:") + expect(result.error_message).to include("stdout: changed") expect(result.actual.stdout).to eq("actual\n") expect(result.expected.stdout).to eq("expected\n") end @@ -264,4 +265,43 @@ Backspin.run(["echo", "hi"], name: "bad_filter_on", filter_on: :verify) end.to raise_error(ArgumentError, /Unknown filter_on/) end + + it "composes a verification error with field summary, context lines, and separators" do + expected_output = (1..15).map { |i| "line #{i}" }.join("\n") + actual_output = (1..15).map { |i| i == 8 ? "CHANGED" : "line #{i}" }.join("\n") + + Backspin.run(["printf", expected_output], name: "full_error_shape", mode: :record) + + expect do + Backspin.run(["printf", actual_output], name: "full_error_shape") + end.to raise_error(Backspin::VerificationError) do |error| + lines = error.message.lines.map(&:chomp) + + # Header + expect(lines[0]).to eq("Backspin verification failed!") + expect(lines[1]).to start_with("Record:") + + # Field summary section + expect(lines).to include("Summary:") + expect(lines).to include(" stdout: changed") + expect(lines).to include(" stderr: unchanged") + expect(lines).to include(" status: unchanged") + + # Diff section header followed by context + change + context + stdout_idx = lines.index("[stdout]") + diff_body = lines[(stdout_idx + 1)..] + + expect(diff_body).to eq([ + " line 5", + " line 6", + " line 7", + "-line 8", + "+CHANGED", + " line 9", + " line 10", + " line 11", + "..." + ]) + end + end end From 1ffdca5bca138b643ee2ac7eca9403c5c7e0f080 Mon Sep 17 00:00:00 2001 From: Rob Sanheim Date: Sat, 21 Feb 2026 10:39:59 -0600 Subject: [PATCH 2/3] Fix lint: add parentheses to ternary condition --- spec/backspin_run_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/backspin_run_spec.rb b/spec/backspin_run_spec.rb index d409c25..afedeaa 100644 --- a/spec/backspin_run_spec.rb +++ b/spec/backspin_run_spec.rb @@ -268,7 +268,7 @@ it "composes a verification error with field summary, context lines, and separators" do expected_output = (1..15).map { |i| "line #{i}" }.join("\n") - actual_output = (1..15).map { |i| i == 8 ? "CHANGED" : "line #{i}" }.join("\n") + actual_output = (1..15).map { |i| (i == 8) ? "CHANGED" : "line #{i}" }.join("\n") Backspin.run(["printf", expected_output], name: "full_error_shape", mode: :record) From 4543153e6024915cd8706942ccba37ce68247d4a Mon Sep 17 00:00:00 2001 From: Rob Sanheim Date: Mon, 23 Feb 2026 01:59:51 -0600 Subject: [PATCH 3/3] Fix newline-only output diffs in CommandDiff --- lib/backspin/command_diff.rb | 18 +++++++++++++----- spec/backspin/command_diff_spec.rb | 26 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/lib/backspin/command_diff.rb b/lib/backspin/command_diff.rb index a5f6bbf..8e87124 100644 --- a/lib/backspin/command_diff.rb +++ b/lib/backspin/command_diff.rb @@ -120,8 +120,8 @@ def stderr_diff(expected, actual) end def generate_line_diff(expected, actual) - expected_lines = (expected || "").lines.map(&:chomp) - actual_lines = (actual || "").lines.map(&:chomp) + expected_lines = split_lines(expected) + actual_lines = split_lines(actual) max_lines = [expected_lines.length, actual_lines.length].max changed = Array.new(max_lines, false) @@ -152,17 +152,25 @@ def generate_line_diff(expected, actual) in_hunk = true if changed[i] - diff_lines << "-#{expected_lines[i]}" if i < expected_lines.length - diff_lines << "+#{actual_lines[i]}" if i < actual_lines.length + diff_lines << "-#{render_line(expected_lines[i])}" if i < expected_lines.length + diff_lines << "+#{render_line(actual_lines[i])}" if i < actual_lines.length else line = expected_lines[i] || actual_lines[i] - diff_lines << " #{line}" + diff_lines << " #{render_line(line)}" end end diff_lines.join("\n") end + def split_lines(value) + (value || "").lines + end + + def render_line(line) + line.to_s.chomp + end + def maybe_truncate(diff_text) return diff_text if full_diff? diff --git a/spec/backspin/command_diff_spec.rb b/spec/backspin/command_diff_spec.rb index 1d68927..e94e5fb 100644 --- a/spec/backspin/command_diff_spec.rb +++ b/spec/backspin/command_diff_spec.rb @@ -210,6 +210,32 @@ expect(diff).to include("-line 18") expect(diff).to include("+CHANGED 18") end + + it "shows line entries for newline-only changes" do + expected_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["test"], + stdout: "line with newline\n", + stderr: "", + status: 0 + ) + actual_snapshot = Backspin::Snapshot.new( + command_type: Open3::Capture3, + args: ["test"], + stdout: "line with newline", + stderr: "", + status: 0 + ) + + diff = described_class.new( + expected: expected_snapshot, + actual: actual_snapshot + ).diff + + expect(diff).to include("[stdout]") + expect(diff).to include("-line with newline") + expect(diff).to include("+line with newline") + end end context "diff truncation" do