From 4a5a94f8188b3cfb4f6f804998793bc0c8e1260c Mon Sep 17 00:00:00 2001 From: mrudzki Date: Thu, 11 May 2023 14:16:23 +0200 Subject: [PATCH 1/7] Add seed as an option --- lib/turbo_tests/cli.rb | 6 ++++++ lib/turbo_tests/reporter.rb | 5 +++++ lib/turbo_tests/runner.rb | 14 ++++++++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/turbo_tests/cli.rb b/lib/turbo_tests/cli.rb index 52beab2..c6b7584 100644 --- a/lib/turbo_tests/cli.rb +++ b/lib/turbo_tests/cli.rb @@ -16,6 +16,7 @@ def run runtime_log = nil verbose = false fail_fast = nil + seed = nil OptionParser.new { |opts| opts.banner = <<~BANNER @@ -76,6 +77,10 @@ def run end fail_fast = n.nil? || n < 1 ? 1 : n end + + opts.on("--seed SEED", ) do |s| + seed = s + end }.parse!(@argv) requires.each { |f| require(f) } @@ -101,6 +106,7 @@ def run verbose: verbose, fail_fast: fail_fast, count: count, + seed: seed ) if success diff --git a/lib/turbo_tests/reporter.rb b/lib/turbo_tests/reporter.rb index 033ffed..4996dca 100644 --- a/lib/turbo_tests/reporter.rb +++ b/lib/turbo_tests/reporter.rb @@ -114,6 +114,11 @@ def finish RSpec::Core::Notifications::NullNotification) end + def seed_notification(seed, seed_used) + puts RSpec::Core::Notifications::SeedNotification.new(seed, seed_used).fully_formatted + puts + end + protected def delegate_to_formatters(method, *args) diff --git a/lib/turbo_tests/runner.rb b/lib/turbo_tests/runner.rb index 31fd36e..ba291ed 100644 --- a/lib/turbo_tests/runner.rb +++ b/lib/turbo_tests/runner.rb @@ -20,6 +20,8 @@ def self.run(opts = {}) verbose = opts.fetch(:verbose, false) fail_fast = opts.fetch(:fail_fast, nil) count = opts.fetch(:count, nil) + seed = opts.fetch(:seed, rand(0xFFFF).to_s) + seed_used = !opts[:seed].nil? if verbose STDERR.puts "VERBOSE" @@ -34,7 +36,9 @@ def self.run(opts = {}) runtime_log: runtime_log, verbose: verbose, fail_fast: fail_fast, - count: count + count: count, + seed: seed, + seed_used: seed_used ).run end @@ -49,6 +53,8 @@ def initialize(opts) @load_time = 0 @load_count = 0 @failure_count = 0 + @seed = opts[:seed] + @seed_used = opts[:seed_used] @messages = Thread::Queue.new @threads = [] @@ -86,6 +92,8 @@ def run report_number_of_tests(tests_in_groups) + @reporter.seed_notification(@seed, @seed_used) + wait_threads = tests_in_groups.map.with_index do |tests, process_id| start_regular_subprocess(tests, process_id + 1, **subprocess_opts) end @@ -97,6 +105,8 @@ def run @threads.each(&:join) @reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?) + + @reporter.seed_notification(@seed, @seed_used) end private @@ -150,7 +160,7 @@ def start_subprocess(env, extra_args, tests, process_id, record_runtime:) command = [ ENV["BUNDLE_BIN_PATH"], "exec", "rspec", *extra_args, - "--seed", rand(0xFFFF).to_s, + "--seed", @seed, "--format", "TurboTests::JsonRowsFormatter", "--out", tmp_filename, *record_runtime_options, From 0caf91e90a3238ebe939d8311c354211ae287cfd Mon Sep 17 00:00:00 2001 From: mrudzki Date: Thu, 11 May 2023 14:52:32 +0200 Subject: [PATCH 2/7] Add seeds to specs --- spec/cli_spec.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/spec/cli_spec.rb b/spec/cli_spec.rb index b25c2a6..2b2836a 100644 --- a/spec/cli_spec.rb +++ b/spec/cli_spec.rb @@ -1,5 +1,5 @@ RSpec.describe TurboTests::CLI do - subject(:output) { `bundle exec turbo_tests -f d #{fixture}`.strip } + subject(:output) { `bundle exec turbo_tests -f d #{fixture} --seed 1234`.strip } before { output } @@ -8,6 +8,9 @@ %( 1 processes for 1 specs, ~ 1 specs per process +Randomized with seed 1234 + + An error occurred while loading #{fixture}. \e[31mFailure/Error: \e[0m\e[1;34m1\e[0m / \e[1;34m0\e[0m\e[0m \e[31m\e[0m @@ -18,6 +21,11 @@ \e[36m# #{fixture}:1:in `'\e[0m ).strip } + let(:expected_end_of_output) do + "0 examples, 0 failures\n"\ + "\n\n"\ + "Randomized with seed 1234" + end let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" } @@ -25,7 +33,7 @@ expect($?.exitstatus).to eql(1) expect(output).to start_with(expected_start_of_output) - expect(output).to end_with("0 examples, 0 failures") + expect(output).to end_with(expected_end_of_output) end end @@ -66,7 +74,7 @@ expect(output).to include(part) end - expect(output).to end_with("3 examples, 0 failures, 3 pending") + expect(output).to end_with("3 examples, 0 failures, 3 pending\n\n\nRandomized with seed 1234") end end From 3a8165caf83afda5a4dde97c295d17ee881742cd Mon Sep 17 00:00:00 2001 From: mrudzki Date: Thu, 11 May 2023 14:53:18 +0200 Subject: [PATCH 3/7] Fix problems: 1. When seed is not sent 2. With response status in cli being 0 even when failures appeared --- lib/turbo_tests/runner.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/turbo_tests/runner.rb b/lib/turbo_tests/runner.rb index ba291ed..12febc1 100644 --- a/lib/turbo_tests/runner.rb +++ b/lib/turbo_tests/runner.rb @@ -20,7 +20,7 @@ def self.run(opts = {}) verbose = opts.fetch(:verbose, false) fail_fast = opts.fetch(:fail_fast, nil) count = opts.fetch(:count, nil) - seed = opts.fetch(:seed, rand(0xFFFF).to_s) + seed = opts.fetch(:seed, nil) || rand(0xFFFF).to_s seed_used = !opts[:seed].nil? if verbose @@ -102,11 +102,11 @@ def run @reporter.finish + @reporter.seed_notification(@seed, @seed_used) + @threads.each(&:join) @reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?) - - @reporter.seed_notification(@seed, @seed_used) end private From 37cceeb0a273539207faf550b4dfd3dfd2ff0527 Mon Sep 17 00:00:00 2001 From: mrudzki Date: Fri, 12 May 2023 09:37:00 +0200 Subject: [PATCH 4/7] Add changes to help and readme --- README.md | 3 ++- lib/turbo_tests/cli.rb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0d22dc4..f5a285a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@
Turbo-Tests - +
@@ -96,6 +96,7 @@ Options: --runtime-log FILE Location of previously recorded test runtimes -v, --verbose More output --fail-fast=[N] + --seed SEED Seed for rspec ``` ## Development diff --git a/lib/turbo_tests/cli.rb b/lib/turbo_tests/cli.rb index c6b7584..ed80ba7 100644 --- a/lib/turbo_tests/cli.rb +++ b/lib/turbo_tests/cli.rb @@ -78,7 +78,7 @@ def run fail_fast = n.nil? || n < 1 ? 1 : n end - opts.on("--seed SEED", ) do |s| + opts.on("--seed SEED", "Seed for rspec") do |s| seed = s end }.parse!(@argv) From 9991acaf3e7a53ffd180b8f9d4c506b691c6a2cd Mon Sep 17 00:00:00 2001 From: mrudzki Date: Fri, 12 May 2023 09:21:31 +0200 Subject: [PATCH 5/7] Add option to print failing group --- lib/turbo_tests/cli.rb | 8 +++++++- lib/turbo_tests/runner.rb | 17 +++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/turbo_tests/cli.rb b/lib/turbo_tests/cli.rb index ed80ba7..c62ffec 100644 --- a/lib/turbo_tests/cli.rb +++ b/lib/turbo_tests/cli.rb @@ -17,6 +17,7 @@ def run verbose = false fail_fast = nil seed = nil + print_failed_group = false OptionParser.new { |opts| opts.banner = <<~BANNER @@ -81,6 +82,10 @@ def run opts.on("--seed SEED", "Seed for rspec") do |s| seed = s end + + opts.on("--print_failed_group") do + print_failed_group = true + end }.parse!(@argv) requires.each { |f| require(f) } @@ -106,7 +111,8 @@ def run verbose: verbose, fail_fast: fail_fast, count: count, - seed: seed + seed: seed, + print_failed_group: print_failed_group ) if success diff --git a/lib/turbo_tests/runner.rb b/lib/turbo_tests/runner.rb index 12febc1..9e7b768 100644 --- a/lib/turbo_tests/runner.rb +++ b/lib/turbo_tests/runner.rb @@ -22,6 +22,7 @@ def self.run(opts = {}) count = opts.fetch(:count, nil) seed = opts.fetch(:seed, nil) || rand(0xFFFF).to_s seed_used = !opts[:seed].nil? + print_failed_group = opts.fetch(:print_failed_group, false) if verbose STDERR.puts "VERBOSE" @@ -38,7 +39,8 @@ def self.run(opts = {}) fail_fast: fail_fast, count: count, seed: seed, - seed_used: seed_used + seed_used: seed_used, + print_failed_group: print_failed_group ).run end @@ -55,10 +57,10 @@ def initialize(opts) @failure_count = 0 @seed = opts[:seed] @seed_used = opts[:seed_used] - @messages = Thread::Queue.new @threads = [] @error = false + @print_failed_group = opts[:print_failed_group] end def run @@ -106,6 +108,8 @@ def run @threads.each(&:join) + report_failed_group(wait_threads, tests_in_groups) if @print_failed_group + @reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?) end @@ -282,5 +286,14 @@ def report_number_of_tests(groups) puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process" end + + def report_failed_group(wait_threads, tests_in_groups) + wait_threads.map(&:value).each_with_index do |value, index| + next if value.success? + + failing_group = tests_in_groups[index].join(" ") + puts "Group that failed: #{failing_group}" + end + end end end From c7b3c1f6e186c38ff29413d4a465aff1f613697a Mon Sep 17 00:00:00 2001 From: mrudzki Date: Fri, 12 May 2023 09:32:48 +0200 Subject: [PATCH 6/7] Add change to readme --- README.md | 1 + lib/turbo_tests/cli.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f5a285a..b976d85 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ Options: -v, --verbose More output --fail-fast=[N] --seed SEED Seed for rspec + --print_failed_group Prints group that had failures in it ``` ## Development diff --git a/lib/turbo_tests/cli.rb b/lib/turbo_tests/cli.rb index c62ffec..99ee7b5 100644 --- a/lib/turbo_tests/cli.rb +++ b/lib/turbo_tests/cli.rb @@ -83,7 +83,7 @@ def run seed = s end - opts.on("--print_failed_group") do + opts.on("--print_failed_group", "Prints group that had failures in it") do print_failed_group = true end }.parse!(@argv) From 152e694ae12edc4a6b32299b2f04cb85e26ef75c Mon Sep 17 00:00:00 2001 From: mrudzki Date: Fri, 12 May 2023 09:38:28 +0200 Subject: [PATCH 7/7] Fix readme formatting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b976d85..5a27f8f 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Options: -v, --verbose More output --fail-fast=[N] --seed SEED Seed for rspec - --print_failed_group Prints group that had failures in it + --print_failed_group Prints group that had failures in it ``` ## Development