diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index e3534a7bc..f06f814d0 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -9,7 +9,7 @@ use std::cmp; use crate::benchmark::executor::BenchmarkIteration; use crate::command::Command; use crate::options::{ - CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption, + CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption, WarmupOption, }; use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD}; use crate::output::format::{format_duration, format_duration_unit}; @@ -53,6 +53,142 @@ impl<'a> Benchmark<'a> { } } + /// Run warmup iterations before the actual benchmark. + fn perform_warmup_runs( + &self, + run_preparation_command: &impl Fn() -> Result>, + run_conclusion_command: &impl Fn() -> Result>, + output_policy: &CommandOutputPolicy, + ) -> Result<()> { + match self.options.warmup { + WarmupOption::Disabled => Ok(()), + WarmupOption::Fixed(count) => self.perform_fixed_warmup_runs( + count, + run_preparation_command, + run_conclusion_command, + output_policy, + ), + WarmupOption::Auto { + stable_window, + stability_threshold, + max_runs, + } => self.perform_auto_warmup_runs( + stable_window, + stability_threshold, + max_runs, + run_preparation_command, + run_conclusion_command, + output_policy, + ), + } + } + + fn perform_fixed_warmup_runs( + &self, + count: u64, + run_preparation_command: &impl Fn() -> Result>, + run_conclusion_command: &impl Fn() -> Result>, + output_policy: &CommandOutputPolicy, + ) -> Result<()> { + let progress_bar = if self.options.output_style != OutputStyleOption::Disabled { + Some(get_progress_bar( + count, + "Performing warmup runs", + self.options.output_style, + )) + } else { + None + }; + + for i in 0..count { + self.run_single_warmup_iteration( + i, + run_preparation_command, + run_conclusion_command, + output_policy, + )?; + if let Some(bar) = progress_bar.as_ref() { + bar.inc(1); + } + } + + if let Some(bar) = progress_bar.as_ref() { + bar.finish_and_clear(); + } + + Ok(()) + } + + fn perform_auto_warmup_runs( + &self, + stable_window: usize, + stability_threshold: f64, + max_runs: u64, + run_preparation_command: &impl Fn() -> Result>, + run_conclusion_command: &impl Fn() -> Result>, + output_policy: &CommandOutputPolicy, + ) -> Result<()> { + let progress_bar = if self.options.output_style != OutputStyleOption::Disabled { + Some(get_progress_bar( + max_runs, + "Performing warmup runs (auto)", + self.options.output_style, + )) + } else { + None + }; + + let mut recent_times = Vec::with_capacity(stable_window); + + for i in 0..max_runs { + let (timing, _) = self.run_single_warmup_iteration( + i, + run_preparation_command, + run_conclusion_command, + output_policy, + )?; + + recent_times.push(timing.time_real + self.executor.time_overhead()); + if recent_times.len() > stable_window { + recent_times.remove(0); + } + + if let Some(bar) = progress_bar.as_ref() { + bar.inc(1); + } + + if recent_times.len() >= stable_window + && crate::options::is_warmup_stable(&recent_times, stability_threshold) + { + break; + } + } + + if let Some(bar) = progress_bar.as_ref() { + bar.finish_and_clear(); + } + + Ok(()) + } + + fn run_single_warmup_iteration( + &self, + iteration: u64, + run_preparation_command: &impl Fn() -> Result>, + run_conclusion_command: &impl Fn() -> Result>, + output_policy: &CommandOutputPolicy, + ) -> Result<(TimingResult, std::process::ExitStatus)> { + let _ = run_preparation_command()?; + let result = self.executor.run_command_and_measure( + self.command, + BenchmarkIteration::Warmup(iteration), + None, + output_policy, + )?; + let _ = run_conclusion_command()?; + Ok(result) + } + /// Run setup, cleanup, or preparation commands fn run_intermediate_command( &self, @@ -199,33 +335,12 @@ impl<'a> Benchmark<'a> { self.run_setup_command(self.command.get_parameters().iter().cloned(), output_policy)?; // Warmup phase - if self.options.warmup_count > 0 { - let progress_bar = if self.options.output_style != OutputStyleOption::Disabled { - Some(get_progress_bar( - self.options.warmup_count, - "Performing warmup runs", - self.options.output_style, - )) - } else { - None - }; - - for i in 0..self.options.warmup_count { - let _ = run_preparation_command()?; - let _ = self.executor.run_command_and_measure( - self.command, - BenchmarkIteration::Warmup(i), - None, - output_policy, - )?; - let _ = run_conclusion_command()?; - if let Some(bar) = progress_bar.as_ref() { - bar.inc(1) - } - } - if let Some(bar) = progress_bar.as_ref() { - bar.finish_and_clear() - } + if self.options.warmup.is_enabled() { + self.perform_warmup_runs( + &run_preparation_command, + &run_conclusion_command, + output_policy, + )?; } // Set up progress bar (and spinner for initial measurement) @@ -410,7 +525,7 @@ impl<'a> Benchmark<'a> { let scores = modified_zscores(×_real); let outlier_warning_options = OutlierWarningOptions { - warmup_in_use: self.options.warmup_count > 0, + warmup_in_use: self.options.warmup.is_enabled(), prepare_in_use: self .options .preparation_command diff --git a/src/cli.rs b/src/cli.rs index b12f6d34c..7ed45727f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -42,7 +42,8 @@ fn build_command() -> Command { .value_name("NUM") .action(ArgAction::Set) .help( - "Perform NUM warmup runs before the actual benchmark. This can be used \ + "Perform NUM warmup runs before the actual benchmark, or 'auto' to keep \ + running warmup until the last 5 runs vary by less than 1%. This can be used \ to fill (disk) caches for I/O-heavy programs.", ), ) diff --git a/src/options.rs b/src/options.rs index 7c83da1c5..36a88c5d7 100644 --- a/src/options.rs +++ b/src/options.rs @@ -194,13 +194,73 @@ impl Default for ExecutorKind { } } +/// How many warmup runs to perform before benchmarking +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum WarmupOption { + Disabled, + Fixed(u64), + Auto { + stable_window: usize, + stability_threshold: f64, + max_runs: u64, + }, +} + +impl Default for WarmupOption { + fn default() -> Self { + Self::Disabled + } +} + +impl WarmupOption { + pub fn is_enabled(&self) -> bool { + !matches!(self, Self::Disabled) + } + + pub const AUTO_STABLE_WINDOW: usize = 5; + pub const AUTO_STABILITY_THRESHOLD: f64 = 0.01; + pub const AUTO_MAX_RUNS: u64 = 100; +} + +pub fn parse_warmup_option<'a>(value: &str) -> Result> { + if value.eq_ignore_ascii_case("auto") { + Ok(WarmupOption::Auto { + stable_window: WarmupOption::AUTO_STABLE_WINDOW, + stability_threshold: WarmupOption::AUTO_STABILITY_THRESHOLD, + max_runs: WarmupOption::AUTO_MAX_RUNS, + }) + } else { + let count = value + .parse::() + .map_err(|error| OptionsError::IntParsingError("warmup", error))?; + Ok(WarmupOption::Fixed(count)) + } +} + +/// Returns true when the relative spread of `times` is at most `threshold`. +pub fn is_warmup_stable(times: &[Second], threshold: f64) -> bool { + if times.len() < 2 { + return false; + } + + let min = times.iter().cloned().fold(f64::INFINITY, f64::min); + let max = times.iter().cloned().fold(f64::NEG_INFINITY, f64::max); + let mean = times.iter().sum::() / times.len() as Second; + + if mean == 0.0 { + max - min == 0.0 + } else { + (max - min) / mean <= threshold + } +} + /// The main settings for a hyperfine benchmark session pub struct Options { /// Upper and lower bound for the number of benchmark runs pub run_bounds: RunBounds, - /// Number of warmup runs - pub warmup_count: u64, + /// Warmup runs to perform before benchmarking + pub warmup: WarmupOption, /// Minimum benchmarking time pub min_benchmarking_time: Second, @@ -252,7 +312,7 @@ impl Default for Options { fn default() -> Options { Options { run_bounds: RunBounds::default(), - warmup_count: 0, + warmup: WarmupOption::default(), min_benchmarking_time: 3.0, command_failure_action: CmdFailureAction::RaiseError, reference_command: None, @@ -285,7 +345,11 @@ impl Options { .transpose() }; - options.warmup_count = param_to_u64("warmup")?.unwrap_or(options.warmup_count); + options.warmup = matches + .get_one::("warmup") + .map(|value| parse_warmup_option(value)) + .transpose()? + .unwrap_or(options.warmup); let mut min_runs = param_to_u64("min-runs")?; let mut max_runs = param_to_u64("max-runs")?; @@ -502,6 +566,27 @@ impl Options { } } +#[test] +fn test_parse_warmup_option() { + assert_eq!(parse_warmup_option("3").unwrap(), WarmupOption::Fixed(3)); + assert_eq!( + parse_warmup_option("auto").unwrap(), + WarmupOption::Auto { + stable_window: WarmupOption::AUTO_STABLE_WINDOW, + stability_threshold: WarmupOption::AUTO_STABILITY_THRESHOLD, + max_runs: WarmupOption::AUTO_MAX_RUNS, + } + ); +} + +#[test] +fn test_is_warmup_stable() { + assert!(!is_warmup_stable(&[1.0], 0.01)); + assert!(is_warmup_stable(&[1.0, 1.0, 1.0, 1.0, 1.0], 0.01)); + assert!(!is_warmup_stable(&[1.0, 1.0, 1.0, 1.0, 1.2], 0.01)); + assert!(is_warmup_stable(&[1.0, 1.005, 0.995, 1.002, 0.998], 0.01)); +} + #[test] fn test_default_shell() { let shell = Shell::default(); diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 6ccc2e5ed..d48cf5b59 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -561,3 +561,14 @@ fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() { .expect_output("cleanup") .run(); } + +#[test] +fn auto_warmup_option_is_accepted() { + hyperfine() + .arg("--warmup=auto") + .arg("--runs=1") + .arg("--style=none") + .arg("true") + .assert() + .success(); +}