feat: add --warmup auto to detect stable warmup count (fixes #848)

src/benchmark/mod.rs

@@ -9,7 +9,7 @@ use std::cmp;
 use crate::benchmark::executor::BenchmarkIteration;
 use crate::command::Command;
 use crate::options::{
-    CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
+    CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption, WarmupOption,
 };
 use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
 use crate::output::format::{format_duration, format_duration_unit};
@@ -53,6 +53,142 @@ impl<'a> Benchmark<'a> {
         }
     }
 
+    /// Run warmup iterations before the actual benchmark.
+    fn perform_warmup_runs(
+        &self,
+        run_preparation_command: &impl Fn() -> Result<Option<TimingResult>>,
+        run_conclusion_command: &impl Fn() -> Result<Option<TimingResult>>,
+        output_policy: &CommandOutputPolicy,
+    ) -> Result<()> {
+        match self.options.warmup {
+            WarmupOption::Disabled => Ok(()),
+            WarmupOption::Fixed(count) => self.perform_fixed_warmup_runs(
+                count,
+                run_preparation_command,
+                run_conclusion_command,
+                output_policy,
+            ),
+            WarmupOption::Auto {
+                stable_window,
+                stability_threshold,
+                max_runs,
+            } => self.perform_auto_warmup_runs(
+                stable_window,
+                stability_threshold,
+                max_runs,
+                run_preparation_command,
+                run_conclusion_command,
+                output_policy,
+            ),
+        }
+    }
+
+    fn perform_fixed_warmup_runs(
+        &self,
+        count: u64,
+        run_preparation_command: &impl Fn() -> Result<Option<TimingResult>>,
+        run_conclusion_command: &impl Fn() -> Result<Option<TimingResult>>,
+        output_policy: &CommandOutputPolicy,
+    ) -> Result<()> {
+        let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
+            Some(get_progress_bar(
+                count,
+                "Performing warmup runs",
+                self.options.output_style,
+            ))
+        } else {
+            None
+        };
+
+        for i in 0..count {
+            self.run_single_warmup_iteration(
+                i,
+                run_preparation_command,
+                run_conclusion_command,
+                output_policy,
+            )?;
+            if let Some(bar) = progress_bar.as_ref() {
+                bar.inc(1);
+            }
+        }
+
+        if let Some(bar) = progress_bar.as_ref() {
+            bar.finish_and_clear();
+        }
+
+        Ok(())
+    }
+
+    fn perform_auto_warmup_runs(
+        &self,
+        stable_window: usize,
+        stability_threshold: f64,
+        max_runs: u64,
+        run_preparation_command: &impl Fn() -> Result<Option<TimingResult>>,
+        run_conclusion_command: &impl Fn() -> Result<Option<TimingResult>>,
+        output_policy: &CommandOutputPolicy,
+    ) -> Result<()> {
+        let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
+            Some(get_progress_bar(
+                max_runs,
+                "Performing warmup runs (auto)",
+                self.options.output_style,
+            ))
+        } else {
+            None
+        };
+
+        let mut recent_times = Vec::with_capacity(stable_window);
+
+        for i in 0..max_runs {
+            let (timing, _) = self.run_single_warmup_iteration(
+                i,
+                run_preparation_command,
+                run_conclusion_command,
+                output_policy,
+            )?;
+
+            recent_times.push(timing.time_real + self.executor.time_overhead());
+            if recent_times.len() > stable_window {
+                recent_times.remove(0);
+            }
+
+            if let Some(bar) = progress_bar.as_ref() {
+                bar.inc(1);
+            }
+
+            if recent_times.len() >= stable_window
+                && crate::options::is_warmup_stable(&recent_times, stability_threshold)
+            {
+                break;
+            }
+        }
+
+        if let Some(bar) = progress_bar.as_ref() {
+            bar.finish_and_clear();
+        }
+
+        Ok(())
+    }
+
+    fn run_single_warmup_iteration(
+        &self,
+        iteration: u64,
+        run_preparation_command: &impl Fn() -> Result<Option<TimingResult>>,
+        run_conclusion_command: &impl Fn() -> Result<Option<TimingResult>>,
+        output_policy: &CommandOutputPolicy,
+    ) -> Result<(TimingResult, std::process::ExitStatus)> {
+        let _ = run_preparation_command()?;
+        let result = self.executor.run_command_and_measure(
+            self.command,
+            BenchmarkIteration::Warmup(iteration),
+            None,
+            output_policy,
+        )?;
+        let _ = run_conclusion_command()?;
+        Ok(result)
+    }
+
     /// Run setup, cleanup, or preparation commands
     fn run_intermediate_command(
         &self,
@@ -199,33 +335,12 @@ impl<'a> Benchmark<'a> {
         self.run_setup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
 
         // Warmup phase
-        if self.options.warmup_count > 0 {
-            let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
-                Some(get_progress_bar(
-                    self.options.warmup_count,
-                    "Performing warmup runs",
-                    self.options.output_style,
-                ))
-            } else {
-                None
-            };
-
-            for i in 0..self.options.warmup_count {
-                let _ = run_preparation_command()?;
-                let _ = self.executor.run_command_and_measure(
-                    self.command,
-                    BenchmarkIteration::Warmup(i),
-                    None,
-                    output_policy,
-                )?;
-                let _ = run_conclusion_command()?;
-                if let Some(bar) = progress_bar.as_ref() {
-                    bar.inc(1)
-                }
-            }
-            if let Some(bar) = progress_bar.as_ref() {
-                bar.finish_and_clear()
-            }
+        if self.options.warmup.is_enabled() {
+            self.perform_warmup_runs(
+                &run_preparation_command,
+                &run_conclusion_command,
+                output_policy,
+            )?;
         }
 
         // Set up progress bar (and spinner for initial measurement)
@@ -410,7 +525,7 @@ impl<'a> Benchmark<'a> {
         let scores = modified_zscores(&times_real);
 
         let outlier_warning_options = OutlierWarningOptions {
-            warmup_in_use: self.options.warmup_count > 0,
+            warmup_in_use: self.options.warmup.is_enabled(),
             prepare_in_use: self
                 .options
                 .preparation_command

src/cli.rs

@@ -42,7 +42,8 @@ fn build_command() -> Command {
                 .value_name("NUM")
                 .action(ArgAction::Set)
                 .help(
-                    "Perform NUM warmup runs before the actual benchmark. This can be used \
+                    "Perform NUM warmup runs before the actual benchmark, or 'auto' to keep \
+                     running warmup until the last 5 runs vary by less than 1%. This can be used \
                      to fill (disk) caches for I/O-heavy programs.",
                 ),
         )

src/options.rs

@@ -194,13 +194,73 @@ impl Default for ExecutorKind {
     }
 }
 
+/// How many warmup runs to perform before benchmarking
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum WarmupOption {
+    Disabled,
+    Fixed(u64),
+    Auto {
+        stable_window: usize,
+        stability_threshold: f64,
+        max_runs: u64,
+    },
+}
+
+impl Default for WarmupOption {
+    fn default() -> Self {
+        Self::Disabled
+    }
+}
+
+impl WarmupOption {
+    pub fn is_enabled(&self) -> bool {
+        !matches!(self, Self::Disabled)
+    }
+
+    pub const AUTO_STABLE_WINDOW: usize = 5;
+    pub const AUTO_STABILITY_THRESHOLD: f64 = 0.01;
+    pub const AUTO_MAX_RUNS: u64 = 100;
+}
+
+pub fn parse_warmup_option<'a>(value: &str) -> Result<WarmupOption, OptionsError<'a>> {
+    if value.eq_ignore_ascii_case("auto") {
+        Ok(WarmupOption::Auto {
+            stable_window: WarmupOption::AUTO_STABLE_WINDOW,
+            stability_threshold: WarmupOption::AUTO_STABILITY_THRESHOLD,
+            max_runs: WarmupOption::AUTO_MAX_RUNS,
+        })
+    } else {
+        let count = value
+            .parse::<u64>()
+            .map_err(|error| OptionsError::IntParsingError("warmup", error))?;
+        Ok(WarmupOption::Fixed(count))
+    }
+}
+
+/// Returns true when the relative spread of `times` is at most `threshold`.
+pub fn is_warmup_stable(times: &[Second], threshold: f64) -> bool {
+    if times.len() < 2 {
+        return false;
+    }
+
+    let min = times.iter().cloned().fold(f64::INFINITY, f64::min);
+    let max = times.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
+    let mean = times.iter().sum::<Second>() / times.len() as Second;
+
+    if mean == 0.0 {
+        max - min == 0.0
+    } else {
+        (max - min) / mean <= threshold
+    }
+}
+
 /// The main settings for a hyperfine benchmark session
 pub struct Options {
     /// Upper and lower bound for the number of benchmark runs
     pub run_bounds: RunBounds,
 
-    /// Number of warmup runs
-    pub warmup_count: u64,
+    /// Warmup runs to perform before benchmarking
+    pub warmup: WarmupOption,
 
     /// Minimum benchmarking time
     pub min_benchmarking_time: Second,
@@ -252,7 +312,7 @@ impl Default for Options {
     fn default() -> Options {
         Options {
             run_bounds: RunBounds::default(),
-            warmup_count: 0,
+            warmup: WarmupOption::default(),
             min_benchmarking_time: 3.0,
             command_failure_action: CmdFailureAction::RaiseError,
             reference_command: None,
@@ -285,7 +345,11 @@ impl Options {
                 .transpose()
         };
 
-        options.warmup_count = param_to_u64("warmup")?.unwrap_or(options.warmup_count);
+        options.warmup = matches
+            .get_one::<String>("warmup")
+            .map(|value| parse_warmup_option(value))
+            .transpose()?
+            .unwrap_or(options.warmup);
 
         let mut min_runs = param_to_u64("min-runs")?;
         let mut max_runs = param_to_u64("max-runs")?;
@@ -502,6 +566,27 @@ impl Options {
     }
 }
 
+#[test]
+fn test_parse_warmup_option() {
+    assert_eq!(parse_warmup_option("3").unwrap(), WarmupOption::Fixed(3));
+    assert_eq!(
+        parse_warmup_option("auto").unwrap(),
+        WarmupOption::Auto {
+            stable_window: WarmupOption::AUTO_STABLE_WINDOW,
+            stability_threshold: WarmupOption::AUTO_STABILITY_THRESHOLD,
+            max_runs: WarmupOption::AUTO_MAX_RUNS,
+        }
+    );
+}
+
+#[test]
+fn test_is_warmup_stable() {
+    assert!(!is_warmup_stable(&[1.0], 0.01));
+    assert!(is_warmup_stable(&[1.0, 1.0, 1.0, 1.0, 1.0], 0.01));
+    assert!(!is_warmup_stable(&[1.0, 1.0, 1.0, 1.0, 1.2], 0.01));
+    assert!(is_warmup_stable(&[1.0, 1.005, 0.995, 1.002, 0.998], 0.01));
+}
+
 #[test]
 fn test_default_shell() {
     let shell = Shell::default();

tests/execution_order_tests.rs

@@ -561,3 +561,14 @@ fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() {
         .expect_output("cleanup")
         .run();
 }
+
+#[test]
+fn auto_warmup_option_is_accepted() {
+    hyperfine()
+        .arg("--warmup=auto")
+        .arg("--runs=1")
+        .arg("--style=none")
+        .arg("true")
+        .assert()
+        .success();
+}