Feature request: save data directories for each run (run index as format specifier in the command)

[stagnation]

Hi

I have a benchmark use-case where I want to save extra data files for each run, for a more detailed analysis later. I want to benchmark the number of garbage collection events in a program profile.

My run-tool can easily take the parameter I'm scanning as one argument and the directory as another. But it is a little convoluted to do today.

---

Attempts

$ hyperfine \
    --runs 5 \
    --parameter-list mem 150,160 \
    -- './benchmark-different-memory {mem}m {???}'

One attempt is to forgo --runs and use a --parameter-scan for the runs

$ hyperfine \
    --parameter-scan run 0 5 \
    --parameter-list mem 150,160 \
    -- './benchmark-different-memory {mem}m {run}'
error: the argument '--parameter-scan <VAR> <MIN> <MAX>' cannot be used with '--parameter-list <VAR> <VALUES>'

---

Working solution as a command executor. Though the time benchmark breaks as we set runs to 1. Else it does both the default number of runs, and the parameter-list for runs.

Using two parameter-lists instead works:

$ hyperfine \
    --parameter-list run $(seq 5 | paste -sd ',') \
    --parameter-list mem 150,160 \
    --runs 1
    -- './benchmark-different-memory {mem}m {run}'

This is okay, as I do not care about warmups today, but ideally a warmup should be used. Then a two-dimensional parameter list could be --parameter-list run WARMUP{1,2} $(seq 5) or something.

---

One can also hack the code to save the index

For the reference: I also hacked in the index into the run function, and the run can be taken from the environment. If someone wants a stricter API.

$ target/debug/hyperfine --runs 3 --parameter-scan x 1 2 env --show-output | grep HYPERFINE_RUN_INDEX
HYPERFINE_RUN_INDEX=0
HYPERFINE_RUN_INDEX=1
HYPERFINE_RUN_INDEX=2

  Warning: Command took less than 5 ms to complete. Note that the results might be inaccurate because hyp
erfine can not calibrate the shell startup time much more precise than this limit. You can try to use the
 `-N`/`--shell=none` option to disable the shell completely.
HYPERFINE_RUN_INDEX=0
HYPERFINE_RUN_INDEX=1
HYPERFINE_RUN_INDEX=2

And my use case then looks like this, the environment variable is expanded in the run-shell.

~/gits/hyperfine/target/debug/hyperfine \
     --runs 5 \
     --parameter-list mem 200,250 \
     -- './benchmark-different-memory {mem}m $(echo $HYPERFINE_RUN_INDEX)'

[stagnation]

Hacky patch to expose the run index as an environment variable

diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs
index 31db47b..57f09b6 100644
--- a/src/benchmark/executor.rs
+++ b/src/benchmark/executor.rs
@@ -20,6 +20,7 @@ pub trait Executor {
         &self,
         command: &Command<'_>,
         command_failure_action: Option<CmdFailureAction>,
+        index: u64,
     ) -> Result<(TimingResult, ExitStatus)>;

     /// Perform a calibration of this executor. For example,
@@ -41,6 +42,7 @@ fn run_command_and_measure_common(
     command_input_policy: &CommandInputPolicy,
     command_output_policy: &CommandOutputPolicy,
     command_name: &str,
+    index: u64,
 ) -> Result<TimerResult> {
     let stdin = command_input_policy.get_stdin()?;
     let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
@@ -50,6 +52,9 @@ fn run_command_and_measure_common(
         "HYPERFINE_RANDOMIZED_ENVIRONMENT_OFFSET",
         randomized_environment_offset::value(),
     );
+    command.env(
+        "HYPERFINE_RUN_INDEX", format!("{}", index),
+    );

     let result = execute_and_measure(command)
         .with_context(|| format!("Failed to run command '{}'", command_name))?;
@@ -83,6 +88,7 @@ impl<'a> Executor for RawExecutor<'a> {
         &self,
         command: &Command<'_>,
         command_failure_action: Option<CmdFailureAction>,
+        index: u64,
     ) -> Result<(TimingResult, ExitStatus)> {
         let result = run_command_and_measure_common(
             command.get_command()?,
@@ -90,6 +96,7 @@ impl<'a> Executor for RawExecutor<'a> {
             &self.options.command_input_policy,
             &self.options.command_output_policy,
             &command.get_command_line(),
+            index,
         )?;

         Ok((
@@ -132,6 +139,7 @@ impl<'a> Executor for ShellExecutor<'a> {
         &self,
         command: &Command<'_>,
         command_failure_action: Option<CmdFailureAction>,
+        index: u64,
     ) -> Result<(TimingResult, ExitStatus)> {
         let mut command_builder = self.shell.command();
         command_builder
@@ -150,6 +158,7 @@ impl<'a> Executor for ShellExecutor<'a> {
             &self.options.command_input_policy,
             &self.options.command_output_policy,
             &command.get_command_line(),
+            index,
         )?;

         // Subtract shell spawning time
@@ -186,9 +195,9 @@ impl<'a> Executor for ShellExecutor<'a> {
         let mut times_user: Vec<Second> = vec![];
         let mut times_system: Vec<Second> = vec![];

-        for _ in 0..COUNT {
+        for i in 0..COUNT {
             // Just run the shell without any command
-            let res = self.run_command_and_measure(&Command::new(None, ""), None);
+            let res = self.run_command_and_measure(&Command::new(None, ""), None, i);

             match res {
                 Err(_) => {
@@ -258,6 +267,7 @@ impl Executor for MockExecutor {
         &self,
         command: &Command<'_>,
         _command_failure_action: Option<CmdFailureAction>,
+        index: u64,
     ) -> Result<(TimingResult, ExitStatus)> {
         #[cfg(unix)]
         let status = {
diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs
index 0699f7d..17c3a3a 100644
--- a/src/benchmark/mod.rs
+++ b/src/benchmark/mod.rs
@@ -57,7 +57,7 @@ impl<'a> Benchmark<'a> {
         error_output: &'static str,
     ) -> Result<TimingResult> {
         self.executor
-            .run_command_and_measure(command, Some(CmdFailureAction::RaiseError))
+            .run_command_and_measure(command, Some(CmdFailureAction::RaiseError), 0)
             .map(|r| r.0)
             .map_err(|_| anyhow!(error_output))
     }
@@ -160,9 +160,9 @@ impl<'a> Benchmark<'a> {
                 None
             };

-            for _ in 0..self.options.warmup_count {
+            for i in 0..self.options.warmup_count {
                 let _ = run_preparation_command()?;
-                let _ = self.executor.run_command_and_measure(self.command, None)?;
+                let _ = self.executor.run_command_and_measure(self.command, None, i)?;
                 if let Some(bar) = progress_bar.as_ref() {
                     bar.inc(1)
                 }
@@ -188,7 +188,7 @@ impl<'a> Benchmark<'a> {
             preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());

         // Initial timing run
-        let (res, status) = self.executor.run_command_and_measure(self.command, None)?;
+        let (res, status) = self.executor.run_command_and_measure(self.command, None, 0)?;
         let success = status.success();

         // Determine number of benchmark runs
@@ -226,7 +226,7 @@ impl<'a> Benchmark<'a> {
         }

         // Gather statistics (perform the actual benchmark)
-        for _ in 0..count_remaining {
+        for i in 0..count_remaining {
             run_preparation_command()?;

             let msg = {
@@ -238,7 +238,7 @@ impl<'a> Benchmark<'a> {
                 bar.set_message(msg.to_owned())
             }

-            let (res, status) = self.executor.run_command_and_measure(self.command, None)?;
+            let (res, status) = self.executor.run_command_and_measure(self.command, None, i + 1)?;
             let success = status.success();

             times_real.push(res.time_real);