@benchmarkset breaks on ^C

[LilithHafner]

When I interrupt @benchmarksuite it is permanently broken:

julia> @benchmarkset "suite" for k in 1:50000
             @case "case $k" rand($k, $k)
       end
^CERROR: InterruptException:
Stacktrace:
 [1] eval
   @ ./boot.jl:373 [inlined]
 [2] generate_benchmark_definition(eval_module::Any, out_vars::Any, setup_vars::Any, quote_vars::Any, quote_vals::Any, core::Any, setup::Any, teardown::Any, params::Any)
   @ BenchmarkTools ~/.julia/packages/BenchmarkTools/SnZV6/src/execution.jl:482

julia> @benchmarkset "suite" for k in 1:5
             @case "case $k" rand($k, $k)
       end

julia> 

[gdalle]

Wow that is a nasty find. Does it happen when you interrupt a BenchmarkGroup being run?

[LilithHafner]

This seems to be fine:

julia> suite = BenchmarkGroup()
0-element BenchmarkTools.BenchmarkGroup:
  tags: []

julia> for k in 1:20
           suite["case $k"] = @benchmarkable rand($k, $k)
       end

julia> run(suite)
^CERROR: InterruptException:
Stacktrace:
  [1] gc (repeats 2 times)
    @ ./gcutils.jl:98 [inlined]
  [2] gcscrub
    @ ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:4 [inlined]
  [3] _run(b::BenchmarkTools.Benchmark, p::BenchmarkTools.Parameters; verbose::Bool, pad::String, kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ BenchmarkTools ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:95
  [4] #invokelatest#2
    @ ./essentials.jl:818 [inlined]
  [5] invokelatest
    @ ./essentials.jl:813 [inlined]
  [6] #run_result#45
    @ ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:34 [inlined]
  [7] run_result
    @ ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:34 [inlined]
  [8] run(b::BenchmarkTools.Benchmark, p::BenchmarkTools.Parameters; progressid::Base.UUID, nleaves::Int64, ndone::Int64, kwargs::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:verbose, :pad), Tuple{Bool, String}}})
    @ BenchmarkTools ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:117
  [9] run (repeats 2 times)
    @ ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:117 [inlined]
 [10] macro expansion
    @ ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:135 [inlined]
 [11] macro expansion
    @ ./timing.jl:393 [inlined]
 [12] (::BenchmarkTools.var"#51#52"{Bool, String, Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, BenchmarkGroup, Tuple{}})(progressid::Base.UUID, nleaves::Int64, ndone::Int64)
    @ BenchmarkTools ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:134
 [13] _withprogress(f::BenchmarkTools.var"#51#52"{Bool, String, Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, BenchmarkGroup, Tuple{}}, name::String, group::BenchmarkGroup; progressid::Nothing, nleaves::Float64, ndone::Float64, #unused#::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ BenchmarkTools ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:79
 [14] _withprogress
    @ ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:63 [inlined]
 [15] #run#50
    @ ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:125 [inlined]
 [16] run(::BenchmarkGroup)
    @ BenchmarkTools ~/.julia/packages/BenchmarkTools/0owsb/src/execution.jl:125

julia> run(suite)
20-element BenchmarkTools.BenchmarkGroup:
  tags: []
  "case 6" => Trial(41.000 ns)
  "case 10" => Trial(83.000 ns)
  "case 7" => Trial(41.000 ns)
  "case 15" => Trial(125.000 ns)
  "case 14" => Trial(125.000 ns)
  "case 11" => Trial(83.000 ns)
  "case 17" => Trial(208.000 ns)
  "case 19" => Trial(250.000 ns)
  "case 13" => Trial(83.000 ns)
  "case 1" => Trial(0.001 ns)
  ⋮

[gdalle]

Any clue where that could come from?

[LilithHafner]

Nope