Type stability and MPI

[ranocha]

Currently, our MPI-aware mesh constructors are not type-stable since they check for a runtime value (whether we run with MPI) to determine the type of the mesh (serial or MPI-parallel). This is bad since it means setting up and running a simulation in a function is not type-stable, requires dynamic dispatch, and cannot be precompiled completely. Some options are

• just live with it
• do not encode MPI in type parameters
• set a default (e.g., no MPI) and check it during initialization of Trixi.jl, overwrite a function if it needs to be changed (we are indeed MPI-parallel)
• ?

julia> @code_warntype TreeMesh(
           (-1.0, -1.0), (1.0, 1.0), 
           n_cells_max=10, initial_refinement_level=1)
MethodInstance for (::Core.var"#Type##kw")(::NamedTuple{(:n_cells_max, :initial_refinement_level), Tuple{Int64, Int64}}, ::Type{TreeMesh}, ::Tuple{Float64, Float64}, ::Tuple{Float64, Float64})
  from (::Core.var"#Type##kw")(::Any, ::Type{TreeMesh}, coordinates_min::Tuple{Vararg{Real, NDIMS}}, coordinates_max::Tuple{Vararg{Real, NDIMS}}) where NDIMS in Trixi at /afs/math.uni-hamburg.de/users/am/bbb3094/.julia/packages/Trixi/zSXyl/src/meshes/tree_mesh.jl:93
Static Parameters
  NDIMS = 2
Arguments
  _::Core.Const(Core.var"#Type##kw"())
  @_2::NamedTuple{(:n_cells_max, :initial_refinement_level), Tuple{Int64, Int64}}
  @_3::Type{TreeMesh}
  coordinates_min::Tuple{Float64, Float64}
  coordinates_max::Tuple{Float64, Float64}
Locals
  n_cells_max::Int64
  periodicity::Bool
  initial_refinement_level::Int64
  refinement_patches::Tuple{}
  coarsening_patches::Tuple{}
  @_11::Int64
  @_12::Bool
  @_13::Int64
  @_14::Tuple{}
  @_15::Tuple{}
Body::TreeMesh{2, _A} where _A<:Trixi.AbstractTree{2}
1 ── %1  = Base.haskey(@_2, :n_cells_max)::Core.Const(true)
│          Core.typeassert(%1, Core.Bool)
│          (@_11 = Base.getindex(@_2, :n_cells_max))
└───       goto #3
2 ──       Core.Const(:(Core.UndefKeywordError(:n_cells_max)))
└───       Core.Const(:(@_11 = Core.throw(%5)))
3 ┄─ %7  = @_11::Int64
│          (n_cells_max = %7)
│    %9  = Base.haskey(@_2, :periodicity)::Core.Const(false)
└───       goto #5 if not %9
4 ──       Core.Const(:(@_12 = Base.getindex(@_2, :periodicity)))
└───       Core.Const(:(goto %14))
5 ┄─       (@_12 = true)
│    %14 = @_12::Core.Const(true)
│          (periodicity = %14)
│    %16 = Base.haskey(@_2, :initial_refinement_level)::Core.Const(true)
│          Core.typeassert(%16, Core.Bool)
│          (@_13 = Base.getindex(@_2, :initial_refinement_level))
└───       goto #7
6 ──       Core.Const(:(Core.UndefKeywordError(:initial_refinement_level)))
└───       Core.Const(:(@_13 = Core.throw(%20)))
7 ┄─ %22 = @_13::Int64
│          (initial_refinement_level = %22)
│    %24 = Base.haskey(@_2, :refinement_patches)::Core.Const(false)
└───       goto #9 if not %24
8 ──       Core.Const(:(@_14 = Base.getindex(@_2, :refinement_patches)))
└───       Core.Const(:(goto %29))
9 ┄─       (@_14 = ())
│    %29 = @_14::Core.Const(())
│          (refinement_patches = %29)
│    %31 = Base.haskey(@_2, :coarsening_patches)::Core.Const(false)
└───       goto #11 if not %31
10 ─       Core.Const(:(@_15 = Base.getindex(@_2, :coarsening_patches)))
└───       Core.Const(:(goto %36))
11 ┄       (@_15 = ())
│    %36 = @_15::Core.Const(())
│          (coarsening_patches = %36)
│    %38 = (:n_cells_max, :periodicity, :initial_refinement_level, :refinement_patches, :coarsening_patches)::Core.Const((:n_cells_max, :periodicity, :initial_refinement_level, :refinement_patches, :coarsening_patches))
│    %39 = Core.apply_type(Core.NamedTuple, %38)::Core.Const(NamedTuple{(:n_cells_max, :periodicity, :initial_refinement_level, :refinement_patches, :coarsening_patches)})
│    %40 = Base.structdiff(@_2, %39)::Core.Const(NamedTuple())
│    %41 = Base.pairs(%40)::Core.Const(Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}())
│    %42 = Base.isempty(%41)::Core.Const(true)
│          Core.typeassert(%42, Core.Bool)
└───       goto #13
12 ─       Core.Const(:(Base.kwerr(@_2, @_3, coordinates_min, coordinates_max)))
13 ┄ %46 = Trixi.:(var"#TreeMesh#151")(n_cells_max, periodicity::Core.Const(true), initial_refinement_level, refinement_patches, coarsening_patches, @_3, coordinates_min, coordinates_max)::TreeMesh{2, _A} where _A<:Trixi.AbstractTree{2}
└───       return %46

julia> @code_warntype P4estMesh((1, 1); 
           coordinates_min=(-1.0, -1.0), coordinates_max=(1.0, 1.0), 
           polydeg=1, initial_refinement_level=1)
MethodInstance for (::Core.var"#Type##kw")(::NamedTuple{(:coordinates_min, :coordinates_max, :polydeg, :initial_refinement_level), Tuple{Tuple{Float64, Float64}, Tuple{Float64, Float64}, Int64, Int64}}, ::Type{P4estMesh}, ::Tuple{Int64, Int64})
  from (::Core.var"#Type##kw")(::Any, ::Type{P4estMesh}, trees_per_dimension) in Trixi at /afs/math.uni-hamburg.de/users/am/bbb3094/.julia/packages/Trixi/zSXyl/src/meshes/p4est_mesh.jl:146
Arguments
  _::Core.Const(Core.var"#Type##kw"())
  @_2::NamedTuple{(:coordinates_min, :coordinates_max, :polydeg, :initial_refinement_level), Tuple{Tuple{Float64, Float64}, Tuple{Float64, Float64}, Int64, Int64}}                                     
  @_3::Type{P4estMesh}
  trees_per_dimension::Tuple{Int64, Int64}
Locals
  polydeg::Int64
  mapping::Nothing
  faces::Nothing
  coordinates_min::Tuple{Float64, Float64}
  coordinates_max::Tuple{Float64, Float64}
  RealT::Type{Float64}
  initial_refinement_level::Int64
  periodicity::Bool
  unsaved_changes::Bool
  @_14::Int64
  @_15::Nothing
  @_16::Nothing
  @_17::Tuple{Float64, Float64}
  @_18::Tuple{Float64, Float64}
  @_19::Type{Float64}
  @_20::Int64
  @_21::Bool
  @_22::Bool
Body::P4estMesh{2, Float64, _A, Ptr{P4est.LibP4est.p4est}, Ptr{P4est.LibP4est.p4est_ghost_t}, 4} where _A
1 ── %1  = Base.haskey(@_2, :polydeg)::Core.Const(true)
│          Core.typeassert(%1, Core.Bool)
│          (@_14 = Base.getindex(@_2, :polydeg))
└───       goto #3
2 ──       Core.Const(:(Core.UndefKeywordError(:polydeg)))
└───       Core.Const(:(@_14 = Core.throw(%5)))
3 ┄─ %7  = @_14::Int64
│          (polydeg = %7)
│    %9  = Base.haskey(@_2, :mapping)::Core.Const(false)
└───       goto #5 if not %9
4 ──       Core.Const(:(@_15 = Base.getindex(@_2, :mapping)))
└───       Core.Const(:(goto %14))
5 ┄─       (@_15 = Trixi.nothing)
│    %14 = @_15::Core.Const(nothing)
│          (mapping = %14)
│    %16 = Base.haskey(@_2, :faces)::Core.Const(false)
└───       goto #7 if not %16
6 ──       Core.Const(:(@_16 = Base.getindex(@_2, :faces)))
└───       Core.Const(:(goto %21))
7 ┄─       (@_16 = Trixi.nothing)
│    %21 = @_16::Core.Const(nothing)
│          (faces = %21)
│    %23 = Base.haskey(@_2, :coordinates_min)::Core.Const(true)
│          Core.typeassert(%23, Core.Bool)
│          (@_17 = Base.getindex(@_2, :coordinates_min))
└───       goto #9
8 ──       Core.Const(:(@_17 = Trixi.nothing))
9 ┄─ %28 = @_17::Tuple{Float64, Float64}
│          (coordinates_min = %28)
│    %30 = Base.haskey(@_2, :coordinates_max)::Core.Const(true)
│          Core.typeassert(%30, Core.Bool)
│          (@_18 = Base.getindex(@_2, :coordinates_max))
└───       goto #11
10 ─       Core.Const(:(@_18 = Trixi.nothing))
11 ┄ %35 = @_18::Tuple{Float64, Float64}
│          (coordinates_max = %35)
│    %37 = Base.haskey(@_2, :RealT)::Core.Const(false)
└───       goto #13 if not %37
12 ─       Core.Const(:(@_19 = Base.getindex(@_2, :RealT)))
└───       Core.Const(:(goto %42))
13 ┄       (@_19 = Trixi.Float64)
│    %42 = @_19::Core.Const(Float64)
│          (RealT = %42)
│    %44 = Base.haskey(@_2, :initial_refinement_level)::Core.Const(true)
│          Core.typeassert(%44, Core.Bool)
│          (@_20 = Base.getindex(@_2, :initial_refinement_level))
└───       goto #15
14 ─       Core.Const(:(@_20 = 0))
15 ┄ %49 = @_20::Int64
│          (initial_refinement_level = %49)
│    %51 = Base.haskey(@_2, :periodicity)::Core.Const(false)
└───       goto #17 if not %51
16 ─       Core.Const(:(@_21 = Base.getindex(@_2, :periodicity)))
└───       Core.Const(:(goto %56))
17 ┄       (@_21 = true)
│    %56 = @_21::Core.Const(true)
│          (periodicity = %56)
│    %58 = Base.haskey(@_2, :unsaved_changes)::Core.Const(false)
└───       goto #19 if not %58
18 ─       Core.Const(:(@_22 = Base.getindex(@_2, :unsaved_changes)))
└───       Core.Const(:(goto %63))
19 ┄       (@_22 = true)
│    %63 = @_22::Core.Const(true)
│          (unsaved_changes = %63)
│    %65 = (:polydeg, :mapping, :faces, :coordinates_min, :coordinates_max, :RealT, :initial_refinement_level, :periodicity, :unsaved_changes)::Core.Const((:polydeg, :mapping, :faces, :coordinates_min, :coordinates_max, :RealT, :initial_refinement_level, :periodicity, :unsaved_changes))
│    %66 = Core.apply_type(Core.NamedTuple, %65)::Core.Const(NamedTuple{(:polydeg, :mapping, :faces, :coordinates_min, :coordinates_max, :RealT, :initial_refinement_level, :periodicity, :unsaved_changes)})
│    %67 = Base.structdiff(@_2, %66)::Core.Const(NamedTuple())
│    %68 = Base.pairs(%67)::Core.Const(Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}())
│    %69 = Base.isempty(%68)::Core.Const(true)
│          Core.typeassert(%69, Core.Bool)
└───       goto #21
20 ─       Core.Const(:(Base.kwerr(@_2, @_3, trees_per_dimension)))
21 ┄ %73 = Trixi.:(var"#P4estMesh#223")(polydeg, mapping, faces, coordinates_min, coordinates_max, RealT::Core.Const(Float64), initial_refinement_level, periodicity::Core.Const(true), unsaved_changes::Core.Const(true), @_3, trees_per_dimension)::P4estMesh{2, Float64, _A, Ptr{P4est.LibP4est.p4est}, Ptr{P4est.LibP4est.p4est_ghost_t}, 4} where _A
└───       return %73