fix buildkite run so that unit tests run on GPU

[kmdeck]

Purpose

Some of our buildkite runs were not running on GPU. Update buildkite so that they are.

To-do

1. Fix bug in experiments/integrated/performance/profile_allocations.jl. Note that the same code runs on the Clima cluster on GPU. This is the error message:

(base) [kdeck@hpc-22-32 ClimaLand.jl]$ julia --project=.buildkite  experiments/integrated/performance/profile_allocations.jl
[ Info: ProfileCanvas found, running with profiler
ERROR: LoadError: InvalidIRError: compiling MethodInstance for ClimaCoreCUDAExt.knl_copyto!(::ClimaCore.DataLayouts.VIJFH{Float64, 5, 2, 600, CUDA.CuDeviceArray{Float64, 5, 1}}, ::Base.Broadcast.Broadcasted{ClimaCore.DataLayouts.VIJFHStyle{5, 2, 600, CUDA.CuArray{Float64, N, CUDA.DeviceMemory} where N}, NTuple{5, Base.OneTo{Int64}}, typeof(ClimaCore.RecursiveApply.radd), Tuple{ClimaCore.DataLayouts.VIJFH{Float64, 5, 2, 600, CUDA.CuDeviceArray{Float64, 5, 1}}, Base.Broadcast.Broadcasted{ClimaCore.DataLayouts.VIJFHStyle{5, 2, 600, CUDA.CuArray{Float64, N, CUDA.DeviceMemory} where N}, Nothing, typeof(ClimaCore.RecursiveApply.rsub), Tuple{Base.Broadcast.Broadcasted{ClimaCore.DataLayouts.VIJFHStyle{5, 2, 600, CUDA.CuArray{Float64, N, CUDA.DeviceMemory} where N}, Nothing, typeof(phase_change_source), Tuple{ClimaCore.DataLayouts.VIJFH{Float64, 5, 2, 600, CUDA.CuDeviceArray{Float64, 5, 1}}, ClimaCore.DataLayouts.VIJFH{Float64, 5, 2, 600, CUDA.CuDeviceArray{Float64, 5, 1}}, ClimaCore.DataLayouts.VIJFH{Float64, 5, 2, 600, CUDA.CuDeviceArray{Float64, 5, 1}}, Base.Broadcast.Broadcasted{ClimaCore.DataLayouts.VIJFHStyle{5, 2, 600, CUDA.CuArray{Float64, N, CUDA.DeviceMemory} where N}, Nothing, typeof(thermal_time), Tuple{Base.Broadcast.Broadcasted{ClimaCore.DataLayouts.VIJFHStyle{5, 2, 600, CUDA.CuArray{Float64, N, CUDA.DeviceMemory} where N}, Nothing, typeof(volumetric_heat_capacity), Tuple{ClimaCore.DataLayouts.VIJFH{Float64, 5, 2, 600, CUDA.CuDeviceArray{Float64, 5, 1}}, ClimaCore.DataLayouts.VIJFH{Float64, 5, 2, 600, CUDA.CuDeviceArray{Float64, 5, 1}}, Float64, Tuple{ClimaLand.Parameters.LandParameters{Float64, Thermodynamics.Parameters.ThermodynamicsParameters{Float64}, SurfaceFluxes.Parameters.SurfaceFluxesParameters{Float64, SurfaceFluxes.UniversalFunctions.BusingerParams{Float64}, Thermodynamics.Parameters.ThermodynamicsParameters{Float64}}, Insolation.Parameters.InsolationParameters{Float64}}}}}, ClimaCore.DataLayouts.VIJFH{Float64, 5, 2, 600, SubArray{Float64, 5, CUDA.CuDeviceArray{Float64, 5, 1}, Tuple{Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, ClimaCore.DataLayouts.VIJFH{Float64, 5, 2, 600, CUDA.CuDeviceArray{Float64, 5, 1}}}}, Float64, Float64, Tuple{vanGenuchten{Float64}}, Tuple{ClimaLand.Parameters.LandParameters{Float64, Thermodynamics.Parameters.ThermodynamicsParameters{Float64}, SurfaceFluxes.Parameters.SurfaceFluxesParameters{Float64, SurfaceFluxes.UniversalFunctions.BusingerParams{Float64}, Thermodynamics.Parameters.ThermodynamicsParameters{Float64}}, Insolation.Parameters.InsolationParameters{Float64}}}}}}}}}) resulted in invalid LLVM IR
Reason: unsupported dynamic function invocation (call to getindex(t::Tuple, i::Int64) @ Base tuple.jl:31)
Stacktrace:
 [1] _getindex (repeats 7 times)
   @ ./broadcast.jl:705
 [2] _broadcast_getindex
   @ ./broadcast.jl:681
 [3] _getindex
   @ ./broadcast.jl:706
 [4] _broadcast_getindex
   @ ./broadcast.jl:681
 [5] _getindex
   @ ./broadcast.jl:706
 [6] _getindex
   @ ./broadcast.jl:705
 [7] _broadcast_getindex
   @ ./broadcast.jl:681
 [8] getindex
   @ ./broadcast.jl:636
 [9] knl_copyto!
   @ ~/.julia/packages/ClimaCore/Zoz8z/ext/cuda/data_layouts_copyto.jl:13
Reason: unsupported dynamic function invocation (call to matric_potential)
Stacktrace:
  [1] phase_change_source
    @ /central/home/kdeck/ClimaLand.jl/src/standalone/Soil/soil_heat_parameterizations.jl:65

/central/home/kdeck/ClimaLand.jl/src/standalone/Soil/soil_heat_parameterizations.jl:65:

function phase_change_source(
    θ_l::FT,
    θ_i::FT,
    T::FT,
    τ::FT,
    ν::FT,
    θ_r::FT,
    hydrology_cm::C,
    earth_param_set::EP,
) where {FT, EP, C}
    _ρ_i = FT(LP.ρ_cloud_ice(earth_param_set))
    _ρ_l = FT(LP.ρ_cloud_liq(earth_param_set))
    _LH_f0 = FT(LP.LH_f0(earth_param_set))
    _T_freeze = FT(LP.T_freeze(earth_param_set))
    _grav = FT(LP.grav(earth_param_set))
    # According to Dall'Amico (text above equation 1), ψw0 corresponds
    # to the matric potential corresponding to the total water content (liquid and ice).
    θtot = min(_ρ_i / _ρ_l * θ_i + θ_l, ν)
    # This is consistent with Equation (22) of Dall'Amico
    ψw0 = matric_potential(hydrology_cm, effective_saturation(ν, θtot, θ_r)) # <---- Line 65

    ψT = _LH_f0 / _T_freeze / _grav * (T - _T_freeze) * heaviside(_T_freeze - T)
    # Equation (23) of Dall'Amico
    θstar = inverse_matric_potential(hydrology_cm, ψw0 + ψT) * (ν - θ_r) + θ_r

    return (θ_l - θstar) / τ
end

Called by:

    params = model.parameters
    (; ν, ρc_ds, θ_r, hydrology_cm, earth_param_set) = params
    _ρ_l = FT(LP.ρ_cloud_liq(earth_param_set))
    _ρ_i = FT(LP.ρ_cloud_ice(earth_param_set))
    Δz = model.domain.fields.Δz # center face distance
    @. dY.soil.ϑ_l +=
        -phase_change_source(
            p.soil.θ_l,
            Y.soil.θ_i,
            p.soil.T,
            thermal_time(
                volumetric_heat_capacity(
                    p.soil.θ_l,
                    Y.soil.θ_i,
                    ρc_ds,
                    earth_param_set,
                ),
                Δz,
                p.soil.κ,
            ),
            ν,
            θ_r,
            hydrology_cm,
            earth_param_set,
        )

Note that typeof(hydrology_cm) <: AbstractSoilHydrologyClosure and we have set Base.broadcastable(x::AbstractSoilHydrologyClosure) = tuple(x)

Content

---

• [ ] I have read and checked the items on the review checklist.