fix buildkite run so that unit tests run on GPU

.buildkite/pipeline.yml

@@ -26,7 +26,7 @@ steps:
       - "julia --project=.buildkite -e 'using Pkg; Pkg.status()'"
 
       - echo "--- Instantiate test"
-      - "julia --project=test -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.instantiate(;verbose=true)'"
+      - "julia --project=test -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.add(\"MPI\"); Pkg.add(\"CUDA\"); Pkg.instantiate(;verbose=true)'"
       - "julia --project=test -e 'using Pkg; Pkg.status()'"
 
       - echo "--- Instantiate lib/ClimaLandSimulations"
@@ -63,15 +63,6 @@ steps:
         command: "julia --color=yes --project=.buildkite experiments/standalone/Soil/richards_comparison.jl"
         artifact_paths: "experiments/standalone/Soil/cpu/comparison*png"
 
-      - label: "Richards comparison to Bonan: GPU"
-        command: "julia --color=yes --project=.buildkite experiments/standalone/Soil/richards_comparison.jl"
-        artifact_paths: "experiments/standalone/Soil/gpu/comparison*png"
-        agents:
-          slurm_ntasks: 1
-          slurm_gres: "gpu:1"
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-
       - label: "vaira_test"
         command: "julia --color=yes --project=.buildkite experiments/integrated/fluxnet/run_fluxnet.jl US-Var"
         artifact_paths: "experiments/integrated/fluxnet/US-Var/out/*png"
@@ -130,7 +121,16 @@ steps:
         artifact_paths: "experiments/standalone/Soil/artifacts/gpu/*png"
         agents:
           slurm_ntasks: 1
-          slurm_gres: "gpu:1"
+          slurm_gpus: 1
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+
+      - label: "Richards comparison to Bonan: GPU"
+        command: "julia --color=yes --project=.buildkite experiments/standalone/Soil/richards_comparison.jl"
+        artifact_paths: "experiments/standalone/Soil/gpu/comparison*png"
+        agents:
+          slurm_ntasks: 1
+          slurm_gpus: 1
         env:
           CLIMACOMMS_DEVICE: "CUDA"
 
@@ -172,21 +172,25 @@ steps:
         command: "julia --color=yes --project=test test/runtests.jl"
         agents:
           slurm_ntasks: 1
-          slurm_gres: "gpu:p100:1"
+          slurm_gpus: 1
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
 
       - label: "soil/canopy lsm performance on GPU"
         command: "julia --color=yes --project=.buildkite experiments/integrated/performance/profile_allocations.jl"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_ntasks: 1
-          slurm_gres: "gpu:p100:1"
+          slurm_gpus: 1
         artifact_paths: "experiments/integrated/performance/flame*html"
 
       - label: "Global Bucket on GPU (functional albedo)"
         key: "global_bucket_function_gpu"
         command: "julia --color=yes --project=.buildkite experiments/standalone/Bucket/global_bucket_function.jl"
         agents:
           slurm_ntasks: 1
-          slurm_gres: "gpu:p100:1"
+          slurm_gpus: 1
         env:
           CLIMACOMMS_DEVICE: "CUDA"
         artifact_paths: "experiments/standalone/Bucket/artifacts/*gpu*"
@@ -196,7 +200,7 @@ steps:
         command: "julia --color=yes --project=.buildkite experiments/standalone/Bucket/bucket_era5.jl"
         agents:
           slurm_ntasks: 1
-          slurm_gres: "gpu:p100:1"
+          slurm_gpus: 1
         env:
           CLIMACOMMS_DEVICE: "CUDA"
         artifact_paths: "experiments/standalone/Bucket/artifacts_staticmap/*gpu*"
@@ -206,7 +210,7 @@ steps:
         command: "julia --color=yes --project=.buildkite experiments/standalone/Bucket/global_bucket_temporalmap.jl"
         agents:
           slurm_ntasks: 1
-          slurm_gres: "gpu:p100:1"
+          slurm_gpus: 1
         env:
           CLIMACOMMS_DEVICE: "CUDA"
         artifact_paths: "experiments/standalone/Bucket/artifacts_temporalmap/*gpu*"

experiments/integrated/global/global_soil_canopy.jl

@@ -145,14 +145,14 @@ function zenith_angle(
     start_date;
     latitude = ClimaCore.Fields.coordinate_field(surface_space).lat,
     longitude = ClimaCore.Fields.coordinate_field(surface_space).long,
-    insol_params::Insolation.Parameters.InsolationParameters{FT} = earth_param_set.insol_params,
-) where {FT}
+    insol_params::Insolation.Parameters.InsolationParameters{_FT} = earth_param_set.insol_params,
+) where {_FT}
     # This should be time in UTC
     current_datetime = start_date + Dates.Second(round(t))
 
     # Orbital Data uses Float64, so we need to convert to our sim FT
     d, δ, η_UTC =
-        FT.(
+        _FT.(
             Insolation.helper_instantaneous_zenith_angle(
                 current_datetime,
                 start_date,
@@ -235,7 +235,8 @@ LAIfunction = TimeVaryingInput(
 )
 ai_parameterization = Canopy.PrescribedSiteAreaIndex{FT}(LAIfunction, SAI, RAI)
 
-function root_distribution(z::T; rooting_depth = rooting_depth) where {T}
+# TODO: use rooting_depth from `global_parameters`
+function root_distribution(z::T; rooting_depth = T(0.5)) where {T}
     return T(1.0 / rooting_depth) * exp(z / T(rooting_depth)) # 1/m
 end
 

experiments/integrated/performance/profile_allocations.jl

@@ -4,6 +4,7 @@ using ClimaCore
 using Dates
 using Insolation
 import ClimaComms
+@static pkgversion(ClimaComms) >= v"0.6" && ClimaComms.@import_required_backends
 import ClimaUtilities.TimeVaryingInputs: TimeVaryingInput
 
 using ClimaLand
@@ -62,8 +63,9 @@ function set_initial_conditions(land, t0)
         )
 
     for i in 1:2
+        S_l = S_l_ini[i]
         Y.canopy.hydraulics.ϑ_l.:($i) .=
-            augmented_liquid_fraction.(canopy_params.ν, S_l_ini[i])
+            augmented_liquid_fraction.(canopy_params.ν, S_l)
     end
 
     Y.canopy.energy.T = FT(297.5)
@@ -137,14 +139,14 @@ function zenith_angle(
     t,
     start_date;
     cd_field = sfc_cds,
-    insol_params::Insolation.Parameters.InsolationParameters{FT} = earth_param_set.insol_params,
-) where {FT}
+    insol_params::Insolation.Parameters.InsolationParameters{_FT} = earth_param_set.insol_params,
+) where {_FT}
     # This should be time in UTC
     current_datetime = start_date + Dates.Second(round(t))
 
     # Orbital Data uses Float64, so we need to convert to our sim FT
     d, δ, η_UTC =
-        FT.(
+        _FT.(
             Insolation.helper_instantaneous_zenith_angle(
                 current_datetime,
                 start_date,

ext/CreateParametersExt.jl

@@ -66,7 +66,6 @@ function LP.LandParameters(toml_dict::CP.AbstractTOMLDict)
         insol_params,
     )
 end
-Base.broadcastable(ps::LP.LandParameters) = tuple(ps)
 
 """
     AutotrophicRespirationParameters(FT; kwargs...)

src/shared_utilities/Parameters.jl

@@ -28,6 +28,7 @@ Base.@kwdef struct LandParameters{FT, TP, SFP, IP} <: ALP
 end
 
 Base.eltype(::LandParameters{FT}) where {FT} = FT
+Base.broadcastable(ps::LandParameters) = tuple(ps)
 
 # wrapper methods:
 P_ref(ps::ALP) = ps.MSLP

src/standalone/Vegetation/radiation.jl

@@ -24,7 +24,7 @@ struct ConstantGFunction{FT} <: AbstractGFunction{FT}
 end
 
 # Make the ConstantGFunction broadcastable
-Base.broadcastable(G::ConstantGFunction) = Ref(G)
+Base.broadcastable(G::ConstantGFunction) = tuple(G)
 
 """
     CLMGFunction
@@ -38,7 +38,7 @@ struct CLMGFunction{FT} <: AbstractGFunction{FT}
 end
 
 # Make the CLMGFunction broadcastable
-Base.broadcastable(G::CLMGFunction) = Ref(G)
+Base.broadcastable(G::CLMGFunction) = tuple(G)
 
 """
     BeerLambertParameters{FT <: AbstractFloat}
@@ -106,7 +106,7 @@ Base.@kwdef struct TwoStreamParameters{
     "Typical wavelength per NIR photon (m)"
     λ_γ_NIR::FT
     "Number of layers to partition the canopy into when integrating the
-    absorption over the canopy vertically. Unrelated to the number of layers in 
+    absorption over the canopy vertically. Unrelated to the number of layers in
     the vertical discretization of the canopy for the plant hydraulics model.
     (Constant, and should eventually move to ClimaParams)"
     n_layers::UInt64