ClimaCore.jl/.buildkite/perf/pipeline.yml at main · CliMA/ClimaCore.jl · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
# A pipeline for downstream performance checks
agents:
  queue: clima
  slurm_time: 14:00:00
  modules: climacommon/2025_05_15

env:
  JULIA_DEPOT_PATH: "${BUILDKITE_BUILD_PATH}/${BUILDKITE_PIPELINE_SLUG}/depot/default"
  JULIA_MAX_NUM_PRECOMPILE_FILES: 100
  GKSwstype: 100
  SLURM_KILL_BAD_EXIT: 1
  COUPLER_PATH: ".buildkite/perf/ClimaCoupler.jl"
  COUPLER_COMMIT: "27f9a4736d3f324a0b35281d31b2b13d64f7d92c"
  CONFIG_PATH: "$COUPLER_PATH/config/nightly_configs"
  BENCHMARK_CONFIG_PATH: "$COUPLER_PATH/config/benchmark_configs"
  LAND_PATH: ".buildkite/perf/ClimaLand.jl"
  LAND_COMMIT: "4009df0ac2a5b62e84c5d21734926aa00338aee0"

timeout_in_minutes: 840

steps:
  - label: "init :GPU:"
    key: "init_gpu_env"
    command:
      - git clone https://github.com/CliMA/ClimaCoupler.jl.git $COUPLER_PATH && bash -c "cd $COUPLER_PATH && git checkout $COUPLER_COMMIT"
      - "echo $$JULIA_DEPOT_PATH"
      - echo "--- Instantiate AMIP env"
      # Instantiate and dev install ClimaCore
      - julia --project=$COUPLER_PATH/experiments/AMIP/ -e 'using Pkg; Pkg.Registry.update(); Pkg.instantiate(;verbose=true)'
      - julia --project=$COUPLER_PATH/experiments/AMIP/ -e 'using Pkg; Pkg.develop(; path = ".")'
      - julia --project=$COUPLER_PATH/experiments/AMIP/ -e 'using Pkg; Pkg.add("MPI")'
      - julia --project=$COUPLER_PATH/experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'
      - julia --project=$COUPLER_PATH/experiments/AMIP/ -e 'using Pkg; Pkg.status()'
    agents:
      slurm_gpus: 1
      slurm_cpus_per_task: 8
    env:
      JULIA_NUM_PRECOMPILE_TASKS: 8
      JULIA_MAX_NUM_PRECOMPILE_FILES: 50

  - label: "Flagship AMIP GPU with prognostic EDMF + 1M + integrated land (16 helems)"
    key: "gpu_amip_progedmf_1M_land_he16"
    depends_on: "init_gpu_env"
    command:
      - "srun julia --threads=3 --color=yes --project=$COUPLER_PATH/experiments/AMIP/ $COUPLER_PATH/experiments/AMIP/run_simulation.jl --config_file $BENCHMARK_CONFIG_PATH/amip_progedmf_1m_land_he16.yml --job_id gpu_amip_progedmf_1M_land_he16"
      - echo "--- Checking SYPD from artifacts"
      - bash .buildkite/perf/check-perf.sh output/gpu_amip_progedmf_1M_land_he16/artifacts/sypd.txt
    artifact_paths: "output/gpu_amip_progedmf_1M_land_he16/artifacts/*"
    env:
      CLIMACOMMS_DEVICE: "CUDA"
      # Fallback used when no passing main build metadata is available yet.
      BASELINE: "0.278"
      MIN_PERCENT_CHANGE: "-1"
    agents:
      slurm_gpus_per_task: 1
      slurm_cpus_per_task: 4
      slurm_ntasks: 1
      slurm_mem: 16GB

  - label: "init land :GPU:"
    key: "init_gpu_land_env"
    command:
      - git clone https://github.com/CliMA/ClimaLand.jl.git $LAND_PATH && bash -c "cd $LAND_PATH && git checkout $LAND_COMMIT"
      - echo "--- Instantiate ClimaLand benchmark env"
      - julia --project=$LAND_PATH -e 'using Pkg; Pkg.Registry.update(); Pkg.instantiate(;verbose=true)'
      # The benchmark scripts run against .buildkite, which devs ClimaLand itself
      - julia --project=$LAND_PATH/.buildkite -e 'using Pkg; Pkg.develop(; path = ENV["LAND_PATH"]); Pkg.develop(; path = pwd()); Pkg.instantiate(;verbose=true)'
      - julia --project=$LAND_PATH/.buildkite -e 'using Pkg; Pkg.precompile()'
      - julia --project=$LAND_PATH/.buildkite -e 'using CUDA; CUDA.precompile_runtime()'
      - julia --project=$LAND_PATH/.buildkite -e 'using Pkg; Pkg.status()'
    agents:
      slurm_gpus: 1
      slurm_cpus_per_task: 8
    env:
      JULIA_NUM_PRECOMPILE_TASKS: 8
      JULIA_MAX_NUM_PRECOMPILE_FILES: 50

  # ClimaLand is the only model that runs with a land/sea mask, so it is the
  # only one whose timings track the masked loop and column operator paths.
  # Both steps run ClimaLand's own benchmark scripts, which sample the
  # simulation repeatedly and report a mean and standard deviation; the model
  # setup therefore lives in ClimaLand rather than being duplicated here.
  - label: "Global soil model GPU (masked columns)"
    key: "gpu_climaland_soil"
    depends_on: "init_gpu_land_env"
    command:
      - "srun bash .buildkite/perf/run-land-benchmark.sh $LAND_PATH soil"
      - echo "--- Checking the benchmark rate"
      - bash .buildkite/perf/check-perf.sh soil_benchmark_gpu/sims_per_second.txt
    artifact_paths: "soil_benchmark_gpu/*"
    env:
      CLIMACOMMS_DEVICE: "CUDA"
      JULIA_CUDA_MEMORY_POOL: "cuda"
      # No fallback yet: the first passing main build records the baseline.
      # Set BASELINE once a few builds show the spread that the script reports
      # alongside the mean.
      MIN_PERCENT_CHANGE: "-5"
    agents:
      slurm_gpus_per_task: 1
      slurm_cpus_per_task: 4
      slurm_ntasks: 1
      slurm_mem: 16GB

  - label: "Flagship snowy land model GPU (soil + canopy + snow)"
    key: "gpu_climaland_snowy_land"
    depends_on: "init_gpu_land_env"
    command:
      - "srun bash .buildkite/perf/run-land-benchmark.sh $LAND_PATH snowy_land"
      - echo "--- Checking the benchmark rate"
      - bash .buildkite/perf/check-perf.sh snowy_land_benchmark_gpu/sims_per_second.txt
    artifact_paths: "snowy_land_benchmark_gpu/*"
    env:
      CLIMACOMMS_DEVICE: "CUDA"
      JULIA_CUDA_MEMORY_POOL: "cuda"
      # No fallback yet: the first passing main build records the baseline.
      MIN_PERCENT_CHANGE: "-5"
    agents:
      slurm_gpus_per_task: 1
      slurm_cpus_per_task: 4
      slurm_ntasks: 1
      slurm_mem: 16GB