From 58fca1668aecd6fb1afd12a441256ad35900e075 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 5 Jul 2024 15:02:23 -0400 Subject: [PATCH] Update (partially) global-workflow for orion+rocky9 (#2741) This PR: - updates a few submodules (GSI, GSI-utils, GSI-monitor, UFS_utils, GFS-utils) to include recent update to their modulefiles for Orion+Rocky9 upgrade - updates the modulefiles in global-workflow to load modules from Orion+Rocky9 paths - updates modulefiles for `gwsetup` and `gwci` as well. - removes NCL and GEMPAK from Orion. NCL is not used and GEMPAK is not installed. - adds `parm/config.gfs/config.resources.ORION` to address GSI performance degradation after Rocky 9 upgrade. This PR: - does not update the build for UPP. Standalone UPP is not available via ufs-weather-model as of #2729 - will need a follow-up update for `prepobs` and `fit2obs` updated locations when they are installed in `glopara` space on Orion. # Type of change - Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO # How has this been tested? This PR is not sufficient for Orion. This PR must be tested on other platforms (Hera, WCOSS2) as this PR updates submodules. # Checklist - [ ] Any dependent changes have been merged and published - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] My changes generate no new warnings - [ ] New and existing tests pass with my changes - [ ] I have made corresponding changes to the documentation if necessary --------- Co-authored-by: Kate Friedman --- modulefiles/module_base.orion.lua | 9 ++++----- modulefiles/module_gwci.orion.lua | 6 +++--- modulefiles/module_gwsetup.orion.lua | 6 +++--- parm/config/gfs/config.resources.ORION | 17 +++++++++++++++++ sorc/gfs_utils.fd | 2 +- sorc/gsi_enkf.fd | 2 +- sorc/gsi_monitor.fd | 2 +- sorc/gsi_utils.fd | 2 +- sorc/ufs_utils.fd | 2 +- ush/detect_machine.sh | 4 ++-- versions/build.orion.ver | 6 +++--- versions/run.orion.ver | 12 +++--------- 12 files changed, 40 insertions(+), 30 deletions(-) create mode 100644 parm/config/gfs/config.resources.ORION diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index 4d747512db..74d6f0aee6 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -9,8 +9,6 @@ load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) load(pathJoin("python", (os.getenv("python_ver") or "None"))) -load(pathJoin("gempak", (os.getenv("gempak_ver") or "None"))) -load(pathJoin("ncl", (os.getenv("ncl_ver") or "None"))) load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) @@ -44,11 +42,12 @@ setenv("WGRIB2","wgrib2") setenv("WGRIB","wgrib") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) +--prepend_path("MODULEPATH", "/work/noaa/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles") +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/kfriedma/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) -prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +--prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/kfriedma/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) whatis("Description: GFS run environment") diff --git a/modulefiles/module_gwci.orion.lua b/modulefiles/module_gwci.orion.lua index cef7acf308..e2213932d7 100644 --- a/modulefiles/module_gwci.orion.lua +++ b/modulefiles/module_gwci.orion.lua @@ -2,10 +2,10 @@ help([[ Load environment to run GFS workflow ci scripts on Orion ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env-rocky9/install/modulefiles/Core") -load(pathJoin("stack-intel", os.getenv("2022.0.2"))) -load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1"))) +load(pathJoin("stack-intel", os.getenv("2021.9.0"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.9.0"))) load(pathJoin("netcdf-c", os.getenv("4.9.2"))) load(pathJoin("netcdf-fortran", os.getenv("4.6.1"))) diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/module_gwsetup.orion.lua index 37cb511659..b8e2fc8a9f 100644 --- a/modulefiles/module_gwsetup.orion.lua +++ b/modulefiles/module_gwsetup.orion.lua @@ -4,11 +4,11 @@ Load environment to run GFS workflow ci scripts on Orion prepend_path("MODULEPATH", "/apps/modulefiles/core") load(pathJoin("contrib","0.1")) -load(pathJoin("rocoto","1.3.3")) +load(pathJoin("rocoto","1.3.7")) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env-rocky9/install/modulefiles/Core") -local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" local python_ver=os.getenv("python_ver") or "3.11.6" load(pathJoin("stack-intel", stack_intel_ver)) diff --git a/parm/config/gfs/config.resources.ORION b/parm/config/gfs/config.resources.ORION new file mode 100644 index 0000000000..e3e81b0182 --- /dev/null +++ b/parm/config/gfs/config.resources.ORION @@ -0,0 +1,17 @@ +#! /usr/bin/env bash + +# Orion-specific job resources + +case ${step} in + "anal") + # TODO: + # On Orion, after Rocky 9 upgrade, GSI performance is degraded. + # Remove this block once GSI issue is resolved + # https://github.com/NOAA-EMC/GSI/pull/764 + # https://github.com/JCSDA/spack-stack/issues/1166 + export wtime_anal_gdas="02:40:00" + export wtime_anal_gfs="02:00:00" + ;; + *) + ;; +esac diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 0cdc279526..02ce084c24 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 0cdc2795260fc1b59e86a873729433a470794a97 +Subproject commit 02ce084c244823e22661d493a50236b7d5eaf70a diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 8e279f9c73..529bb796be 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 8e279f9c734097f673b07e80f385b2623d13ba4a +Subproject commit 529bb796bea0e490f186729cd168a91c034bb12d diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index f9d6f5f744..e1f9f21af1 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit f9d6f5f744462a449e70abed8c5860b1c4564ad8 +Subproject commit e1f9f21af16ce912fdc2cd75c5b27094a550a0c5 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index 4332814529..9382fd01c2 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit 4332814529465ab8eb58e43a38227b952ebfca49 +Subproject commit 9382fd01c2a626c8934c3f553d420a45de2b4dec diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 2794d413d0..3ef2e6bd72 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 2794d413d083b43d9ba37a15375d5c61b610d29e +Subproject commit 3ef2e6bd725d2662fd6ee95897cb7bac222e5144 diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index cfd0fa97e2..b049a6040e 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -45,7 +45,7 @@ case $(hostname -f) in *) MACHINE_ID=UNKNOWN ;; # Unknown platform esac -if [[ ${MACHINE_ID} == "UNKNOWN" ]]; then +if [[ ${MACHINE_ID} == "UNKNOWN" ]]; then case ${PW_CSP:-} in "aws" | "google" | "azure") MACHINE_ID=noaacloud ;; *) PW_CSP="UNKNOWN" @@ -75,7 +75,7 @@ elif [[ -d /scratch1 ]]; then MACHINE_ID=hera elif [[ -d /work ]]; then # We are on MSU Orion or Hercules - mount=$(findmnt -n -o SOURCE /home) + mount=$(findmnt -n -o SOURCE /home) if [[ ${mount} =~ "hercules" ]]; then MACHINE_ID=hercules else diff --git a/versions/build.orion.ver b/versions/build.orion.ver index df7856110d..834ecfc166 100644 --- a/versions/build.orion.ver +++ b/versions/build.orion.ver @@ -1,5 +1,5 @@ -export stack_intel_ver=2022.0.2 -export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-env +export stack_intel_ver=2021.9.0 +export stack_impi_ver=2021.9.0 +export spack_env=gsi-addon-env-rocky9 source "${HOMEgfs:-}/versions/build.spack.ver" export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.orion.ver b/versions/run.orion.ver index 2fdeae8888..112636fb20 100644 --- a/versions/run.orion.ver +++ b/versions/run.orion.ver @@ -1,12 +1,6 @@ -export stack_intel_ver=2022.0.2 -export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-env - -export ncl_ver=6.6.2 -export gempak_ver=7.5.1 +export stack_intel_ver=2021.9.0 +export stack_impi_ver=2021.9.0 +export spack_env=gsi-addon-env-rocky9 source "${HOMEgfs:-}/versions/run.spack.ver" export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" - -#cdo is older on Orion -export cdo_ver=2.0.5