Temporary fix for Rocky8 (rt_dev1) (NOAA-GSL#58)

* Temporary fix for Rocky 8 * replace gfs_utils.fd pointer to KaYee * Rocky8 pointer update * Increase more wallclock time to run 120hrs
kayeekayee · May 17, 2024 · 7085f9f · 7085f9f
1 parent a33ce41
commit 7085f9f
Show file tree

Hide file tree

Showing 15 changed files with 26 additions and 21 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -7,7 +7,7 @@
 	url = https://github.com/NOAA-EMC/wxflow
 [submodule "sorc/gfs_utils.fd"]
 	path = sorc/gfs_utils.fd
-	url = https://github.com/NOAA-EMC/gfs-utils
+	url = https://github.com/kayeekayee/gfs-utils
 [submodule "sorc/ufs_utils.fd"]
 	path = sorc/ufs_utils.fd
 	url = https://github.com/NOAA-GSL/UFS_UTILS.git

diff --git a/INFO b/INFO
@@ -1,3 +1,7 @@
+04-10-24
+========
+   temporary fix for Rocky 8
+
 03-28-24
 ========
   change dt_inner to 50 for aerosol aware Thompson

diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
@@ -21,8 +21,8 @@ status=$?
 [[ $status -ne 0 ]] && exit $status
 
 #JKH load ufs-utils modules
-module use /scratch1/BMC/gsd-fv3-dev/Judy.K.Henderson/test/gsl_ufs_dev/sorc/ufs_utils.fd/modulefiles
-module load build.hera.intel
+module use /scratch1/BMC/gsd-fv3/rtruns/modulefiles
+module load build.hera.gfsinit
 module list
 
 ###############################################################

diff --git a/jobs/rocoto/remapgrib.ksh b/jobs/rocoto/remapgrib.ksh
@@ -4,8 +4,8 @@
 #    236 201 244 130 224 242
 
 # initialize
-module load intel/2022.1.2
-module load wgrib2/2.0.8
+module load gnu/13.2.0 intel/2023.2.0 netcdf/4.7.0 wgrib2/3.1.2_ncep
+module list
 ECHO=echo
 MKDIR=mkdir
 LN=ln
@@ -20,6 +20,7 @@ echo "fcst        = ${fcst}"
 echo "GRID_NAMES  = ${GRID_NAMES}"
 echo
 
+echo `which wgrib2`
 #########################
 # Grid Definitions
 #########################

diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua
@@ -2,9 +2,8 @@ help([[
 Load environment to run GFS on Hera
 ]])
 
-spack_stack_ver=(os.getenv("spack_stack_ver") or "None")
-spack_env=(os.getenv("spack_env") or "None")
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core")
+local spack_mod_path=(os.getenv("spack_mod_path") or "None")
+prepend_path("MODULEPATH", spack_mod_path)
 
 load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None")))
 load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None")))
@@ -42,7 +41,7 @@ setenv("WGRIB2","wgrib2")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
 --prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
-prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
+prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/dev-gfsv17/modulefiles"))
 load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
 
 prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))

diff --git a/modulefiles/module_gwci.hera.lua b/modulefiles/module_gwci.hera.lua
@@ -2,13 +2,13 @@ help([[
 Load environment to run GFS workflow setup scripts on Hera
 ]])
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core")
 
 load(pathJoin("stack-intel", os.getenv("2021.5.0")))
 load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1")))
 
 load(pathJoin("netcdf-c", os.getenv("4.9.2")))
-load(pathJoin("netcdf-fortran", os.getenv("4.6.0")))
+load(pathJoin("netcdf-fortran", os.getenv("4.6.1")))
 load(pathJoin("nccmp","1.9.0.1"))
 load(pathJoin("wgrib2", "2.0.8"))
 

diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua
@@ -4,10 +4,10 @@ Load environment to run GFS workflow setup scripts on Hera
 
 load(pathJoin("rocoto"))
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core")
 
 local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
-local python_ver=os.getenv("python_ver") or "3.10.8"
+local python_ver=os.getenv("python_ver") or "3.11.6"
 
 load(pathJoin("stack-intel", stack_intel_ver))
 load(pathJoin("python", python_ver))

diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd
diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd
diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd
diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
@@ -16,7 +16,8 @@ source "${HOMEgfs}/modulefiles/module-setup.sh.inc"
 source "${HOMEgfs}/versions/run.ver"
 
 # Load our modules:
-module use "${HOMEgfs}/modulefiles"
+#KYWmodule use "${HOMEgfs}/modulefiles"
+module use "/scratch1/BMC/gsd-fv3/rtruns/modulefiles/"
 
 if [[ -d /lfs/f1 ]]; then
   # We are on WCOSS2 (Cactus or Dogwood)

diff --git a/versions/build.hera.ver b/versions/build.hera.ver
@@ -1,5 +1,5 @@
 export stack_intel_ver=2021.5.0
 export stack_impi_ver=2021.5.1
-export spack_env=gsi-addon-dev
+export spack_env=gsi-addon-dev-rocky8
 source "${HOMEgfs:-}/versions/build.spack.ver"
 export spack_mod_path="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"
diff --git a/versions/run.hera.ver b/versions/run.hera.ver
@@ -1,6 +1,6 @@
 export stack_intel_ver=2021.5.0
 export stack_impi_ver=2021.5.1
-export spack_env=gsi-addon-dev
+export spack_env=gsi-addon-dev-rocky8
 
 export hpss_ver=hpss
 export ncl_ver=6.6.2

diff --git a/versions/run.spack.ver b/versions/run.spack.ver
@@ -1,5 +1,5 @@
 export spack_stack_ver=1.5.1
-export spack_env=gsi-addon
+export spack_env=gsi-addon-dev-rocky8
 export python_ver=3.10.8
 
 export jasper_ver=2.0.32

diff --git a/workflow/gsl_template_hera.xml b/workflow/gsl_template_hera.xml
@@ -80,7 +80,7 @@
 	<account>gsd-fv3</account>
 	<queue>batch</queue>
 	<partition>hera</partition>
-	<walltime>04:10:00</walltime>
+	<walltime>06:00:00</walltime>
 <!--	<nodes>101:ppn=40:tpp=1</nodes> -->    <!-- 16x16, 2th, 2wg, 40wt -->
 	<nodes>56:ppn=40:tpp=1</nodes>         <!-- 12x12, 2th, 1wg, 40wt -->
         <native>&NATIVE_STR;</native>