Merge branch 'NOAA-EMC:develop' into feature/issue_2475

.gitignore

@@ -48,6 +48,8 @@ parm/gdas/io
 parm/gdas/ioda
 parm/gdas/snow
 parm/gdas/soca
+parm/gdas/jcb-gdas
+parm/gdas/jcb-algorithms
 parm/monitor
 parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
@@ -195,3 +197,8 @@ versions/run.ver
 ush/python/wxflow
 workflow/wxflow
 ci/scripts/wxflow
+
+# jcb checkout and symlinks
+ush/python/jcb
+workflow/jcb
+ci/scripts/jcb

.gitmodules

@@ -30,3 +30,7 @@
   path = sorc/upp.fd
   url = https://github.com/NOAA-EMC/UPP.git
   ignore = dirty
+[submodule "sorc/jcb"]
+  path = sorc/jcb
+  url = https://github.com/noaa-emc/jcb
+  fetchRecurseSubmodules = false

ci/Jenkinsfile

@@ -149,7 +149,7 @@ pipeline {
                                                 }
                                             }
                                             sh(script: './link_workflow.sh')
-                                            sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+                                            //sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
                                         }
                                     }
                                     if (env.CHANGE_ID && system == 'gfs') {

jobs/JGLOBAL_FORECAST

@@ -99,6 +99,6 @@ fi
 # Remove the Temporary working directory
 ##########################################
 cd "${DATAROOT}" || true
-[[ "${KEEPDATA}" == "NO" ]] && rm -rf "${DATA} ${DATArestart}"  # do not remove DATAjob. It contains DATAoutput
+[[ "${KEEPDATA}" == "NO" ]] && rm -rf "${DATA}" "${DATArestart}"  # do not remove DATAjob. It contains DATAoutput
 
 exit 0

modulefiles/module_base.jet.lua

@@ -39,6 +39,9 @@ load(pathJoin("met", (os.getenv("met_ver") or "None")))
 load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
 load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 
+-- Adding perl as a module; With Rocky8, perl packages will not be from the OS
+load(pathJoin("perl", (os.getenv("perl_ver") or "None")))
+
 setenv("WGRIB2","wgrib2")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 

modulefiles/module_gwsetup.jet.lua

@@ -4,7 +4,7 @@ Load environment to run GFS workflow setup scripts on Jet
 
 load(pathJoin("rocoto"))
 
-prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core")
 
 local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
 local python_ver=os.getenv("python_ver") or "3.11.6"

parm/config/gefs/config.fcst

@@ -6,6 +6,7 @@
 echo "BEGIN: config.fcst"
 
 export USE_ESMF_THREADING="YES"  # Toggle to use ESMF-managed threading or traditional threading in UFSWM
+export COPY_FINAL_RESTARTS="NO" # Toggle to copy restarts from the end of GFS/GEFS Run (GDAS is handled seperately)
 
 # Turn off waves if not used for this CDUMP
 case ${WAVE_CDUMP} in
@@ -108,18 +109,10 @@ if (( gwd_opt == 2 )); then
 fi
 
 # Sponge layer settings
-export tau=0.
-export rf_cutoff=10.
 export d2_bg_k1=0.20
 export d2_bg_k2=0.04
 export dz_min=6
 export n_sponge=42
-if (( LEVS == 128 )) && [[ "${CDUMP}" =~ "gdas" ]]; then
-   export tau=5.0
-   export rf_cutoff=1.0e3
-   export d2_bg_k1=0.20
-   export d2_bg_k2=0.0
-fi
 
 # PBL/turbulance schemes
 export hybedmf=".false."

parm/config/gefs/config.resources

@@ -252,31 +252,45 @@ case ${step} in
     export memory_wavepostsbs="10GB"
     ;;
 
+  # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes.
+  # Limit the number of tasks/node to 40.
   "wavepostbndpnt")
     export wtime_wavepostbndpnt="01:00:00"
     export npe_wavepostbndpnt=240
     export nth_wavepostbndpnt=1
     export npe_node_wavepostbndpnt=$(( npe_node_max / nth_wavepostbndpnt ))
-    export NTASKS=${npe_wavepostbndpnt}
     export is_exclusive=True
+    if [[ ${npe_node_wavepostbndpnt} -gt 40 ]]; then
+        export npe_node_wavepostbndpnt=40
+        export is_exclusive=False
+    fi
+    export NTASKS=${npe_wavepostbndpnt}
     ;;
 
   "wavepostbndpntbll")
     export wtime_wavepostbndpntbll="01:00:00"
     export npe_wavepostbndpntbll=448
     export nth_wavepostbndpntbll=1
     export npe_node_wavepostbndpntbll=$(( npe_node_max / nth_wavepostbndpntbll ))
-    export NTASKS=${npe_wavepostbndpntbll}
     export is_exclusive=True
+    if [[ ${npe_node_wavepostbndpntbll} -gt 40 ]]; then
+        export npe_node_wavepostbndpntbll=40
+        export is_exclusive=False
+    fi
+    export NTASKS=${npe_wavepostbndpntbll}
     ;;
 
   "wavepostpnt")
     export wtime_wavepostpnt="04:00:00"
     export npe_wavepostpnt=200
     export nth_wavepostpnt=1
     export npe_node_wavepostpnt=$(( npe_node_max / nth_wavepostpnt ))
-    export NTASKS=${npe_wavepostpnt}
     export is_exclusive=True
+    if [[ ${npe_node_wavepostpnt} -gt 40 ]]; then
+        export npe_node_wavepostpnt=40
+        export is_exclusive=False
+    fi
+    export NTASKS=${npe_wavepostpnt}
     ;;
 
   *)

parm/config/gefs/config.ufs

@@ -80,8 +80,14 @@ case "${fv3_res}" in
         export nthreads_fv3_gfs=1
         export nthreads_ufs=1
         export nthreads_ufs_gfs=1
-        export cdmbgwd="0.071,2.1,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+	      export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction
+	      export cdmbgwd="0.071,2.1,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="40.0,1.77,1.0,1.0"   # settings for GSL drag suite
+        export k_split=1
+        export n_split=4
+        export tau=10.0
+        export rf_cutoff=100.0
+        export fv_sg_adj=3600
         export knob_ugwp_tauamp=6.0e-3      # setting for UGWPv1 non-stationary GWD
         export WRITE_GROUP=1
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1
@@ -98,16 +104,22 @@ case "${fv3_res}" in
         export nthreads_fv3_gfs=1
         export nthreads_ufs=1
         export nthreads_ufs_gfs=1
-        export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+	      export xr_cnvcld=".false."  # Do not pass conv. clouds to Xu-Randall cloud fraction
+	      export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="20.0,2.5,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=3.0e-3      # setting for UGWPv1 non-stationary GWD
+        export k_split=1
+        export n_split=4
+        export tau=8.0
+        export rf_cutoff=100.0
+        export fv_sg_adj=1800
         export WRITE_GROUP=1
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1
         export WRITE_GROUP_GFS=1
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=1
         ;;
     "C192")
-        export DELTIM=450
+        export DELTIM=600
         export layout_x=4
         export layout_y=6
         export layout_x_gfs=4
@@ -119,6 +131,11 @@ case "${fv3_res}" in
         export cdmbgwd="0.23,1.5,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="10.0,3.5,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=1.5e-3      # setting for UGWPv1 non-stationary GWD
+        export k_split=2
+        export n_split=4
+        export tau=6.0
+        export rf_cutoff=100.0
+        export fv_sg_adj=1800
         export WRITE_GROUP=1
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
         export WRITE_GROUP_GFS=2
@@ -137,6 +154,11 @@ case "${fv3_res}" in
         export cdmbgwd="1.1,0.72,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="5.0,5.0,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=0.8e-3      # setting for UGWPv1 non-stationary GWD
+        export k_split=2
+        export n_split=4
+        export tau=4.0
+        export rf_cutoff=100.0
+        export fv_sg_adj=900
         export WRITE_GROUP=2
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=8
         export WRITE_GROUP_GFS=2
@@ -155,13 +177,18 @@ case "${fv3_res}" in
         export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="2.5,7.5,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=0.5e-3      # setting for UGWPv1 non-stationary GWD
+        export k_split=2
+        export n_split=4
+        export tau=3.0
+        export rf_cutoff=100.0
+        export fv_sg_adj=450
         export WRITE_GROUP=2
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
         export WRITE_GROUP_GFS=4
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20 #Note this should be 10 for WCOSS2
         ;;
     "C1152")
-        export DELTIM=120
+        export DELTIM=150
         export layout_x=8
         export layout_y=16
         export layout_x_gfs=8
@@ -173,6 +200,11 @@ case "${fv3_res}" in
         export cdmbgwd="4.0,0.10,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="1.67,8.8,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=0.35e-3      # setting for UGWPv1 non-stationary GWD
+        export k_split=2
+        export n_split=6
+        export tau=2.5
+        export rf_cutoff=100.0
+        export fv_sg_adj=450
         export WRITE_GROUP=4
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10  # TODO: refine these numbers when a case is available
         export WRITE_GROUP_GFS=4
@@ -191,6 +223,11 @@ case "${fv3_res}" in
         export cdmbgwd="4.0,0.05,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="0.625,14.1,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=0.13e-3      # setting for UGWPv1 non-stationary GWD
+        export k_split=4
+        export n_split=5
+        export tau=0.5
+        export rf_cutoff=100.0
+        export fv_sg_adj=300
         export WRITE_GROUP=4
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10  # TODO: refine these numbers when a case is available
         export WRITE_GROUP_GFS=4

parm/config/gfs/config.atmanl

@@ -5,18 +5,20 @@
 
 echo "BEGIN: config.atmanl"
 
-export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/gdas_prototype_3d.yaml.j2"
-export JEDIYAML="${PARMgfs}/gdas/atm/variational/3dvar_drpcg.yaml.j2"
+export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2"
+export JCB_ALGO_YAML="${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2"
+
 export STATICB_TYPE="gsibec"
+export LOCALIZATION_TYPE="bump"
 export INTERP_METHOD='barycentric'
 
 if [[ ${DOHYBVAR} = "YES" ]]; then
     # shellcheck disable=SC2153
     export CASE_ANL=${CASE_ENS}
-    export BERROR_YAML="${PARMgfs}/gdas/atm/berror/hybvar_${STATICB_TYPE}.yaml.j2"
+    export BERROR_YAML="background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}"
 else
     export CASE_ANL=${CASE}
-    export BERROR_YAML="${PARMgfs}/gdas/atm/berror/staticb_${STATICB_TYPE}.yaml.j2"
+    export BERROR_YAML="background_error_static_${STATICB_TYPE}"
 fi
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"

parm/config/gfs/config.atmanlfv3inc

@@ -8,7 +8,7 @@ echo "BEGIN: config.atmanlfv3inc"
 # Get task specific resources
 . "${EXPDIR}/config.resources" atmanlfv3inc
 
-export JEDIYAML=${PARMgfs}/gdas/atm/utils/fv3jedi_fv3inc_variational.yaml.j2
+export JCB_ALGO=fv3jedi_fv3inc_variational
 export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x
 
 echo "END: config.atmanlfv3inc"

parm/config/gfs/config.atmensanl

@@ -5,8 +5,9 @@
 
 echo "BEGIN: config.atmensanl"
 
-export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/lgetkf_prototype.yaml.j2"
-export JEDIYAML="${PARMgfs}/gdas/atm/lgetkf/lgetkf.yaml.j2"
+export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2"
+export JCB_ALGO_YAML="${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2"
+
 export INTERP_METHOD='barycentric'
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"

parm/config/gfs/config.fcst

@@ -6,6 +6,7 @@
 echo "BEGIN: config.fcst"
 
 export USE_ESMF_THREADING="YES"  # Toggle to use ESMF-managed threading or traditional threading in UFSWM
+export COPY_FINAL_RESTARTS="NO" # Toggle to copy restarts from the end of GFS/GEFS Run (GDAS is handled seperately)
 
 # Turn off waves if not used for this CDUMP
 case ${WAVE_CDUMP} in
@@ -120,18 +121,10 @@ if (( gwd_opt == 2 )); then
 fi
 
 # Sponge layer settings
-export tau=0.
-export rf_cutoff=10.
 export d2_bg_k1=0.20
 export d2_bg_k2=0.04
 export dz_min=6
 export n_sponge=42
-if (( LEVS == 128 )) && [[ "${CDUMP}" =~ "gdas" ]]; then
-   export tau=5.0
-   export rf_cutoff=1.0e3
-   export d2_bg_k1=0.20
-   export d2_bg_k2=0.0
-fi
 
 # PBL/turbulance schemes
 export hybedmf=".false."

parm/config/gfs/config.resources

@@ -140,31 +140,45 @@ case ${step} in
     export memory_wavepostsbs_gfs="10GB"
     ;;
 
+  # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes.
+  # Limit the number of tasks/node to 40.
   "wavepostbndpnt")
     export wtime_wavepostbndpnt="01:00:00"
     export npe_wavepostbndpnt=240
     export nth_wavepostbndpnt=1
     export npe_node_wavepostbndpnt=$(( npe_node_max / nth_wavepostbndpnt ))
-    export NTASKS=${npe_wavepostbndpnt}
     export is_exclusive=True
+    if [[ ${npe_node_wavepostbndpnt} -gt 40 ]]; then
+        export npe_node_wavepostbndpnt=40
+        export is_exclusive=False
+    fi
+    export NTASKS=${npe_wavepostbndpnt}
     ;;
 
   "wavepostbndpntbll")
     export wtime_wavepostbndpntbll="01:00:00"
     export npe_wavepostbndpntbll=448
     export nth_wavepostbndpntbll=1
     export npe_node_wavepostbndpntbll=$(( npe_node_max / nth_wavepostbndpntbll ))
-    export NTASKS=${npe_wavepostbndpntbll}
     export is_exclusive=True
+    if [[ ${npe_node_wavepostbndpntbll} -gt 40 ]]; then
+        export npe_node_wavepostbndpntbll=40
+        export is_exclusive=False
+    fi
+    export NTASKS=${npe_wavepostbndpntbll}
     ;;
 
   "wavepostpnt")
     export wtime_wavepostpnt="04:00:00"
     export npe_wavepostpnt=200
     export nth_wavepostpnt=1
     export npe_node_wavepostpnt=$(( npe_node_max / nth_wavepostpnt ))
-    export NTASKS=${npe_wavepostpnt}
     export is_exclusive=True
+    if [[ ${npe_node_wavepostpnt} -gt 40 ]]; then
+        export npe_node_wavepostpnt=40
+        export is_exclusive=False
+    fi
+    export NTASKS=${npe_wavepostpnt}
     ;;
 
   "wavegempak")
@@ -699,7 +713,7 @@ case ${step} in
 
     case "${CASE}" in
       "C48" | "C96" | "C192")
-        declare -x "wtime_${step}"="00:15:00"
+        declare -x "wtime_${step}"="00:20:00"
         declare -x "wtime_${step}_gfs"="03:00:00"
         ;;
       "C384")
@@ -747,6 +761,7 @@ case ${step} in
         exit 4
       ;;
     esac
+    if [[ ${machine} == "JET" ]]; then unset memory_upp ; fi
     export npe_node_upp=${npe_upp}
 
     export nth_upp=1
@@ -1032,7 +1047,7 @@ case ${step} in
     export npe_node_esfc=$(( npe_node_max / nth_esfc ))
     export nth_cycle=${nth_esfc}
     export npe_node_cycle=$(( npe_node_max / nth_cycle ))
-    export memory_esfc="80GB"
+    if [[ ${machine} != "JET" ]]; then export memory_esfc="80G" ; fi
     ;;
 
   "epos")