Merge branch 'NOAA-EMC:develop' into feature/issue_2475

ci/cases/gfsv17/ocnanal.yaml

@@ -17,7 +17,7 @@ base:
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
 
 ocnanal:
-  SOCA_INPUT_FIX_DIR: {{ FIXgfs }}/gdas/soca/1440x1080x75/soca
+  SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
   SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
   SOCA_NINNER: 100
 

ci/scripts/check_ci.sh

@@ -50,14 +50,14 @@ fi
 export GH
 
 rocotostat=$(command -v rocotostat)
-if [[ -z ${rocotostat+x} ]]; then
+if [[ -z ${rocotostat} ]]; then
   echo "rocotostat not found on system"
   exit 1
 else
   echo "rocotostat being used from ${rocotostat}"
 fi
 rocotocheck=$(command -v rocotocheck)
-if [[ -z ${rocotocheck+x} ]]; then
+if [[ -z ${rocotocheck} ]]; then
   echo "rocotocheck not found on system"
   exit 1
 else
@@ -70,7 +70,7 @@ pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
   pr_list=$("${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true
 fi
-if [[ -z "${pr_list+x}" ]]; then
+if [[ -z "${pr_list}" ]]; then
   echo "no PRs open and ready to run cases on .. exiting"
   exit 0
 fi
@@ -124,7 +124,7 @@ for pr in ${pr_list}; do
 
   for pslot_dir in "${pr_dir}/RUNTESTS/EXPDIR/"*; do
     pslot=$(basename "${pslot_dir}") || true
-    if [[ -z "${pslot+x}" ]]; then
+    if [[ -z "${pslot}" ]]; then
       echo "No experiments found in ${pslot_dir} .. exiting"
       exit 0
     fi

ci/scripts/driver.sh

@@ -77,8 +77,9 @@ pr_list=$(${GH} pr list --repo "${REPO_URL}" --label "CI-${MACHINE_ID^}-Ready" -
 
 for pr in ${pr_list}; do
   pr_dir="${GFS_CI_ROOT}/PR/${pr}"
+  [[ ! -d ${pr_dir} ]] && mkdir -p "${pr_dir}"
   db_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}")
-  output_ci_single="${GFS_CI_ROOT}/PR/${pr}/output_single.log"
+  output_ci_single="${pr_dir}/output_single.log"
   #############################################################
   # Check if a Ready labeled PR has changed back from once set
   # and in that case completely kill the previose driver.sh cron
@@ -107,7 +108,9 @@ for pr in ${pr_list}; do
            echo -e "${pstree_out}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
         fi
       else
-        ssh "${driver_HOST}" 'pstree -A -p "${driver_PID}" | grep -Eow "[0-9]+" | xargs kill'
+        # Check if the driver is still running on the head node; if so, kill it and all child processes
+        #shellcheck disable=SC2029
+        ssh "${driver_HOST}" "pstree -A -p \"${driver_PID}\" | grep -Eow \"[0-9]+\" | xargs kill || echo \"Failed to kill process with PID: ${driver_PID}, it may not be valid.\""
       fi
       {
         echo "Driver PID: Requested termination of ${driver_PID} and children on ${driver_HOST}"
@@ -141,7 +144,7 @@ pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
   pr_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Ready) || true
 fi
-if [[ -z "${pr_list+x}" ]]; then
+if [[ -z "${pr_list}" ]]; then
   echo "no PRs open and ready for checkout/build .. exiting"
   exit 0
 fi
@@ -155,7 +158,7 @@ fi
 for pr in ${pr_list}; do
   # Skip pr's that are currently Building for when overlapping driver scripts are being called from within cron
   pr_building=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | grep Building) || true
-  if [[ -z "${pr_building+x}" ]]; then
+  if [[ -n "${pr_building}" ]]; then
       continue
   fi
   id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id')

jobs/rocoto/arch_test.sh

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+###############################################################
+exit 0

jobs/rocoto/prepoceanobs.sh

@@ -14,8 +14,10 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for class defs and utils
-
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush"
+# shellcheck disable=SC2311
+pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/"
+PYTHONPATH="${pyiodaPATH}:${PYTHONPATH}"
+export PYTHONPATH
 
 ###############################################################
 # Execute the JJOB

parm/config/gefs/config.arch

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+########## config.arch ##########
+# Archive specific
+
+echo "BEGIN: config.arch"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" arch
+
+export ARCH_GAUSSIAN="YES"
+export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS}
+export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS}
+
+echo "END: config.arch"

parm/config/gefs/config.resources

@@ -280,6 +280,13 @@ case ${step} in
     export is_exclusive=False
     ;;
 
+  "arch")
+    export walltime="06:00:00"
+    export ntasks=1
+    export tasks_per_node=1
+    export threads_per_task=1
+    export memory="4096M"
+    ;;
   *)
     echo "FATAL ERROR: Invalid job ${step} passed to ${BASH_SOURCE[0]}"
     exit 1

parm/config/gfs/config.resources

@@ -544,7 +544,7 @@ case ${step} in
     case ${OCNRES} in
       "025")
         memory="128GB"
-        ntasks=40;;
+        ntasks=16;;
       "050")
         memory="32GB"
         ntasks=16;;

parm/gdas/staging/atm_lgetkf_bkg.yaml.j2

@@ -1,6 +1,3 @@
-{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
-{% set time_list = [current_cycle] %}
-
 mkdir:
 {% for imem in range(1,NMEM_ENS+1) %}
    {% set memchar = 'mem%03d' | format(imem) %}
@@ -14,19 +11,13 @@ mkdir:
 - '{{ COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}'
 {% endfor %}
 copy:
-{% for time in time_list %}
-   {% for imem in range(1,NMEM_ENS+1) %}
-      {% set memchar = 'mem%03d' | format(imem) %}
-      {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
-      	      	      	    '${RUN}': 'enkfgdas',
-                            '${YMD}': previous_cycle | to_YMD,	
-                            '${HH}': previous_cycle | strftime('%H'),
-                            '${MEMDIR}': memchar }) %}
-- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/{{ memchar }}/']
-      {% for ftype in ftype_list %}
-         {% for itile in range(1,7) %}
-- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/{{ memchar }}/']
-         {% endfor %}
-      {% endfor %}
-   {% endfor %}
+{% for imem in range(1,NMEM_ENS+1) %}
+   {% set memchar = 'mem%03d' | format(imem) %}
+   {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
+                         '${RUN}': 'enkfgdas',
+                         '${YMD}': previous_cycle | to_YMD,	
+                         '${HH}': previous_cycle | strftime('%H'),
+                         '${MEMDIR}': memchar }) %}
+- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/bkg/{{ memchar }}/']
+- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/bkg/{{ memchar }}/']
 {% endfor %}

parm/gdas/staging/atm_var_bkg.yaml.j2

@@ -1,14 +1,5 @@
-{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
-{% set time_list = [current_cycle] %}
-
 mkdir:
 - '{{ DATA }}/bkg'
 copy:
-{% for time in time_list %}
-- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/']
-   {% for ftype in ftype_list %}
-      {% for itile in range(1,ntiles+1) %}
-- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/']
-      {% endfor %}
-   {% endfor %}
-{% endfor %}
+- ['{{ COM_ATMOS_HISTORY_PREV }}/{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/bkg/']
+- ['{{ COM_ATMOS_HISTORY_PREV }}/{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/bkg/']

parm/gdas/staging/atm_var_fv3ens.yaml.j2

@@ -1,24 +1,15 @@
-{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
-{% set time_list = [current_cycle] %}
-
 mkdir:
 {% for imem in range(1,NMEM_ENS+1) %}
 - '{{ DATA }}/ens/{{ 'mem%03d' | format(imem) }}'
 {% endfor %}
 copy:
-{% for time in time_list %}
-   {% for imem in range(1,NMEM_ENS+1) %}
-      {% set memchar = 'mem%03d' | format(imem) %}
-      {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
-                            '${RUN}': 'enkfgdas',
-                            '${YMD}': previous_cycle | to_YMD,
-                            '${HH}': previous_cycle | strftime('%H'),
-                            '${MEMDIR}': memchar }) %}
-- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/ens/{{ memchar }}/']
-      {% for ftype in ftype_list %}
-         {% for itile in range(1,ntiles+1) %}
-- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/ens/{{ memchar }}/']
-         {% endfor %}
-      {% endfor %}
-   {% endfor %}
+{% for imem in range(1,NMEM_ENS+1) %}
+   {% set memchar = 'mem%03d' | format(imem) %}
+   {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
+                         '${RUN}': 'enkfgdas',
+                         '${YMD}': previous_cycle | to_YMD,
+                         '${HH}': previous_cycle | strftime('%H'),
+                         '${MEMDIR}': memchar }) %}
+- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/ens/{{ memchar }}/']
+- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/ens/{{ memchar }}/']
 {% endfor %}

ush/forecast_postdet.sh

@@ -209,6 +209,8 @@ EOF
       ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FH3}.nc"
       ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FH3}.nc"
       ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FH3}"
+      ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_atmf${FH3}.nc" "cubed_sphere_grid_atmf${FH3}.nc"
+      ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_sfcf${FH3}.nc" "cubed_sphere_grid_sfcf${FH3}.nc"
       if [[ "${WRITE_DOPOST}" == ".true." ]]; then
         ${NLN} "${COMOUT_ATMOS_MASTER}/${RUN}.t${cyc}z.master.grb2f${FH3}" "GFSPRS.GrbF${FH2}"
         ${NLN} "${COMOUT_ATMOS_MASTER}/${RUN}.t${cyc}z.sfluxgrbf${FH3}.grib2" "GFSFLX.GrbF${FH2}"

ush/parsing_model_configure_FV3.sh

@@ -31,15 +31,15 @@ local WRITE_GROUP=${WRITE_GROUP:-1}
 local WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP:-24}
 local ITASKS=1
 local OUTPUT_HISTORY=${OUTPUT_HISTORY:-".true."}
-local HISTORY_FILE_ON_NATIVE_GRID=".false."
+local HISTORY_FILE_ON_NATIVE_GRID=".true."
 local WRITE_DOPOST=${WRITE_DOPOST:-".false."}
 local WRITE_NSFLIP=${WRITE_NSFLIP:-".false."}
 local NUM_FILES=${NUM_FILES:-2}
 local FILENAME_BASE="'atm' 'sfc'"
 # OUTPUT_GRID
 local OUTPUT_FILE="'${OUTPUT_FILETYPE_ATM}' '${OUTPUT_FILETYPE_SFC}'"
 local ZSTANDARD_LEVEL=0
-local IDEFLATE=1  # netCDF zlib lossless compression (0-9); 0: no compression
+local IDEFLATE=0  # netCDF zlib lossless compression (0-9); 0: no compression
 local QUANTIZE_NSD=${QUANTIZE_NSD:-0}  # netCDF compression
 local ICHUNK2D=$((4*restile))
 local JCHUNK2D=$((2*restile))

ush/python/pygfs/task/marine_bmat.py

@@ -43,7 +43,7 @@ def __init__(self, config):
                 'MARINE_WINDOW_END': _window_end,
                 'MARINE_WINDOW_MIDDLE': self.task_config.current_cycle,
                 'BERROR_YAML_DIR': os.path.join(_home_gdas, 'parm', 'soca', 'berror'),
-                'GRID_GEN_YAML': os.path.join(_home_gdas, 'parm', 'soca', 'gridgen', 'gridgen.yaml'),
+                'UTILITY_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'soca_utils_stage.yaml.j2'),
                 'MARINE_ENSDA_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'ensda', 'stage_ens_mem.yaml.j2'),
                 'MARINE_DET_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'soca_det_bkg_stage.yaml.j2'),
                 'ENSPERT_RELPATH': _enspert_relpath,
@@ -82,12 +82,11 @@ def initialize(self: Task) -> None:
         # TODO(G): Check ocean backgrounds dates for consistency
         bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config)
         FileHandler(bkg_list).sync()
-        for cice_fname in ['./INPUT/cice.res.nc', './bkg/ice.bkg.f006.nc', './bkg/ice.bkg.f009.nc']:
-            mdau.cice_hist2fms(cice_fname, cice_fname)
 
-        # stage the grid generation yaml
-        FileHandler({'copy': [[self.task_config.GRID_GEN_YAML,
-                               os.path.join(self.task_config.DATA, 'gridgen.yaml')]]}).sync()
+        # stage the soca utility yamls (gridgen, fields and ufo mapping yamls)
+        logger.info(f"Staging SOCA utility yaml files from {self.task_config.HOMEgfs}/parm/gdas/soca")
+        soca_utility_list = parse_j2yaml(self.task_config.UTILITY_YAML_TMPL, self.task_config)
+        FileHandler(soca_utility_list).sync()
 
         # generate the variance partitioning YAML file
         logger.debug("Generate variance partitioning YAML file")

ush/python/pygfs/utils/marine_da_utils.py

@@ -61,29 +61,6 @@ def prep_input_nml(task_config: AttrDict) -> None:
         nml.write('mom_input.nml')
 
 
-@logit(logger)
-def cice_hist2fms(input_filename: str, output_filename: str) -> None:
-    """ Reformat the CICE history file so it can be read by SOCA/FMS
-    Simple reformatting utility to allow soca/fms to read the CICE history files
-    """
-
-    # open the CICE history file
-    ds = xr.open_dataset(input_filename)
-
-    if 'aicen' in ds.variables and 'hicen' in ds.variables and 'hsnon' in ds.variables:
-        logger.info(f"*** Already reformatted, skipping.")
-        return
-
-    # rename the dimensions to xaxis_1 and yaxis_1
-    ds = ds.rename({'ni': 'xaxis_1', 'nj': 'yaxis_1'})
-
-    # rename the variables
-    ds = ds.rename({'aice_h': 'aicen', 'hi_h': 'hicen', 'hs_h': 'hsnon'})
-
-    # Save the new netCDF file
-    ds.to_netcdf(output_filename, mode='w')
-
-
 @logit(logger)
 def stage_ens_mem(task_config: AttrDict) -> None:
     """ Copy the ensemble members to the DATA directory

versions/fix.ver

@@ -9,7 +9,7 @@ export cpl_ver=20230526
 export datm_ver=20220805
 export gdas_crtm_ver=20220805
 export gdas_fv3jedi_ver=20220805
-export gdas_soca_ver=20240624
+export gdas_soca_ver=20240802
 export gdas_gsibec_ver=20240416
 export gdas_obs_ver=20240213
 export glwu_ver=20220805

workflow/applications/gefs.py

@@ -14,7 +14,7 @@ def _get_app_configs(self):
         """
         Returns the config_files that are involved in gefs
         """
-        configs = ['stage_ic', 'fcst', 'atmos_products']
+        configs = ['stage_ic', 'fcst', 'atmos_products', 'arch']
 
         if self.nens > 0:
             configs += ['efcs', 'atmos_ensstat']
@@ -79,4 +79,6 @@ def get_task_names(self):
         if self.do_extractvars:
             tasks += ['extractvars']
 
+        tasks += ['arch']
+
         return {f"{self._base['RUN']}": tasks}

workflow/rocoto/gefs_tasks.py

@@ -488,3 +488,44 @@ def extractvars(self):
         task = rocoto.create_task(member_metatask_dict)
 
         return task
+
+    def arch(self):
+        deps = []
+        dep_dict = {'type': 'metatask', 'name': 'atmos_prod'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dep_dict = {'type': 'metatask', 'name': 'atmos_ensstat'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        if self.app_config.do_ice:
+            dep_dict = {'type': 'metatask', 'name': 'ice_prod'}
+            deps.append(rocoto.add_dependency(dep_dict))
+        if self.app_config.do_ocean:
+            dep_dict = {'type': 'metatask', 'name': 'ocean_prod'}
+            deps.append(rocoto.add_dependency(dep_dict))
+        if self.app_config.do_wave:
+            dep_dict = {'type': 'metatask', 'name': 'wave_post_grid'}
+            deps.append(rocoto.add_dependency(dep_dict))
+            dep_dict = {'type': 'metatask', 'name': 'wave_post_pnt'}
+            deps.append(rocoto.add_dependency(dep_dict))
+            if self.app_config.do_wave_bnd:
+                dep_dict = {'type': 'metatask', 'name': 'wave_post_bndpnt'}
+                deps.append(rocoto.add_dependency(dep_dict))
+                dep_dict = {'type': 'metatask', 'name': 'wave_post_bndpnt_bull'}
+                deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps, dep_condition='and')
+
+        resources = self.get_resource('arch')
+        task_name = 'arch'
+        task_dict = {'task_name': task_name,
+                     'resources': resources,
+                     'envars': self.envars,
+                     'cycledef': 'gefs',
+                     'dependency': dependencies,
+                     'command': f'{self.HOMEgfs}/jobs/rocoto/arch_test.sh',
+                     'job_name': f'{self.pslot}_{task_name}_@H',
+                     'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log',
+                     'maxtries': '&MAXTRIES;'
+                     }
+
+        task = rocoto.create_task(task_dict)
+
+        return task