[develop] Make get_obs tasks day-dependent in workflow; other improvements and bug fixes

doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst

@@ -168,48 +168,6 @@ These settings define platform-specific run commands. Users should set run comma
 ``PRE_TASK_CMDS``: (Default: "")
    Pre-task commands such as ``ulimit`` needed by tasks. For example: ``'{ ulimit -s unlimited; ulimit -a; }'``
 
-METplus Parameters
-----------------------
-
-:ref:`METplus <MetplusComponent>` is a scientific verification framework that spans a wide range of temporal and spatial scales. Many of the METplus parameters are described below, but additional documentation for the METplus components is available on the `METplus website <https://dtcenter.org/community-code/metplus>`__. 
-
-.. _METParamNote:
-
-.. note::
-   Where a date field is required: 
-      * ``YYYY`` refers to the 4-digit valid year
-      * ``MM`` refers to the 2-digit valid month
-      * ``DD`` refers to the 2-digit valid day of the month
-      * ``HH`` refers to the 2-digit valid hour of the day
-      * ``mm`` refers to the 2-digit valid minutes of the hour
-      * ``SS`` refers to the two-digit valid seconds of the hour
-
-``CCPA_OBS_DIR``: (Default: ``"{{ workflow.EXPTDIR }}/obs_data/ccpa/proc"``)
-   User-specified location of the directory where :term:`CCPA` hourly precipitation files used by METplus are located (or, if retrieved by the workflow, where they will be placed). See comments in file ``scripts/exregional_get_verif_obs.sh`` for more details about files and directory structure, as well as important caveats about errors in the metadata and file names. 
-
-   .. attention:: 
-      Do not set this to the same path as other ``*_OBS_DIR`` variables; otherwise unexpected results and data loss may occur.
-
-``NOHRSC_OBS_DIR``: (Default: ``"{{ workflow.EXPTDIR }}/obs_data/nohrsc/proc"``)
-   User-specified location of top-level directory where NOHRSC 6- and 24-hour snowfall accumulation files used by METplus are located (or, if retrieved by the workflow, where they will be placed). See comments in file scripts/exregional_get_verif_obs.sh for more details about files and directory structure 
-
-   .. attention:: 
-      Do not set this to the same path as other ``*_OBS_DIR`` variables; otherwise unexpected results and data loss may occur. 
-
-   .. note::
-      Due to limited availability of NOHRSC observation data on NOAA :term:`HPSS` and the likelihood that snowfall accumulation verification will not be desired outside of winter cases, this verification option is currently not present in the workflow by default. In order to use it, the verification environment variable ``VX_FIELDS`` should be updated to include ``ASNOW``. This will allow the related workflow tasks to be run.
-
-``MRMS_OBS_DIR``: (Default: ``"{{ workflow.EXPTDIR }}/obs_data/mrms/proc"``)
-   User-specified location of the directory where :term:`MRMS` composite reflectivity and echo top files used by METplus are located (or, if retrieved by the workflow, where they will be placed). See comments in the ``scripts/exregional_get_verif_obs.sh`` for more details about files and directory structure. 
-
-   .. attention:: 
-      Do not set this to the same path as other ``*_OBS_DIR`` variables; otherwise unexpected results and data loss may occur.
-
-``NDAS_OBS_DIR``: (Default: ``"{{ workflow.EXPTDIR }}/obs_data/ndas/proc"``)
-   User-specified location of top-level directory where :term:`NDAS` prepbufr files used by METplus are located (or, if retrieved by the workflow, where they will be placed). See comments in file ``scripts/exregional_get_verif_obs.sh`` for more details about files and directory structure. 
-
-   .. attention:: 
-      Do not set this to the same path as other ``*_OBS_DIR`` variables; otherwise unexpected results and data loss may occur.
 
 Other Platform-Specific Directories
 --------------------------------------
@@ -1635,6 +1593,49 @@ General Verification Parameters
 ``METPLUS_VERBOSITY_LEVEL``: (Default: ``2``)
    Logging verbosity level used by METplus verification tools. Valid values: 0 to 5, with 0 quiet and 5 loud. 
 
+METplus Parameters
+----------------------
+
+:ref:`METplus <MetplusComponent>` is a scientific verification framework that spans a wide range of temporal and spatial scales. Many of the METplus parameters are described below, but additional documentation for the METplus components is available on the `METplus website <https://dtcenter.org/community-code/metplus>`__. 
+
+.. _METParamNote:
+
+.. note::
+   Where a date field is required: 
+      * ``YYYY`` refers to the 4-digit valid year
+      * ``MM`` refers to the 2-digit valid month
+      * ``DD`` refers to the 2-digit valid day of the month
+      * ``HH`` refers to the 2-digit valid hour of the day
+      * ``mm`` refers to the 2-digit valid minutes of the hour
+      * ``SS`` refers to the two-digit valid seconds of the hour
+
+``CCPA_OBS_DIR``: (Default: ``"{{ workflow.EXPTDIR }}/obs_data/ccpa/proc"``)
+   User-specified location of the directory where :term:`CCPA` hourly precipitation files used by METplus are located (or, if retrieved by the workflow, where they will be placed). See comments in file ``scripts/exregional_get_verif_obs.sh`` for more details about files and directory structure, as well as important caveats about errors in the metadata and file names. 
+
+   .. attention:: 
+      Do not set this to the same path as other ``*_OBS_DIR`` variables; otherwise unexpected results and data loss may occur.
+
+``NOHRSC_OBS_DIR``: (Default: ``"{{ workflow.EXPTDIR }}/obs_data/nohrsc/proc"``)
+   User-specified location of top-level directory where NOHRSC 6- and 24-hour snowfall accumulation files used by METplus are located (or, if retrieved by the workflow, where they will be placed). See comments in file scripts/exregional_get_verif_obs.sh for more details about files and directory structure 
+
+   .. attention:: 
+      Do not set this to the same path as other ``*_OBS_DIR`` variables; otherwise unexpected results and data loss may occur. 
+
+   .. note::
+      Due to limited availability of NOHRSC observation data on NOAA :term:`HPSS` and the likelihood that snowfall accumulation verification will not be desired outside of winter cases, this verification option is currently not present in the workflow by default. In order to use it, the verification environment variable ``VX_FIELDS`` should be updated to include ``ASNOW``. This will allow the related workflow tasks to be run.
+
+``MRMS_OBS_DIR``: (Default: ``"{{ workflow.EXPTDIR }}/obs_data/mrms/proc"``)
+   User-specified location of the directory where :term:`MRMS` composite reflectivity and echo top files used by METplus are located (or, if retrieved by the workflow, where they will be placed). See comments in the ``scripts/exregional_get_verif_obs.sh`` for more details about files and directory structure. 
+
+   .. attention:: 
+      Do not set this to the same path as other ``*_OBS_DIR`` variables; otherwise unexpected results and data loss may occur.
+
+``NDAS_OBS_DIR``: (Default: ``"{{ workflow.EXPTDIR }}/obs_data/ndas/proc"``)
+   User-specified location of top-level directory where :term:`NDAS` prepbufr files used by METplus are located (or, if retrieved by the workflow, where they will be placed). See comments in file ``scripts/exregional_get_verif_obs.sh`` for more details about files and directory structure. 
+
+   .. attention:: 
+      Do not set this to the same path as other ``*_OBS_DIR`` variables; otherwise unexpected results and data loss may occur.
+
 Templates for Observation Files
 ---------------------------------
 

jobs/JREGIONAL_GET_VERIF_OBS

@@ -69,7 +69,6 @@ This is the J-job script for the task that checks, pulls, and stages
 observation data for verification purposes.
 ========================================================================"
 
-#
 #
 #-----------------------------------------------------------------------
 #

parm/data_locations.yml

@@ -319,18 +319,15 @@ CCPA_obs:
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
     archive_internal_dir:
-      - "./00"
-      - "./06"
-      - "./12"
-      - "./18"
+      - "./{hh}"
     archive_file_names:
       - "com2_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar"
       - "gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar"
       - "com_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar"
       - "com_ccpa_v4.2_ccpa.{yyyy}{mm}{dd}.tar"
     file_names:
       obs:
-        - "ccpa.t{hh}z.01h.hrap.conus.gb2"
+        - "ccpa.t*z.01h.hrap.conus.gb2"
 
 MRMS_obs:
   hpss:
@@ -365,11 +362,13 @@ NDAS_obs:
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
+      - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}        
     archive_file_names:
       - "com2_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
       - "gpfs_dell1_nco_ops_com_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
       - "com_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
       - "com_obsproc_v1.1_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
+      - "com_obsproc_v1.2_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
     file_names:
       obs:
         - "./nam.t{hh}z.prepbufr.tm*.nr"
@@ -386,6 +385,6 @@ NOHRSC_obs:
       - "dcom_{yyyy}{mm}{dd}.tar"
     file_names:
       obs:
-        - "sfav2_CONUS_*h_{yyyy}{mm}{dd}{hh}_grid184.grb2"
+        - "sfav2_CONUS_6h_{yyyy}{mm}{dd}*_grid184.grb2"
     archive_internal_dir:
       - ./wgrbbul/nohrsc_snowfall/

parm/metplus/EnsembleStat.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST
@@ -516,15 +516,8 @@ PcpCombine tool.  In that file, the field name consists of the observation
 field name here (field_obs) with the accumulation period appended to it
 (separated by an underscore), so we must do the same here to get an exact
 match.
-
-Note:
-Turns out for ASNOW, PcpCombine is not run for obs, so we exclude that
-from the "if" clause here (so it goes into the "else").  For workflow
-behavior uniformity between APCP and ASNOW, consider running PcpCombine
-for ASNOW observations as well (just as it's run for APCP observations).
-      {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 #}
-      {%- if (input_field_group in ['APCP']) %}
+      {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}_{{accum_hh}}
       {%- else %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}

parm/metplus/GenEnsProd.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST

parm/metplus/GridStat_ensmean.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST
@@ -400,15 +400,8 @@ PcpCombine tool.  In that file, the field name consists of the observation
 field name here (field_obs) with the accumulation period appended to it
 (separated by an underscore), so we must do the same here to get an exact
 match.
-
-Note:
-Turns out for ASNOW, PcpCombine is not run for obs, so we exclude that
-from the "if" clause here (so it goes into the "else").  For workflow
-behavior uniformity between APCP and ASNOW, consider running PcpCombine
-for ASNOW observations as well (just as it's run for APCP observations).
-      {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 #}
-      {%- if (input_field_group in ['APCP']) %}
+      {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}_{{accum_hh}}
       {%- else %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}

parm/metplus/GridStat_ensprob.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST
@@ -354,15 +354,8 @@ PcpCombine tool.  In that file, the field name consists of the observation
 field name here (field_obs) with the accumulation period appended to it
 (separated by an underscore), so we must do the same here to get an exact
 match.
-
-Note:
-Turns out for ASNOW, PcpCombine is not run for obs, so we exclude that
-from the "if" clause here (so it goes into the "else").  For workflow
-behavior uniformity between APCP and ASNOW, consider running PcpCombine
-for ASNOW observations as well (just as it's run for APCP observations).
-            {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 #}
-            {%- if (input_field_group in ['APCP']) %}
+            {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}_{{accum_hh}}
             {%- else %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}

parm/metplus/GridStat_or_PointStat.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST
@@ -93,7 +93,7 @@ METPLUS_CONF = {{ '{' ~ METPLUS_TOOL_NAME ~ '_OUTPUT_DIR}' }}/metplus_final.{{me
 
 {%- if (METPLUS_TOOL_NAME == 'GRID_STAT') %}
 
-  {%- if (input_field_group == 'APCP') %}
+  {%- if input_field_group in ['APCP', 'ASNOW'] %}
 
 #{{METPLUS_TOOL_NAME}}_INTERP_FIELD = BOTH
 #{{METPLUS_TOOL_NAME}}_INTERP_VLD_THRESH = 1.0
@@ -539,15 +539,8 @@ PcpCombine tool.  In that file, the field name consists of the observation
 field name here (field_obs) with the accumulation period appended to it
 (separated by an underscore), so we must do the same here to get an exact
 match.
-
-Note:
-Turns out for ASNOW, PcpCombine is not run for obs, so we exclude that
-from the "if" clause here (so it goes into the "else").  For workflow
-behavior uniformity between APCP and ASNOW, consider running PcpCombine
-for ASNOW observations as well (just as it's run for APCP observations).
-  {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 #}
-  {%- if (input_field_group in ['APCP']) %}
+  {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}_{{accum_hh}}
   {%- else %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}

parm/metplus/Pb2nc_obs.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{leadhr_list}}
 
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST

parm/metplus/PcpCombine.conf

@@ -35,7 +35,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST
@@ -79,17 +79,26 @@ FCST_PCP_COMBINE_RUN = False
 #
 {{FCST_OR_OBS}}_PCP_COMBINE_METHOD = ADD
 
-{%- if (FCST_OR_OBS == 'FCST') and (input_field_group == 'ASNOW') %}
+{%- if (FCST_OR_OBS == 'FCST') %}
+  {%- if (input_field_group == 'ASNOW') %}
 #
 # Specify name of variable for Snowfall Accumulation.
-# NOTE: Currently TSNOWP is used which is a constant-density estimate of snowfall accumulation.
-# In future RRFS development, a GSL product with variable-density snowfall accumulation
-# is planned for UPP. When that is included and turned on in post, this variable may be changed
-# to ASNOW.
 #
-FCST_PCP_COMBINE_INPUT_NAMES = TSNOWP
-
-FCST_PCP_COMBINE_INPUT_LEVELS = A01
+# NOTE:
+# For forecasts, currently TSNOWP is used which is a constant-density
+# estimate of snowfall accumulation.  In future RRFS development, a GSL
+# product with variable-density snowfall accumulation is planned for UPP.
+# When that is included and turned on in post, this variable may be
+# changed to ASNOW.
+#
+{{FCST_OR_OBS}}_PCP_COMBINE_INPUT_NAMES = TSNOWP
+{{FCST_OR_OBS}}_PCP_COMBINE_INPUT_LEVELS = A{{input_accum_hh}}
+  {%- endif %}
+{%- elif (FCST_OR_OBS == 'OBS') %}
+  {%- if (input_field_group == 'ASNOW') %}
+{{FCST_OR_OBS}}_PCP_COMBINE_INPUT_NAMES = ASNOW
+{{FCST_OR_OBS}}_PCP_COMBINE_INPUT_LEVELS = A{{input_accum_hh}}
+  {%- endif %}
 {%- endif %}
 #
 # Specify how to name the array in the NetCDF file that PcpCombine
@@ -98,7 +107,7 @@ FCST_PCP_COMBINE_INPUT_LEVELS = A01
 # For accumulation variables (which is the only type of variable that we
 # run PcpCombine on), we add the accumulation period to the variable name
 # because this is how METplus normally sets names.  This is because,
-# epending on the settings in the METplus configuration file, it is
+# depending on the settings in the METplus configuration file, it is
 # possible for a single NetCDF output file to contain output for multiple
 # accumulations, so even though the "level" attribute of each accumulation
 # variable in the output file will contain the level (e.g. "A1" or "A3"),
@@ -110,18 +119,18 @@ FCST_PCP_COMBINE_INPUT_LEVELS = A01
 # the output NetCDF file).
 #
 {%- if (input_field_group in ['APCP', 'ASNOW']) %}
-{{FCST_OR_OBS}}_PCP_COMBINE_OUTPUT_NAME = {{fieldname_in_met_output}}_{{accum_hh}}
+{{FCST_OR_OBS}}_PCP_COMBINE_OUTPUT_NAME = {{fieldname_in_met_output}}_{{output_accum_hh}}
 {%- else %}
 {{FCST_OR_OBS}}_PCP_COMBINE_OUTPUT_NAME = {{fieldname_in_met_output}}
 {%- endif %}
 #
 # Accumulation interval available in the input data.
 #
-{{FCST_OR_OBS}}_PCP_COMBINE_INPUT_ACCUMS = 01
+{{FCST_OR_OBS}}_PCP_COMBINE_INPUT_ACCUMS = {{input_accum_hh}}
 #
 # Accumulation interval to generate in the output file.
 #
-{{FCST_OR_OBS}}_PCP_COMBINE_OUTPUT_ACCUM = {{accum_hh}}
+{{FCST_OR_OBS}}_PCP_COMBINE_OUTPUT_ACCUM = {{output_accum_hh}}
 #
 # If the output NetCDF file already exists, specify whether or not to
 # skip the call to PcpCombine.
@@ -177,7 +186,7 @@ FCST_PCP_COMBINE_CONSTANT_INIT = True
 #
 # Name to identify observation data in output.
 #
-OBTYPE = CCPA
+OBTYPE = {{obtype}}
 {%- endif %}
 #
 # Specify file type of input data.

parm/metplus/PointStat_ensmean.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST

parm/metplus/PointStat_ensprob.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST

parm/metplus/vx_config_det.yaml

@@ -50,6 +50,7 @@ APCP:
 ASNOW:
     ASNOW:
         A6: ['gt0.0', 'ge2.54', 'ge5.08', 'ge10.16', 'ge20.32']
+        A24: ['gt0.0', 'ge2.54', 'ge10.16', 'ge20.32', 'ge30.48']
 REFC:
     REFC%%MergedReflectivityQCComposite:
         L0%%Z500: ['ge20', 'ge30', 'ge40', 'ge50']

parm/metplus/vx_config_ens.yaml

@@ -14,6 +14,7 @@ APCP:
 ASNOW:
     ASNOW:
         A6: ['gt0.0', 'ge2.54', 'ge5.08', 'ge10.16', 'ge20.32']
+        A24: ['gt0.0', 'ge2.54', 'ge10.16', 'ge20.32', 'ge30.48']
 REFC:
     REFC%%MergedReflectivityQCComposite:
         L0%%Z500: ['ge20', 'ge30', 'ge40', 'ge50']