From 20fb88c057b22991ed3b889c7388a271cc5db99d Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 28 Aug 2024 17:01:06 +0000 Subject: [PATCH 01/70] replace separate JEDI radiance bias correction files with tarball (#2862) --- parm/archive/gdas_restarta.yaml.j2 | 2 ++ parm/stage/analysis.yaml.j2 | 9 +---- ush/python/pygfs/task/analysis.py | 36 ++++++++++---------- ush/python/pygfs/task/atm_analysis.py | 48 ++++++++++----------------- 4 files changed, 38 insertions(+), 57 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 9d86292065..7a011671be 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -32,6 +32,8 @@ gdas_restarta: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + {% else %} + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radbcor" {% endif %} # Snow surface data diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index d30389644a..4068f1e928 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -10,17 +10,10 @@ analysis: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc"] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "radbcor"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} {% endfor %} - {% if DO_JEDIATMVAR %} - {% for ftype in ["satbias.nc", "satbias_cov.nc", "tlapse.txt"] %} - {% for file in glob(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z.atms_*." ~ ftype) %} - - ["{{ file }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] - {% endfor %} - {% endfor %} - {% endif %} {% endfor %} # mem loop {% endif %} diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index bf47b9a950..b8e7809263 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -42,8 +42,7 @@ def initialize(self) -> None: FileHandler(obs_dict).sync() # some analyses need to stage bias corrections - bias_dict = self.get_bias_dict() - FileHandler(bias_dict).sync() + self.get_bias() # link jedi executable to run directory self.link_jediexe() @@ -128,25 +127,21 @@ def get_obs_dict(self) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias_dict(self) -> Dict[str, Any]: - """Compile a dictionary of observation files to copy + def get_bias(self) -> None: + """Stage radiance bias correciton files - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation bias correction files that are to be copied to the run directory - from the component directory. - TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in - `analysis.py` and should be implemented in the component where this is applicable. + This method stages radiance bias correction files in the obs sub-diretory of the run directory Parameters ---------- + Task: GDAS task Returns ---------- - bias_dict: Dict - a dictionary containing the list of observation bias files to copy for FileHandler + None """ - logger.info(f"Extracting a list of bias correction files from Jedi config file") + logger.info(f"Copy radiance bias correction tarball if Jedi config processes bias corrected radiances") observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') logger.debug(f"observations:\n{pformat(observations)}") @@ -156,17 +151,22 @@ def get_bias_dict(self) -> Dict[str, Any]: obfile = ob['obs bias']['input file'] obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-2]) - for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: - bfile = f"{prefix}.{file}" - copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - # TODO: Why is this specific to ATMOS? + prefix = '.'.join(basename.split('.')[:-3]) + bfile = f"{prefix}.radbcor" + copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + break bias_dict = { 'mkdir': [os.path.join(self.task_config.DATA, 'bc')], 'copy': copylist } - return bias_dict + FileHandler(bias_dict).sync() + + radtar = os.path.join(obdir, bfile) + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() @logit(logger) def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None: diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 4e9d37335c..e32dcdf815 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -207,37 +207,23 @@ def finalize(self: Analysis) -> None: } FileHandler(yaml_copy).sync() - # copy bias correction files to ROTDIR - logger.info("Copy bias correction files from DATA/ to COM/") - biasdir = os.path.join(self.task_config.DATA, 'bc') - biasls = os.listdir(biasdir) - biaslist = [] - for bfile in biasls: - src = os.path.join(biasdir, bfile) - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) - biaslist.append([src, dest]) - - gprefix = f"{self.task_config.GPREFIX}" - gsuffix = f"{to_YMDH(self.task_config.previous_cycle)}" + ".txt" - aprefix = f"{self.task_config.APREFIX}" - asuffix = f"{to_YMDH(self.task_config.current_cycle)}" + ".txt" - - logger.info(f"Copying {gprefix}*{gsuffix} from DATA/ to COM/ as {aprefix}*{asuffix}") - obsdir = os.path.join(self.task_config.DATA, 'obs') - obsls = os.listdir(obsdir) - for ofile in obsls: - if ofile.endswith(".txt"): - src = os.path.join(obsdir, ofile) - tfile = ofile.replace(gprefix, aprefix) - tfile = tfile.replace(gsuffix, asuffix) - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, tfile) - biaslist.append([src, dest]) - - bias_copy = { - 'mkdir': [self.task_config.COM_ATMOS_ANALYSIS], - 'copy': biaslist, - } - FileHandler(bias_copy).sync() + # path of output radiance bias correction tarfile + bfile = f"{self.task_config.APREFIX}radbcor" + radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) + + # get lists of radiance bias correction files to put in tarball + satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) + tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + + # tar radiance bias correction files to ROTDIR + logger.info(f"Creating radiance bias correction tar file {radtar}") + with tarfile.open(radtar, 'w') as radbcor: + for satfile in satlist: + radbcor.add(satfile, arcname=os.path.basename(satfile)) + for tlapfile in tlaplist: + radbcor.add(tlapfile, arcname=os.path.basename(tlapfile)) + logger.info(f"Add {radbcor.getnames()}") + radbcor.close() # Copy FV3 atm increment to comrot directory logger.info("Copy UFS model readable atm increment file") From 73aa233f743ccd9232481e34fd5a3c72ace17a2f Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 29 Aug 2024 10:26:14 +0000 Subject: [PATCH 02/70] rename JEDI radiance bias correction tarball file to be more self-describing (#2862) --- parm/archive/gdas_restarta.yaml.j2 | 2 +- parm/stage/analysis.yaml.j2 | 2 +- ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/atm_analysis.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 7a011671be..fc5ce9478d 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -33,7 +33,7 @@ gdas_restarta: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radbcor" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}rad_varbc_params.tar" {% endif %} # Snow surface data diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index 4068f1e928..2d4b7d2854 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -10,7 +10,7 @@ analysis: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "radbcor"] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "rad_varbc_params.tar"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index b8e7809263..ee674f8ef0 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -152,7 +152,7 @@ def get_bias(self) -> None: obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.radbcor" + bfile = f"{prefix}.rad_varbc_params.tar" copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) break diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index e32dcdf815..99c6e600cb 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -208,7 +208,7 @@ def finalize(self: Analysis) -> None: FileHandler(yaml_copy).sync() # path of output radiance bias correction tarfile - bfile = f"{self.task_config.APREFIX}radbcor" + bfile = f"{self.task_config.APREFIX}rad_varbc_params.tar" radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) # get lists of radiance bias correction files to put in tarball From 1203866f068eea2a9ccd28ad9d65b4c6c7d62311 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 7 Sep 2024 11:36:35 +0000 Subject: [PATCH 03/70] update gdas.cd hash to include recent commits to GDASApp develop (#2862) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index faa95efb18..554c55a6ad 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit faa95efb18f0f52acab2cf09b17f78406f9b48b1 +Subproject commit 554c55a6ad1a4fc7d5868122a9b0147af3b300a2 From 5985e9f718ded5316007f6d38ac16b5abb2c9649 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 9 Sep 2024 14:22:13 +0000 Subject: [PATCH 04/70] update pygfs jedi class and atmospheric tasks to extract JEDI radiance bias correction files from tarball (#2862) --- ush/python/pygfs/jedi/jedi.py | 9 +++-- ush/python/pygfs/task/analysis.py | 45 ------------------------ ush/python/pygfs/task/atm_analysis.py | 16 +++++++++ ush/python/pygfs/task/atmens_analysis.py | 16 +++++++++ 4 files changed, 36 insertions(+), 50 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 62dcb517ca..dace86c4fe 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -216,11 +216,10 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: obfile = ob['obs bias']['input file'] obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-2]) - for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: - bfile = f"{prefix}.{file}" - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - # TODO: Why is this specific to ATMOS? + prefix = '.'.join(basename.split('.')[:-3]) + bfile = f"{prefix}.rad_varbc_params.tar" + copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + break bias_dict = { 'mkdir': [os.path.join(task_config.DATA, 'bc')], diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index d9cd358219..1d8b38483b 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -41,9 +41,6 @@ def initialize(self) -> None: obs_dict = self.get_obs_dict() FileHandler(obs_dict).sync() - # some analyses need to stage bias corrections - self.get_bias() - # link jedi executable to run directory self.link_jediexe() @@ -126,48 +123,6 @@ def get_obs_dict(self) -> Dict[str, Any]: } return obs_dict - @logit(logger) - def get_bias(self) -> None: - """Stage radiance bias correciton files - - This method stages radiance bias correction files in the obs sub-diretory of the run directory - - Parameters - ---------- - Task: GDAS task - - Returns - ---------- - None - """ - - logger.info(f"Copy radiance bias correction tarball if Jedi config processes bias corrected radiances") - observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') - logger.debug(f"observations:\n{pformat(observations)}") - - copylist = [] - for ob in observations['observers']: - if 'obs bias' in ob.keys(): - obfile = ob['obs bias']['input file'] - obdir = os.path.dirname(obfile) - basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.rad_varbc_params.tar" - copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - break - - bias_dict = { - 'mkdir': [os.path.join(self.task_config.DATA, 'bc')], - 'copy': copylist - } - FileHandler(bias_dict).sync() - - radtar = os.path.join(obdir, bfile) - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() - @logit(logger) def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None: """Add cubed-sphere increments to cubed-sphere backgrounds diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index ad082063bb..36f768fa13 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import re import glob import gzip import tarfile @@ -143,6 +144,21 @@ def initialize_analysis(self) -> None: FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # Extract radiance bias correction files from tarball + for action, filelist in bias_dict.items(): + if 'copy' in action: + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + if re.search(".tar", src): + radtar = src + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 55e72702b1..7004bf059c 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import re import glob import gzip import tarfile @@ -142,6 +143,21 @@ def initialize_analysis(self) -> None: FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # Extract radiance bias correction files from tarball + for action, filelist in bias_dict.items(): + if 'copy' in action: + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + if re.search(".tar", src): + radtar = src + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) From 73e56a7fa14c553f0d64003b88c8e9118f088bea Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 10 Sep 2024 10:50:33 +0000 Subject: [PATCH 05/70] move radiance bias correction staging to jedi class (#2868) --- ush/python/pygfs/jedi/jedi.py | 29 ++++++++++++++++-------- ush/python/pygfs/task/atm_analysis.py | 20 +--------------- ush/python/pygfs/task/atmens_analysis.py | 20 +--------------- 3 files changed, 21 insertions(+), 48 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index dace86c4fe..4e527460ab 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 import os +import tarfile from logging import getLogger +from pprint import pformat from typing import List, Dict, Any, Optional from jcb import render from wxflow import (AttrDict, + FileHandler, chdir, rm_p, parse_j2yaml, logit, @@ -188,14 +191,10 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: + def get_bias(self, task_config: AttrDict) -> Dict[str, Any]: """Compile a dictionary of observation files to copy - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation bias correction files that are to be copied to the run directory - from the component directory. - TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in - `analysis.py` and should be implemented in the component where this is applicable. + This method stages radiance bias correction files in the obs sub-diretory of the run directory Parameters ---------- @@ -204,8 +203,7 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: Returns ---------- - bias_dict: Dict - a dictionary containing the list of observation bias files to copy for FileHandler + None """ observations = find_value_in_nested_dict(self.config, 'observations') @@ -218,14 +216,25 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) bfile = f"{prefix}.rad_varbc_params.tar" - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + radtar = os.path.join(obdir, bfile) + copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), radtar]) break bias_dict = { 'mkdir': [os.path.join(task_config.DATA, 'bc')], 'copy': copylist } - return bias_dict + + # stage bias corrections + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract radiance bias correction files from tarball + radtar = os.path.join(obdir, bfile) + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 36f768fa13..c1a6558ade 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import re import glob import gzip import tarfile @@ -140,24 +139,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - bias_dict = self.jedi.get_bias_dict(self.task_config) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # Extract radiance bias correction files from tarball - for action, filelist in bias_dict.items(): - if 'copy' in action: - for sublist in filelist: - if len(sublist) != 2: - raise Exception( - f"List must be of the form ['src', 'dest'], not {sublist}") - src = sublist[0] - if re.search(".tar", src): - radtar = src - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + self.jedi.get_bias(self.task_config) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 7004bf059c..1f3a065133 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import re import glob import gzip import tarfile @@ -139,24 +138,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - bias_dict = self.jedi.get_bias_dict(self.task_config) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # Extract radiance bias correction files from tarball - for action, filelist in bias_dict.items(): - if 'copy' in action: - for sublist in filelist: - if len(sublist) != 2: - raise Exception( - f"List must be of the form ['src', 'dest'], not {sublist}") - src = sublist[0] - if re.search(".tar", src): - radtar = src - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + self.jedi.get_bias(self.task_config) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 666d7d6d89c8c17ef9627367a3efdb76454d07f7 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 10 Sep 2024 15:03:43 +0000 Subject: [PATCH 06/70] initial attempt to generalize processing of variational bias correction files using jedi class (#2862) --- ush/python/pygfs/jedi/jedi.py | 6 +++--- ush/python/pygfs/task/atm_analysis.py | 6 ++++-- ush/python/pygfs/task/atmens_analysis.py | 6 ++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 4e527460ab..2b899df4e3 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -191,7 +191,7 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias(self, task_config: AttrDict) -> Dict[str, Any]: + def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: """Compile a dictionary of observation files to copy This method stages radiance bias correction files in the obs sub-diretory of the run directory @@ -215,9 +215,9 @@ def get_bias(self, task_config: AttrDict) -> Dict[str, Any]: obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.rad_varbc_params.tar" + bfile = f"{prefix}.{bias_file}" radtar = os.path.join(obdir, bfile) - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), radtar]) + copylist.append([os.path.join(task_config.VarBcDir, bfile), radtar]) break bias_dict = { diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index c1a6558ade..454d557e87 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -66,7 +66,8 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", 'atm_obsdataout_path': f"{self.task_config.DATA}/diags/", - 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications + 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications + 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", } ) @@ -139,7 +140,8 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - self.jedi.get_bias(self.task_config) + bias_file = f"rad_varbc_params.tar" + self.jedi.get_bias(self.task_config, bias_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1f3a065133..01fc4faac0 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -66,7 +66,8 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"./obs/", 'atm_obsdataout_path': f"./diags/", - 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications + 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications + 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", } ) @@ -138,7 +139,8 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - self.jedi.get_bias(self.task_config) + bias_file = f"rad_varbc_params.tar" + self.jedi.get_bias(self.task_config, bias_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 1e4a28434dc475cf533250219383bf20c347977e Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 10 Sep 2024 17:09:42 +0000 Subject: [PATCH 07/70] move VarBcDir inside initialize_analysis method (#2862) --- ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 454d557e87..2cc738008a 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -66,8 +66,7 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", 'atm_obsdataout_path': f"{self.task_config.DATA}/diags/", - 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications - 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", + 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications } ) @@ -140,6 +139,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") + self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" self.jedi.get_bias(self.task_config, bias_file) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 01fc4faac0..b57c80e22c 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -66,8 +66,7 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"./obs/", 'atm_obsdataout_path': f"./diags/", - 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications - 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", + 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications } ) @@ -139,6 +138,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") + self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" self.jedi.get_bias(self.task_config, bias_file) From b48e126341795a11ad3941e0beb56c95644fac99 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 11 Sep 2024 18:29:24 +0000 Subject: [PATCH 08/70] update sorc/gdas.cd hash to bring in radiance tarball ctest changes (#2862) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 554c55a6ad..032b708f6a 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 554c55a6ad1a4fc7d5868122a9b0147af3b300a2 +Subproject commit 032b708f6a476ae4726d1533b82feb21fc8daa92 From ba77d4ec1954968de8e88ebc002073ff85ab6953 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Thu, 12 Sep 2024 16:05:43 +0000 Subject: [PATCH 09/70] add extract_tar method to jedi class, use extract_tar in atm and atmens analysis scripts (#2862) --- ush/python/pygfs/jedi/jedi.py | 51 +++++++++++++++++------- ush/python/pygfs/task/atm_analysis.py | 8 +++- ush/python/pygfs/task/atmens_analysis.py | 8 +++- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 2b899df4e3..b5fdb42c7c 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -3,7 +3,6 @@ import os import tarfile from logging import getLogger -from pprint import pformat from typing import List, Dict, Any, Optional from jcb import render from wxflow import (AttrDict, @@ -191,19 +190,24 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: + def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: """Compile a dictionary of observation files to copy - This method stages radiance bias correction files in the obs sub-diretory of the run directory + This method extracts 'observers' from the JEDI yaml and determines from that list + if bias correction tar files are to be copied to the run directory + from the component directory. Parameters ---------- task_config: AttrDict Attribute-dictionary of all configuration variables associated with a GDAS task. + bias_file + name of bias correction tar file Returns ---------- - None + bias_dict: Dict + a dictionary containing the list of observation bias files to copy for FileHandler """ observations = find_value_in_nested_dict(self.config, 'observations') @@ -216,8 +220,8 @@ def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) bfile = f"{prefix}.{bias_file}" - radtar = os.path.join(obdir, bfile) - copylist.append([os.path.join(task_config.VarBcDir, bfile), radtar]) + tar_file = os.path.join(obdir, bfile) + copylist.append([os.path.join(task_config.VarBcDir, bfile), tar_file]) break bias_dict = { @@ -225,16 +229,33 @@ def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: 'copy': copylist } - # stage bias corrections - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + return bias_dict + + @logit(logger) + def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: + """Extract files from list of tarfiles + + This method extract bias correction files from tarball(s) + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + tar_dict + a dictionary containing the list of tar files + + Returns + ---------- + None + """ - # extract radiance bias correction files from tarball - radtar = os.path.join(obdir, bfile) - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + # extract bias correction files from tar file + for tar_file in tar_dict['copy']: + if ".tar" in tar_file[1]: + with tarfile.open(tar_file[1], "r") as tarball: + tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) + logger.info(f"Extract {tarball.getnames()}") + tarball.close() @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 2cc738008a..900e78ba1c 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -141,7 +141,13 @@ def initialize_analysis(self) -> None: logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" - self.jedi.get_bias(self.task_config, bias_file) + bias_dict = self.jedi.get_bias_dict(self.task_config, bias_file) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract bias corrections + logger.info(f"Extract bias correction files from tarball") + self.jedi.extract_tar(self.task_config, bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index b57c80e22c..a02932d8f5 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -140,7 +140,13 @@ def initialize_analysis(self) -> None: logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" - self.jedi.get_bias(self.task_config, bias_file) + bias_dict = self.jedi.get_bias(self.task_config, bias_file) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract bias corrections + logger.info(f"Extract bias correction files from tarball") + self.jedi.extract_tar(self.task_config, bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From bc3a6ddab05faaea4466719e96bc36133901a199 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Thu, 12 Sep 2024 16:15:21 +0000 Subject: [PATCH 10/70] correct typo in atmens_analysis.py (2862) --- ush/python/pygfs/task/atmens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index a02932d8f5..8bab631495 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -140,7 +140,7 @@ def initialize_analysis(self) -> None: logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" - bias_dict = self.jedi.get_bias(self.task_config, bias_file) + bias_dict = self.jedi.get_bias_dict(self.task_config, bias_file) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") From 0ab7bdd6083051f5cb9b9bc687d2a4bbbe5e2e34 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 17 Sep 2024 18:20:47 +0000 Subject: [PATCH 11/70] improve error handling for jedi class tarfile.extractall (#2862) --- ush/python/pygfs/jedi/jedi.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index b5fdb42c7c..08cfbc02af 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -252,10 +252,19 @@ def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: # extract bias correction files from tar file for tar_file in tar_dict['copy']: if ".tar" in tar_file[1]: - with tarfile.open(tar_file[1], "r") as tarball: - tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) - logger.info(f"Extract {tarball.getnames()}") - tarball.close() + try: + with tarfile.open(tar_file[1], "r") as tarball: + tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) + logger.info(f"Extract {tarball.getnames()}") + except tarfile.ReadError as err: + if tarfile.is_tarfile(tarfile[1]): + logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") + else: + logger.info() + except tarfile.ExtractError as err: + logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") @logit(logger) From 8b627a159fdef4bc67e5877b8d75f9a696699675 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 17 Sep 2024 18:23:52 +0000 Subject: [PATCH 12/70] remove unnecessary close following tarfile add (#2862) --- ush/python/pygfs/task/atm_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 900e78ba1c..27243159fa 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -287,7 +287,6 @@ def finalize(self) -> None: for tlapfile in tlaplist: radbcor.add(tlapfile, arcname=os.path.basename(tlapfile)) logger.info(f"Add {radbcor.getnames()}") - radbcor.close() # Copy FV3 atm increment to comrot directory logger.info("Copy UFS model readable atm increment file") From eb4fc374b83b693ac767587452a8d63aca77675f Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 17 Sep 2024 18:45:14 +0000 Subject: [PATCH 13/70] use endswith to clean up scripting (#2862) --- ush/python/pygfs/jedi/jedi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 08cfbc02af..61ad820302 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -251,7 +251,7 @@ def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: # extract bias correction files from tar file for tar_file in tar_dict['copy']: - if ".tar" in tar_file[1]: + if tar_file[1].endswith('.tar'): try: with tarfile.open(tar_file[1], "r") as tarball: tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) From ed52b724ef8762f9c1eea2c347a5276f49c9469b Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 18 Sep 2024 17:13:24 +0000 Subject: [PATCH 14/70] refactor jedi class method extract_tar (#2862) --- ush/python/pygfs/jedi/jedi.py | 43 +++++++++++------------- ush/python/pygfs/task/atm_analysis.py | 5 +-- ush/python/pygfs/task/atmens_analysis.py | 5 +-- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 61ad820302..9e69065bb2 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -232,39 +232,36 @@ def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: return bias_dict @logit(logger) - def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: - """Extract files from list of tarfiles + def extract_tar(self, tar_file: str) -> None: + """Extract bias correction files from a tarball - This method extract bias correction files from tarball(s) + This method extract files from a tarball Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. - tar_dict - a dictionary containing the list of tar files + tar_file + path/name of tarball Returns ---------- None """ - # extract bias correction files from tar file - for tar_file in tar_dict['copy']: - if tar_file[1].endswith('.tar'): - try: - with tarfile.open(tar_file[1], "r") as tarball: - tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) - logger.info(f"Extract {tarball.getnames()}") - except tarfile.ReadError as err: - if tarfile.is_tarfile(tarfile[1]): - logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") - raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") - else: - logger.info() - except tarfile.ExtractError as err: - logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") - raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") + # extract files from tar file + tar_path = os.path.dirname(tar_file) + try: + with tarfile.open(tar_file, "r") as tarball: + tarball.extractall(path=tar_path) + logger.info(f"Extract {tarball.getnames()}") + except tarfile.ReadError as err: + if tarfile.is_tarfile(tarfile[1]): + logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") + else: + logger.info() + except tarfile.ExtractError as err: + logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 27243159fa..badb01a74a 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -146,8 +146,9 @@ def initialize_analysis(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - logger.info(f"Extract bias correction files from tarball") - self.jedi.extract_tar(self.task_config, bias_dict) + tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") + logger.info(f"Extract bias correction files from {tar_file}") + self.jedi.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 8bab631495..4b2f8ebbf4 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -145,8 +145,9 @@ def initialize_analysis(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - logger.info(f"Extract bias correction files from tarball") - self.jedi.extract_tar(self.task_config, bias_dict) + tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") + logger.info(f"Extract bias correction files from {tar_file}") + self.jedi.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 80305b67cfbf941e40604d71b7f1413106b2d807 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 18 Sep 2024 18:04:24 +0000 Subject: [PATCH 15/70] update sorc/gdas.cd to current head of GDASApp develop (#2862) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 032b708f6a..55e895f1dc 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 032b708f6a476ae4726d1533b82feb21fc8daa92 +Subproject commit 55e895f1dcf4e6be36eb0eb4c8a7995d429157e0 From 821ee760945bbd38b5a4c6c7db28d9fa08de7fb2 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 18 Sep 2024 18:21:10 +0000 Subject: [PATCH 16/70] make jedi method extract_tar static, clean up comments, correct typo (#2862) --- ush/python/pygfs/jedi/jedi.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 9e69065bb2..415a0a3c08 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -231,9 +231,10 @@ def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: return bias_dict + @staticmethod @logit(logger) - def extract_tar(self, tar_file: str) -> None: - """Extract bias correction files from a tarball + def extract_tar(tar_file: str) -> None: + """Extract files from a tarball This method extract files from a tarball @@ -254,14 +255,14 @@ def extract_tar(self, tar_file: str) -> None: tarball.extractall(path=tar_path) logger.info(f"Extract {tarball.getnames()}") except tarfile.ReadError as err: - if tarfile.is_tarfile(tarfile[1]): - logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") - raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") + if tarfile.is_tarfile(tar_file): + logger.error(f"FATAL ERROR: {tar_file} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tar_file}") else: logger.info() except tarfile.ExtractError as err: - logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") - raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") + logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") @logit(logger) From 042ec5094cd62dff9d5abc1e4770dd2451284d0c Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 19 Sep 2024 18:17:17 +0000 Subject: [PATCH 17/70] correctly prefix tlapse radiance bias correction files for cycling (#2862) --- ush/python/pygfs/task/atm_analysis.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index badb01a74a..5f67ea9d72 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -276,9 +276,21 @@ def finalize(self) -> None: bfile = f"{self.task_config.APREFIX}rad_varbc_params.tar" radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) - # get lists of radiance bias correction files to put in tarball + # rename and copy tlapse radiance bias correction files from obs to bc + tlapobs = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + copylist = [] + for tlapfile in tlapobs: + obsfile = os.path.basename(tlapfile).split('.', 2) + newfile = f"{self.task_config.APREFIX}{obsfile[2]}" + copylist.append([tlapfile, os.path.join(self.task_config.DATA, 'bc', newfile)]) + tlapse_dict = { + 'copy': copylist + } + FileHandler(tlapse_dict).sync() + + # get lists of radiance bias correction files to add to tarball satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) - tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*tlapse.txt')) # tar radiance bias correction files to ROTDIR logger.info(f"Creating radiance bias correction tar file {radtar}") From 5306b069727777d814af4433c9b57d46c47f2269 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 20 Sep 2024 10:01:29 +0000 Subject: [PATCH 18/70] extend duration of C96C48_ufs_hybatmDA CI by one cycle to 2024022406 (#2862) --- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index 0b5aa7b6ac..b1566d77a0 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -11,7 +11,7 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2024022318 - edate: 2024022400 + edate: 2024022406 nens: 2 gfs_cyc: 1 start: warm From 3a812884ab7b65fee4c966700088cab82778db0c Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 27 Sep 2024 12:54:34 +0000 Subject: [PATCH 19/70] Initial commit --- ush/python/pygfs/task/atm_analysis.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 8d340a5b73..62de16ffbe 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -129,8 +129,24 @@ def initialize_analysis(self) -> None: ---------- None """ - super().initialize() + # stage observations + logger.info(f"Staging list of observation files generated from JEDI config") + jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) + jcb_config.update(parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config)) + jcb_config['algorithm'] = 'atm_obs_staging' + obs_dict = render(jcb_config) + FileHandler(obs_dict).sync() + logger.debug(f"Observation files:\n{pformat(obs_dict)}") + + # Test + jcb_config = {} + jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) + jcb_config.update(parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config)) + jcb_config['algorithm'] = 'atm_bias_staging' + bias_dict = render(jcb_config) + logger.debug(f"foo:\n{pformat(bias_dict)}") + # stage observations logger.info(f"Staging list of observation files generated from JEDI config") obs_dict = self.jedi.get_obs_dict(self.task_config) From 43c8075b39e9134c0567dd7a4c06c9879a0010a1 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 27 Sep 2024 13:10:12 +0000 Subject: [PATCH 20/70] correct ORION.env typo, adjust JEDI ORION job configurations (#2862) --- env/ORION.env | 2 +- parm/config/gfs/config.resources.ORION | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/env/ORION.env b/env/ORION.env index 1bc7eb60d4..3b8053d060 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -142,7 +142,7 @@ elif [[ "${step}" = "marineanlchkpt" ]]; then export NTHREADS_OCNANAL=${NTHREADSmax} - export APRUN_MARINEANLCHKPT="${APRUN} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}" elif [[ "${step}" = "ocnanalecen" ]]; then diff --git a/parm/config/gfs/config.resources.ORION b/parm/config/gfs/config.resources.ORION index d761df7b73..461b6f14f7 100644 --- a/parm/config/gfs/config.resources.ORION +++ b/parm/config/gfs/config.resources.ORION @@ -23,6 +23,16 @@ case ${step} in # Remove this block once the GSI issue is resolved. export walltime="00:45:00" ;; + "atmanlvar") + # Run on 8 nodes for memory requirement + export tasks_per_node=8 + export walltime="00:45:00" + ;; + "atmensanlobs") + # Run on 8 nodes for memory requirement + export tasks_per_node=8 + export walltime="00:45:00" + ;; *) ;; esac From 4f0446a08488a48bd1f82ae04b0c7e42331ae20b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 27 Sep 2024 18:01:00 +0000 Subject: [PATCH 21/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 55e895f1dc..d39bf61570 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 55e895f1dcf4e6be36eb0eb4c8a7995d429157e0 +Subproject commit d39bf61570394730e17cd6508307ff7a624cd3cd From 7c30e482e1fe1d939ec53c1bec5e203739298466 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 4 Oct 2024 16:31:45 +0000 Subject: [PATCH 22/70] Update --- .../exglobal_atm_analysis_fv3_increment.py | 3 +- scripts/exglobal_atm_analysis_initialize.py | 4 +- scripts/exglobal_atm_analysis_variational.py | 2 +- .../exglobal_atmens_analysis_initialize.py | 4 +- scripts/exglobal_atmens_analysis_obs.py | 2 +- scripts/exglobal_atmens_analysis_sol.py | 2 +- ush/python/pygfs/jedi/jedi.py | 203 +++++++----------- ush/python/pygfs/task/atm_analysis.py | 96 ++------- ush/python/pygfs/task/atmens_analysis.py | 76 ++----- 9 files changed, 118 insertions(+), 274 deletions(-) diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index 72413ddbd4..f1422cca89 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -21,5 +21,4 @@ AtmAnl = AtmAnalysis(config, 'atmanlfv3inc') # Initialize and execute FV3 increment converter - AtmAnl.initialize_jedi() - AtmAnl.execute(config.APRUN_ATMANLFV3INC) + AtmAnl.jedi.execute(config.APRUN_ATMANLFV3INC) diff --git a/scripts/exglobal_atm_analysis_initialize.py b/scripts/exglobal_atm_analysis_initialize.py index 9deae07bb3..d9af271235 100755 --- a/scripts/exglobal_atm_analysis_initialize.py +++ b/scripts/exglobal_atm_analysis_initialize.py @@ -23,5 +23,5 @@ AtmAnl = AtmAnalysis(config, 'atmanlvar') # Initialize JEDI variational analysis - AtmAnl.initialize_jedi() - AtmAnl.initialize_analysis() + AtmAnl.jedi.initialize(AtmAnl.task_config) + AtmAnl.initialize() diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index 8359532069..cba7a33a5d 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config, 'atmanlvar') # Execute JEDI variational analysis - AtmAnl.execute(config.APRUN_ATMANLVAR, ['fv3jedi', 'variational']) + AtmAnl.jedi.execute(config.APRUN_ATMANLVAR, ['fv3jedi', 'variational']) diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index 326fe80628..26bb9a6dab 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -26,5 +26,5 @@ AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') # Initialize JEDI ensemble DA analysis - AtmEnsAnl.initialize_jedi() - AtmEnsAnl.initialize_analysis() + AtmEnsAnl.jedi.initialize(AtmEnsAnl.task_config) + AtmEnsAnl.initialize() diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index c701f8cb4e..ac3271272e 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.execute(config.APRUN_ATMENSANLOBS, ['fv3jedi', 'localensembleda']) + AtmEnsAnl.jedi.execute(config.APRUN_ATMENSANLOBS, ['fv3jedi', 'localensembleda']) diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index be78e694b1..d93c42ddef 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -21,5 +21,5 @@ AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlsol') # Initialize and execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.initialize_jedi() + AtmEnsAnl.jedi.initialize(AtmEnsAnl.task_config) AtmEnsAnl.execute(config.APRUN_ATMENSANLSOL, ['fv3jedi', 'localensembleda']) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 415a0a3c08..366f0f7470 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -4,14 +4,12 @@ import tarfile from logging import getLogger from typing import List, Dict, Any, Optional +from pprint import pformat from jcb import render -from wxflow import (AttrDict, - FileHandler, +from wxflow import (AttrDict, FileHandler, Task, Executable, chdir, rm_p, - parse_j2yaml, + parse_j2yaml, save_as_yaml, logit, - Task, - Executable, WorkflowException) logger = getLogger(__name__.split('.')[-1]) @@ -22,7 +20,7 @@ class Jedi: Class for initializing and executing JEDI applications """ @logit(logger, name="Jedi") - def __init__(self, task_config: AttrDict, yaml_name: Optional[str] = None) -> None: + def __init__(self, DATA: str, JEDIEXE: str, yaml_name: Optional[str]) -> None: """Constructor for JEDI objects This method will construct a Jedi object. @@ -44,69 +42,51 @@ def __init__(self, task_config: AttrDict, yaml_name: Optional[str] = None) -> No None """ - # For provenance, save incoming task_config as a private attribute of JEDI object - self._task_config = task_config + _exe_name = os.path.basename(JEDIEXE) - _exe_name = os.path.basename(task_config.JEDIEXE) - - self.exe = os.path.join(task_config.DATA, _exe_name) + self.exe_src = JEDIEXE + self.rundir = DATA + self.exe = os.path.join(DATA, _exe_name) if yaml_name: - self.yaml = os.path.join(task_config.DATA, yaml_name + '.yaml') + self.yaml = os.path.join(DATA, yaml_name + '.yaml') else: - self.yaml = os.path.join(task_config.DATA, os.path.splitext(_exe_name)[0] + '.yaml') + self.yaml = os.path.join(DATA, os.path.splitext(_exe_name)[0] + '.yaml') + + # Initialize empty JEDI input config attribute-dictionary self.config = AttrDict() - self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') + +# self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') @logit(logger) - def set_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> AttrDict: - """Compile a JEDI configuration dictionary from a template file and save to a YAML file + def initialize(self, task_config: AttrDict) -> None: + """Initialize JEDI application - Parameters - ---------- - task_config : AttrDict - Dictionary of all configuration variables associated with a GDAS task. - algorithm (optional) : str - Name of the algorithm used to generate the JEDI configuration dictionary. - It will override the algorithm set in the task_config.JCB_<>_YAML file. - - Returns - ---------- - None + This method will initialize a JEDI application. + This includes: + - generating JEDI YAML config + - saving JEDI YAML config to run directory + - linking the JEDI executable to run directory """ - if 'JCB_BASE_YAML' in task_config.keys(): - # Step 1: Fill templates of the JCB base YAML file - jcb_config = parse_j2yaml(task_config.JCB_BASE_YAML, task_config) - - # Step 2: If algorithm is present then override the algorithm in the JEDI - # config. Otherwise, if the algorithm J2-YAML is present, fill - # its templates and merge. - if algorithm: - jcb_config['algorithm'] = algorithm - elif 'JCB_ALGO' in task_config.keys(): - jcb_config['algorithm'] = task_config.JCB_ALGO - elif 'JCB_ALGO_YAML' in task_config.keys(): - jcb_algo_config = parse_j2yaml(task_config.JCB_ALGO_YAML, task_config) - jcb_config.update(jcb_algo_config) - - # Step 3: Generate the JEDI YAML using JCB - self.config = render(jcb_config) - elif 'JEDIYAML' in task_config.keys(): - # Generate JEDI YAML without using JCB - self.config = parse_j2yaml(task_config.JEDIYAML, task_config, - searchpath=self.j2tmpl_dir) - else: - logger.exception(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - raise KeyError(f"FATAL ERROR: Task config must contain JCB_BASE_YAML or JEDIYAML") + # Render JEDI config dictionary + logger.info(f"Generating JEDI YAML config: {self.yaml}") + self.config = self.get_config(task_config) + logger.debug(f"JEDI config:\n{pformat(self.config)}") + + # Save JEDI config dictionary to YAML in run directory + logger.debug(f"Writing JEDI YAML config to: {self.yaml}") + save_as_yaml(self.config, self.yaml) + # Link JEDI executable to run directory + logger.info(f"Linking JEDI executable {self.exe_src} to {self.exe}") + self.link_exe() + @logit(logger) - def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: + def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: """Execute JEDI application Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. aprun_cmd: str String comprising the run command for the JEDI executable. jedi_args (optional): List @@ -118,7 +98,7 @@ def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[Lis Attribute-dictionary of JEDI configuration rendered from a template. """ - chdir(task_config.DATA) + chdir(self.rundir) exec_cmd = Executable(aprun_cmd) exec_cmd.add_default_arg(self.exe) @@ -127,6 +107,7 @@ def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[Lis exec_cmd.add_default_arg(arg) exec_cmd.add_default_arg(self.yaml) + logger.info(f"Executing {exec_cmd}") try: exec_cmd() except OSError: @@ -134,103 +115,82 @@ def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[Lis except Exception: raise WorkflowException(f"FATAL ERROR: An error occurred during execution of {exec_cmd}") - @staticmethod @logit(logger) - def link_exe(task_config: AttrDict) -> None: - """Link JEDI executable to run directory + def get_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> AttrDict: + """Compile a JEDI configuration dictionary from a template file and save to a YAML file Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. + task_config : AttrDict + Dictionary of all configuration variables associated with a GDAS task. + algorithm (optional) : str + Name of the algorithm used to generate the JEDI configuration dictionary. + It will override the algorithm set in the task_config.JCB_ALGO_YAML file. Returns ---------- None """ - # TODO: linking is not permitted per EE2. - # Needs work in JEDI to be able to copy the exec. [NOAA-EMC/GDASApp#1254] - logger.warn("Linking is not permitted per EE2.") - exe_dest = os.path.join(task_config.DATA, os.path.basename(task_config.JEDIEXE)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(task_config.JEDIEXE, exe_dest) + # Fill JCB base YAML template and build JCB config dictionary + jcb_config = parse_j2yaml(task_config.JCB_BASE_YAML, task_config) + + # Add JCB algorithm YAML, if it exists, to JCB config dictionary + if 'JCB_ALGO_YAML' in task_config.keys(): + jcb_config.update(parse_j2yaml(task_config.JCB_ALGO_YAML, task_config)) - @logit(logger) - def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: - """Compile a dictionary of observation files to copy + # Set algorithm in JCB config dictionary or override the one set by JCB_ALGO_YAML + if algorithm: + jcb_config['algorithm'] = algorithm - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation files that are to be copied to the run directory - from the observation input directory + # Generate JEDI YAML config by rendering JCB config dictionary + jedi_config = render(jcb_config) + + return jedi_config + + @logit(logger) + def link_exe(self) -> None: + """Link JEDI executable to run directory Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. + None Returns ---------- - obs_dict: Dict - a dictionary containing the list of observation files to copy for FileHandler + None """ - observations = find_value_in_nested_dict(self.config, 'observations') - - copylist = [] - for ob in observations['observers']: - obfile = ob['obs space']['obsdatain']['engine']['obsfile'] - basename = os.path.basename(obfile) - copylist.append([os.path.join(task_config.COM_OBS, basename), obfile]) - obs_dict = { - 'mkdir': [os.path.join(task_config.DATA, 'obs')], - 'copy': copylist - } - return obs_dict + # TODO: linking is not permitted per EE2. + # Needs work in JEDI to be able to copy the exec. [NOAA-EMC/GDASApp#1254] + logger.warn("Linking is not permitted per EE2.") + if os.path.exists(self.exe): + rm_p(self.exe) + os.symlink(self.exe_src, self.exe) + @staticmethod @logit(logger) - def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: - """Compile a dictionary of observation files to copy - - This method extracts 'observers' from the JEDI yaml and determines from that list - if bias correction tar files are to be copied to the run directory - from the component directory. + def remove_redundant(input_list: List) -> List: + """Remove reduncancies from list with possible redundant, non-mutable elements Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. - bias_file - name of bias correction tar file + input_list : List + List with possible redundant, non-mutable elements Returns ---------- - bias_dict: Dict - a dictionary containing the list of observation bias files to copy for FileHandler + output_list : List + Input list but with redundancies removed """ - observations = find_value_in_nested_dict(self.config, 'observations') - - copylist = [] - for ob in observations['observers']: - if 'obs bias' in ob.keys(): - obfile = ob['obs bias']['input file'] - obdir = os.path.dirname(obfile) - basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.{bias_file}" - tar_file = os.path.join(obdir, bfile) - copylist.append([os.path.join(task_config.VarBcDir, bfile), tar_file]) - break - - bias_dict = { - 'mkdir': [os.path.join(task_config.DATA, 'bc')], - 'copy': copylist - } - - return bias_dict + output_list = [] + for item in input_list: + if item not in output_list: + output_list.append(item); + return output_list + @staticmethod @logit(logger) def extract_tar(tar_file: str) -> None: @@ -264,7 +224,6 @@ def extract_tar(tar_file: str) -> None: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") - @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: """ diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 6582db100d..8a21b1552f 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -6,8 +6,7 @@ import tarfile from logging import getLogger from pprint import pformat -from typing import Optional, Dict, Any - +from typing import Any, Dict, List, Optional from wxflow import (AttrDict, FileHandler, add_to_datetime, to_fv3time, to_timedelta, to_YMDH, @@ -74,45 +73,15 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI object - self.jedi = Jedi(self.task_config, yaml_name) - - @logit(logger) - def initialize_jedi(self): - """Initialize JEDI application - - This method will initialize a JEDI application used in the global atm analysis. - This includes: - - generating and saving JEDI YAML config - - linking the JEDI executable - - Parameters - ---------- - None - - Returns - ---------- - None - """ - - # get JEDI-to-FV3 increment converter config and save to YAML file - logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") - self.jedi.set_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") - - # save JEDI config to YAML file - logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") - save_as_yaml(self.jedi.config, self.jedi.yaml) - - # link JEDI executable - logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") - self.jedi.link_exe(self.task_config) + self.jedi = Jedi(self.task_config.DATA, self.task_config.JEDIEXE, yaml_name) @logit(logger) - def initialize_analysis(self) -> None: + def initialize(self) -> None: """Initialize a global atm analysis This method will initialize a global atm analysis. This includes: + - initializing JEDI variational application - staging observation files - staging bias correction files - staging CRTM fix files @@ -130,36 +99,30 @@ def initialize_analysis(self) -> None: None """ - # stage observations - logger.info(f"Staging list of observation files generated from JEDI config") - jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) - jcb_config.update(parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config)) - jcb_config['algorithm'] = 'atm_obs_staging' - obs_dict = render(jcb_config) - FileHandler(obs_dict).sync() - logger.debug(f"Observation files:\n{pformat(obs_dict)}") - - # Test - jcb_config = {} - jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) - jcb_config.update(parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config)) - jcb_config['algorithm'] = 'atm_bias_staging' - bias_dict = render(jcb_config) - logger.debug(f"foo:\n{pformat(bias_dict)}") + # initialize JEDI variational application + logger.info(f"Initializing JEDI ensemble DA application") + self.jedi.initialize(self.task_config) # stage observations - logger.info(f"Staging list of observation files generated from JEDI config") - obs_dict = self.jedi.get_obs_dict(self.task_config) + logger.info(f"Staging list of observation files") + obs_dict = self.jedi.get_config(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") + # stage bias corrections + logger.info(f"Staging list of bias correction files") + bias_dict = self.jedi.get_config(self.task_config, 'atm_bias_staging') + bias_dict['copy'] = jedi.remove_redundant(bias_dict['copy']) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" bias_dict = self.jedi.get_bias_dict(self.task_config, bias_file) FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + logger.debug(f"Bias correction files bar2:\n{pformat(bias_dict)}") # extract bias corrections tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") @@ -208,31 +171,6 @@ def initialize_analysis(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() - @logit(logger) - def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: - """Run JEDI executable - - This method will run JEDI executables for the global atm analysis - - Parameters - ---------- - aprun_cmd : str - Run command for JEDI application on HPC system - jedi_args : List - List of additional optional arguments for JEDI application - - Returns - ---------- - None - """ - - if jedi_args: - logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") - else: - logger.info(f"Executing {self.jedi.exe} {self.jedi.yaml}") - - self.jedi.execute(self.task_config, aprun_cmd, jedi_args) - @logit(logger) def finalize(self) -> None: """Finalize a global atm analysis diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 4b2f8ebbf4..73cf5ea8c8 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -74,45 +74,15 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI object - self.jedi = Jedi(self.task_config, yaml_name) + self.jedi = Jedi(self.task_config.DATA, self.task_config.JEDIEXE, yaml_name) @logit(logger) - def initialize_jedi(self): - """Initialize JEDI application - - This method will initialize a JEDI application used in the global atmens analysis. - This includes: - - generating and saving JEDI YAML config - - linking the JEDI executable - - Parameters - ---------- - None - - Returns - ---------- - None - """ - - # get JEDI config and save to YAML file - logger.info(f"Generating JEDI config: {self.jedi.yaml}") - self.jedi.set_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") - - # save JEDI config to YAML file - logger.info(f"Writing JEDI config to YAML file: {self.jedi.yaml}") - save_as_yaml(self.jedi.config, self.jedi.yaml) - - # link JEDI-to-FV3 increment converter executable - logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") - self.jedi.link_exe(self.task_config) - - @logit(logger) - def initialize_analysis(self) -> None: + def initialize(self) -> None: """Initialize a global atmens analysis This method will initialize a global atmens analysis. This includes: + - initialize JEDI ensemble DA application - staging observation files - staging bias correction files - staging CRTM fix files @@ -128,19 +98,21 @@ def initialize_analysis(self) -> None: ---------- None """ - super().initialize() + + # initialize JEDI ensemble DA application + logger.info(f"Initializing JEDI ensemble DA application") + self.jedi.initialize(self.task_config) # stage observations - logger.info(f"Staging list of observation files generated from JEDI config") - obs_dict = self.jedi.get_obs_dict(self.task_config) + logger.info(f"Staging list of observation files") + obs_dict = self.jedi.get_config(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections - logger.info(f"Staging list of bias correction files generated from JEDI config") - self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" - bias_file = f"rad_varbc_params.tar" - bias_dict = self.jedi.get_bias_dict(self.task_config, bias_file) + logger.info(f"Staging list of bias correction files") + bias_dict = self.jedi.get_config(self.task_config, 'atm_bias_staging') + bias_dict['copy'] = jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") @@ -175,30 +147,6 @@ def initialize_analysis(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() - @logit(logger) - def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: - """Run JEDI executable - - This method will run JEDI executables for the global atmens analysis - - Parameters - ---------- - aprun_cmd : str - Run command for JEDI application on HPC system - jedi_args : List - List of additional optional arguments for JEDI application - Returns - ---------- - None - """ - - if jedi_args: - logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") - else: - logger.info(f"Executing {self.jedi.exe} {self.jedi.yaml}") - - self.jedi.execute(self.task_config, aprun_cmd, jedi_args) - @logit(logger) def finalize(self) -> None: """Finalize a global atmens analysis From f18d4ce5e585053a64c6b3a933997a472f09feae Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Oct 2024 17:48:53 +0000 Subject: [PATCH 23/70] Initialize all JEDI applications on the initialize jobs --- parm/config/gfs/config.atmanl | 6 +- parm/config/gfs/config.atmanlfv3inc | 3 - parm/config/gfs/config.atmensanl | 13 +-- parm/config/gfs/config.atmensanlfv3inc | 3 - parm/config/gfs/config.atmensanlobs | 2 - parm/config/gfs/config.atmensanlsol | 2 - parm/config/gfs/yaml/defaults.yaml | 9 +- .../exglobal_atm_analysis_fv3_increment.py | 4 +- scripts/exglobal_atm_analysis_initialize.py | 3 +- scripts/exglobal_atm_analysis_variational.py | 4 +- .../exglobal_atmens_analysis_fv3_increment.py | 5 +- .../exglobal_atmens_analysis_initialize.py | 10 +-- scripts/exglobal_atmens_analysis_letkf.py | 9 +- scripts/exglobal_atmens_analysis_obs.py | 4 +- scripts/exglobal_atmens_analysis_sol.py | 5 +- ush/python/pygfs/jedi/jedi.py | 75 ++++++++-------- ush/python/pygfs/task/atm_analysis.py | 51 +++++++++-- ush/python/pygfs/task/atmens_analysis.py | 89 +++++++++++++++++-- 18 files changed, 195 insertions(+), 102 deletions(-) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index 9a06088ecc..a2baadde7b 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -6,7 +6,8 @@ echo "BEGIN: config.atmanl" export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" -export JCB_ALGO_YAML=@JCB_ALGO_YAML@ +export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@ +export JCB_ALGO_FV3INC="fv3jedi_fv3inc_variational" export STATICB_TYPE=@STATICB_TYPE@ export LOCALIZATION_TYPE="bump" @@ -33,6 +34,7 @@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE=${EXECgfs}/gdas.x +export JEDIEXE_VAR="${EXECgfs}/gdas.x" +export JEDIEXE_FV3INC="${EXECgfs}/fv3jedi_fv3inc.x" echo "END: config.atmanl" diff --git a/parm/config/gfs/config.atmanlfv3inc b/parm/config/gfs/config.atmanlfv3inc index ab7efa3a60..4e7714628e 100644 --- a/parm/config/gfs/config.atmanlfv3inc +++ b/parm/config/gfs/config.atmanlfv3inc @@ -8,7 +8,4 @@ echo "BEGIN: config.atmanlfv3inc" # Get task specific resources . "${EXPDIR}/config.resources" atmanlfv3inc -export JCB_ALGO=fv3jedi_fv3inc_variational -export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x - echo "END: config.atmanlfv3inc" diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index f5a1278248..2c57525834 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -6,11 +6,11 @@ echo "BEGIN: config.atmensanl" export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" -if [[ ${lobsdiag_forenkf} = ".false." ]] ; then - export JCB_ALGO_YAML=@JCB_ALGO_YAML_LETKF@ -else - export JCB_ALGO_YAML=@JCB_ALGO_YAML_OBS@ -fi + +export JCB_ALGO_YAML_LETKF=@JCB_ALGO_YAML_LETKF@ +export JCB_ALGO_YAML_OBS=@JCB_ALGO_YAML_OBS@ +export JCB_ALGO_YAML_SOL=@JCB_ALGO_YAML_SOL@ +export JCB_ALGO_FV3INC="fv3jedi_fv3inc_lgetkf" export INTERP_METHOD='barycentric' @@ -24,6 +24,7 @@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE=${EXECgfs}/gdas.x +export JEDIEXE_LETKF=${EXECgfs}/gdas.x +export JEDIEXE_FV3INC=${EXECgfs}/fv3jedi_fv3inc.x echo "END: config.atmensanl" diff --git a/parm/config/gfs/config.atmensanlfv3inc b/parm/config/gfs/config.atmensanlfv3inc index 2dc73f3f6e..fe3337e5a2 100644 --- a/parm/config/gfs/config.atmensanlfv3inc +++ b/parm/config/gfs/config.atmensanlfv3inc @@ -8,7 +8,4 @@ echo "BEGIN: config.atmensanlfv3inc" # Get task specific resources . "${EXPDIR}/config.resources" atmensanlfv3inc -export JCB_ALGO=fv3jedi_fv3inc_lgetkf -export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x - echo "END: config.atmensanlfv3inc" diff --git a/parm/config/gfs/config.atmensanlobs b/parm/config/gfs/config.atmensanlobs index dff3fa3095..c7e050b009 100644 --- a/parm/config/gfs/config.atmensanlobs +++ b/parm/config/gfs/config.atmensanlobs @@ -8,6 +8,4 @@ echo "BEGIN: config.atmensanlobs" # Get task specific resources . "${EXPDIR}/config.resources" atmensanlobs -export JCB_ALGO_YAML=@JCB_ALGO_YAML@ - echo "END: config.atmensanlobs" diff --git a/parm/config/gfs/config.atmensanlsol b/parm/config/gfs/config.atmensanlsol index dac161373b..8ef905d1bd 100644 --- a/parm/config/gfs/config.atmensanlsol +++ b/parm/config/gfs/config.atmensanlsol @@ -8,6 +8,4 @@ echo "BEGIN: config.atmensanlsol" # Get task specific resources . "${EXPDIR}/config.resources" atmensanlsol -export JCB_ALGO_YAML=@JCB_ALGO_YAML@ - echo "END: config.atmensanlsol" diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index dfc67d1237..caadf23464 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -23,7 +23,7 @@ base: FHMAX_ENKF_GFS: 12 atmanl: - JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2" + JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2" STATICB_TYPE: "gsibec" LAYOUT_X_ATMANL: 8 LAYOUT_Y_ATMANL: 8 @@ -33,16 +33,11 @@ atmanl: atmensanl: JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" + JCB_ALGO_YAML_SOL: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2" LAYOUT_X_ATMENSANL: 8 LAYOUT_Y_ATMENSANL: 8 IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 - -atmensanlobs: - JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" - -atmensanlsol: - JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2" aeroanl: IO_LAYOUT_X: 1 diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index f1422cca89..d49dbe103c 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -18,7 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis object - AtmAnl = AtmAnalysis(config, 'atmanlfv3inc') + AtmAnl = AtmAnalysis(config) # Initialize and execute FV3 increment converter - AtmAnl.jedi.execute(config.APRUN_ATMANLFV3INC) + AtmAnl.jedi_fv3inc.execute() diff --git a/scripts/exglobal_atm_analysis_initialize.py b/scripts/exglobal_atm_analysis_initialize.py index d9af271235..444a532f6c 100755 --- a/scripts/exglobal_atm_analysis_initialize.py +++ b/scripts/exglobal_atm_analysis_initialize.py @@ -20,8 +20,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis task - AtmAnl = AtmAnalysis(config, 'atmanlvar') + AtmAnl = AtmAnalysis(config) # Initialize JEDI variational analysis - AtmAnl.jedi.initialize(AtmAnl.task_config) AtmAnl.initialize() diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index cba7a33a5d..1200fe147c 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -18,7 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis task - AtmAnl = AtmAnalysis(config, 'atmanlvar') + AtmAnl = AtmAnalysis(config) # Execute JEDI variational analysis - AtmAnl.jedi.execute(config.APRUN_ATMANLVAR, ['fv3jedi', 'variational']) + AtmAnl.jedi_var.execute() diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index 48eb6a6a1e..dab8803d9c 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -18,8 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis object - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlfv3inc') + AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI FV3 increment converter - AtmEnsAnl.initialize_jedi() - AtmEnsAnl.execute(config.APRUN_ATMENSANLFV3INC) + AtmEnsAnl.jedi_fv3inc.execute() diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index 26bb9a6dab..010a6f1075 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -20,11 +20,11 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - if not config.lobsdiag_forenkf: - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') - else: - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') + AtmEnsAnl = AtmEnsAnalysis(config) +# if not config.lobsdiag_forenkf: +# AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') +# else: +# AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') # Initialize JEDI ensemble DA analysis - AtmEnsAnl.jedi.initialize(AtmEnsAnl.task_config) AtmEnsAnl.initialize() diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 45b06524fe..33d2fd3b8f 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -18,7 +18,12 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') + AtmEnsAnl = AtmEnsAnalysis(config) + # Initalize JEDI ensemble DA application + # Note: This is normally done in AtmEnsAnl.initialize(), but the that now + # initializes the split observer-solver. This case is just for testing. + AtmEnsAnl.jedi_letkf.initialize() + # Execute the JEDI ensemble DA analysis - AtmEnsAnl.execute(config.APRUN_ATMENSANLLETKF, ['fv3jedi', 'localensembleda']) + AtmEnsAnl.jedi_letkf.execute() diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index ac3271272e..660276d3e4 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -18,7 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') + AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.jedi.execute(config.APRUN_ATMENSANLOBS, ['fv3jedi', 'localensembleda']) + AtmEnsAnl.jedi_letkf_obs.execute() diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index d93c42ddef..a1e71dc1cf 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -18,8 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlsol') + AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.jedi.initialize(AtmEnsAnl.task_config) - AtmEnsAnl.execute(config.APRUN_ATMENSANLSOL, ['fv3jedi', 'localensembleda']) + AtmEnsAnl.jedi_letkf_sol.execute() diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 366f0f7470..a57871d619 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -20,12 +20,11 @@ class Jedi: Class for initializing and executing JEDI applications """ @logit(logger, name="Jedi") - def __init__(self, DATA: str, JEDIEXE: str, yaml_name: Optional[str]) -> None: + def __init__(self, config) -> None: """Constructor for JEDI objects This method will construct a Jedi object. This includes: - - save a copy of task_config for provenance - set the default JEDI YAML and executable names - set an empty AttrDict for the JEDI config - set the default directory for J2-YAML templates @@ -34,28 +33,23 @@ def __init__(self, DATA: str, JEDIEXE: str, yaml_name: Optional[str]) -> None: ---------- task_config: AttrDict Attribute-dictionary of all configuration variables associated with a GDAS task. - yaml_name: str, optional - Name of YAML file for JEDI configuration Returns ---------- None """ - - _exe_name = os.path.basename(JEDIEXE) - - self.exe_src = JEDIEXE - self.rundir = DATA - self.exe = os.path.join(DATA, _exe_name) - if yaml_name: - self.yaml = os.path.join(DATA, yaml_name + '.yaml') - else: - self.yaml = os.path.join(DATA, os.path.splitext(_exe_name)[0] + '.yaml') - - # Initialize empty JEDI input config attribute-dictionary - self.config = AttrDict() -# self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') + # Create the configuration dictionary for JEDI object + self.jedi_config = config.deepcopy() + + local_dict = AttrDict( + { + 'exe': os.path.join(self.config.run_dir, os.path.basename(self.config.exe_src)), + 'yaml': os.path.join(DATA, config.yaml_name + '.yaml'), + 'input_config': None + } + ) + self.jedi_config.update(local_dict) @logit(logger) def initialize(self, task_config: AttrDict) -> None: @@ -69,16 +63,16 @@ def initialize(self, task_config: AttrDict) -> None: """ # Render JEDI config dictionary - logger.info(f"Generating JEDI YAML config: {self.yaml}") - self.config = self.get_config(task_config) - logger.debug(f"JEDI config:\n{pformat(self.config)}") + logger.info(f"Generating JEDI YAML config: {self.jedi_config.yaml}") + self.jedi_config.input_config = self.render_jcb(task_config) + logger.debug(f"JEDI config:\n{pformat(self.jedi_config.input_config)}") # Save JEDI config dictionary to YAML in run directory - logger.debug(f"Writing JEDI YAML config to: {self.yaml}") - save_as_yaml(self.config, self.yaml) + logger.debug(f"Writing JEDI YAML config to: {self.jedi_config.yaml}") + save_as_yaml(self.jedi_config.input_config, self.jedi_config.yaml) # Link JEDI executable to run directory - logger.info(f"Linking JEDI executable {self.exe_src} to {self.exe}") + logger.info(f"Linking JEDI executable {self.jedi_config.exe_src} to {self.jedi_config.exe}") self.link_exe() @logit(logger) @@ -100,12 +94,12 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: chdir(self.rundir) - exec_cmd = Executable(aprun_cmd) - exec_cmd.add_default_arg(self.exe) - if jedi_args: - for arg in jedi_args: + exec_cmd = Executable(self.jedi_config.aprun_cmd) + exec_cmd.add_default_arg(self.jedi_config.exe) + if self.jedi_config.jedi_args: + for arg in self.jedi_config.jedi_args: exec_cmd.add_default_arg(arg) - exec_cmd.add_default_arg(self.yaml) + exec_cmd.add_default_arg(self.jedi_config.yaml) logger.info(f"Executing {exec_cmd}") try: @@ -116,7 +110,7 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: raise WorkflowException(f"FATAL ERROR: An error occurred during execution of {exec_cmd}") @logit(logger) - def get_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> AttrDict: + def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> AttrDict: """Compile a JEDI configuration dictionary from a template file and save to a YAML file Parameters @@ -125,19 +119,24 @@ def get_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> Dictionary of all configuration variables associated with a GDAS task. algorithm (optional) : str Name of the algorithm used to generate the JEDI configuration dictionary. - It will override the algorithm set in the task_config.JCB_ALGO_YAML file. + It will override the algorithm set in the jedi_config.jcb_algo_yaml file. Returns ---------- None - """ + """ + if not self.jedi_config.jcb_algo_yaml and not.algorithm: + logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") + logger.error(f"FATAL ERROR: JEDI config must contain jcb_algo_yaml or algorithm be + specified as an input to jedi.render_jcb") + # Fill JCB base YAML template and build JCB config dictionary - jcb_config = parse_j2yaml(task_config.JCB_BASE_YAML, task_config) + jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) # Add JCB algorithm YAML, if it exists, to JCB config dictionary - if 'JCB_ALGO_YAML' in task_config.keys(): - jcb_config.update(parse_j2yaml(task_config.JCB_ALGO_YAML, task_config)) + if self.jedi_config.jcb_algo_yaml: + jcb_config.update(parse_j2yaml(self.jedi_config.jcb_algo_yaml, task_config)) # Set algorithm in JCB config dictionary or override the one set by JCB_ALGO_YAML if algorithm: @@ -164,9 +163,8 @@ def link_exe(self) -> None: # TODO: linking is not permitted per EE2. # Needs work in JEDI to be able to copy the exec. [NOAA-EMC/GDASApp#1254] logger.warn("Linking is not permitted per EE2.") - if os.path.exists(self.exe): - rm_p(self.exe) - os.symlink(self.exe_src, self.exe) + if not os.path.exists(self.jedi_config.exe): + os.symlink(self.jedi_config.exe_src, self.jedi_config.exe) @staticmethod @logit(logger) @@ -224,6 +222,7 @@ def extract_tar(tar_file: str) -> None: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") +# TODO: remove since no longer used @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: """ diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index f9c81d5456..0e038165f7 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -23,7 +23,7 @@ class AtmAnalysis(Task): Class for JEDI-based global atm analysis tasks """ @logit(logger, name="AtmAnalysis") - def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): + def __init__(self, config: Dict[str, Any]): """Constructor global atm analysis task This method will construct a global atm analysis task. @@ -35,8 +35,6 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): ---------- config: Dict dictionary object containing task configuration - yaml_name: str, optional - name of YAML file for JEDI configuration Returns ---------- @@ -72,8 +70,35 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - # Create JEDI object - self.jedi = Jedi(self.task_config.DATA, self.task_config.JEDIEXE, yaml_name) + # Create JEDI variational object + jedi_config = AttrDict( + { + 'exe_src': self.task_config.JEDIEXE_VAR, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': None, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_VAR, + 'rundir': self.task_config.DATA, + 'aprun_cmd': self.task_config.APRUN_ATMANLVAR, + 'yaml_name': 'atmanlvar', + 'jedi_args': ['fv3jedi', 'variational'] + } + ) + self.jedi_var = Jedi(jedi_config) + + # Create JEDI FV3 increment converter object + jedi_config = AttrDict( + { + 'exe_src': self.task_config.JEDIEXE_FV3INC, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': self.task_config.JCB_ALGO_FV3INC + 'jcb_algo_yaml': None, + 'rundir': self.task_config.DATA, + 'aprun_cmd': self.task_config.APRUN_ATMANLFV3INC, + 'yaml_name': 'atmanlfv3inc', + 'jedi_args': None + } + ) + self.jedi_fv3inc = Jedi(jedi_config) @logit(logger) def initialize(self) -> None: @@ -81,7 +106,7 @@ def initialize(self) -> None: This method will initialize a global atm analysis. This includes: - - initializing JEDI variational application + - initialize JEDI applications - staging observation files - staging bias correction files - staging CRTM fix files @@ -98,16 +123,24 @@ def initialize(self) -> None: ---------- None """ + + # initialize JEDI variational application + logger.info(f"Initializing JEDI variational DA application") + self.jedi_var.initialize() + + # initialize JEDI FV3 increment conversion application + logger.info(f"Initializing JEDI FV3 increment conversion application") + self.jedi_fv3inc.initialize() # stage observations logger.info(f"Staging list of observation files") - obs_dict = self.jedi.get_config(self.task_config, 'atm_obs_staging') + obs_dict = self.jedi_var.render_jcb(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi.get_config(self.task_config, 'atm_bias_staging') + bias_dict = self.jedi_var.render_jcb(self.task_config, 'atm_bias_staging') bias_dict['copy'] = jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") @@ -115,7 +148,7 @@ def initialize(self) -> None: # extract bias corrections tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") logger.info(f"Extract bias correction files from {tar_file}") - self.jedi.extract_tar(tar_file) + self.jedi_var.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 859372e6e5..1ca1248451 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -28,7 +28,7 @@ class AtmEnsAnalysis(Task): Class for JEDI-based global atmens analysis tasks """ @logit(logger, name="AtmEnsAnalysis") - def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): + def __init__(self, config: Dict[str, Any]): """Constructor global atmens analysis task This method will construct a global atmens analysis task. @@ -40,8 +40,6 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): ---------- config: Dict dictionary object containing task configuration - yaml_name: str, optional - name of YAML file for JEDI configuration Returns ---------- @@ -73,15 +71,76 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - # Create JEDI object - self.jedi = Jedi(self.task_config.DATA, self.task_config.JEDIEXE, yaml_name) + # Create JEDI LETKF observer object + jedi_config = AttrDict( + { + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_LETKF, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': None + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_OBS + 'aprun_cmd': self.task_config.APRUN_ATMENSANLOBS, + 'yaml_name': 'atmensanlobs', + 'jedi_args': ['fv3jedi', 'localensembleda'] + } + ) + self.jedi_letkf_obs = Jedi(jedi_config) + # Create JEDI LETKF solver object + jedi_config = AttrDict( + { + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_LETKF, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': None + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_SOL + 'aprun_cmd': self.task_config.APRUN_ATMENSANLSOL, + 'yaml_name': 'atmensanlsol', + 'jedi_args': ['fv3jedi', 'localensembleda'] + } + ) + self.jedi_letkf_sol = Jedi(jedi_config) + + # Create JEDI FV3 increment converter + jedi_config = AttrDict( + { + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_FV3INC, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': self.task_config.JCB_ALGO_FV3INC + 'jcb_algo_yaml': None + 'aprun_cmd': self.task_config.APRUN_ATMENSANLFV3INC, + 'yaml_name': 'atmensanlfv3inc', + 'jedi_args': None + } + ) + self.jedi_fv3inc = Jedi(jedi_config) + + # Note: Since we now use the split observer-solvers, the following + # is only for testing. + + # Create JEDI LETKF object + jedi_config = AttrDict( + { + 'exe_src': self.task_config.JEDIEXE_LETKF, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': None + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF + 'rundir': self.task_config.DATA, + 'aprun_cmd': self.task_config.APRUN_ATMENSANLLETKF, + 'yaml_name': 'atmensanlletkf', + 'jedi_args': ['fv3jedi', 'localensembleda'] + } + ) + self.jedi_letkf = Jedi(jedi_config) + @logit(logger) def initialize(self) -> None: """Initialize a global atmens analysis This method will initialize a global atmens analysis. This includes: + - initialize JEDI applications - staging observation files - staging bias correction files - staging CRTM fix files @@ -98,23 +157,35 @@ def initialize(self) -> None: None """ + # initialize JEDI LETKF observer application + logger.info(f"Initializing JEDI LETKF observer application") + self.jedi_letkf_obs.initialize() + + # initialize JEDI LETKF solver application + logger.info(f"Initializing JEDI LETKF solver application") + self.jedi_letkf_sol.initialize() + + # initialize JEDI FV3 increment conversion application + logger.info(f"Initializing JEDI FV3 increment conversion application") + self.jedi_fv3inc.initialize() + # stage observations logger.info(f"Staging list of observation files") - obs_dict = self.jedi.get_config(self.task_config, 'atm_obs_staging') + obs_dict = self.jedi_letkf_obs.render_jcb(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi.get_config(self.task_config, 'atm_bias_staging') - bias_dict['copy'] = jedi.remove_redundant(bias_dict['copy']) + bias_dict = self.jedi_letkf_obs.render_jcb(self.task_config, 'atm_bias_staging') + bias_dict['copy'] = jedi_letkf_obs.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") logger.info(f"Extract bias correction files from {tar_file}") - self.jedi.extract_tar(tar_file) + self.jedi_letkf_obs..extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From d5e476b7a8a8ba4406e3cbb62dd13463c0708508 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Oct 2024 17:57:38 +0000 Subject: [PATCH 24/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index d39bf61570..12633b1fdd 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit d39bf61570394730e17cd6508307ff7a624cd3cd +Subproject commit 12633b1fddd230bb16a7d88b438542c88b4623f2 From cec645fca86e57d540eb789d1727dec586832533 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Oct 2024 22:08:46 +0000 Subject: [PATCH 25/70] pynorms --- ush/python/pygfs/jedi/jedi.py | 27 ++++++++++++------------ ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 8 +++---- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index a57871d619..ff92569229 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -38,10 +38,10 @@ def __init__(self, config) -> None: ---------- None """ - + # Create the configuration dictionary for JEDI object self.jedi_config = config.deepcopy() - + local_dict = AttrDict( { 'exe': os.path.join(self.config.run_dir, os.path.basename(self.config.exe_src)), @@ -74,7 +74,7 @@ def initialize(self, task_config: AttrDict) -> None: # Link JEDI executable to run directory logger.info(f"Linking JEDI executable {self.jedi_config.exe_src} to {self.jedi_config.exe}") self.link_exe() - + @logit(logger) def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: """Execute JEDI application @@ -101,7 +101,7 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: exec_cmd.add_default_arg(arg) exec_cmd.add_default_arg(self.jedi_config.yaml) - logger.info(f"Executing {exec_cmd}") + logger.info(f"Executing {exec_cmd}") try: exec_cmd() except OSError: @@ -124,16 +124,16 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> Returns ---------- None - """ + """ - if not self.jedi_config.jcb_algo_yaml and not.algorithm: + if not self.jedi_config.jcb_algo_yaml and not algorithm: logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") logger.error(f"FATAL ERROR: JEDI config must contain jcb_algo_yaml or algorithm be - specified as an input to jedi.render_jcb") - + specified as an input to jedi.render_jcb()") + # Fill JCB base YAML template and build JCB config dictionary jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) - + # Add JCB algorithm YAML, if it exists, to JCB config dictionary if self.jedi_config.jcb_algo_yaml: jcb_config.update(parse_j2yaml(self.jedi_config.jcb_algo_yaml, task_config)) @@ -146,7 +146,7 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> jedi_config = render(jcb_config) return jedi_config - + @logit(logger) def link_exe(self) -> None: """Link JEDI executable to run directory @@ -185,10 +185,10 @@ def remove_redundant(input_list: List) -> List: output_list = [] for item in input_list: if item not in output_list: - output_list.append(item); + output_list.append(item) return output_list - + @staticmethod @logit(logger) def extract_tar(tar_file: str) -> None: @@ -222,7 +222,8 @@ def extract_tar(tar_file: str) -> None: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") -# TODO: remove since no longer used + +# TODO: remove since no longer used @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: """ diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 0e038165f7..b5fedbf994 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -130,8 +130,8 @@ def initialize(self) -> None: # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi_fv3inc.initialize() - + self.jedi_fv3inc.initialize() + # stage observations logger.info(f"Staging list of observation files") obs_dict = self.jedi_var.render_jcb(self.task_config, 'atm_obs_staging') diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1ca1248451..ebd1aede62 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -118,7 +118,7 @@ def __init__(self, config: Dict[str, Any]): # Note: Since we now use the split observer-solvers, the following # is only for testing. - + # Create JEDI LETKF object jedi_config = AttrDict( { @@ -133,7 +133,7 @@ def __init__(self, config: Dict[str, Any]): } ) self.jedi_letkf = Jedi(jedi_config) - + @logit(logger) def initialize(self) -> None: """Initialize a global atmens analysis @@ -157,11 +157,11 @@ def initialize(self) -> None: None """ - # initialize JEDI LETKF observer application + # initialize JEDI LETKF observer application logger.info(f"Initializing JEDI LETKF observer application") self.jedi_letkf_obs.initialize() - # initialize JEDI LETKF solver application + # initialize JEDI LETKF solver application logger.info(f"Initializing JEDI LETKF solver application") self.jedi_letkf_sol.initialize() From bc7fbbd9414ba8ffc461346e94d07d644eb45435 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Oct 2024 22:11:33 +0000 Subject: [PATCH 26/70] pynorms #2 --- ush/python/pygfs/jedi/jedi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index ff92569229..47d4ba0993 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -129,7 +129,7 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> if not self.jedi_config.jcb_algo_yaml and not algorithm: logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") logger.error(f"FATAL ERROR: JEDI config must contain jcb_algo_yaml or algorithm be - specified as an input to jedi.render_jcb()") + specified as an input to jedi.render_jcb()") # Fill JCB base YAML template and build JCB config dictionary jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) From b3d323994e92290f64e03cc5cbf8db7a74111e7a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Oct 2024 22:14:06 +0000 Subject: [PATCH 27/70] pynorms #3 --- scripts/exglobal_atmens_analysis_letkf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 33d2fd3b8f..b6b47a2264 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -21,9 +21,9 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initalize JEDI ensemble DA application - # Note: This is normally done in AtmEnsAnl.initialize(), but the that now + # Note: This is normally done in AtmEnsAnl.initialize(), but that method now # initializes the split observer-solver. This case is just for testing. AtmEnsAnl.jedi_letkf.initialize() - + # Execute the JEDI ensemble DA analysis AtmEnsAnl.jedi_letkf.execute() From afada5d224d9b0873b60688481e180018f56f7d2 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 00:21:57 +0000 Subject: [PATCH 28/70] Fix some python bugs --- ush/python/pygfs/jedi/jedi.py | 19 +++++++++++++------ ush/python/pygfs/task/atm_analysis.py | 2 +- ush/python/pygfs/task/atmens_analysis.py | 18 +++++++++--------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 47d4ba0993..ec0cd8050f 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -126,13 +126,12 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> None """ - if not self.jedi_config.jcb_algo_yaml and not algorithm: - logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - logger.error(f"FATAL ERROR: JEDI config must contain jcb_algo_yaml or algorithm be - specified as an input to jedi.render_jcb()") - # Fill JCB base YAML template and build JCB config dictionary - jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) + if self.jedi_config.jcb_base_yaml: + jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) + else: + logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") + logger.error(f"FATAL ERROR: JEDI config must contain jcb_base_yaml.") # Add JCB algorithm YAML, if it exists, to JCB config dictionary if self.jedi_config.jcb_algo_yaml: @@ -141,6 +140,14 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> # Set algorithm in JCB config dictionary or override the one set by JCB_ALGO_YAML if algorithm: jcb_config['algorithm'] = algorithm + elif self.jedi_config.jcb_algo: + jcb_config['algorithm'] = self.jedi_config.jcb_algo + elif 'algorithm' in jcb_config: + pass + else: + logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") + logger.error(f"FATAL ERROR: algorithm must be specified in JEDI config, " + + "JCB algorithm YAML, or as input to jedi.render_jcb()") # Generate JEDI YAML config by rendering JCB config dictionary jedi_config = render(jcb_config) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index b5fedbf994..02aeb47e47 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -90,7 +90,7 @@ def __init__(self, config: Dict[str, Any]): { 'exe_src': self.task_config.JEDIEXE_FV3INC, 'jcb_base_yaml': self.task_config.jcb_base_yaml, - 'jcb_algo': self.task_config.JCB_ALGO_FV3INC + 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, 'jcb_algo_yaml': None, 'rundir': self.task_config.DATA, 'aprun_cmd': self.task_config.APRUN_ATMANLFV3INC, diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index ebd1aede62..410244138b 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -77,8 +77,8 @@ def __init__(self, config: Dict[str, Any]): 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.jcb_base_yaml, - 'jcb_algo': None - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_OBS + 'jcb_algo': None, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_OBS, 'aprun_cmd': self.task_config.APRUN_ATMENSANLOBS, 'yaml_name': 'atmensanlobs', 'jedi_args': ['fv3jedi', 'localensembleda'] @@ -92,8 +92,8 @@ def __init__(self, config: Dict[str, Any]): 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.jcb_base_yaml, - 'jcb_algo': None - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_SOL + 'jcb_algo': None, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_SOL, 'aprun_cmd': self.task_config.APRUN_ATMENSANLSOL, 'yaml_name': 'atmensanlsol', 'jedi_args': ['fv3jedi', 'localensembleda'] @@ -107,8 +107,8 @@ def __init__(self, config: Dict[str, Any]): 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_FV3INC, 'jcb_base_yaml': self.task_config.jcb_base_yaml, - 'jcb_algo': self.task_config.JCB_ALGO_FV3INC - 'jcb_algo_yaml': None + 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, + 'jcb_algo_yaml': None, 'aprun_cmd': self.task_config.APRUN_ATMENSANLFV3INC, 'yaml_name': 'atmensanlfv3inc', 'jedi_args': None @@ -124,8 +124,8 @@ def __init__(self, config: Dict[str, Any]): { 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.jcb_base_yaml, - 'jcb_algo': None - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF + 'jcb_algo': None, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, 'rundir': self.task_config.DATA, 'aprun_cmd': self.task_config.APRUN_ATMENSANLLETKF, 'yaml_name': 'atmensanlletkf', @@ -185,7 +185,7 @@ def initialize(self) -> None: # extract bias corrections tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") logger.info(f"Extract bias correction files from {tar_file}") - self.jedi_letkf_obs..extract_tar(tar_file) + self.jedi_letkf_obs.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 9e57e52ae0e4e78f169e75109c4fa2fb65629e9d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 01:12:30 +0000 Subject: [PATCH 29/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 12633b1fdd..3bf7b50f32 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 12633b1fddd230bb16a7d88b438542c88b4623f2 +Subproject commit 3bf7b50f324d4073cb357d0088bd11d324038593 From 25fad0e4587f3de7faa6af1c7de62d00bcbab5b7 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 03:05:50 +0000 Subject: [PATCH 30/70] Fixing bugs --- .../exglobal_atm_analysis_fv3_increment.py | 2 +- scripts/exglobal_atm_analysis_variational.py | 2 +- .../exglobal_atmens_analysis_fv3_increment.py | 2 +- scripts/exglobal_atmens_analysis_letkf.py | 4 +-- scripts/exglobal_atmens_analysis_obs.py | 2 +- scripts/exglobal_atmens_analysis_sol.py | 2 +- ush/python/pygfs/jedi/jedi.py | 27 ++++++++-------- ush/python/pygfs/task/atm_analysis.py | 21 +++++++------ ush/python/pygfs/task/atmens_analysis.py | 31 +++++++++---------- 9 files changed, 46 insertions(+), 47 deletions(-) diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index d49dbe103c..594ae6e7f7 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Initialize and execute FV3 increment converter - AtmAnl.jedi_fv3inc.execute() + AtmAnl.jedi_fv3inc.execute(config.APRUN_ATMANLFV3INC) diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index 1200fe147c..c7929f6b19 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Execute JEDI variational analysis - AtmAnl.jedi_var.execute() + AtmAnl.jedi_var.execute(config.APRUN_ATMANLVAR) diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index dab8803d9c..42d0afceed 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI FV3 increment converter - AtmEnsAnl.jedi_fv3inc.execute() + AtmEnsAnl.jedi_fv3inc.execute(config.APRUN_ATMENSANLFV3INC) diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index b6b47a2264..050449334e 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -23,7 +23,7 @@ # Initalize JEDI ensemble DA application # Note: This is normally done in AtmEnsAnl.initialize(), but that method now # initializes the split observer-solver. This case is just for testing. - AtmEnsAnl.jedi_letkf.initialize() + AtmEnsAnl.jedi_letkf.initialize(AtmEnsAnl.task_config) # Execute the JEDI ensemble DA analysis - AtmEnsAnl.jedi_letkf.execute() + AtmEnsAnl.jedi_letkf.execute(config.APRUN_ATMENSANLLETKF) diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index 660276d3e4..6cd961c99f 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.jedi_letkf_obs.execute() + AtmEnsAnl.jedi_letkf_obs.execute(config.APRUN_ATMENSANLOBS) diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index a1e71dc1cf..dab5206daf 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.jedi_letkf_sol.execute() + AtmEnsAnl.jedi_letkf_sol.execute(config.APRUN_ATMENSANLSOL) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index ec0cd8050f..dfbd167407 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -40,16 +40,17 @@ def __init__(self, config) -> None: """ # Create the configuration dictionary for JEDI object - self.jedi_config = config.deepcopy() - local_dict = AttrDict( { - 'exe': os.path.join(self.config.run_dir, os.path.basename(self.config.exe_src)), - 'yaml': os.path.join(DATA, config.yaml_name + '.yaml'), + 'exe': os.path.join(config.rundir, os.path.basename(config.exe_src)), + 'yaml': os.path.join(config.rundir, config.yaml_name + '.yaml'), 'input_config': None } ) - self.jedi_config.update(local_dict) + self.jedi_config = AttrDict(**config, **local_dict) + + # Save a copy of jedi_config + self._jedi_config = self.jedi_config.deepcopy() @logit(logger) def initialize(self, task_config: AttrDict) -> None: @@ -76,15 +77,13 @@ def initialize(self, task_config: AttrDict) -> None: self.link_exe() @logit(logger) - def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: + def execute(self, aprun_cmd: str) -> None: """Execute JEDI application Parameters ---------- aprun_cmd: str String comprising the run command for the JEDI executable. - jedi_args (optional): List - List of strings comprising optional input arguments for the JEDI executable. Returns ---------- @@ -92,9 +91,9 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: Attribute-dictionary of JEDI configuration rendered from a template. """ - chdir(self.rundir) + chdir(self.jedi_config.rundir) - exec_cmd = Executable(self.jedi_config.aprun_cmd) + exec_cmd = Executable(aprun_cmd) exec_cmd.add_default_arg(self.jedi_config.exe) if self.jedi_config.jedi_args: for arg in self.jedi_config.jedi_args: @@ -131,13 +130,13 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) else: logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - logger.error(f"FATAL ERROR: JEDI config must contain jcb_base_yaml.") + logger.error(f"FATAL ERROR: JEDI configuration dictionary must contain jcb_base_yaml.") # Add JCB algorithm YAML, if it exists, to JCB config dictionary if self.jedi_config.jcb_algo_yaml: jcb_config.update(parse_j2yaml(self.jedi_config.jcb_algo_yaml, task_config)) - # Set algorithm in JCB config dictionary or override the one set by JCB_ALGO_YAML + # Set algorithm in JCB config dictionary if algorithm: jcb_config['algorithm'] = algorithm elif self.jedi_config.jcb_algo: @@ -146,8 +145,8 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> pass else: logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - logger.error(f"FATAL ERROR: algorithm must be specified in JEDI config, " + - "JCB algorithm YAML, or as input to jedi.render_jcb()") + logger.error(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + + "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") # Generate JEDI YAML config by rendering JCB config dictionary jedi_config = render(jcb_config) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 02aeb47e47..04f161c81d 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -74,11 +74,10 @@ def __init__(self, config: Dict[str, Any]): jedi_config = AttrDict( { 'exe_src': self.task_config.JEDIEXE_VAR, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_VAR, 'rundir': self.task_config.DATA, - 'aprun_cmd': self.task_config.APRUN_ATMANLVAR, 'yaml_name': 'atmanlvar', 'jedi_args': ['fv3jedi', 'variational'] } @@ -89,11 +88,10 @@ def __init__(self, config: Dict[str, Any]): jedi_config = AttrDict( { 'exe_src': self.task_config.JEDIEXE_FV3INC, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, 'jcb_algo_yaml': None, 'rundir': self.task_config.DATA, - 'aprun_cmd': self.task_config.APRUN_ATMANLFV3INC, 'yaml_name': 'atmanlfv3inc', 'jedi_args': None } @@ -126,11 +124,11 @@ def initialize(self) -> None: # initialize JEDI variational application logger.info(f"Initializing JEDI variational DA application") - self.jedi_var.initialize() + self.jedi_var.initialize(self.task_config) # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi_fv3inc.initialize() + self.jedi_fv3inc.initialize(self.task_config) # stage observations logger.info(f"Staging list of observation files") @@ -141,14 +139,17 @@ def initialize(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files") bias_dict = self.jedi_var.render_jcb(self.task_config, 'atm_bias_staging') - bias_dict['copy'] = jedi.remove_redundant(bias_dict['copy']) + bias_dict['copy'] = self.jedi_var.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") - logger.info(f"Extract bias correction files from {tar_file}") - self.jedi_var.extract_tar(tar_file) + for item in bias_dict['copy']: + bias_file = os.path.basename(item[0]) + if os.path.splitext(bias_file)[1] == '.tar': + tar_file = f"{os.path.dirname(item[1])}/{bias_file}" + logger.info(f"Extract bias correction files from {tar_file}") + self.jedi_var.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 410244138b..332172fa8d 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -76,10 +76,9 @@ def __init__(self, config: Dict[str, Any]): { 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_OBS, - 'aprun_cmd': self.task_config.APRUN_ATMENSANLOBS, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_OBS, 'yaml_name': 'atmensanlobs', 'jedi_args': ['fv3jedi', 'localensembleda'] } @@ -91,10 +90,9 @@ def __init__(self, config: Dict[str, Any]): { 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_SOL, - 'aprun_cmd': self.task_config.APRUN_ATMENSANLSOL, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_SOL, 'yaml_name': 'atmensanlsol', 'jedi_args': ['fv3jedi', 'localensembleda'] } @@ -106,10 +104,9 @@ def __init__(self, config: Dict[str, Any]): { 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_FV3INC, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, 'jcb_algo_yaml': None, - 'aprun_cmd': self.task_config.APRUN_ATMENSANLFV3INC, 'yaml_name': 'atmensanlfv3inc', 'jedi_args': None } @@ -123,11 +120,10 @@ def __init__(self, config: Dict[str, Any]): jedi_config = AttrDict( { 'exe_src': self.task_config.JEDIEXE_LETKF, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, 'rundir': self.task_config.DATA, - 'aprun_cmd': self.task_config.APRUN_ATMENSANLLETKF, 'yaml_name': 'atmensanlletkf', 'jedi_args': ['fv3jedi', 'localensembleda'] } @@ -159,15 +155,15 @@ def initialize(self) -> None: # initialize JEDI LETKF observer application logger.info(f"Initializing JEDI LETKF observer application") - self.jedi_letkf_obs.initialize() + self.jedi_letkf_obs.initialize(self.task_config) # initialize JEDI LETKF solver application logger.info(f"Initializing JEDI LETKF solver application") - self.jedi_letkf_sol.initialize() + self.jedi_letkf_sol.initialize(self.task_config) # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi_fv3inc.initialize() + self.jedi_fv3inc.initialize(self.task_config) # stage observations logger.info(f"Staging list of observation files") @@ -183,9 +179,12 @@ def initialize(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") - logger.info(f"Extract bias correction files from {tar_file}") - self.jedi_letkf_obs.extract_tar(tar_file) + for item in bias_dict['copy']: + bias_file = os.path.basename(item[0]) + if os.path.splitext(bias_file)[1] == '.tar': + tar_file = f"{os.path.dirname(item[1])}/{bias_file}" + logger.info(f"Extract bias correction files from {tar_file}") + self.jedi_var.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 78bf67d1fe991d71c22adaad9d9da0741d56a569 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 03:12:37 +0000 Subject: [PATCH 31/70] Bug fix --- ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 04f161c81d..60c904396f 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -139,7 +139,7 @@ def initialize(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files") bias_dict = self.jedi_var.render_jcb(self.task_config, 'atm_bias_staging') - bias_dict['copy'] = self.jedi_var.remove_redundant(bias_dict['copy']) + bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") @@ -149,7 +149,7 @@ def initialize(self) -> None: if os.path.splitext(bias_file)[1] == '.tar': tar_file = f"{os.path.dirname(item[1])}/{bias_file}" logger.info(f"Extract bias correction files from {tar_file}") - self.jedi_var.extract_tar(tar_file) + Jedi.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 332172fa8d..178a8ee097 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -174,7 +174,7 @@ def initialize(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files") bias_dict = self.jedi_letkf_obs.render_jcb(self.task_config, 'atm_bias_staging') - bias_dict['copy'] = jedi_letkf_obs.remove_redundant(bias_dict['copy']) + bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") @@ -184,7 +184,7 @@ def initialize(self) -> None: if os.path.splitext(bias_file)[1] == '.tar': tar_file = f"{os.path.dirname(item[1])}/{bias_file}" logger.info(f"Extract bias correction files from {tar_file}") - self.jedi_var.extract_tar(tar_file) + Jedi.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From dadda23f2a5f6cff16ec5556b03e80ebda471cd9 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 16:37:15 +0000 Subject: [PATCH 32/70] Remove redundant code for tarball extraction, update comments, and update GDAS hash --- sorc/gdas.cd | 2 +- ush/python/pygfs/jedi/jedi.py | 107 ++++++++++++++++------- ush/python/pygfs/task/atm_analysis.py | 9 +- ush/python/pygfs/task/atmens_analysis.py | 9 +- 4 files changed, 79 insertions(+), 48 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 3bf7b50f32..8a501f7905 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 3bf7b50f324d4073cb357d0088bd11d324038593 +Subproject commit 8a501f7905497f5b65649fd05c825f5869b2a6c8 diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index dfbd167407..bda70e86d4 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -25,14 +25,13 @@ def __init__(self, config) -> None: This method will construct a Jedi object. This includes: - - set the default JEDI YAML and executable names - - set an empty AttrDict for the JEDI config - - set the default directory for J2-YAML templates + - create the jedi_config AttrDict and extend it with additional required entries + - save a coy of jedi_config Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. + config: AttrDict + Attribute-dictionary of all configuration variables required for the Jedi class Returns ---------- @@ -58,9 +57,18 @@ def initialize(self, task_config: AttrDict) -> None: This method will initialize a JEDI application. This includes: - - generating JEDI YAML config - - saving JEDI YAML config to run directory + - generating JEDI input YAML config + - saving JEDI input YAML config to run directory - linking the JEDI executable to run directory + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + + Returns + ---------- + None """ # Render JEDI config dictionary @@ -87,8 +95,7 @@ def execute(self, aprun_cmd: str) -> None: Returns ---------- - jedi_config: AttrDict - Attribute-dictionary of JEDI configuration rendered from a template. + None """ chdir(self.jedi_config.rundir) @@ -122,7 +129,8 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> Returns ---------- - None + jedi_input_config: AttrDict + Attribute-dictionary of JEDI configuration rendered from a template. """ # Fill JCB base YAML template and build JCB config dictionary @@ -149,9 +157,9 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") # Generate JEDI YAML config by rendering JCB config dictionary - jedi_config = render(jcb_config) + jedi_input_config = render(jcb_config) - return jedi_config + return jedi_input_config @logit(logger) def link_exe(self) -> None: @@ -197,37 +205,70 @@ def remove_redundant(input_list: List) -> List: @staticmethod @logit(logger) - def extract_tar(tar_file: str) -> None: - """Extract files from a tarball - - This method extract files from a tarball + def extract_tar_from_fh_dict(fh_dict) -> None: + """Extract tarballs from FileHandler input dictionary + This method extracts files from tarballs specified in a FileHander + input dictionary for the 'copy' action. + Parameters ---------- - tar_file - path/name of tarball + fh_dict + Input dictionary for FileHandler Returns ---------- None """ - - # extract files from tar file - tar_path = os.path.dirname(tar_file) - try: - with tarfile.open(tar_file, "r") as tarball: - tarball.extractall(path=tar_path) - logger.info(f"Extract {tarball.getnames()}") - except tarfile.ReadError as err: - if tarfile.is_tarfile(tar_file): - logger.error(f"FATAL ERROR: {tar_file} could not be read") - raise tarfile.ReadError(f"FATAL ERROR: unable to read {tar_file}") + + for item in fh_dict['copy']: + # Use the filename from the destination entry if it's a file path + # Otherwise, it's a directory, so use the source entry filename + if os.path.isfile(item[1]): + filename = os.path.basename(item[1]) else: - logger.info() - except tarfile.ExtractError as err: - logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") - raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") + filename = os.path.basename(item[0]) + + # Extract if file is a tarball + if os.path.splitext(filename)[1] == '.tar': + tar_file = f"{os.path.dirname(item[1])}/{filename}" + if os.path.isfile(tar_file): + logger.info(f"Extract files from {tar_file}") + extract_tar(tar_file) + else: + logger.error(f"FATAL ERROR: {tar_file} could not be read") + logger.error(f"FATAL ERROR: {tar_file} does not exist!") +@logit(logger) +def extract_tar(tar_file: str) -> None: + """Extract files from a tarball + + This method extract files from a tarball + + Parameters + ---------- + tar_file + path/name of tarball + Returns + ---------- + None + """ + + # extract files from tar file + tar_path = os.path.dirname(tar_file) + try: + with tarfile.open(tar_file, "r") as tarball: + tarball.extractall(path=tar_path) + logger.info(f"Extract {tarball.getnames()}") + except tarfile.ReadError as err: + if tarfile.is_tarfile(tar_file): + logger.error(f"FATAL ERROR: {tar_file} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tar_file}") + else: + logger.info() + except tarfile.ExtractError as err: + logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") # TODO: remove since no longer used @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 60c904396f..20d5264420 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -29,7 +29,7 @@ def __init__(self, config: Dict[str, Any]): This method will construct a global atm analysis task. This includes: - extending the task_config attribute AttrDict to include parameters required for this task - - instantiate the Jedi attribute object + - instantiate the Jedi attribute objects Parameters ---------- @@ -144,12 +144,7 @@ def initialize(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - for item in bias_dict['copy']: - bias_file = os.path.basename(item[0]) - if os.path.splitext(bias_file)[1] == '.tar': - tar_file = f"{os.path.dirname(item[1])}/{bias_file}" - logger.info(f"Extract bias correction files from {tar_file}") - Jedi.extract_tar(tar_file) + Jedi.extract_tar_from_fh_dict(bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 178a8ee097..b00dc4d9e8 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -34,7 +34,7 @@ def __init__(self, config: Dict[str, Any]): This method will construct a global atmens analysis task. This includes: - extending the task_config attribute AttrDict to include parameters required for this task - - instantiate the Jedi attribute object + - instantiate the Jedi attribute objects Parameters ---------- @@ -179,12 +179,7 @@ def initialize(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - for item in bias_dict['copy']: - bias_file = os.path.basename(item[0]) - if os.path.splitext(bias_file)[1] == '.tar': - tar_file = f"{os.path.dirname(item[1])}/{bias_file}" - logger.info(f"Extract bias correction files from {tar_file}") - Jedi.extract_tar(tar_file) + Jedi.extract_tar_from_fh_dict(bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From ee22db3861f99e063141b9b25982038dc68c2a55 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 16:39:54 +0000 Subject: [PATCH 33/70] pynorms --- ush/python/pygfs/jedi/jedi.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index bda70e86d4..f64dacf9bb 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -208,9 +208,9 @@ def remove_redundant(input_list: List) -> List: def extract_tar_from_fh_dict(fh_dict) -> None: """Extract tarballs from FileHandler input dictionary - This method extracts files from tarballs specified in a FileHander + This method extracts files from tarballs specified in a FileHander input dictionary for the 'copy' action. - + Parameters ---------- fh_dict @@ -220,7 +220,7 @@ def extract_tar_from_fh_dict(fh_dict) -> None: ---------- None """ - + for item in fh_dict['copy']: # Use the filename from the destination entry if it's a file path # Otherwise, it's a directory, so use the source entry filename @@ -238,6 +238,8 @@ def extract_tar_from_fh_dict(fh_dict) -> None: else: logger.error(f"FATAL ERROR: {tar_file} could not be read") logger.error(f"FATAL ERROR: {tar_file} does not exist!") + + @logit(logger) def extract_tar(tar_file: str) -> None: """Extract files from a tarball @@ -254,7 +256,7 @@ def extract_tar(tar_file: str) -> None: None """ - # extract files from tar file + # extract files from tar file tar_path = os.path.dirname(tar_file) try: with tarfile.open(tar_file, "r") as tarball: From 057a5e95a61109bf4c5fd583e3c0903cedace5b9 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 16:41:56 +0000 Subject: [PATCH 34/70] pynorms #2 --- ush/python/pygfs/jedi/jedi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index f64dacf9bb..b062abb43f 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -229,7 +229,7 @@ def extract_tar_from_fh_dict(fh_dict) -> None: else: filename = os.path.basename(item[0]) - # Extract if file is a tarball + # Extract if file is a tarball if os.path.splitext(filename)[1] == '.tar': tar_file = f"{os.path.dirname(item[1])}/{filename}" if os.path.isfile(tar_file): @@ -272,6 +272,7 @@ def extract_tar(tar_file: str) -> None: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") + # TODO: remove since no longer used @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: From ccd11fc3b8eccee88668a967babde0020f0f5c53 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 9 Oct 2024 15:27:10 +0000 Subject: [PATCH 35/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 8a501f7905..29696c82ab 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 8a501f7905497f5b65649fd05c825f5869b2a6c8 +Subproject commit 29696c82ab994e02dcfbeefc4b30953de352605b From c8f38bed573ba21138ca5fbfb0be45abcb1bb0e0 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 9 Oct 2024 23:56:36 +0000 Subject: [PATCH 36/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 29696c82ab..140dc0e6a6 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 29696c82ab994e02dcfbeefc4b30953de352605b +Subproject commit 140dc0e6a6550321b6cf1db6630f3d6ca067dc12 From 700204d6c0c5cf77f36a4c66939d449f7d22c77e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 10 Oct 2024 01:40:13 +0000 Subject: [PATCH 37/70] Address Cory's comments --- ush/python/pygfs/jedi/jedi.py | 20 ++++++++++---------- ush/python/pygfs/task/atm_analysis.py | 2 +- ush/python/pygfs/task/atmens_analysis.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index b062abb43f..6a1ec3855c 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -205,7 +205,7 @@ def remove_redundant(input_list: List) -> List: @staticmethod @logit(logger) - def extract_tar_from_fh_dict(fh_dict) -> None: + def extract_tar_from_filehandler_dict(filehandler_dict) -> None: """Extract tarballs from FileHandler input dictionary This method extracts files from tarballs specified in a FileHander @@ -213,7 +213,7 @@ def extract_tar_from_fh_dict(fh_dict) -> None: Parameters ---------- - fh_dict + filehandler_dict Input dictionary for FileHandler Returns @@ -221,7 +221,7 @@ def extract_tar_from_fh_dict(fh_dict) -> None: None """ - for item in fh_dict['copy']: + for item in filehandler_dict['copy']: # Use the filename from the destination entry if it's a file path # Otherwise, it's a directory, so use the source entry filename if os.path.isfile(item[1]): @@ -229,16 +229,13 @@ def extract_tar_from_fh_dict(fh_dict) -> None: else: filename = os.path.basename(item[0]) - # Extract if file is a tarball + # Check if file is a tar ball if os.path.splitext(filename)[1] == '.tar': tar_file = f"{os.path.dirname(item[1])}/{filename}" - if os.path.isfile(tar_file): - logger.info(f"Extract files from {tar_file}") - extract_tar(tar_file) - else: - logger.error(f"FATAL ERROR: {tar_file} could not be read") - logger.error(f"FATAL ERROR: {tar_file} does not exist!") + # Extract tarball + logger.info(f"Extract files from {tar_file}") + extract_tar(tar_file) @logit(logger) def extract_tar(tar_file: str) -> None: @@ -262,6 +259,9 @@ def extract_tar(tar_file: str) -> None: with tarfile.open(tar_file, "r") as tarball: tarball.extractall(path=tar_path) logger.info(f"Extract {tarball.getnames()}") + except tarfile.FileExistsError as err: + logger.exception(f"FATAL ERROR: {tar_file} does not exist") + raise tarfile.FileExistsError(f"FATAL ERROR: {tar_file} does not exist") except tarfile.ReadError as err: if tarfile.is_tarfile(tar_file): logger.error(f"FATAL ERROR: {tar_file} could not be read") diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 20d5264420..7b91ecc909 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -144,7 +144,7 @@ def initialize(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - Jedi.extract_tar_from_fh_dict(bias_dict) + Jedi.extract_tar_from_filehandler_dict(bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index b00dc4d9e8..c532dcb9f5 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -179,7 +179,7 @@ def initialize(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - Jedi.extract_tar_from_fh_dict(bias_dict) + Jedi.extract_tar_from_filehandler_dict(bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 62783ce5af36226efb59f9ddf10b90beda92e7db Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 10 Oct 2024 01:44:29 +0000 Subject: [PATCH 38/70] pynorms --- ush/python/pygfs/jedi/jedi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 6a1ec3855c..c9871e4ae3 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -237,6 +237,7 @@ def extract_tar_from_filehandler_dict(filehandler_dict) -> None: logger.info(f"Extract files from {tar_file}") extract_tar(tar_file) + @logit(logger) def extract_tar(tar_file: str) -> None: """Extract files from a tarball From f2a7a5585d27521d93352d53eb38de809b405394 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 12:46:21 +0000 Subject: [PATCH 39/70] Slight change for readability --- ush/python/pygfs/task/atm_analysis.py | 14 ++++++-------- ush/python/pygfs/task/atmens_analysis.py | 20 ++++++++------------ 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 7b91ecc909..6d7b1145ff 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -71,32 +71,30 @@ def __init__(self, config: Dict[str, Any]): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI variational object - jedi_config = AttrDict( + self.jedi_var = Jedi(AttrDict( { + 'yaml_name': 'atmanlvar', + 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_VAR, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_VAR, - 'rundir': self.task_config.DATA, - 'yaml_name': 'atmanlvar', 'jedi_args': ['fv3jedi', 'variational'] } ) - self.jedi_var = Jedi(jedi_config) # Create JEDI FV3 increment converter object - jedi_config = AttrDict( + self.jedi_fv3inc = Jedi(AttrDict( { + 'yaml_name': 'atmanlfv3inc', + 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_FV3INC, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, 'jcb_algo_yaml': None, - 'rundir': self.task_config.DATA, - 'yaml_name': 'atmanlfv3inc', 'jedi_args': None } ) - self.jedi_fv3inc = Jedi(jedi_config) @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index c532dcb9f5..334b2622c3 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -72,63 +72,59 @@ def __init__(self, config: Dict[str, Any]): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI LETKF observer object - jedi_config = AttrDict( + self.jedi_letkf_obs = Jedi(AttrDict( { + 'yaml_name': 'atmensanlobs', 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_OBS, - 'yaml_name': 'atmensanlobs', 'jedi_args': ['fv3jedi', 'localensembleda'] } ) - self.jedi_letkf_obs = Jedi(jedi_config) # Create JEDI LETKF solver object - jedi_config = AttrDict( + self.jedi_letkf_sol = Jedi(AttrDict( { + 'yaml_name': 'atmensanlsol', 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_SOL, - 'yaml_name': 'atmensanlsol', 'jedi_args': ['fv3jedi', 'localensembleda'] } ) - self.jedi_letkf_sol = Jedi(jedi_config) # Create JEDI FV3 increment converter - jedi_config = AttrDict( + self.jedi_fv3inc = Jedi(AttrDict( { + 'yaml_name': 'atmensanlfv3inc', 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_FV3INC, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, 'jcb_algo_yaml': None, - 'yaml_name': 'atmensanlfv3inc', 'jedi_args': None } ) - self.jedi_fv3inc = Jedi(jedi_config) # Note: Since we now use the split observer-solvers, the following # is only for testing. # Create JEDI LETKF object - jedi_config = AttrDict( + self.jedi_letkf = Jedi(AttrDict( { + 'yaml_name': 'atmensanlletkf', 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, 'rundir': self.task_config.DATA, - 'yaml_name': 'atmensanlletkf', 'jedi_args': ['fv3jedi', 'localensembleda'] } ) - self.jedi_letkf = Jedi(jedi_config) @logit(logger) def initialize(self) -> None: From 4baa1d53f9ce6fc9d42bc51c3bbc149b2618a730 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 13:28:52 +0000 Subject: [PATCH 40/70] Add key checking to JEDI class constructor --- ush/python/pygfs/jedi/jedi.py | 7 ++++++- ush/python/pygfs/task/atm_analysis.py | 15 +++++++++------ ush/python/pygfs/task/atmens_analysis.py | 23 +++++++++++++---------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index c9871e4ae3..fea1c176d7 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -38,6 +38,11 @@ def __init__(self, config) -> None: None """ + _key_list = ['yaml_name', 'rundir', 'exe_src', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml', 'jedi_args'] + for key in _key_list: + if key not in config.keys: + raise KeyError(f"Key '{key}' not found in the nested dictionary") + # Create the configuration dictionary for JEDI object local_dict = AttrDict( { @@ -46,7 +51,7 @@ def __init__(self, config) -> None: 'input_config': None } ) - self.jedi_config = AttrDict(**config, **local_dict) + self.jedi_config.update(local_dict) # Save a copy of jedi_config self._jedi_config = self.jedi_config.deepcopy() diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 6d7b1145ff..f08c7e5589 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -70,8 +70,11 @@ def __init__(self, config: Dict[str, Any]): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create dictionary of JEDI objects + self.jedi = AttrDict() + # Create JEDI variational object - self.jedi_var = Jedi(AttrDict( + self.jedi['atmanlvar'] = Jedi(AttrDict( { 'yaml_name': 'atmanlvar', 'rundir': self.task_config.DATA, @@ -84,7 +87,7 @@ def __init__(self, config: Dict[str, Any]): ) # Create JEDI FV3 increment converter object - self.jedi_fv3inc = Jedi(AttrDict( + self.jedi['atmanlfv3inc'] = Jedi(AttrDict( { 'yaml_name': 'atmanlfv3inc', 'rundir': self.task_config.DATA, @@ -122,21 +125,21 @@ def initialize(self) -> None: # initialize JEDI variational application logger.info(f"Initializing JEDI variational DA application") - self.jedi_var.initialize(self.task_config) + self.jedi['atmanlvar'].initialize(self.task_config) # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi_fv3inc.initialize(self.task_config) + self.jedi['atmanlfv3inc'].initialize(self.task_config) # stage observations logger.info(f"Staging list of observation files") - obs_dict = self.jedi_var.render_jcb(self.task_config, 'atm_obs_staging') + obs_dict = self.jedi['atmanlvar'].render_jcb(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi_var.render_jcb(self.task_config, 'atm_bias_staging') + bias_dict = self.jedi['atmanlvar'].render_jcb(self.task_config, 'atm_bias_staging') bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 334b2622c3..dc4075971a 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -71,8 +71,11 @@ def __init__(self, config: Dict[str, Any]): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create dictionary of JEDI objects + self.jedi = AttrDict() + # Create JEDI LETKF observer object - self.jedi_letkf_obs = Jedi(AttrDict( + self.jedi['atmensanlobs'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlobs', 'rundir': self.task_config.DATA, @@ -85,7 +88,7 @@ def __init__(self, config: Dict[str, Any]): ) # Create JEDI LETKF solver object - self.jedi_letkf_sol = Jedi(AttrDict( + self.jedi['atmensanlsol'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlsol', 'rundir': self.task_config.DATA, @@ -98,7 +101,7 @@ def __init__(self, config: Dict[str, Any]): ) # Create JEDI FV3 increment converter - self.jedi_fv3inc = Jedi(AttrDict( + self.jedi['atmensanlfv3inc'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlfv3inc', 'rundir': self.task_config.DATA, @@ -114,14 +117,14 @@ def __init__(self, config: Dict[str, Any]): # is only for testing. # Create JEDI LETKF object - self.jedi_letkf = Jedi(AttrDict( + self.jedi['atmensanlletkf'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlletkf', + 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, - 'rundir': self.task_config.DATA, 'jedi_args': ['fv3jedi', 'localensembleda'] } ) @@ -151,25 +154,25 @@ def initialize(self) -> None: # initialize JEDI LETKF observer application logger.info(f"Initializing JEDI LETKF observer application") - self.jedi_letkf_obs.initialize(self.task_config) + self.jedi['atmensanlobs'].initialize(self.task_config) # initialize JEDI LETKF solver application logger.info(f"Initializing JEDI LETKF solver application") - self.jedi_letkf_sol.initialize(self.task_config) + self.jedi['atmensanlsol'].initialize(self.task_config) # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi_fv3inc.initialize(self.task_config) + self.jedi['atmensanlfv3inc'].initialize(self.task_config) # stage observations logger.info(f"Staging list of observation files") - obs_dict = self.jedi_letkf_obs.render_jcb(self.task_config, 'atm_obs_staging') + obs_dict = self.jedi['atmensanlobs'].render_jcb(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi_letkf_obs.render_jcb(self.task_config, 'atm_bias_staging') + bias_dict = self.jedi['atmensanlobs'].render_jcb(self.task_config, 'atm_bias_staging') bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") From 2f72ecccfd1c09702faeeefdfeac9e73d54e3945 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 13:33:19 +0000 Subject: [PATCH 41/70] Update --- ush/python/pygfs/task/atm_analysis.py | 8 ++++---- ush/python/pygfs/task/atmens_analysis.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index f08c7e5589..698567cb60 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -73,7 +73,7 @@ def __init__(self, config: Dict[str, Any]): # Create dictionary of JEDI objects self.jedi = AttrDict() - # Create JEDI variational object + # atmanlvar self.jedi['atmanlvar'] = Jedi(AttrDict( { 'yaml_name': 'atmanlvar', @@ -84,9 +84,9 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_VAR, 'jedi_args': ['fv3jedi', 'variational'] } - ) + )) - # Create JEDI FV3 increment converter object + # atmanlfv3inc self.jedi['atmanlfv3inc'] = Jedi(AttrDict( { 'yaml_name': 'atmanlfv3inc', @@ -97,7 +97,7 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index dc4075971a..e3196b7e98 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -74,7 +74,7 @@ def __init__(self, config: Dict[str, Any]): # Create dictionary of JEDI objects self.jedi = AttrDict() - # Create JEDI LETKF observer object + # atmensanlobs self.jedi['atmensanlobs'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlobs', @@ -85,9 +85,9 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_OBS, 'jedi_args': ['fv3jedi', 'localensembleda'] } - ) + )) - # Create JEDI LETKF solver object + # atmensanlsol self.jedi['atmensanlsol'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlsol', @@ -98,9 +98,9 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_SOL, 'jedi_args': ['fv3jedi', 'localensembleda'] } - ) + )) - # Create JEDI FV3 increment converter + # atmensanlfv3inc self.jedi['atmensanlfv3inc'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlfv3inc', @@ -111,12 +111,12 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) # Note: Since we now use the split observer-solvers, the following # is only for testing. - # Create JEDI LETKF object + # atmensanlletkf self.jedi['atmensanlletkf'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlletkf', @@ -127,7 +127,7 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, 'jedi_args': ['fv3jedi', 'localensembleda'] } - ) + )) @logit(logger) def initialize(self) -> None: From e3ffaf0b68b181d2819b54a0611a520da4717839 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 13:49:16 +0000 Subject: [PATCH 42/70] Update --- scripts/exglobal_atm_analysis_fv3_increment.py | 6 +++--- scripts/exglobal_atm_analysis_initialize.py | 4 ++-- scripts/exglobal_atm_analysis_variational.py | 4 ++-- scripts/exglobal_atmens_analysis_fv3_increment.py | 4 ++-- scripts/exglobal_atmens_analysis_initialize.py | 8 ++------ scripts/exglobal_atmens_analysis_letkf.py | 8 ++++---- scripts/exglobal_atmens_analysis_obs.py | 4 ++-- scripts/exglobal_atmens_analysis_sol.py | 4 ++-- 8 files changed, 19 insertions(+), 23 deletions(-) diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index 594ae6e7f7..39a9b4e25a 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atm_analysis_fv3_increment.py # This script creates an AtmAnalysis object -# and runs the initialize_fv3inc and execute methods -# which convert the JEDI increment into an FV3 increment +# and runs the execute method of its Jedi +# object attribute import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Initialize and execute FV3 increment converter - AtmAnl.jedi_fv3inc.execute(config.APRUN_ATMANLFV3INC) + AtmAnl.jedi['atmanlfv3inc'].execute(config.APRUN_ATMANLFV3INC) diff --git a/scripts/exglobal_atm_analysis_initialize.py b/scripts/exglobal_atm_analysis_initialize.py index 444a532f6c..749d320111 100755 --- a/scripts/exglobal_atm_analysis_initialize.py +++ b/scripts/exglobal_atm_analysis_initialize.py @@ -2,8 +2,8 @@ # exglobal_atm_analysis_initialize.py # This script creates an AtmAnalysis class # and runs the initialize method -# which create and stage the runtime directory -# and create the YAML configuration +# which creates and stages the runtime directory +# and creates the YAML configuration # for a global atm variational analysis import os diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index c7929f6b19..21d99da3a2 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atm_analysis_variational.py # This script creates an AtmAnalysis object -# and runs the execute method +# and runs the execute method of its Jedi object attribute # which executes the global atm variational analysis import os @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Execute JEDI variational analysis - AtmAnl.jedi_var.execute(config.APRUN_ATMANLVAR) + AtmAnl.jedi['atmanlvar'].execute(config.APRUN_ATMANLVAR) diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index 42d0afceed..288c043adc 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_fv3_increment.py # This script creates an AtmEnsAnalysis object -# and runs the initialize_fv3inc and execute methods +# and runs the execute method of its Jedi object attribute # which convert the JEDI increment into an FV3 increment import os @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI FV3 increment converter - AtmEnsAnl.jedi_fv3inc.execute(config.APRUN_ATMENSANLFV3INC) + AtmEnsAnl.jedi['atmensanlfv3inc'].execute(config.APRUN_ATMENSANLFV3INC) diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index 010a6f1075..124e755594 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -2,8 +2,8 @@ # exglobal_atmens_analysis_initialize.py # This script creates an AtmEnsAnalysis class # and runs the initialize method -# which create and stage the runtime directory -# and create the YAML configuration +# which creates and stages the runtime directory +# and creates the YAML configuration # for a global atm local ensemble analysis import os @@ -21,10 +21,6 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) -# if not config.lobsdiag_forenkf: -# AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') -# else: -# AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') # Initialize JEDI ensemble DA analysis AtmEnsAnl.initialize() diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 050449334e..c0516003d6 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_letkf.py # This script creates an AtmEnsAnalysis object -# and runs the execute method which executes -# the global atm local ensemble analysis +# and runs the execute method of its Jedi object attribute +# which executes the global atm local ensemble analysis import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -23,7 +23,7 @@ # Initalize JEDI ensemble DA application # Note: This is normally done in AtmEnsAnl.initialize(), but that method now # initializes the split observer-solver. This case is just for testing. - AtmEnsAnl.jedi_letkf.initialize(AtmEnsAnl.task_config) + AtmEnsAnl.jedi['atmensanlletkf'].initialize(AtmEnsAnl.task_config) # Execute the JEDI ensemble DA analysis - AtmEnsAnl.jedi_letkf.execute(config.APRUN_ATMENSANLLETKF) + AtmEnsAnl.jedi['atmensanlletkf'].execute(config.APRUN_ATMENSANLLETKF) diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index 6cd961c99f..84b1f28096 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_obs.py # This script creates an AtmEnsAnalysis object -# and runs the execute method +# and runs the execute method of its Jedi object attribute # which executes the global atm local ensemble analysis in observer mode import os @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.jedi_letkf_obs.execute(config.APRUN_ATMENSANLOBS) + AtmEnsAnl.jedi['atmensanlobs'].execute(config.APRUN_ATMENSANLOBS) diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index dab5206daf..e6c112f97f 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_sol.py # This script creates an AtmEnsAnalysis object -# and runs the execute method +# and runs the execute method of its Jedi object attribute # which executes the global atm local ensemble analysis in solver mode import os @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.jedi_letkf_sol.execute(config.APRUN_ATMENSANLSOL) + AtmEnsAnl.jedi['atmensanlsol'].execute(config.APRUN_ATMENSANLSOL) From 955f19112700b31422634f46d355c7953821d481 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 13:53:18 +0000 Subject: [PATCH 43/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 140dc0e6a6..c71d3b1721 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 140dc0e6a6550321b6cf1db6630f3d6ca067dc12 +Subproject commit c71d3b17214007d59f91cc9c0988d4a8a279b22d From e8baba30c32bee0d01fcb0b3d1e48bce1b6be23e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 13:57:03 +0000 Subject: [PATCH 44/70] pynorms --- ush/python/pygfs/jedi/jedi.py | 2 +- ush/python/pygfs/task/atm_analysis.py | 2 +- ush/python/pygfs/task/atmens_analysis.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index fea1c176d7..89c81c93dc 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -42,7 +42,7 @@ def __init__(self, config) -> None: for key in _key_list: if key not in config.keys: raise KeyError(f"Key '{key}' not found in the nested dictionary") - + # Create the configuration dictionary for JEDI object local_dict = AttrDict( { diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 698567cb60..5ddb678036 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -90,7 +90,7 @@ def __init__(self, config: Dict[str, Any]): self.jedi['atmanlfv3inc'] = Jedi(AttrDict( { 'yaml_name': 'atmanlfv3inc', - 'rundir': self.task_config.DATA, + 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_FV3INC, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index e3196b7e98..46072dfe8b 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -73,7 +73,7 @@ def __init__(self, config: Dict[str, Any]): # Create dictionary of JEDI objects self.jedi = AttrDict() - + # atmensanlobs self.jedi['atmensanlobs'] = Jedi(AttrDict( { From 0805a08e9ceb1f75162394b6f23c58d0d5b1bfe3 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 18:29:14 +0000 Subject: [PATCH 45/70] Initial commit --- parm/config/gfs/config.marineanl | 21 +- parm/config/gfs/config.marinebmat | 8 - ush/python/pygfs/task/marine_bmat.py | 282 ++++++++++++--------------- 3 files changed, 144 insertions(+), 167 deletions(-) diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index a19fc015e2..cf056387ca 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -15,6 +15,25 @@ export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage_${OCNRES}.yaml.j2 export MARINE_UTILITY_YAML_TMPL="${PARMgfs}/gdas/soca/soca_utils_stage.yaml.j2" export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_mem.yaml.j2" export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" -export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" + +export JCB_BASE_YAML="${PARMgfs}/gdas/soca/marine-jcb-base.yaml" +#export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" + +export JCB_ALGO_GRIDGEN="soca_gridgen" +export JCB_ALGO_DIAGB="soca_diagb" +export JCB_ALGO_SETCORSCALES="soca_setcorscales" +export JCB_ALGO_PARAMETERS_DIFFUSION_HZ="soca_parameters_diffusion_hz" +export JCB_ALGO_VTSCALES="soca_vtscales" +export JCB_ALGO_PARAMETERS_DIFFUSION_VT="soca_parameters_diffusion_vt" +export JCB_ALGO_ENSB="soca_ensb" +export JCB_ALGO_ENSWEIGHTS="soca_ensweights" + +export JEDIEXE_GRIDGEN="${EXECgfs}/gdas_soca_gridgen.x" +export JEDIEXE_DIAGB="${EXECgfs}/gdas_soca_diagb.x" +export JEDIEXE_SETCORSCALES="${EXECgfs}/soca_setcorscales.x" +export JEDIEXE_PARAMETERS_DIFFUSION_HZ="${EXECgfs}/gdas_soca_error_covariance_toolbox.x" +export JEDIEXE_PARAMETERS_DIFFUSION_VT="${EXECgfs}/gdas_soca_error_covariance_toolbox.x" +export JEDIEXE_ENSB="${EXECgfs}/gdas_ens_handler.x" +export JEDIEXE_ENSWEIGHTS="${EXECgfs}/gdas_socahybridweights.x" echo "END: config.marineanl" diff --git a/parm/config/gfs/config.marinebmat b/parm/config/gfs/config.marinebmat index 00352737d0..d88739dced 100644 --- a/parm/config/gfs/config.marinebmat +++ b/parm/config/gfs/config.marinebmat @@ -8,12 +8,4 @@ echo "BEGIN: config.marinebmat" # Get task specific resources . "${EXPDIR}/config.resources" marinebmat -export BERROR_DIAGB_YAML="${PARMgfs}/gdas/soca/berror/soca_diagb.yaml.j2" -export BERROR_VTSCALES_YAML="${PARMgfs}/gdas/soca/berror/soca_vtscales.yaml.j2" -export BERROR_DIFFV_YAML="${PARMgfs}/gdas/soca/berror/soca_parameters_diffusion_vt.yaml.j2" -export BERROR_HZSCALES_YAML="${PARMgfs}/gdas/soca/berror/soca_setcorscales.yaml" -export BERROR_DIFFH_YAML="${PARMgfs}/gdas/soca/berror/soca_parameters_diffusion_hz.yaml.j2" -export BERROR_ENS_RECENTER_YAML="${PARMgfs}/gdas/soca/berror/soca_ensb.yaml.j2" -export BERROR_HYB_WEIGHTS_YAML="${PARMgfs}/gdas/soca/berror/soca_ensweights.yaml.j2" - echo "END: config.marinebmat" diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 93329f05ac..b316b221f9 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -50,6 +50,100 @@ def __init__(self, config): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create dictionary of Jedi objects + self.jedi = AttrDict() + + # gridgen + self.jedi['gridgen'] = Jedi(AttrDict( + { + 'yaml_name': 'gridgen', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_GRIDGEN, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_GRIDGEN, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + )) + + # soca_diagb + self.jedi['soca_diagb'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_diagb', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_DIAGB, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_DIAGB, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + )) + + # soca_parameters_diffusion_vt + self.jedi['soca_parameters_diffusion_vt'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_parameters_diffusion_vt', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_PARAMETERS_DIFFUSION_VT, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_PARAMETERS_DIFFUSION_VT, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + ) + + # soca_setcorscales + self.jedi['soca_setcorscales'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_setcorscales', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_SETCORSCALES, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_SETCORSCALES, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + ) + + # soca_parameters_diffusion_hz + self.jedi['soca_parameters_diffusion_hz'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_parameters_diffusion_hz', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_PARAMETERS_DIFFUSION_HZ, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_PARAMETERS_DIFFUSION_HZ, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + ) + + # soca_ensb + self.jedi['soca_ensb'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_ensb', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_ENSB, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_ENSB, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + ) + + # soca_ensweights + self.jedi['soca_ensb'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_ensb', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_ENSWEIGHTS, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_ENSWEIGHTS, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + ) + @logit(logger) def initialize(self: Task) -> None: """Initialize a global B-matrix @@ -63,7 +157,6 @@ def initialize(self: Task) -> None: - generating the YAML files for the JEDI and GDASApp executables - creating output directories """ - super().initialize() # stage fix files logger.info(f"Staging SOCA fix files from {self.task_config.SOCA_INPUT_FIX_DIR}") @@ -78,54 +171,27 @@ def initialize(self: Task) -> None: bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) FileHandler(bkg_list).sync() - # stage the soca utility yamls (gridgen, fields and ufo mapping yamls) - logger.info(f"Staging SOCA utility yaml files") - soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) - FileHandler(soca_utility_list).sync() - - # generate the variance partitioning YAML file - logger.info(f"Generate variance partitioning YAML file from {self.task_config.BERROR_DIAGB_YAML}") - diagb_config = parse_j2yaml(path=self.task_config.BERROR_DIAGB_YAML, data=self.task_config) - diagb_config.save(os.path.join(self.task_config.DATA, 'soca_diagb.yaml')) - - # generate the vertical decorrelation scale YAML file - logger.info(f"Generate the vertical correlation scale YAML file from {self.task_config.BERROR_VTSCALES_YAML}") - vtscales_config = parse_j2yaml(path=self.task_config.BERROR_VTSCALES_YAML, data=self.task_config) - vtscales_config.save(os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) - - # generate vertical diffusion scale YAML file - logger.info(f"Generate vertical diffusion YAML file from {self.task_config.BERROR_DIFFV_YAML}") - diffvz_config = parse_j2yaml(path=self.task_config.BERROR_DIFFV_YAML, data=self.task_config) - diffvz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_vt.yaml')) - - # generate the horizontal diffusion YAML files - if True: # TODO(G): skip this section once we have optimized the scales - # stage the correlation scale configuration - logger.info(f"Generate correlation scale YAML file from {self.task_config.BERROR_HZSCALES_YAML}") - FileHandler({'copy': [[self.task_config.BERROR_HZSCALES_YAML, - os.path.join(self.task_config.DATA, 'soca_setcorscales.yaml')]]}).sync() - - # generate horizontal diffusion scale YAML file - logger.info(f"Generate horizontal diffusion scale YAML file from {self.task_config.BERROR_DIFFH_YAML}") - diffhz_config = parse_j2yaml(path=self.task_config.BERROR_DIFFH_YAML, data=self.task_config) - diffhz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_hz.yaml')) - - # hybrid EnVAR case + # initialize vtscales python script + vtscales_config = self.render_jcb(task_config, 'vtscales') + save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml') + FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), + os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() + + # initialize JEDI applications + self.jedi['gridgen'].initialize(self.task_config) + self.jedi['soca_diagb'].initialize(self.task_config) + self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) + self.jedi['soca_setcorscales'].initialize(self.task_config) + self.jedi['soca_parameters_diffusion_hz'].initialize(self.task_config) + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: + self.jedi['soca_ensb'.initialize(self.task_config) + self.jedi['soca_ensweights'].initialize(self.task_config) + + # stage ensemble members for the hybrid background error if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - # stage ensemble membersfiles for use in hybrid background error logger.debug(f"Stage ensemble members for the hybrid background error") mdau.stage_ens_mem(self.task_config) - - # generate ensemble recentering/rebalancing YAML file - logger.debug("Generate ensemble recentering YAML file") - ensrecenter_config = parse_j2yaml(path=self.task_config.BERROR_ENS_RECENTER_YAML, data=self.task_config) - ensrecenter_config.save(os.path.join(self.task_config.DATA, 'soca_ensb.yaml')) - - # generate ensemble weights YAML file - logger.debug("Generate hybrid-weigths YAML file") - hybridweights_config = parse_j2yaml(path=self.task_config.BERROR_HYB_WEIGHTS_YAML, data=self.task_config) - hybridweights_config.save(os.path.join(self.task_config.DATA, 'soca_ensweights.yaml')) - + # create the symbolic link to the static B-matrix directory link_target = os.path.join(self.task_config.DATAstaticb) link_name = os.path.join(self.task_config.DATA, 'staticb') @@ -134,130 +200,30 @@ def initialize(self: Task) -> None: os.symlink(link_target, link_name) @logit(logger) - def gridgen(self: Task) -> None: - # link gdas_soca_gridgen.x - mdau.link_executable(self.task_config, 'gdas_soca_gridgen.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_gridgen.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('gridgen.yaml') - - mdau.run(exec_cmd) - - @logit(logger) - def variance_partitioning(self: Task) -> None: - # link the variance partitioning executable, gdas_soca_diagb.x - mdau.link_executable(self.task_config, 'gdas_soca_diagb.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_diagb.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_diagb.yaml') - - mdau.run(exec_cmd) - - @logit(logger) - def horizontal_diffusion(self: Task) -> None: - """Generate the horizontal diffusion coefficients - """ - # link the executable that computes the correlation scales, gdas_soca_setcorscales.x, - # and prepare the command to run it - mdau.link_executable(self.task_config, 'gdas_soca_setcorscales.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_setcorscales.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_setcorscales.yaml') - - # create a files containing the correlation scales - mdau.run(exec_cmd) - - # link the executable that computes the correlation scales, gdas_soca_error_covariance_toolbox.x, - # and prepare the command to run it - mdau.link_executable(self.task_config, 'gdas_soca_error_covariance_toolbox.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_error_covariance_toolbox.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_parameters_diffusion_hz.yaml') - - # compute the coefficients of the diffusion operator - mdau.run(exec_cmd) - - @logit(logger) - def vertical_diffusion(self: Task) -> None: + def execute_vtscales(self: Task) -> None: """Generate the vertical diffusion coefficients """ - # compute the vertical correlation scales based on the MLD - FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), - os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() + # compute the vertical correlation scales based on the MLD exec_cmd = Executable("python") exec_name = os.path.join(self.task_config.DATA, 'calc_scales.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('soca_vtscales.yaml') - mdau.run(exec_cmd) - - # link the executable that computes the correlation scales, gdas_soca_error_covariance_toolbox.x, - # and prepare the command to run it - mdau.link_executable(self.task_config, 'gdas_soca_error_covariance_toolbox.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_error_covariance_toolbox.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_parameters_diffusion_vt.yaml') - - # compute the coefficients of the diffusion operator - mdau.run(exec_cmd) - - @logit(logger) - def ensemble_perturbations(self: Task) -> None: - """Generate the 3D ensemble of perturbation for the 3DEnVAR - - This method will generate ensemble perturbations re-balanced w.r.t the - deterministic background. - This includes: - - computing a storing the unbalanced ensemble perturbations' statistics - - recentering the ensemble members around the deterministic background and - accounting for the nonlinear steric recentering - - saving the recentered ensemble statistics - """ - mdau.link_executable(self.task_config, 'gdas_ens_handler.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_ens_handler.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_ensb.yaml') - - # generate the ensemble perturbations - mdau.run(exec_cmd) - - @logit(logger) - def hybrid_weight(self: Task) -> None: - """Generate the hybrid weights for the 3DEnVAR - - This method will generate the 3D fields hybrid weights for the 3DEnVAR for each - variables. - TODO(G): Currently implemented for the specific case of the static ensemble members only - """ - mdau.link_executable(self.task_config, 'gdas_socahybridweights.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_socahybridweights.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_ensweights.yaml') - - # compute the ensemble weights - mdau.run(exec_cmd) - + mdau.run(exec_cmd) + @logit(logger) - def execute(self: Task) -> None: + def execute(self, aprun_cmd: str) -> None: """Generate the full B-matrix This method will generate the full B-matrix according to the configuration. """ - chdir(self.task_config.DATA) - self.gridgen() # TODO: This should be optional in case the geometry file was staged - self.variance_partitioning() - self.horizontal_diffusion() # TODO: Make this optional once we've converged on an acceptable set of scales - self.vertical_diffusion() - # hybrid EnVAR case + self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged + self.execute_vtscales() + self.jedi['soca_parameters_diffusion_vt'].execute(aprun_cmd) + self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.ensemble_perturbations() # TODO: refactor this from the old scripts - self.hybrid_weight() # TODO: refactor this from the old scripts + self.jedi['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts + self.jedi['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts @logit(logger) def finalize(self: Task) -> None: From 8cd354a496360230126f4fce7cfcfdafda9bdfaa Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 18:35:10 +0000 Subject: [PATCH 46/70] Debug --- ush/python/pygfs/jedi/jedi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 89c81c93dc..f873be1077 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -40,7 +40,7 @@ def __init__(self, config) -> None: _key_list = ['yaml_name', 'rundir', 'exe_src', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml', 'jedi_args'] for key in _key_list: - if key not in config.keys: + if key not in config: raise KeyError(f"Key '{key}' not found in the nested dictionary") # Create the configuration dictionary for JEDI object @@ -51,7 +51,7 @@ def __init__(self, config) -> None: 'input_config': None } ) - self.jedi_config.update(local_dict) + self.jedi_config = AttrDict(**config, **local_dict) # Save a copy of jedi_config self._jedi_config = self.jedi_config.deepcopy() From 8bf1a4293a329f145e921374a060cf6bc78f0521 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 20:25:25 +0000 Subject: [PATCH 47/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c71d3b1721..6e85be032c 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c71d3b17214007d59f91cc9c0988d4a8a279b22d +Subproject commit 6e85be032c18419fa245d7d0743264c70bd0592c From 8d53fe663e3abe8bde4aa7d5f63705da2cb75799 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 15 Oct 2024 01:22:53 +0000 Subject: [PATCH 48/70] Saving progress --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_analysis.py | 2 +- ush/python/pygfs/task/marine_bmat.py | 18 ++++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6e85be032c..b70652625a 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6e85be032c18419fa245d7d0743264c70bd0592c +Subproject commit b70652625a7e87f277e8dc5c0b08588e61ee95ff diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 4e4311b906..964ea4a0d3 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -200,7 +200,7 @@ def _prep_variational_yaml(self: Task) -> None: # Add the things to the envconfig in order to template JCB files envconfig_jcb['PARMgfs'] = self.task_config.PARMgfs - envconfig_jcb['nmem_ens'] = self.task_config.NMEM_ENS + envconfig_jcb['NMEM_ENS'] = self.task_config.NMEM_ENS envconfig_jcb['berror_model'] = 'marine_background_error_static_diffusion' if self.task_config.NMEM_ENS > 3: envconfig_jcb['berror_model'] = 'marine_background_error_hybrid_diffusion_diffusion' diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index b316b221f9..dee992dc0e 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -14,6 +14,8 @@ Executable, Task) +from pygfs.jedi import Jedi + logger = getLogger(__name__.split('.')[-1]) @@ -90,7 +92,7 @@ def __init__(self, config): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) # soca_setcorscales self.jedi['soca_setcorscales'] = Jedi(AttrDict( @@ -103,7 +105,7 @@ def __init__(self, config): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) # soca_parameters_diffusion_hz self.jedi['soca_parameters_diffusion_hz'] = Jedi(AttrDict( @@ -116,7 +118,7 @@ def __init__(self, config): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) # soca_ensb self.jedi['soca_ensb'] = Jedi(AttrDict( @@ -129,7 +131,7 @@ def __init__(self, config): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) # soca_ensweights self.jedi['soca_ensb'] = Jedi(AttrDict( @@ -142,7 +144,7 @@ def __init__(self, config): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) @logit(logger) def initialize(self: Task) -> None: @@ -172,8 +174,8 @@ def initialize(self: Task) -> None: FileHandler(bkg_list).sync() # initialize vtscales python script - vtscales_config = self.render_jcb(task_config, 'vtscales') - save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml') + vtscales_config = self.jedi['soca_parameters_diffusion_vt'].render_jcb(self.task_config, 'soca_vtscales') + save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() @@ -184,7 +186,7 @@ def initialize(self: Task) -> None: self.jedi['soca_setcorscales'].initialize(self.task_config) self.jedi['soca_parameters_diffusion_hz'].initialize(self.task_config) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.jedi['soca_ensb'.initialize(self.task_config) + self.jedi['soca_ensb'].initialize(self.task_config) self.jedi['soca_ensweights'].initialize(self.task_config) # stage ensemble members for the hybrid background error From 694ad55b656761992f6ff307a5f2ee903d64ffbb Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 15 Oct 2024 02:27:29 +0000 Subject: [PATCH 49/70] Saving progress --- scripts/exglobal_marinebmat.py | 2 +- sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_bmat.py | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/exglobal_marinebmat.py b/scripts/exglobal_marinebmat.py index e285e646ac..fd8770c18e 100755 --- a/scripts/exglobal_marinebmat.py +++ b/scripts/exglobal_marinebmat.py @@ -20,5 +20,5 @@ # Create an instance of the MarineBMat task marineBMat = MarineBMat(config) marineBMat.initialize() - marineBMat.execute() + marineBMat.execute(config.APRUN_MARINEBMAT) marineBMat.finalize() diff --git a/sorc/gdas.cd b/sorc/gdas.cd index b70652625a..52bc8d5ef7 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit b70652625a7e87f277e8dc5c0b08588e61ee95ff +Subproject commit 52bc8d5ef7ad436e07da6baf5f15bbd8dac03104 diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index dee992dc0e..a6384f38b9 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -9,7 +9,7 @@ FileHandler, add_to_datetime, to_timedelta, chdir, - parse_j2yaml, + parse_j2yaml, save_as_yaml, logit, Executable, Task) @@ -40,12 +40,14 @@ def __init__(self, config): local_dict = AttrDict( { 'PARMsoca': os.path.join(self.task_config.PARMgfs, 'gdas', 'soca'), + 'CALC_SCALE_EXEC': _calc_scale_exec, 'MARINE_WINDOW_BEGIN': _window_begin, - 'MARINE_WINDOW_END': _window_end, 'MARINE_WINDOW_MIDDLE': self.task_config.current_cycle, + 'MARINE_WINDOW_END': _window_end, + 'MARINE_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", 'ENSPERT_RELPATH': _enspert_relpath, - 'CALC_SCALE_EXEC': _calc_scale_exec, - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z." + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z." } ) From cc4a0d80a46714b028d41d82d3e41ffb950a8804 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 15 Oct 2024 13:10:59 +0000 Subject: [PATCH 50/70] Update GDAS hash to develop --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c71d3b1721..e024564f72 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c71d3b17214007d59f91cc9c0988d4a8a279b22d +Subproject commit e024564f72e8b8b617e2a6a1cc06053e6dfb5786 From 9d8aa9382569daea83fb61e52f426c457e30de87 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 15 Oct 2024 15:03:14 +0000 Subject: [PATCH 51/70] Clean up exception handling --- ush/python/pygfs/jedi/jedi.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index f873be1077..f4692e82d5 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -142,8 +142,7 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> if self.jedi_config.jcb_base_yaml: jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) else: - logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - logger.error(f"FATAL ERROR: JEDI configuration dictionary must contain jcb_base_yaml.") + raise KeyError(f"FATAL ERROR: JEDI configuration dictionary must contain jcb_base_yaml.") # Add JCB algorithm YAML, if it exists, to JCB config dictionary if self.jedi_config.jcb_algo_yaml: @@ -157,9 +156,8 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> elif 'algorithm' in jcb_config: pass else: - logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - logger.error(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + - "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") + raise KeyError(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + + "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") # Generate JEDI YAML config by rendering JCB config dictionary jedi_input_config = render(jcb_config) From b83ff6e59779cc68d7e143d2404891b717ff510f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 16 Oct 2024 19:18:54 +0000 Subject: [PATCH 52/70] Save progress --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_bmat.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 52bc8d5ef7..9b2eabd108 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 52bc8d5ef7ad436e07da6baf5f15bbd8dac03104 +Subproject commit 9b2eabd10879357c3354623306b2d516becdab03 diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index a6384f38b9..748197a02c 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -182,7 +182,8 @@ def initialize(self: Task) -> None: os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() # initialize JEDI applications - self.jedi['gridgen'].initialize(self.task_config) + self.jedi['gridgen'].initialize(self.task_config) + logger.error('foobar') # Test self.jedi['soca_diagb'].initialize(self.task_config) self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) self.jedi['soca_setcorscales'].initialize(self.task_config) @@ -221,10 +222,11 @@ def execute(self, aprun_cmd: str) -> None: This method will generate the full B-matrix according to the configuration. """ self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged - self.execute_vtscales() - self.jedi['soca_parameters_diffusion_vt'].execute(aprun_cmd) + self.jedi['soca_diagb'].execute(aprun_cmd) self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales self.jedi['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.execute_vtscales() + self.jedi['soca_parameters_diffusion_vt'].execute(aprun_cmd) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: self.jedi['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts self.jedi['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts From 00adfeadda43450713e3604d44ec6b17cff82d46 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 22 Oct 2024 13:03:47 +0000 Subject: [PATCH 53/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index e024564f72..93e7ec60bb 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit e024564f72e8b8b617e2a6a1cc06053e6dfb5786 +Subproject commit 93e7ec60bbc354a3db42d174eb59f8ed1a170f48 From 3c98b82b4e6ba05a5cbe9187f15e6d86fd47b961 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 22 Oct 2024 13:28:45 +0000 Subject: [PATCH 54/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 9b2eabd108..136bcc09a4 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 9b2eabd10879357c3354623306b2d516becdab03 +Subproject commit 136bcc09a4994b7b15dd67aa83ae26543c7f9fae From cc74aa4be96e87041206bc6f703d8278728b99ad Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 15:59:22 +0000 Subject: [PATCH 55/70] Bugfixes --- parm/config/gfs/config.marineanl | 2 +- ush/python/pygfs/task/marine_bmat.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index cf056387ca..273dec9178 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -30,7 +30,7 @@ export JCB_ALGO_ENSWEIGHTS="soca_ensweights" export JEDIEXE_GRIDGEN="${EXECgfs}/gdas_soca_gridgen.x" export JEDIEXE_DIAGB="${EXECgfs}/gdas_soca_diagb.x" -export JEDIEXE_SETCORSCALES="${EXECgfs}/soca_setcorscales.x" +export JEDIEXE_SETCORSCALES="${EXECgfs}/gdas_soca_setcorscales.x" export JEDIEXE_PARAMETERS_DIFFUSION_HZ="${EXECgfs}/gdas_soca_error_covariance_toolbox.x" export JEDIEXE_PARAMETERS_DIFFUSION_VT="${EXECgfs}/gdas_soca_error_covariance_toolbox.x" export JEDIEXE_ENSB="${EXECgfs}/gdas_ens_handler.x" diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 748197a02c..69bdbfbedd 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -175,6 +175,11 @@ def initialize(self: Task) -> None: bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) FileHandler(bkg_list).sync() + # stage the soca utility yamls (fields and ufo mapping yamls) + logger.info(f"Staging SOCA utility yaml files") + soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) + FileHandler(soca_utility_list).sync() + # initialize vtscales python script vtscales_config = self.jedi['soca_parameters_diffusion_vt'].render_jcb(self.task_config, 'soca_vtscales') save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) @@ -183,7 +188,6 @@ def initialize(self: Task) -> None: # initialize JEDI applications self.jedi['gridgen'].initialize(self.task_config) - logger.error('foobar') # Test self.jedi['soca_diagb'].initialize(self.task_config) self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) self.jedi['soca_setcorscales'].initialize(self.task_config) From c45544580a069aadb1e332aec037cc52459e62a4 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 15:59:47 +0000 Subject: [PATCH 56/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 136bcc09a4..a69df434d6 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 136bcc09a4994b7b15dd67aa83ae26543c7f9fae +Subproject commit a69df434d678419ca36b813f4f4c2836db5d28d5 From 2dc75c72aed7227d34793a1417aa16c25e07bb94 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 16:06:32 +0000 Subject: [PATCH 57/70] pynorms --- ush/python/pygfs/task/marine_bmat.py | 38 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 69bdbfbedd..f4490227ec 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -56,11 +56,11 @@ def __init__(self, config): # Create dictionary of Jedi objects self.jedi = AttrDict() - + # gridgen self.jedi['gridgen'] = Jedi(AttrDict( { - 'yaml_name': 'gridgen', + 'yaml_name': 'gridgen', 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_GRIDGEN, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, @@ -95,7 +95,7 @@ def __init__(self, config): 'jedi_args': None } )) - + # soca_setcorscales self.jedi['soca_setcorscales'] = Jedi(AttrDict( { @@ -134,7 +134,7 @@ def __init__(self, config): 'jedi_args': None } )) - + # soca_ensweights self.jedi['soca_ensb'] = Jedi(AttrDict( { @@ -175,23 +175,23 @@ def initialize(self: Task) -> None: bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) FileHandler(bkg_list).sync() - # stage the soca utility yamls (fields and ufo mapping yamls) + # stage the soca utility yamls (fields and ufo mapping yamls) logger.info(f"Staging SOCA utility yaml files") soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) FileHandler(soca_utility_list).sync() - + # initialize vtscales python script vtscales_config = self.jedi['soca_parameters_diffusion_vt'].render_jcb(self.task_config, 'soca_vtscales') save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() - + # initialize JEDI applications self.jedi['gridgen'].initialize(self.task_config) - self.jedi['soca_diagb'].initialize(self.task_config) - self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) + self.jedi['soca_diagb'].initialize(self.task_config) + self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) self.jedi['soca_setcorscales'].initialize(self.task_config) - self.jedi['soca_parameters_diffusion_hz'].initialize(self.task_config) + self.jedi['soca_parameters_diffusion_hz'].initialize(self.task_config) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: self.jedi['soca_ensb'].initialize(self.task_config) self.jedi['soca_ensweights'].initialize(self.task_config) @@ -200,7 +200,7 @@ def initialize(self: Task) -> None: if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: logger.debug(f"Stage ensemble members for the hybrid background error") mdau.stage_ens_mem(self.task_config) - + # create the symbolic link to the static B-matrix directory link_target = os.path.join(self.task_config.DATAstaticb) link_name = os.path.join(self.task_config.DATA, 'staticb') @@ -212,28 +212,28 @@ def initialize(self: Task) -> None: def execute_vtscales(self: Task) -> None: """Generate the vertical diffusion coefficients """ - # compute the vertical correlation scales based on the MLD + # compute the vertical correlation scales based on the MLD exec_cmd = Executable("python") exec_name = os.path.join(self.task_config.DATA, 'calc_scales.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('soca_vtscales.yaml') - mdau.run(exec_cmd) - + mdau.run(exec_cmd) + @logit(logger) def execute(self, aprun_cmd: str) -> None: """Generate the full B-matrix This method will generate the full B-matrix according to the configuration. """ - self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged + self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged self.jedi['soca_diagb'].execute(aprun_cmd) - self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales - self.jedi['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales self.execute_vtscales() self.jedi['soca_parameters_diffusion_vt'].execute(aprun_cmd) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.jedi['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts - self.jedi['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts + self.jedi['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts + self.jedi['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts @logit(logger) def finalize(self: Task) -> None: From ac3d385cceab8c670c48c5517ff627bf10cc0e59 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 16:18:36 +0000 Subject: [PATCH 58/70] Remove comment --- parm/config/gfs/config.marineanl | 1 - 1 file changed, 1 deletion(-) diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index 273dec9178..fc3719f106 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -17,7 +17,6 @@ export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_me export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" export JCB_BASE_YAML="${PARMgfs}/gdas/soca/marine-jcb-base.yaml" -#export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" export JCB_ALGO_GRIDGEN="soca_gridgen" export JCB_ALGO_DIAGB="soca_diagb" From 4cd585c218dd9af0a6790f33f700e1522e0d5cea Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 17:20:52 +0000 Subject: [PATCH 59/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a69df434d6..b8ce7e1666 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a69df434d678419ca36b813f4f4c2836db5d28d5 +Subproject commit b8ce7e16662eef62eec57244322aeac3d7e74d9a From 7e4defade6ed7cdaccb5b674c7a694ddf5717d5f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 17:24:10 +0000 Subject: [PATCH 60/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index b8ce7e1666..a00b9191ce 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit b8ce7e16662eef62eec57244322aeac3d7e74d9a +Subproject commit a00b9191ce5253e984539024ab21ab3c44ece7f4 From 1c0fcf1fd5b43edaf0e79057298ab46408f897ce Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 17:38:48 +0000 Subject: [PATCH 61/70] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 93e7ec60bb..f1222ec379 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 93e7ec60bbc354a3db42d174eb59f8ed1a170f48 +Subproject commit f1222ec37924d567a8d935f0cad1a6a705045e4e From 97bf8e83c1693ddd78d53b8102f593958478eee1 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 18:08:49 +0000 Subject: [PATCH 62/70] Add comment blocks to methods --- ush/python/pygfs/task/marine_bmat.py | 62 +++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index f4490227ec..59515bd15a 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -25,7 +25,24 @@ class MarineBMat(Task): """ @logit(logger, name="MarineBMat") def __init__(self, config): + """Constructor for marine B-matrix task + + This method will construct the marine B-matrix task object + This includes: + - extending the task_config AttrDict to include parameters required for this task + - instantiate the Jedi attribute objects + + Parameters + ---------- + config: Dict + dictionary object containing task configuration + + Returns + ---------- + None + """ super().__init__(config) + _home_gdas = os.path.join(self.task_config.HOMEgfs, 'sorc', 'gdas.cd') _calc_scale_exec = os.path.join(self.task_config.HOMEgfs, 'ush', 'soca', 'calc_scales.py') _window_begin = add_to_datetime(self.task_config.current_cycle, @@ -158,8 +175,17 @@ def initialize(self: Task) -> None: - staging SOCA fix files - staging static ensemble members (optional) - staging ensemble members (optional) - - generating the YAML files for the JEDI and GDASApp executables + - initializing the soca_vtscales Python script + - initializing the JEDI applications - creating output directories + + Parameters + ---------- + None + + Returns + ---------- + None """ # stage fix files @@ -210,7 +236,20 @@ def initialize(self: Task) -> None: @logit(logger) def execute_vtscales(self: Task) -> None: - """Generate the vertical diffusion coefficients + """Execute vertical diffusion coefficients generator + + This method will execute a Python script which generatres the vertical diffusion coefficients + This includes: + - constructing the executable object + - running the executable object + + Parameters + ---------- + None + + Returns + ---------- + None """ # compute the vertical correlation scales based on the MLD exec_cmd = Executable("python") @@ -224,7 +263,19 @@ def execute(self, aprun_cmd: str) -> None: """Generate the full B-matrix This method will generate the full B-matrix according to the configuration. + This includes: + - running all JEDI application and Python scripts required to generate the B-matrix + + Parameters + ---------- + aprun_cmd: str + String comprising the run command for the JEDI executable. + + Returns + ---------- + None """ + self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged self.jedi['soca_diagb'].execute(aprun_cmd) self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales @@ -246,6 +297,13 @@ def finalize(self: Task) -> None: - keep the re-balanced ensemble perturbation files in DATAenspert - ... + Parameters + ---------- + None + + Returns + ---------- + None """ # Copy the soca grid if it was created grid_file = os.path.join(self.task_config.DATA, 'soca_gridspec.nc') From 66129a297e695bf4a205f6fe662016ea79f54300 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 18:17:29 +0000 Subject: [PATCH 63/70] pynorms --- ush/python/pygfs/task/marine_bmat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 59515bd15a..ca8730a706 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -31,7 +31,7 @@ def __init__(self, config): This includes: - extending the task_config AttrDict to include parameters required for this task - instantiate the Jedi attribute objects - + Parameters ---------- config: Dict @@ -42,7 +42,7 @@ def __init__(self, config): None """ super().__init__(config) - + _home_gdas = os.path.join(self.task_config.HOMEgfs, 'sorc', 'gdas.cd') _calc_scale_exec = os.path.join(self.task_config.HOMEgfs, 'ush', 'soca', 'calc_scales.py') _window_begin = add_to_datetime(self.task_config.current_cycle, From 74e5efa27034f01971892ae5297dd4093180a2f7 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 20:14:07 +0000 Subject: [PATCH 64/70] Make jcb rendering method save copies of jcb_config and task_ocnfig --- ush/python/pygfs/jedi/jedi.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index f4692e82d5..f4e9ac67c4 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -56,6 +56,14 @@ def __init__(self, config) -> None: # Save a copy of jedi_config self._jedi_config = self.jedi_config.deepcopy() + # Create a dictionary of dictionaries for saving copies of the jcb_config + # associated with each algorithm + self._jcb_configs = AttrDict() + + # Create a dictionary of dictionaries for saving copies of the task_config + # used to render each JCB template + self._task_configs_for_jcb = AttrDict() + @logit(logger) def initialize(self, task_config: AttrDict) -> None: """Initialize JEDI application @@ -156,9 +164,13 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> elif 'algorithm' in jcb_config: pass else: - raise KeyError(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + + raise Exception(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") + # Save copies of the task_config and jcb_config used to render this JCB template + self._task_configs_for_jcb[jcb_config['algorithm']] = task_config.deepcopy() + self._jcb_configs[jcb_config['algorithm']] = jcb_config.deepcopy() + # Generate JEDI YAML config by rendering JCB config dictionary jedi_input_config = render(jcb_config) From 371c4c097c04436b6af8a9675cdea8fb86f64e11 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 21:06:47 +0000 Subject: [PATCH 65/70] pynorms --- ush/python/pygfs/jedi/jedi.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index f4e9ac67c4..67b4213305 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -58,11 +58,11 @@ def __init__(self, config) -> None: # Create a dictionary of dictionaries for saving copies of the jcb_config # associated with each algorithm - self._jcb_configs = AttrDict() + self._jcb_config_dict = AttrDict() # Create a dictionary of dictionaries for saving copies of the task_config # used to render each JCB template - self._task_configs_for_jcb = AttrDict() + self._task_config_dict = AttrDict() @logit(logger) def initialize(self, task_config: AttrDict) -> None: @@ -165,15 +165,15 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> pass else: raise Exception(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + - "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") + "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") - # Save copies of the task_config and jcb_config used to render this JCB template - self._task_configs_for_jcb[jcb_config['algorithm']] = task_config.deepcopy() - self._jcb_configs[jcb_config['algorithm']] = jcb_config.deepcopy() - # Generate JEDI YAML config by rendering JCB config dictionary jedi_input_config = render(jcb_config) + # Save copies of the task_config and jcb_config used to render this JCB template + self._task_config_dict[jcb_config['algorithm']] = task_config.deepcopy() + self._jcb_config_dict[jcb_config['algorithm']] = jcb_config.deepcopy() + return jedi_input_config @logit(logger) From f710d0ffbec0be6904c7c7985f4d6a4444b9ef17 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 24 Oct 2024 15:51:48 +0000 Subject: [PATCH 66/70] Fix to make sure bkg_list.yaml isn't loaded directly in marine JCB base --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_analysis.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a00b9191ce..1660f1ae50 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a00b9191ce5253e984539024ab21ab3c44ece7f4 +Subproject commit 1660f1ae50da47a1f6bc0c31c625bd3d185d93ca diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 964ea4a0d3..f9fdbb61bb 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -15,7 +15,7 @@ from wxflow import (AttrDict, FileHandler, add_to_datetime, to_timedelta, to_YMD, - parse_j2yaml, + parse_j2yaml, parse_yaml, logit, Executable, Task, @@ -192,7 +192,7 @@ def _prep_variational_yaml(self: Task) -> None: mdau.gen_bkg_list(bkg_path='./bkg', window_begin=self.task_config.MARINE_WINDOW_BEGIN, yaml_name='bkg_list.yaml') - + # Make a copy of the env config before modifying to avoid breaking something else envconfig_jcb = copy.deepcopy(self.task_config) logger.info(f"---------------- Prepare the yaml configuration") @@ -225,7 +225,7 @@ def _prep_variational_yaml(self: Task) -> None: jcb_algo_config = YAMLFile(path=jcb_algo_yaml) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) - + # Override base with the application specific config jcb_config = {**jcb_base_config, **jcb_algo_config} @@ -233,6 +233,9 @@ def _prep_variational_yaml(self: Task) -> None: jcb_config['window_begin'] = self.task_config.MARINE_WINDOW_BEGIN.strftime('%Y-%m-%dT%H:%M:%SZ') jcb_config['window_middle'] = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') + # Current hack so that this is not done directly in the JCB base yaml + jcb_config['marine_pseudo_model_states'] = parse_yaml('bkg_list.yaml') + # Render the full JEDI configuration file using JCB jedi_config = render(jcb_config) From 215188aeedc246ecab77e63d5a4ba45cb1971f20 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 24 Oct 2024 15:53:38 +0000 Subject: [PATCH 67/70] pynorms --- ush/python/pygfs/task/marine_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index f9fdbb61bb..0cffb3fdcc 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -192,7 +192,7 @@ def _prep_variational_yaml(self: Task) -> None: mdau.gen_bkg_list(bkg_path='./bkg', window_begin=self.task_config.MARINE_WINDOW_BEGIN, yaml_name='bkg_list.yaml') - + # Make a copy of the env config before modifying to avoid breaking something else envconfig_jcb = copy.deepcopy(self.task_config) logger.info(f"---------------- Prepare the yaml configuration") From cbf1c6f61fca90c5e5cdb5761b211862df8ef558 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 24 Oct 2024 15:54:31 +0000 Subject: [PATCH 68/70] pynorms --- ush/python/pygfs/task/marine_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 0cffb3fdcc..71aa6ea67f 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -225,7 +225,7 @@ def _prep_variational_yaml(self: Task) -> None: jcb_algo_config = YAMLFile(path=jcb_algo_yaml) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) - + # Override base with the application specific config jcb_config = {**jcb_base_config, **jcb_algo_config} From 54223bf620cc4b4a16b33957b74aa0ed207407c2 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 24 Oct 2024 18:18:11 +0000 Subject: [PATCH 69/70] pynorms --- ush/python/pygfs/task/marine_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 71aa6ea67f..b311c6874f 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -235,7 +235,7 @@ def _prep_variational_yaml(self: Task) -> None: # Current hack so that this is not done directly in the JCB base yaml jcb_config['marine_pseudo_model_states'] = parse_yaml('bkg_list.yaml') - + # Render the full JEDI configuration file using JCB jedi_config = render(jcb_config) From abdbe31115b34857970c233d67241a2989bc21b5 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 24 Oct 2024 19:27:01 +0000 Subject: [PATCH 70/70] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 1660f1ae50..12643982fa 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 1660f1ae50da47a1f6bc0c31c625bd3d185d93ca +Subproject commit 12643982fade77a47e573da41a90198457e356be