Merge branch 'develop' into feature/gaea-c5

.github/workflows/hera.yaml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Passes on Hera 
 
     steps:

.github/workflows/norms.yaml

@@ -3,7 +3,7 @@ on: [push]
 
 jobs:
   check_pynorms:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Check coding norms with pycodestyle and cpplint
 
     steps:

.github/workflows/orion.yaml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Passes on Orion
 
     steps:

.github/workflows/unittests.yaml

@@ -3,7 +3,7 @@ on: [push, pull_request]
 
 jobs:
   ctests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Run Unit Tests with ctest
 
     steps:

.github/workflows/unittests_g-w.yaml

@@ -3,7 +3,7 @@ on: [push, pull_request]
 
 jobs:
   ctests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Run Unit Tests inside global-workflow with ctest
 
     steps:

mains/gdas.cc

@@ -16,6 +16,7 @@
 
 #include "oops/runs/ConvertToStructuredGrid.h"
 #include "oops/runs/ConvertState.h"
+#include "oops/runs/EnsMeanAndVariance.h"
 #include "oops/runs/HofX4D.h"
 #include "oops/runs/LocalEnsembleDA.h"
 #include "oops/runs/Run.h"
@@ -59,6 +60,9 @@ int runApp(int argc, char** argv, const std::string traits, const std::string ap
   apps["convertstate"] = []() {
       return std::make_unique<oops::ConvertState<Traits>>();
   };
+  apps["ensmean"] = []() {
+      return std::make_unique<oops::EnsMeanAndVariance<Traits>>();
+  };
   apps["hofx4d"] = []() {
       return std::make_unique<oops::HofX4D<Traits, ufo::ObsTraits>>();
   };

modulefiles/GDAS/hercules.gnu.lua

@@ -0,0 +1,93 @@
+help([[
+Load environment for running the GDAS application with gnu compilers and MPI.
+]])
+
+local pkgName    = myModuleName()
+local pkgVersion = myModuleVersion()
+local pkgNameVer = myModuleFullName()
+
+prepend_path("MODULEPATH", '/work/noaa/epic/role-epic/spack-stack/hercules/modulefiles')
+prepend_path("MODULEPATH", '/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core')
+prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack')
+
+
+---- below two lines get us access to the spack-stack modules
+load("stack-gcc/12.2.0")
+load("stack-openmpi/4.1.6")
+
+load("cmake/3.23.1")
+load("curl/8.4.0")
+load("zlib/1.2.13")
+load("git/2.31.1")
+--load("pkg-config/0.27.1")
+load("hdf5/1.14.3")
+load("parallel-netcdf/1.12.3")
+load("netcdf-c/4.9.2")
+load("nccmp/1.9.0.1")
+load("netcdf-fortran/4.6.1")
+load("nco/5.1.6")
+load("parallelio/2.6.2")
+load("wget/1.21.1")
+load("boost/1.84.0")
+load("bufr/12.0.1")
+load("git-lfs/3.1.2")
+load("ecbuild/3.7.2")
+load("openjpeg/2.3.1")
+load("eccodes/2.33.0")
+load("eigen/3.4.0")
+load("openblas/0.3.24")
+load("eckit/1.24.5")
+load("fftw/3.3.10")
+load("fckit/0.11.0")
+load("fiat/1.2.0")
+load("ectrans/1.2.0")
+load("fms/2023.04")
+load("esmf/8.6.1")
+load("atlas/0.36.0")
+load("sp/2.5.0")
+load("gsl-lite/0.37.0")
+load("libjpeg/2.1.0")
+load("krb5/1.20.1")
+load("libtirpc/1.3.3")
+load("hdf/4.2.15")
+load("jedi-cmake/1.4.0")
+load("libpng/1.6.37")
+load("libxt/1.3.0")
+load("libxmu/1.1.4")
+load("libxpm/3.5.17")
+load("libxaw/1.0.15")
+load("udunits/2.2.28")
+load("ncview/2.1.9")
+load("netcdf-cxx4/4.3.1")
+load("py-pybind11/2.11.0")
+--load("crtm/v2.4_jedi")
+load("contrib/0.1")
+load("noaatools/3.1")
+load("rocoto/1.3.7")
+
+load("hpc/1.2.0")
+unload("python/3.10.13")
+unload("py-numpy/1.22.3")
+load("miniconda3/4.6.14")
+load("gdasapp/1.0.0")
+-- below is a hack because of cmake finding the wrong python...
+setenv("CONDA_PREFIX", "/work2/noaa/da/python/opt/core/miniconda3/4.6.14/envs/gdasapp/")
+
+setenv("CC","mpicc")
+setenv("FC","mpifort")
+setenv("CXX","mpicxx")
+local mpiexec = '/opt/slurm/bin/srun'
+local mpinproc = '-n'
+setenv('MPIEXEC_EXEC', mpiexec)
+setenv('MPIEXEC_NPROC', mpinproc)
+
+setenv("CRTM_FIX","/work2/noaa/da/role-da/GDASApp/fix/crtm/2.4.0")
+setenv("GDASAPP_TESTDATA","/work2/noaa/da/role-da/GDASApp/testdata")
+setenv("GDASAPP_UNIT_TEST_DATA_PATH", "/work2/noaa/da/role-da/GDASApp/unittestdata")
+
+execute{cmd="ulimit -s unlimited",modeA={"load"}}
+
+whatis("Name: ".. pkgName)
+whatis("Version: ".. pkgVersion)
+whatis("Category: GDASApp")
+whatis("Description: Load all libraries needed for GDASApp")

modulefiles/GDAS/hercules.intel.lua

@@ -6,43 +6,44 @@ local pkgName    = myModuleName()
 local pkgVersion = myModuleVersion()
 local pkgNameVer = myModuleFullName()
 
-prepend_path("MODULEPATH", '/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core')
+prepend_path("MODULEPATH", '/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core')
 prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack')
 
 -- below two lines get us access to the spack-stack modules
 load("stack-intel/2021.9.0")
 load("stack-intel-oneapi-mpi/2021.9.0")
 --load("stack-python/3.10.8")
 -- JCSDA has 'jedi-fv3-env/unified-dev', but we should load these manually as needed
+
 load("cmake/3.23.1")
 load("curl/8.4.0")
 load("zlib/1.2.13")
 load("git/2.31.1")
 --load("pkg-config/0.27.1")
-load("hdf5/1.14.0")
-load("parallel-netcdf/1.12.2")
+load("hdf5/1.14.3")
+load("parallel-netcdf/1.12.3")
 load("netcdf-c/4.9.2")
 load("nccmp/1.9.0.1")
 load("netcdf-fortran/4.6.1")
-load("nco/5.0.6")
-load("parallelio/2.5.10")
+load("nco/5.1.6")
+load("parallelio/2.6.2")
 load("wget/1.21.1")
-load("boost/1.83.0")
+load("boost/1.84.0")
 load("bufr/12.0.1")
 load("git-lfs/3.1.2")
 load("ecbuild/3.7.2")
-load("openjpeg/2.3.1")
-load("eccodes/2.32.0")
+load("openjpeg/2.4.0")
+load("eccodes/2.33.0")
 load("eigen/3.4.0")
-load("openblas/0.3.24")
+load("openblas/0.3.27")
 load("eckit/1.24.5")
 load("fftw/3.3.10")
 load("fckit/0.11.0")
 load("fiat/1.2.0")
 load("ectrans/1.2.0")
 load("fms/2023.04")
-load("esmf/8.6.0")
-load("atlas/0.35.1")
+load("esmf/8.6.1")
+load("atlas/0.36.0")
 load("sp/2.5.0")
 load("gsl-lite/0.37.0")
 load("libjpeg/2.1.0")
@@ -51,10 +52,10 @@ load("libtirpc/1.3.3")
 load("hdf/4.2.15")
 load("jedi-cmake/1.4.0")
 load("libpng/1.6.37")
-load("libxt/1.1.5")
+load("libxt/1.3.0")
 load("libxmu/1.1.4")
-load("libxpm/4.11.0")
-load("libxaw/1.0.13")
+load("libxpm/3.5.17")
+load("libxaw/1.0.15")
 load("udunits/2.2.28")
 load("ncview/2.1.9")
 load("netcdf-cxx4/4.3.1")
@@ -69,7 +70,6 @@ unload("python/3.10.13")
 unload("py-numpy/1.22.3")
 load("miniconda3/4.6.14")
 load("gdasapp/1.0.0")
-
 -- below is a hack because of cmake finding the wrong python...
 setenv("CONDA_PREFIX", "/work2/noaa/da/python/opt/core/miniconda3/4.6.14/envs/gdasapp/")
 

parm/aero/obs/config/viirs_n20_aod.yaml.j2

@@ -3,7 +3,7 @@
     obsdatain:
       engine:
         type: H5File
-        obsfile: "{{ DATA }}/obs/{{ OPREFIX }}viirs_n20.{{ current_cycle | to_YMDH }}.nc4"
+        obsfile: "{{ DATA }}/obs/{{ OPREFIX }}viirs_n20_aod.tm00.nc"
     obsdataout:
       engine:
         type: H5File

parm/aero/obs/config/viirs_n21_aod.yaml.j2

@@ -3,7 +3,7 @@
     obsdatain:
       engine:
         type: H5File
-        obsfile: "{{ DATA }}/obs/{{ OPREFIX }}viirs_n21.{{ current_cycle | to_YMDH }}.nc4"
+        obsfile: "{{ DATA }}/obs/{{ OPREFIX }}viirs_n21_aod.tm00.nc"
     obsdataout:
       engine:
         type: H5File

parm/aero/obs/config/viirs_npp_aod.yaml.j2

@@ -3,7 +3,7 @@
     obsdatain:
       engine:
         type: H5File
-        obsfile: "{{ DATA }}/obs/{{ OPREFIX }}viirs_npp.{{ current_cycle | to_YMDH }}.nc4"
+        obsfile: "{{ DATA }}/obs/{{ OPREFIX }}viirs_npp_aod.tm00.nc"
     obsdataout:
       engine:
         type: H5File

parm/atm/jcb-base.yaml.j2

@@ -71,6 +71,8 @@ observations: all_observations
 crtm_coefficient_path: "{{ DATA }}/crtm/"
 
 # Naming conventions for observational files
+atmosphere_obsdataroot_path: "{{COM_OBS}}"
+
 atmosphere_obsdatain_path: "{{atm_obsdatain_path}}"
 atmosphere_obsdatain_prefix: "{{OPREFIX}}"
 atmosphere_obsdatain_suffix: ".tm00.nc"
@@ -80,6 +82,8 @@ atmosphere_obsdataout_prefix: diag_
 atmosphere_obsdataout_suffix: "_{{ current_cycle | to_YMDH }}.nc"
 
 # Naming conventions for bias correction files
+atmosphere_obsbiasroot_path: "{{COM_ATMOS_ANALYSIS_PREV}}"
+
 atmosphere_obsbiasin_path: "{{DATA}}/obs/"
 atmosphere_obsbiasin_prefix: "{{GPREFIX}}"
 atmosphere_obsbiasin_suffix: ".satbias.nc"
@@ -94,6 +98,16 @@ atmosphere_obsbiasout_suffix: ".satbias.nc"
 atmosphere_obsbiascovout_prefix: "{{APREFIX}}"
 atmosphere_obsbiascovout_suffix: ".satbias_cov.nc"
 
+bias_files:
+  atms_n20: rad_varbc_params.tar
+  atms_npp: rad_varbc_params.tar
+  mtiasi_metop-a: rad_varbc_params.tar
+  mtiasi_metop-b: rad_varbc_params.tar
+  amsua_n19: rad_varbc_params.tar
+  ssmis_f17: rad_varbc_params.tar
+  ssmis_f18: rad_varbc_params.tar
+  cris-fsr_n20: rad_varbc_params.tar
+  cris-fsr_npp: rad_varbc_params.tar
 
 # Local Ensemble DA (LETKF)
 # -------------------------

test/atm/global-workflow/config.yaml

@@ -10,7 +10,7 @@ base:
   PTMP: "@bindir@/test/atm/global-workflow/testrun"
 
 atmanl:
-  JCB_ALGO_YAML: "@srcdir@/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2"
+  JCB_ALGO_YAML_VAR: "@srcdir@/test/atm/global-workflow/jcb-prototype_3dvar.yaml.j2"
   STATICB_TYPE: "identity"
   ATMRES_ANL: "C48"
   LAYOUT_X_ATMANL: 1
@@ -19,11 +19,6 @@ atmanl:
 atmensanl:
   JCB_ALGO_YAML_LETKF: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2"
   JCB_ALGO_YAML_OBS: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2"
+  JCB_ALGO_YAML_SOL: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2"
   LAYOUT_X_ATMENSANL: 1
   LAYOUT_Y_ATMENSANL: 1
-
-atmensanlobs:
-  JCB_ALGO_YAML: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2"
-
-atmensanlsol:
-  JCB_ALGO_YAML: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2"

test/marine/CMakeLists.txt

@@ -88,7 +88,7 @@ CHECK_AND_SET_PATH(
 	BUFR_TEST_DIR
 )
 if (NOT BUFR_TEST_DIR_EXISTS)
-	message(STATUS "BUFR test file directory not found -- bufr to ioda tests not generted.")
+	message(WARNING "BUFR test file directory not found -- bufr to ioda tests not generated.")
 	set(GENERATE_BUFR2IODA_TESTS FALSE)
 else()
     # message(STATUS "Found bufr test directory: ${BUFR_TEST_DIR}")
@@ -105,7 +105,7 @@ else()
 		OCEAN_BASIN_FILE
 	)
 	if (NOT OCEAN_BASIN_FILE_EXISTS)
-		message("Ocean basin data file not found -- bufr to ioda tests not generated.")
+		message(WARNING "Ocean basin data file not found -- bufr to ioda tests not generated.")
 		set(GENERATE_BUFR2IODA_TESTS FALSE)
 	endif()
 	# message(STATUS "Found ocean basin data in ${OCEAN_BASIN_FILE}")
@@ -116,17 +116,26 @@ endif()
 function(ADD_INSITU_TEST testname testbufr)
 	# set(CONFIG_TYPE "json")
 	set(CONFIG_TYPE "yaml")
-	set(DATE "2021063006")
+
+	if (testbufr STREQUAL "dbuoy")
+		set(DATE "2019010700")
+		set(CYCLE "00")
+	else()
+		set(DATE "2021063006")
+		set(CYCLE "06")
+	endif()
+
 	set(TEST "bufr2ioda_insitu_${testname}")
 
 	set(TESTREF_FILE "${TEST}_${DATE}.ref")
 
 	# stage the input file to directory ${BUFR_INPUT_DIR}
 	set(BUFR_INPUT_DIR ${TEST_WORKING_DIR})
-	set(BUFR_TEST_FILE "${DATE}-gdas.t06z.${testbufr}.tm00.bufr_d")
+	set(BUFR_TEST_FILE "${DATE}-gdas.t${CYCLE}z.${testbufr}.tm00.bufr_d")
 	set(BUFR_FILE "${BUFR_TEST_DIR}/${BUFR_TEST_FILE}")
 	if (NOT EXISTS ${BUFR_FILE})
-		message(FATAL_ERROR "BUFR file ${BUFR_FILE} not found")
+		message(WARNING "BUFR file ${BUFR_FILE} not found, test not generated")
+		return()
 	endif()
 	file(COPY ${BUFR_FILE} DESTINATION ${BUFR_INPUT_DIR})
 
@@ -144,7 +153,7 @@ function(ADD_INSITU_TEST testname testbufr)
 	)
 
 	add_test(
-		NAME test_${TEST}
+		NAME test_gdasapp_${TEST}
 		COMMAND ${MARINE_BUFR2IODA_DIR}/${TEST}.py -c ${CONFIG_FILE} -t ${TESTREF_DIR}/${TESTREF_FILE}
 		WORKING_DIRECTORY ${TEST_WORKING_DIR}
 	)
@@ -156,6 +165,8 @@ if (GENERATE_BUFR2IODA_TESTS)
 	ADD_INSITU_TEST("profile_bathy" "bathy")
 	ADD_INSITU_TEST("profile_glider" "subpfl")
 	ADD_INSITU_TEST("profile_tesac" "tesac")
+	ADD_INSITU_TEST("profile_tropical" "dbuoy")
 	ADD_INSITU_TEST("profile_xbtctd" "xbtctd")
+	ADD_INSITU_TEST("surface_drifter" "dbuoy")
 	ADD_INSITU_TEST("surface_trkob" "trkob")
 endif()