Merge branch 'develop' into feature/jdas_enkf_diag_com_in_out

mingshichen-noaa · Oct 18, 2024 · c9a50a4 · c9a50a4
2 parents aa7d8a2 + 82710f4
commit c9a50a4
Show file tree

Hide file tree

Showing 4 changed files with 279 additions and 22 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,210 @@
+# global-workflow is a collaborative space where contributions come from a variety of sources
+# This file is to ensure that new development gets reviewed by the appropriate SME
+
+# global-workflow default owners (not a complete list)
+@KateFriedman-NOAA
+@WalterKolczynski-NOAA
+@DavidHuber-NOAA
+
+# Specific directory owners
+/ci/ @TerrenceMcGuinness-NOAA @WalterKolczynski-NOAA
+
+/ecf/ @lgannoaa
+
+/workflow/ @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA
+
+# Specific file owners
+# build scripts
+sorc/build_*.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA
+sorc/link_workflow.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA
+
+# jobs
+jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX @CoryMartin-NOAA
+jobs/JGDAS_ATMOS_ANALYSIS_DIAG @RussTreadon-NOAA @CoryMartin-NOAA
+jobs/JGDAS_ATMOS_CHGRES_FORENKF @RussTreadon-NOAA @CoryMartin-NOAA
+jobs/JGDAS_ATMOS_GEMPAK @GwenChen-NOAA
+jobs/JGDAS_ATMOS_GEMPAK_META_NCDC @GwenChen-NOAA
+jobs/JGDAS_ATMOS_VERFOZN @EdwardSafford-NOAA
+jobs/JGDAS_ATMOS_VERFRAD @EdwardSafford-NOAA
+jobs/JGDAS_ENKF_* @RussTreadon-NOAA @CoryMartin-NOAA @CatherineThomas-NOAA
+jobs/JGDAS_FIT2OBS @jack-woollen 
+jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN @guillaumevernieres
+jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY @guillaumevernieres
+jobs/JGFS_ATMOS_AWIPS_20KM_1P0DEG @GwenChen-NOAA
+jobs/JGFS_ATMOS_CYCLONE_GENESIS @JiayiPeng-NOAA 
+jobs/JGFS_ATMOS_CYCLONE_TRACKER @JiayiPeng-NOAA 
+jobs/JGFS_ATMOS_FBWIND @GwenChen-NOAA
+jobs/JGFS_ATMOS_FSU_GENESIS
+jobs/JGFS_ATMOS_GEMPAK @GwenChen-NOAA
+jobs/JGFS_ATMOS_GEMPAK_META @GwenChen-NOAA
+jobs/JGFS_ATMOS_GEMPAK_NCDC_UPAPGIF @GwenChen-NOAA
+jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC @GwenChen-NOAA
+jobs/JGFS_ATMOS_PGRB2_SPEC_NPOESS @WenMeng-NOAA
+jobs/JGFS_ATMOS_POSTSND @BoCui-NOAA
+jobs/JGFS_ATMOS_VERIFICATION
+jobs/JGLOBAL_AERO_ANALYSIS_* @CoryMartin-NOAA
+jobs/JGLOBAL_ARCHIVE @DavidHuber-NOAA
+jobs/JGLOBAL_ATMENS_ANALYSIS_* @RussTreadon-NOAA @CoryMartin-NOAA @DavidNew-NOAA
+jobs/JGLOBAL_ATMOS_ANALYSIS @RussTreadon-NOAA @CatherineThomas-NOAA
+jobs/JGLOBAL_ATMOS_ANALYSIS_CALC @RussTreadon-NOAA @CatherineThomas-NOAA @CoryMartin-NOAA
+jobs/JGLOBAL_ATMOS_EMCSFC_SFC_PREP @GeorgeGayno-NOAA
+jobs/JGLOBAL_ATMOS_ENSSTAT
+jobs/JGLOBAL_ATMOS_POST_MANAGER
+jobs/JGLOBAL_ATMOS_PRODUCTS @WenMeng-NOAA
+jobs/JGLOBAL_ATMOS_SFCANL @GeorgeGayno-NOAA
+jobs/JGLOBAL_ATMOS_TROPCY_QC_RELOC
+jobs/JGLOBAL_ATMOS_UPP @WenMeng-NOAA
+jobs/JGLOBAL_ATMOS_VMINMON @EdwardSafford-NOAA
+jobs/JGLOBAL_ATM_* @RussTreadon-NOAA @DavidNew-NOAA @CoryMartin-NOAA
+jobs/JGLOBAL_CLEANUP @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA
+jobs/JGLOBAL_EXTRACTVARS @EricSinsky-NOAA
+jobs/JGLOBAL_FORECAST @aerorahul
+jobs/JGLOBAL_MARINE_* @guillaumevernieres @AndrewEichmann-NOAA
+jobs/JGLOBAL_OCEANICE_PRODUCTS @GwenChen-NOAA
+jobs/JGLOBAL_PREP_EMISSIONS @bbakernoaa
+jobs/JGLOBAL_PREP_OBS_AERO @CoryMartin-NOAA
+jobs/JGLOBAL_PREP_OCEAN_OBS @guillaumevernieres @AndrewEichmann-NOAA
+jobs/JGLOBAL_*SNOW* @jiaruidong2017
+jobs/JGLOBAL_STAGE_IC @KateFriedman-NOAA
+jobs/JGLOBAL_WAVE_* @JessicaMeixner-NOAA @sbanihash
+jobs/rocoto/* @WalterKolczynski-NOAA @KateFriedman-NOAA @DavidHuber-NOAA
+
+# scripts
+scripts/exgdas_aero_analysis_generate_bmatrix.py @CoryMartin-NOAA
+scripts/exgdas_atmos_chgres_forenkf.sh @RussTreadon-NOAA @CoryMartin-NOAA
+scripts/exgdas_atmos_gempak_gif_ncdc.sh @GwenChen-NOAA
+scripts/exgdas_atmos_nawips.sh @GwenChen-NOAA
+scripts/exgdas_atmos_verfozn.sh @EdwardSafford-NOAA
+scripts/exgdas_atmos_verfrad.sh @EdwardSafford-NOAA
+scripts/exgdas_enkf_earc.py @DavidHuber-NOAA
+scripts/exgdas_enkf_ecen.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA
+scripts/exgdas_enkf_post.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA
+scripts/exgdas_enkf_select_obs.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA
+scripts/exgdas_enkf_sfc.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA
+scripts/exgdas_enkf_snow_recenter.py @jiaruidong2017
+scripts/exgdas_enkf_update.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA
+scripts/exgdas_global_marine_analysis_letkf.py @guillaumevernieres @AndrewEichmann-NOAA
+scripts/exgfs_aero_init_aerosol.py @WalterKolczynski-NOAA
+scripts/exgfs_atmos_awips_20km_1p0deg.sh @GwenChen-NOAA
+scripts/exgfs_atmos_fbwind.sh @GwenChen-NOAA
+scripts/exgfs_atmos_gempak_gif_ncdc_skew_t.sh @GwenChen-NOAA
+scripts/exgfs_atmos_gempak_meta.sh @GwenChen-NOAA
+scripts/exgfs_atmos_goes_nawips.sh @GwenChen-NOAA
+scripts/exgfs_atmos_grib2_special_npoess.sh @WenMeng-NOAA
+scripts/exgfs_atmos_nawips.sh @GwenChen-NOAA
+scripts/exgfs_atmos_postsnd.sh @BoCui-NOAA
+scripts/exgfs_pmgr.sh
+scripts/exgfs_prdgen_manager.sh
+scripts/exgfs_wave_* @JessicaMeixner-NOAA @sbanihash
+scripts/exglobal_aero_analysis_* @CoryMartin-NOAA
+scripts/exglobal_archive.py @DavidHuber-NOAA
+scripts/exglobal_atm_analysis_* @RussTreadon-NOAA @DavidNew-NOAA
+scripts/exglobal_atmens_analysis_* @RussTreadon-NOAA @DavidNew-NOAA
+scripts/exglobal_atmos_analysis*.sh @RussTreadon-NOAA @CoryMartin-NOAA
+scripts/exglobal_atmos_ensstat.sh @RussTreadon-NOAA
+scripts/exglobal_atmos_pmgr.sh
+scripts/exglobal_atmos_products.sh @WenMeng-NOAA
+scripts/exglobal_atmos_sfcanl.sh @GeorgeGayno-NOAA
+scripts/exglobal_atmos_tropcy_qc_reloc.sh
+scripts/exglobal_atmos_upp.py @WenMeng-NOAA
+scripts/exglobal_atmos_vminmon.sh @EdwardSafford-NOAA
+scripts/exglobal_cleanup.sh @DavidHuber-NOAA
+scripts/exglobal_diag.sh @RussTreadon-NOAA @CoryMartin-NOAA
+scripts/exglobal_extractvars.sh @EricSinsky-NOAA
+scripts/exglobal_forecast.py @aerorahul
+scripts/exglobal_forecast.sh @aerorahul @WalterKolczynski-NOAA
+scripts/exglobal_marine_analysis_* @guillaumevernieres @AndrewEichmann-NOAA
+scripts/exglobal_marinebmat.py @guillaumevernieres @AndrewEichmann-NOAA
+scripts/exglobal_oceanice_products.py @GwenChen-NOAA
+scripts/exglobal_prep_emissions.py @bbakernoaa
+scripts/exglobal_prep_obs_aero.py @CoryMartin-NOAA
+scripts/exglobal_prep_snow_obs.py @jiaruidong2017
+scripts/exglobal_snow_analysis.py @jiaruidong2017
+scripts/exglobal_stage_ic.py @KateFriedman-NOAA
+
+# ush
+WAM_XML_to_ASCII.pl
+atmos_ensstat.sh
+atmos_extractvars.sh @EricSinsky-NOAA
+bash_utils.sh @WalterKolczynski-NOAA
+calcanl_gfs.py @CoryMartin-NOAA
+calcinc_gfs.py @CoryMartin-NOAA
+compare_f90nml.py @WalterKolczynski-NOAA @aerorahul
+detect_machine.sh @WalterKolczynski-NOAA
+extractvars_tools.sh @EricSinsky-NOAA
+file_utils.sh @WalterKolczynski-NOAA
+forecast_det.sh @aerorahul @WalterKolczynski-NOAA
+forecast_postdet.sh @aerorahul @WalterKolczynski-NOAA
+forecast_predet.sh @aerorahul @WalterKolczynski-NOAA
+fv3gfs_remap_weights.sh
+gaussian_sfcanl.sh @GeorgeGayno-NOAA
+getdump.sh @WalterKolczynski-NOAA @KateFriedman-NOAA
+getges.sh @WalterKolczynski-NOAA @KateFriedman-NOAA
+getgfsnctime @CoryMartin-NOAA
+getncdimlen @CoryMartin-NOAA
+gfs_bfr2gpk.sh @GwenChen-NOAA
+gfs_bufr.sh @GwenChen-NOAA
+gfs_bufr_netcdf.sh @GwenChen-NOAA
+gfs_sndp.sh @BoCui-NOAA
+gfs_truncate_enkf.sh @CoryMartin-NOAA
+global_savefits.sh
+gsi_utils.py @CoryMartin-NOAA
+interp_atmos_master.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA
+interp_atmos_sflux.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA
+jjob_header.sh @WalterKolczynski-NOAA
+link_crtm_fix.sh @WalterKolczynski-NOAA
+load_fv3gfs_modules.sh @WalterKolczynski-NOAA @aerorahul
+load_ufsda_modules.sh @WalterKolczynski-NOAA @aerorahul @CoryMartin-NOAA
+load_ufswm_modules.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA
+merge_fv3_aerosol_tile.py @WalterKolczynski-NOAA
+minmon_xtrct_*.pl @EdwardSafford-NOAA
+module-setup.sh @WalterKolczynski-NOAA @aerorahul
+oceanice_nc2grib2.sh @GwenChen-NOAA
+ocnice_extractvars.sh @EricSinsky-NOAA
+ozn_xtrct.sh @EdwardSafford-NOAA
+parse-storm-type.pl
+parsing_model_configure_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa
+parsing_namelists_CICE.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @DeniseWorthen
+parsing_namelists_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa
+parsing_namelists_FV3_nest.sh @guoqing-noaa
+parsing_namelists_MOM6.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @jiandewang
+parsing_namelists_WW3.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA @sbanihash
+parsing_ufs_configure.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa
+preamble.sh @WalterKolczynski-NOAA
+product_functions.sh @WalterKolczynski-NOAA @aerorahul
+radmon_*.sh @EdwardSafford-NOAA
+rstprod.sh @WalterKolczynski-NOAA @DavidHuber-NOAA
+run_mpmd.sh @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA
+syndat_getjtbul.sh @JiayiPeng-NOAA 
+syndat_qctropcy.sh @JiayiPeng-NOAA 
+tropcy_relocate.sh @JiayiPeng-NOAA 
+tropcy_relocate_extrkr.sh @JiayiPeng-NOAA 
+wave_*.sh @JessicaMeixner-NOAA @sbanihash
+
+# ush/python
+ush/python/pygfs/jedi/__init__.py @aerorahul @DavidNew-NOAA
+ush/python/pygfs/jedi/jedi.py @DavidNew-NOAA
+ush/python/pygfs/task/__init__.py @aerorahul
+ush/python/pygfs/task/aero_analysis.py @DavidNew-NOAA @CoryMartin-NOAA
+ush/python/pygfs/task/aero_bmatrix.py @DavidNew-NOAA @CoryMartin-NOAA
+ush/python/pygfs/task/aero_emissions.py @bbakernoaa
+ush/python/pygfs/task/aero_prepobs.py @CoryMartin-NOAA 
+ush/python/pygfs/task/analysis.py @DavidNew-NOAA @RussTreadon-NOAA
+ush/python/pygfs/task/archive.py @DavidHuber-NOAA
+ush/python/pygfs/task/atm_analysis.py @DavidNew-NOAA @RussTreadon-NOAA
+ush/python/pygfs/task/atmens_analysis.py @DavidNew-NOAA @RussTreadon-NOAA
+ush/python/pygfs/task/bmatrix.py @DavidNew-NOAA
+ush/python/pygfs/task/gfs_forecast.py @aerorahul
+ush/python/pygfs/task/marine_analysis.py @guillaumevernieres @AndrewEichmann-NOAA
+ush/python/pygfs/task/marine_bmat.py @guillaumevernieres @AndrewEichmann-NOAA
+ush/python/pygfs/task/marine_letkf.py @guillaumevernieres @AndrewEichmann-NOAA
+ush/python/pygfs/task/oceanice_products.py @aerorahul @GwenChen-NOAA
+ush/python/pygfs/task/snow_analysis.py @jiaruidong2017
+ush/python/pygfs/task/snowens_analysis.py @jiaruidong2017
+ush/python/pygfs/task/stage_ic.py @KateFriedman-NOAA
+ush/python/pygfs/task/upp.py @aerorahul @WenMeng-NOAA
+ush/python/pygfs/ufswm/__init__.py @aerorahul
+ush/python/pygfs/ufswm/gfs.py @aerorahul
+ush/python/pygfs/ufswm/ufs.py @aerorahul
+ush/python/pygfs/utils/__init__.py @aerorahul
+ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA
diff --git a/.github/workflows/globalworkflow-ci.yaml → .github/workflows/pw_aws_centos.yaml b/.github/workflows/globalworkflow-ci.yaml → .github/workflows/pw_aws_centos.yaml
@@ -1,4 +1,4 @@
-name: gw-ci-orion
+name: gw-ci-aws-centos
 
 on: [workflow_dispatch]
 
@@ -15,72 +15,75 @@ on: [workflow_dispatch]
 #         └── ${pslot}
 env:
   TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
-  MACHINE_ID: orion
+  MACHINE_ID: noaacloud
 
 jobs:
-  checkout-build-link:
-    runs-on: [self-hosted, orion-ready]
+  checkout:
+    runs-on: [self-hosted, aws, parallelworks, centos]
     timeout-minutes: 600
 
     steps:
+
     - name: Checkout global-workflow
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
-        path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
+        path: ${{ github.run_id }}/HOMEgfs
+        submodules: 'recursive'
+        ref: ${{ github.event.pull_request.head.ref }}
 
-    - name: Checkout components
-      run: |
-        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./checkout.sh -c -g  # Options e.g. -u can be added late
+  build-link:      
+    runs-on: [self-hosted, aws, parallelworks, centos]
+    needs: checkout
+
+    steps:
 
     - name: Build components
       run: |
         cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./build_all.sh
+        ./build_all.sh -j 8
 
     - name: Link artifacts
       run: |
         cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
         ./link_workflow.sh
 
   create-experiments:
-    needs: checkout-build-link
-    runs-on: [self-hosted, orion-ready]
+    needs: checkout
+    runs-on: [self-hosted, aws, parallelworks, centos]
     strategy:
       matrix:
-        case: ["C48_S2S", "C96_atm3DVar"]
+        case: ["C48_ATM"]
 
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
-          HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs
           RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
           pslot: ${{ matrix.case }}.${{ github.run_id }}
         run: |
+          mkdir -p ${{ env.RUNTESTS }}
           cd ${{ env.TEST_DIR }}/HOMEgfs
           source workflow/gw_setup.sh
-          source ci/platforms/orion.sh
-          ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml --dir ${{ env.HOMEgfs_PR }}
+          source ci/platforms/config.noaacloud
+          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite
 
   run-experiments:
     needs: create-experiments
-    runs-on: [self-hosted, orion-ready]
+    runs-on: [self-hosted, aws, parallelworks, centos]
     strategy:
       max-parallel: 2
       matrix:
-        case: ["C48_S2S", "C96_atm3DVar"]
+        case: ["C48_ATM"]
     steps:
       - name: Run Experiment ${{ matrix.case }}
         run: |
           cd ${{ env.TEST_DIR }}/HOMEgfs
-          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }}
+          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
 
   clean-up:
     needs: run-experiments
-    runs-on: [self-hosted, orion-ready]
+    runs-on: [self-hosted, aws, parallelworks, centos]
     steps:
       - name: Clean-up
         run: |
           cd ${{ github.workspace }}
           rm -rf ${{ github.run_id }}
-
diff --git a/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt b/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt
@@ -0,0 +1,5 @@
+sudo yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
+sudo yum -y install git
+/contrib/Terry.McGuinness/SETUP/provision_runner.sh
+ALLNODES
+/contrib/Terry.McGuinness/SETUP/mount-epic-contrib.sh
diff --git a/ci/scripts/utils/parallel_works/provision_runner.sh b/ci/scripts/utils/parallel_works/provision_runner.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# This script provisions a GitHub Actions runner on a Rocky or CentOS system.
+# It performs the following steps:
+# 1. Checks the operating system from /etc/os-release.
+# 2. Verifies if the operating system is either Rocky or CentOS.
+# 3. Checks if an actions-runner process is already running for the current user.
+# 4. Copies the actions-runner tar file from a specified directory to the home directory.
+# 5. Extracts the tar file and starts the actions-runner in the background.
+#
+# The actions-runner tar file contains the necessary binaries and scripts to run
+# a GitHub Actions runner. It is specific to the operating system and is expected
+# to be located in the /contrib/${CI_USER}/SETUP/ directory.
+
+CI_USER="Terry.McGuinness"
+
+# Get the Operating System name from /etc/os-release
+OS_NAME=$(grep -E '^ID=' /etc/os-release | sed -E 's/ID="?([^"]*)"?/\1/') || true
+
+# Check if the OS is Rocky or CentOS
+if [[ "${OS_NAME}" == "rocky" || "${OS_NAME}" == "centos" ]]; then
+  echo "Operating System is ${OS_NAME}"
+else
+  echo "Unsupported Operating System: ${OS_NAME}"
+  exit 1
+fi
+
+running=$(pgrep -u "${USER}" run-helper -c) || true
+if [[ "${running}" -gt 0 ]]; then
+   echo "actions-runner is already running"
+   exit
+fi
+
+cp "/contrib/${CI_USER}/SETUP/actions-runner_${OS_NAME}.tar.gz" "${HOME}"
+cd "${HOME}" || exit
+tar -xf "actions-runner_${OS_NAME}.tar.gz"
+cd actions-runner || exit
+d=$(date +%Y-%m-%d-%H:%M)
+nohup ./run.sh >& "run_nohup${d}.log" &