Skip to content

Commit

Permalink
Merge branch 'NOAA-EMC:develop' into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
jderrico-noaa authored Feb 29, 2024
2 parents e2d88dd + 5166593 commit 4038e8a
Show file tree
Hide file tree
Showing 191 changed files with 817 additions and 910 deletions.
51 changes: 31 additions & 20 deletions Jenkinsfile → ci/Jenkinsfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
def Machine = 'none'
def machine = 'none'
def HOME = 'none'
def localworkspace = 'none'
def commonworkspace = 'none'
def caseList = ''
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']

pipeline {
agent { label 'built-in' }
Expand All @@ -20,7 +20,6 @@ pipeline {
agent { label 'built-in' }
steps {
script {
localworkspace = env.WORKSPACE
machine = 'none'
for (label in pullRequest.labels) {
echo "Label: ${label}"
Expand All @@ -42,14 +41,16 @@ pipeline {
agent { label "${machine}-emc" }
steps {
script {
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
HOME = "${WORKSPACE}/TESTDIR"
commonworkspace = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS")
pullRequest.addLabel("CI-${Machine}-Building")
if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
pullRequest.removeLabel("CI-${Machine}-Ready")
ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
HOME = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
pullRequest.addLabel("CI-${Machine}-Building")
if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
pullRequest.removeLabel("CI-${Machine}-Ready")
}
}
pullRequest.comment("Building and running on ${Machine} in directory ${HOME}")
}
}
}
Expand All @@ -73,17 +74,15 @@ pipeline {
def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
sh(script: "mkdir -p ${HOMEgfs}")
ws(HOMEgfs) {
env.MACHINE_ID = machine // MACHINE_ID is used in the build scripts to determine the machine and is added to the shell environment
if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}<br>Still doing a checkout to get the latest changes")
sh(script: 'source workflow/gw_setup.sh; git pull --recurse-submodules')
checkout scm
dir('sorc') {
sh(script: './link_workflow.sh')
}
} else {
checkout scm
sh(script: 'source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive')
def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
def build_args_list = builds_file['builds']
def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
Expand All @@ -99,6 +98,9 @@ pipeline {
}
pullRequest.addLabel("CI-${Machine}-Running")
}
if (system == 'gfs') {
caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
}
}
}
}
Expand All @@ -114,11 +116,15 @@ pipeline {
axis {
name 'Case'
// TODO add dynamic list of cases from env vars (needs addtional plugins)
values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA', 'C96_atmsnowDA' // 'C48mx500_3DVarAOWCDA'
values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA'
}
}
stages {
stage('Create Experiment') {

stage('Create Experiments') {
when {
expression { return caseList.contains(Case) }
}
steps {
script {
sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp")
Expand All @@ -130,15 +136,20 @@ pipeline {
}
}
}

stage('Run Experiments') {
when {
expression { return caseList.contains(Case) }
}
steps {
script {
HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
ws(HOMEgfs) {
pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
// pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
err = sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
if (err != 0) {
pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
try {
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
} catch (Exception e) {
pullRequest.comment("**FAILURE** running experiment: ${Case} on ${Machine}")
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS")
ws(HOME) {
Expand All @@ -153,12 +164,12 @@ pipeline {
}
error("Failed to run experiments ${Case} on ${Machine}")
}
// pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
}
pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
}

}
}

}
}
}
Expand Down
5 changes: 0 additions & 5 deletions ci/cases/pr/C48_S2SWA_gefs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,3 @@ arguments:
idate: 2021032312
edate: 2021032312
yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml

skip_ci_on_hosts:
- hera
- orion
- hercules
2 changes: 1 addition & 1 deletion ci/cases/yamls/atmsnowDA_defaults_ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ defaults:
!INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
base:
DOIAU: "NO"
DO_JEDILANDDA: "YES"
DO_JEDISNOWDA: "YES"
ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
8 changes: 7 additions & 1 deletion ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,16 @@ base:
DO_JEDIATMVAR: "YES"
DO_JEDIATMENS: "YES"
ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
atmanl:
LAYOUT_X_ATMANL: 1
LAYOUT_Y_ATMANL: 1
atmensanl:
LAYOUT_X_ATMENSANL: 1
LAYOUT_Y_ATMENSANL: 1
esfc:
DONST: "NO"
nsst:
NST_MODEL: "1"
sfcanl:
DONST: "NO"


32 changes: 32 additions & 0 deletions ci/scripts/utils/get_host_case_list.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
import os
from os.path import basename, splitext
import sys
import glob
from wxflow import parse_j2yaml
from wxflow import AttrDict

_here = os.path.dirname(__file__)
_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..'))

if __name__ == '__main__':

if len(sys.argv) < 2:
print('Usage: get_host_case_list.py <host_name>')
sys.exit(1)

host = sys.argv[1]

case_list = []
HOMEgfs = _top
data = AttrDict(HOMEgfs=_top)
data.update(os.environ)

case_files = glob.glob(f'{HOMEgfs}/ci/cases/pr/*.yaml')
for case_yaml in case_files:
case_conf = parse_j2yaml(path=case_yaml, data=data)
if 'skip_ci_on_hosts' in case_conf:
if host.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]:
continue
case_list.append(splitext(basename(case_yaml))[0])
print(' '.join(case_list))
1 change: 1 addition & 0 deletions ci/scripts/utils/wxflow
2 changes: 1 addition & 1 deletion env/AWSPW.env
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlrun atmensanlrun aeroanlrun landanlrun"
echo "atmanlrun atmensanlrun aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
Expand Down
2 changes: 1 addition & 1 deletion env/CONTAINER.env
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlrun atmensanlrun aeroanlrun landanl"
echo "atmanlrun atmensanlrun aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
Expand Down
14 changes: 7 additions & 7 deletions env/HERA.env
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlrun atmensanlrun aeroanlrun landanl"
echo "atmanlrun atmensanlrun aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
Expand Down Expand Up @@ -42,7 +42,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
export sys_tp="HERA"
export launcher_PREP="srun"

elif [[ "${step}" = "preplandobs" ]]; then
elif [[ "${step}" = "prepsnowobs" ]]; then

export APRUN_CALCFIMS="${launcher} -n 1"

Expand Down Expand Up @@ -79,13 +79,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
[[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"

elif [[ "${step}" = "landanl" ]]; then
elif [[ "${step}" = "snowanl" ]]; then

nth_max=$((npe_node_max / npe_node_landanl))
nth_max=$((npe_node_max / npe_node_snowanl))

export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
[[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
export APRUN_LANDANL="${launcher} -n ${npe_landanl} --cpus-per-task=${NTHREADS_LANDANL}"
export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
[[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
export APRUN_SNOWANL="${launcher} -n ${npe_snowanl} --cpus-per-task=${NTHREADS_SNOWANL}"

export APRUN_APPLY_INCR="${launcher} -n 6"

Expand Down
12 changes: 6 additions & 6 deletions env/HERCULES.env
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ case ${step} in
export sys_tp="HERCULES"
export launcher_PREP="srun"
;;
"preplandobs")
"prepsnowobs")

export APRUN_CALCFIMS="${launcher} -n 1"
;;
Expand Down Expand Up @@ -79,13 +79,13 @@ case ${step} in
[[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
;;
"landanl")
"snowanl")

nth_max=$((npe_node_max / npe_node_landanl))
nth_max=$((npe_node_max / npe_node_snowanl))

export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
[[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
export APRUN_LANDANL="${launcher} -n ${npe_landanl} --cpus-per-task=${NTHREADS_LANDANL}"
export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
[[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
export APRUN_SNOWANL="${launcher} -n ${npe_snowanl} --cpus-per-task=${NTHREADS_SNOWANL}"

export APRUN_APPLY_INCR="${launcher} -n 6"
;;
Expand Down
14 changes: 7 additions & 7 deletions env/JET.env
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlrun atmensanlrun aeroanlrun landanl"
echo "atmanlrun atmensanlrun aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
Expand Down Expand Up @@ -33,7 +33,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
export sys_tp="JET"
export launcher_PREP="srun"

elif [[ "${step}" = "preplandobs" ]]; then
elif [[ "${step}" = "prepsnowobs" ]]; then

export APRUN_CALCFIMS="${launcher} -n 1"

Expand Down Expand Up @@ -70,13 +70,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
[[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"

elif [[ "${step}" = "landanl" ]]; then
elif [[ "${step}" = "snowanl" ]]; then

nth_max=$((npe_node_max / npe_node_landanl))
nth_max=$((npe_node_max / npe_node_snowanl))

export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
[[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
export APRUN_LANDANL="${launcher} -n ${npe_landanl}"
export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
[[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
export APRUN_SNOWANL="${launcher} -n ${npe_snowanl}"

export APRUN_APPLY_INCR="${launcher} -n 6"

Expand Down
14 changes: 7 additions & 7 deletions env/ORION.env
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlrun atmensanlrun aeroanlrun landanl"
echo "atmanlrun atmensanlrun aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
Expand Down Expand Up @@ -40,7 +40,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
export sys_tp="ORION"
export launcher_PREP="srun"

elif [[ "${step}" = "preplandobs" ]]; then
elif [[ "${step}" = "prepsnowobs" ]]; then

export APRUN_CALCFIMS="${launcher} -n 1"

Expand Down Expand Up @@ -78,13 +78,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
[[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"

elif [[ "${step}" = "landanl" ]]; then
elif [[ "${step}" = "snowanl" ]]; then

nth_max=$((npe_node_max / npe_node_landanl))
nth_max=$((npe_node_max / npe_node_snowanl))

export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
[[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
export APRUN_LANDANL="${launcher} -n ${npe_landanl} --cpus-per-task=${NTHREADS_LANDANL}"
export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
[[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
export APRUN_SNOWANL="${launcher} -n ${npe_snowanl} --cpus-per-task=${NTHREADS_SNOWANL}"

export APRUN_APPLY_INCR="${launcher} -n 6"

Expand Down
14 changes: 7 additions & 7 deletions env/S4.env
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlrun atmensanlrun aeroanlrun landanl"
echo "atmanlrun atmensanlrun aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
Expand Down Expand Up @@ -33,7 +33,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
export sys_tp="S4"
export launcher_PREP="srun"

elif [[ "${step}" = "preplandobs" ]]; then
elif [[ "${step}" = "prepsnowobs" ]]; then

export APRUN_CALCFIMS="${launcher} -n 1"

Expand Down Expand Up @@ -70,13 +70,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
[[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"

elif [[ "${step}" = "landanl" ]]; then
elif [[ "${step}" = "snowanl" ]]; then

nth_max=$((npe_node_max / npe_node_landanl))
nth_max=$((npe_node_max / npe_node_snowanl))

export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
[[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
export APRUN_LANDANL="${launcher} -n ${npe_landanl}"
export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
[[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
export APRUN_SNOWANL="${launcher} -n ${npe_snowanl}"

export APRUN_APPLY_INCR="${launcher} -n 6"

Expand Down
Loading

0 comments on commit 4038e8a

Please sign in to comment.