Skip to content

Commit

Permalink
Replace Jinja namespaces with replace_tmpl filter and disable ACCOUNT…
Browse files Browse the repository at this point in the history
…_SERVICE option (NOAA-EMC#2775)

Removes the namespace-based construction of EnKF member COM directories
in the enkf archive template.
  • Loading branch information
DavidHuber-NOAA authored Jul 23, 2024
1 parent c45b961 commit 65a7ab7
Show file tree
Hide file tree
Showing 12 changed files with 15 additions and 38 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci_unit_tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
run: |
sudo mkdir -p /scratch1/NCEPDEV
cd $GITHUB_WORKSPACE/sorc
git submodule update --init --recursive
git submodule update --init
./link_workflow.sh
cd $GITHUB_WORKSPACE/ci/scripts/tests
ln -s ../wxflow
Expand Down
40 changes: 13 additions & 27 deletions parm/archive/master_enkf.yaml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -52,35 +52,21 @@ datasets:
{% for mem in range(first_group_mem, last_group_mem + 1) %}

# Declare a dict of search and replace terms to run on each template
{% set tmpl_dict = {'ROTDIR':ROTDIR,
'RUN':RUN,
'YMD':cycle_YMD,
'HH':cycle_HH,
'MEMDIR':"mem" + '%03d' % mem} %}

# Replace template variables with tmpl_dict, one key at a time
# This must be done in a namespace to overcome jinja scoping
# Variables set inside of a for loop are lost at the end of the loop
# unless they are part of a namespace
{% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL,
COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL,
COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %}

{% for key in tmpl_dict.keys() %}
{% set search_term = '${' + key + '}' %}
{% set replace_term = tmpl_dict[key] %}
{% set com_ns.COMIN_ATMOS_ANALYSIS_MEM =
com_ns.COMIN_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %}
{% set com_ns.COMIN_ATMOS_HISTORY_MEM =
com_ns.COMIN_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %}
{% set com_ns.COMIN_ATMOS_RESTART_MEM =
com_ns.COMIN_ATMOS_RESTART_MEM.replace(search_term, replace_term) %}
{% endfor %}
{% set mem_char = 'mem%03d' | format(mem) %}
{% set tmpl_dict = ({ '${ROTDIR}':ROTDIR,
'${RUN}':RUN,
'${YMD}':cycle_YMD,
'${HH}':cycle_HH,
'${MEMDIR}': mem_char }) %}

{% set COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %}
{% set COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) %}
{% set COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) %}

# Append the member COM directories
{% do COMIN_ATMOS_ANALYSIS_MEM_list.append(com_ns.COMIN_ATMOS_ANALYSIS_MEM)%}
{% do COMIN_ATMOS_HISTORY_MEM_list.append(com_ns.COMIN_ATMOS_HISTORY_MEM)%}
{% do COMIN_ATMOS_RESTART_MEM_list.append(com_ns.COMIN_ATMOS_RESTART_MEM)%}
{% do COMIN_ATMOS_ANALYSIS_MEM_list.append(COMIN_ATMOS_ANALYSIS_MEM)%}
{% do COMIN_ATMOS_HISTORY_MEM_list.append(COMIN_ATMOS_HISTORY_MEM)%}
{% do COMIN_ATMOS_RESTART_MEM_list.append(COMIN_ATMOS_RESTART_MEM)%}

{% endfor %}

Expand Down
1 change: 0 additions & 1 deletion workflow/hosts/awspw.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ STMP: '/lustre/${USER}/stmp2/'
PTMP: '/lustre/${USER}/stmp4/'
NOSCRUB: ${HOMEDIR}
ACCOUNT: hwufscpldcld
ACCOUNT_SERVICE: hwufscpldcld
SCHEDULER: slurm
QUEUE: batch
QUEUE_SERVICE: batch
Expand Down
1 change: 0 additions & 1 deletion workflow/hosts/container.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ PTMP: '/home/${USER}'
NOSCRUB: $HOMEDIR
SCHEDULER: none
ACCOUNT: ''
ACCOUNT_SERVICE: ''
QUEUE: ''
QUEUE_SERVICE: ''
PARTITION_BATCH: ''
Expand Down
1 change: 0 additions & 1 deletion workflow/hosts/gaea.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ STMP: '/gpfs/f5/ufs-ard/scratch/${USER}'
PTMP: '/gpfs/f5/ufs-ard/scratch/${USER}'
NOSCRUB: $HOMEDIR
ACCOUNT: ufs-ard
ACCOUNT_SERVICE: ufs-ard
SCHEDULER: slurm
QUEUE: normal
QUEUE_SERVICE: normal
Expand Down
1 change: 0 additions & 1 deletion workflow/hosts/hera.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ STMP: '/scratch1/NCEPDEV/stmp2/${USER}'
PTMP: '/scratch1/NCEPDEV/stmp4/${USER}'
NOSCRUB: $HOMEDIR
ACCOUNT: fv3-cpu
ACCOUNT_SERVICE: fv3-cpu
SCHEDULER: slurm
QUEUE: batch
QUEUE_SERVICE: batch
Expand Down
1 change: 0 additions & 1 deletion workflow/hosts/hercules.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ PTMP: '/work/noaa/stmp/${USER}/HERCULES'
NOSCRUB: $HOMEDIR
SCHEDULER: slurm
ACCOUNT: fv3-cpu
ACCOUNT_SERVICE: fv3-cpu
QUEUE: batch
QUEUE_SERVICE: batch
PARTITION_BATCH: hercules
Expand Down
1 change: 0 additions & 1 deletion workflow/hosts/jet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ STMP: '/lfs4/HFIP/hfv3gfs/${USER}/stmp'
PTMP: '/lfs4/HFIP/hfv3gfs/${USER}/ptmp'
NOSCRUB: $HOMEDIR
ACCOUNT: hfv3gfs
ACCOUNT_SERVICE: hfv3gfs
SCHEDULER: slurm
QUEUE: batch
QUEUE_SERVICE: batch
Expand Down
1 change: 0 additions & 1 deletion workflow/hosts/orion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ PTMP: '/work/noaa/stmp/${USER}/ORION'
NOSCRUB: $HOMEDIR
SCHEDULER: slurm
ACCOUNT: fv3-cpu
ACCOUNT_SERVICE: fv3-cpu
QUEUE: batch
QUEUE_SERVICE: batch
PARTITION_BATCH: orion
Expand Down
1 change: 0 additions & 1 deletion workflow/hosts/s4.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ STMP: '/scratch/users/${USER}'
PTMP: '/scratch/users/${USER}'
NOSCRUB: ${HOMEDIR}
ACCOUNT: star
ACCOUNT_SERVICE: star
SCHEDULER: slurm
QUEUE: s4
QUEUE_SERVICE: serial
Expand Down
1 change: 0 additions & 1 deletion workflow/hosts/wcoss2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ STMP: '/lfs/h2/emc/stmp/${USER}'
PTMP: '/lfs/h2/emc/ptmp/${USER}'
NOSCRUB: $HOMEDIR
ACCOUNT: 'GFS-DEV'
ACCOUNT_SERVICE: 'GFS-DEV'
SCHEDULER: pbspro
QUEUE: 'dev'
QUEUE_SERVICE: 'dev_transfer'
Expand Down
2 changes: 1 addition & 1 deletion workflow/rocoto/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ def get_resource(self, task_name):

task_config = self._configs[task_name]

account = task_config['ACCOUNT_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_config['ACCOUNT']
account = task_config['ACCOUNT']

if f'wtime_{task_name}_{self.cdump}' in task_config:
walltime = task_config[f'wtime_{task_name}_{self.cdump}']
Expand Down

0 comments on commit 65a7ab7

Please sign in to comment.