Add the capability to use slurm reservation nodes (NOAA-EMC#2627)

Add the capability to use slurm reservation nodes Add "ACCOUNT_SERVICE" for jobs to run in PARTITION_SERVICE Resolves NOAA-EMC#2626
KateFriedman-NOAA · May 30, 2024 · 4422550 · 4422550
1 parent a54153f
commit 4422550
Show file tree

Hide file tree

Showing 11 changed files with 23 additions and 1 deletion.
diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
@@ -13,10 +13,12 @@ export RUN_ENVIR="emc"
 
 # Account, queue, etc.
 export ACCOUNT="@ACCOUNT@"
+export ACCOUNT_SERVICE="@ACCOUNT_SERVICE@"
 export QUEUE="@QUEUE@"
 export QUEUE_SERVICE="@QUEUE_SERVICE@"
 export PARTITION_BATCH="@PARTITION_BATCH@"
 export PARTITION_SERVICE="@PARTITION_SERVICE@"
+export RESERVATION="@RESERVATION@"
 
 # Project to use in mass store:
 export HPSS_PROJECT="@HPSS_PROJECT@"

diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
@@ -13,10 +13,12 @@ export RUN_ENVIR="emc"
 
 # Account, queue, etc.
 export ACCOUNT="@ACCOUNT@"
+export ACCOUNT_SERVICE="@ACCOUNT_SERVICE@"
 export QUEUE="@QUEUE@"
 export QUEUE_SERVICE="@QUEUE_SERVICE@"
 export PARTITION_BATCH="@PARTITION_BATCH@"
 export PARTITION_SERVICE="@PARTITION_SERVICE@"
+export RESERVATION="@RESERVATION@"
 
 # Project to use in mass store:
 export HPSS_PROJECT="@HPSS_PROJECT@"

diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml
@@ -7,11 +7,13 @@ STMP: '/lustre/${USER}/stmp2/'
 PTMP: '/lustre/${USER}/stmp4/'
 NOSCRUB: ${HOMEDIR}
 ACCOUNT: hwufscpldcld
+ACCOUNT_SERVICE: hwufscpldcld
 SCHEDULER: slurm
 QUEUE: batch
 QUEUE_SERVICE: batch
 PARTITION_BATCH: compute
 PARTITION_SERVICE: compute
+RESERVATION: ''
 CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
 HPSSARCH: 'YES'

diff --git a/workflow/hosts/container.yaml b/workflow/hosts/container.yaml
@@ -8,10 +8,12 @@ PTMP: '/home/${USER}'
 NOSCRUB: $HOMEDIR
 SCHEDULER: none
 ACCOUNT: ''
+ACCOUNT_SERVICE: ''
 QUEUE: ''
 QUEUE_SERVICE: ''
 PARTITION_BATCH: ''
 PARTITION_SERVICE: ''
+RESERVATION: ''
 CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod'
 HPSSARCH: 'NO'

diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml
@@ -8,11 +8,13 @@ STMP: '/scratch1/NCEPDEV/stmp2/${USER}'
 PTMP: '/scratch1/NCEPDEV/stmp4/${USER}'
 NOSCRUB: $HOMEDIR
 ACCOUNT: fv3-cpu
+ACCOUNT_SERVICE: fv3-cpu
 SCHEDULER: slurm
 QUEUE: batch
 QUEUE_SERVICE: batch
 PARTITION_BATCH: hera
 PARTITION_SERVICE: service
+RESERVATION: ''
 CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod'
 HPSSARCH: 'YES'

diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml
@@ -9,10 +9,12 @@ PTMP: '/work/noaa/stmp/${USER}/HERCULES'
 NOSCRUB: $HOMEDIR
 SCHEDULER: slurm
 ACCOUNT: fv3-cpu
+ACCOUNT_SERVICE: fv3-cpu
 QUEUE: batch
 QUEUE_SERVICE: batch
 PARTITION_BATCH: hercules
 PARTITION_SERVICE: service
+RESERVATION: ''
 CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod'
 HPSSARCH: 'NO'

diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml
@@ -8,11 +8,13 @@ STMP: '/lfs4/HFIP/hfv3gfs/${USER}/stmp'
 PTMP: '/lfs4/HFIP/hfv3gfs/${USER}/ptmp'
 NOSCRUB: $HOMEDIR
 ACCOUNT: hfv3gfs
+ACCOUNT_SERVICE: hfv3gfs
 SCHEDULER: slurm
 QUEUE: batch
 QUEUE_SERVICE: batch
 PARTITION_BATCH: kjet
 PARTITION_SERVICE: service
+RESERVATION: ''
 CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod'
 HPSSARCH: 'YES'

diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml
@@ -9,10 +9,12 @@ PTMP: '/work/noaa/stmp/${USER}/ORION'
 NOSCRUB: $HOMEDIR
 SCHEDULER: slurm
 ACCOUNT: fv3-cpu
+ACCOUNT_SERVICE: fv3-cpu
 QUEUE: batch
 QUEUE_SERVICE: batch
 PARTITION_BATCH: orion
 PARTITION_SERVICE: service
+RESERVATION: ''
 CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod'
 HPSSARCH: 'NO'

diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml
@@ -8,11 +8,13 @@ STMP: '/scratch/users/${USER}'
 PTMP: '/scratch/users/${USER}'
 NOSCRUB: ${HOMEDIR}
 ACCOUNT: star
+ACCOUNT_SERVICE: star
 SCHEDULER: slurm
 QUEUE: s4
 QUEUE_SERVICE: serial
 PARTITION_BATCH: s4
 PARTITION_SERVICE: serial
+RESERVATION: ''
 CHGRP_RSTPROD: 'NO'
 CHGRP_CMD: 'ls'
 HPSSARCH: 'NO'

diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml
@@ -8,11 +8,13 @@ STMP: '/lfs/h2/emc/stmp/${USER}'
 PTMP: '/lfs/h2/emc/ptmp/${USER}'
 NOSCRUB: $HOMEDIR
 ACCOUNT: 'GFS-DEV'
+ACCOUNT_SERVICE: 'GFS-DEV'
 SCHEDULER: pbspro
 QUEUE: 'dev'
 QUEUE_SERVICE: 'dev_transfer'
 PARTITION_BATCH: ''
 PARTITION_SERVICE: ''
+RESERVATION: ''
 CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod'
 HPSSARCH: 'NO'

diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py
@@ -169,7 +169,7 @@ def get_resource(self, task_name):
 
         task_config = self._configs[task_name]
 
-        account = task_config['ACCOUNT']
+        account = task_config['ACCOUNT_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_config['ACCOUNT']
 
         walltime = task_config[f'wtime_{task_name}']
         if self.cdump in ['gfs'] and f'wtime_{task_name}_gfs' in task_config.keys():
@@ -208,6 +208,8 @@ def get_resource(self, task_name):
                 native += ':shared'
         elif scheduler in ['slurm']:
             native = '--export=NONE'
+            if task_config['RESERVATION'] != "":
+                native += '' if task_name in Tasks.SERVICE_TASKS else ' --reservation=' + task_config['RESERVATION']
 
         queue = task_config['QUEUE_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_config['QUEUE']