From 9b1f9c9c69d04c72838bce173b8ab5c76491798d Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Tue, 25 Apr 2023 09:57:47 -0400 Subject: [PATCH 01/88] temporary checkin --- pre/remap_restart/remap_restarts.py | 75 +++++++++++++++++++++++------ pre/remap_restart/remap_utils.py | 43 +++++++++++++++++ 2 files changed, 104 insertions(+), 14 deletions(-) diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index d9a0958..6597e1b 100644 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -47,10 +47,55 @@ def parse_args(): ''') parser = argparse.ArgumentParser(description='Remap restarts',epilog=program_description,formatter_class=argparse.RawDescriptionHelpFormatter) - # define a mutually exclusive group of arguments - group = parser.add_mutually_exclusive_group() - group.add_argument('-c', '--config_file', help='YAML config file') - group.add_argument('-o', '--flattened_yaml', help='Flattened YAML config', metavar='input:air:drymass=1 input:air:hydrostatic=0 ...') + + p_sub = parser.add_subparsers(help = 'Sub command help') + p_config = p_sub.add_parser( + 'config', + help = "Use config file as input", + ) + p_command = p_sub.add_parser( + 'command_line', + help = "Use command line as input", + ) + + p_config.add_argument('-c', '--config_file', help='YAML config file') + + p_command.add_argument('-merra2', action='store_true', help='use merra2 restarts') + p_command.add_argument('-ymd', help='yyyymmdd year mone date of input and output restarts') + p_command.add_argument('-hr', help='hh hours of input and output restarts') + p_command.add_argument('-grout', help='Grid ID of the output restart') + p_command.add_argument('-levsout', help='levels of output restarts') + + p_command.add_argument('-outdir', help='directory for output restarts') + p_command.add_argument('-d', help='directory for input restarts') + + p_command.add_argument('-expid', help='restart id for input restarts') + p_command.add_argument('-newid', help='restart id for output restarts') + + p_command.add_argument('-tagin', help='GCM or DAS tag associated with inputs') + p_command.add_argument('-tagout', help='GCM or DAS tag associated with outputs') + + p_command.add_argument('-wemin', help='minimum water snow water equivalent for input catch/cn') + p_command.add_argument('-wemout', help='minimum water snow water equivalent for output catch/cn') + + p_command.add_argument('-oceanin', default='c', help='ocean horizontal grid of inputs') + p_command.add_argument('-oceanout', default='c', help='ocean horizontal grid of outputs') + + p_command.add_argument('-ocnmdlin', default='data', help='ocean input model') + p_command.add_argument('-ocnmdlout',default='data', help='ocean output model') + + p_command.add_argument('-nobkg', action='store_true', help="Don't remap bkg files") + p_command.add_argument('-nolbl', action='store_true', help="label final restarts with 'tagID.gridID' extension") + p_command.add_argument('-nolcv', action='store_true', help="Don't remap lcv files") + p_command.add_argument('-bkg', action='store_true', help="Don't remap bkg files") + p_command.add_argument('-lbl', action='store_true', help="label final restarts with 'tagID.gridID' extension") + p_command.add_argument('-lcv', action='store_true', help="Don't remap lcv files") + p_command.add_argument('-altbcs', help= "use user's boundary condition files") + + p_command.add_argument('-rs', default=3, help='flag indicating which restarts to regrid') + + +#$GEOSBIN/regrid.pl -np -ymd ${year}${month}${day} -hr 21 -grout C${AGCM_IM} -levsout ${AGCM_LM} -outdir . -d . -expid $RSTID -tagin @EMIP_BCS_IN -oceanin e -i -nobkg -lbl -nolcv -tagout @LSMBCS -rs 3 -oceanout @OCEANOUT # Parse using parse_known_args so we can pass the rest to the remap scripts # If config_file is used, then extra_args will be empty @@ -64,15 +109,16 @@ def main(): config = '' # Parse the command line arguments from parse_args() capturing the arguments and the rest - command_line_args, extra_args = parse_args() - config_yaml = command_line_args.config_file - flattened_yaml = command_line_args.flattened_yaml - - if config_yaml: - config = yaml_to_config(config_yaml) - elif flattened_yaml: - config_yaml = 'remap_params.yaml' - config = args_to_config(extra_args) + args, extra_args = parse_args() + + if (len(sys.argv) > 1) : + args, extra_args = parse_args() + print(args) + if sys.argv[1] == 'config': + config = yaml_to_config(args.config_yaml) + if sys.argv[1] == 'command_line': + raw_config = get_config_from_command_line(args) + print(raw_config) else: raw_config = get_config_from_questionary() params = remap_params(raw_config) @@ -80,6 +126,7 @@ def main(): question_flag = True config_yaml = 'remap_params.yaml' + exit() print('\n') print_config(config) @@ -117,4 +164,4 @@ def main(): if __name__ == '__main__' : main() - + diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 3f71429..9c99509 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -117,6 +117,49 @@ def merra2_expid(config): return config +def get_config_from_command_line(cml): + + gridID = {} + gridID['c'] = '360X180' + gridID['e'] = '1440x720' + gridID['f'] = '2880x1440' + gridID['CS'] = 'CS' + gridID['aa'] = '72x36' + gridID['bb'] = '360x200' + gridID['dd'] = '720x410' + gridID['ee'] = '1440x1080' + + answers = {} + answers["input:shared:MERRA-2"] = cml.merra2 + answers["input:shared:yyyymmddhh"] = cml.ymd + cml.hr + answers["output:shared:agrid"] = cml.grout + answers["output:air:nlevel"] = cml.levsout + answers["output:shared:out_dir"] = cml.outdir + '/' + answers["input:shared:rst_dir"] = cml.d + '/' + answers["output:shared:expid"] = cml.newid + + answers["input:shared:tag"] = cml.tagin + answers["output:shared:tag"] = cml.tagout + + answers["input:shared:model"] = cml.ocnmdlin + answers["output:shared:model"] = cml.ocnmdlout + answers["input:shared:ogrid"] = gridID[cml.oceanin] + answers["output:shared:ogrid"] = gridID[cml.oceanout] + + answers["output:analysis:bkg"] = cml.bkg + answers["output:analysis:lcv"] = cml.lcv + + if cml.rs == 1: + answers["output:air:remap"] = True + if cml.rs == 2: + answers["output:surface:remap"] = True + if cml.rs == 3: + answers["output:surface:remap"] = True + answers["output:air:remap"] = True + + + return flatten + if __name__ == '__main__' : config = yaml_to_config('c24Toc12.yaml') print_config(config) From 08994116905dd77f25373f563d08db90e98a82b1 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 4 May 2023 14:40:20 -0400 Subject: [PATCH 02/88] stage --- pre/remap_restart/remap_restarts.py | 10 ++++-- pre/remap_restart/remap_utils.py | 51 +++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index 6597e1b..e1c9455 100644 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -28,14 +28,16 @@ def parse_args(): There are three ways to use this script to remap restarts. 1. Use an existing config file to remap: - ./remap_restarts.py -c my_config.yaml + ./remap_restarts.py config -c my_config.yaml 2. Use questionary to convert template remap_params.tpl to remap_params.yaml and then remap: ./remap_restarts.py 3. Use command line to input a flattened yaml file: - ./remap_restarts.py -o input:air:drymass=1 input:air:hydrostatic=0 ... + ./remap_restarts.py command_line -ymd 20040414 -hr 21 .... + To get help, please use + ./remap_restarts.py command_line -h NOTE: Each individual script can be executed independently 1. remap_questions.py generates raw_answer.yaml @@ -109,7 +111,6 @@ def main(): config = '' # Parse the command line arguments from parse_args() capturing the arguments and the rest - args, extra_args = parse_args() if (len(sys.argv) > 1) : args, extra_args = parse_args() @@ -118,6 +119,9 @@ def main(): config = yaml_to_config(args.config_yaml) if sys.argv[1] == 'command_line': raw_config = get_config_from_command_line(args) + params = remap_params(raw_config) + config = params.config + question_flag = True print(raw_config) else: raw_config = get_config_from_questionary() diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 9c99509..be956c3 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -119,15 +119,15 @@ def merra2_expid(config): def get_config_from_command_line(cml): - gridID = {} - gridID['c'] = '360X180' - gridID['e'] = '1440x720' - gridID['f'] = '2880x1440' - gridID['CS'] = 'CS' - gridID['aa'] = '72x36' - gridID['bb'] = '360x200' - gridID['dd'] = '720x410' - gridID['ee'] = '1440x1080' + ogridID = {} + ogridID['c'] = '360X180' + ogridID['e'] = '1440x720' + ogridID['f'] = '2880x1440' + ogridID['CS'] = 'CS' + ogridID['aa'] = '72x36' + ogridID['bb'] = '360x200' + ogridID['dd'] = '720x410' + ogridID['ee'] = '1440x1080' answers = {} answers["input:shared:MERRA-2"] = cml.merra2 @@ -143,8 +143,8 @@ def get_config_from_command_line(cml): answers["input:shared:model"] = cml.ocnmdlin answers["output:shared:model"] = cml.ocnmdlout - answers["input:shared:ogrid"] = gridID[cml.oceanin] - answers["output:shared:ogrid"] = gridID[cml.oceanout] + answers["input:shared:ogrid"] = ogridID[cml.oceanin] + answers["output:shared:ogrid"] = ogridID[cml.oceanout] answers["output:analysis:bkg"] = cml.bkg answers["output:analysis:lcv"] = cml.lcv @@ -157,8 +157,33 @@ def get_config_from_command_line(cml): answers["output:surface:remap"] = True answers["output:air:remap"] = True - - return flatten + if not answers.get('input:shared:model') : + answers['input:shared:model'] = 'data' + if answers['input:shared:MERRA-2']: + answers['input:shared:rst_dir'] = tmp_merra2_dir(answers) + if answers.get('output:shared:ogrid') == 'CS': + answers['output:shared:ogrid'] = answers['output:shared:agrid'] + answers['input:shared:rst_dir'] = os.path.abspath(answers['input:shared:rst_dir']) + answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) + + config = {} + config['input'] = {} + config['input']['shared'] = {} + config['input']['surface'] = {} + config['output'] = {} + config['output']['shared'] = {} + config['output']['air'] = {} + config['output']['surface'] = {} + config['output']['analysis'] = {} + config['slurm'] = {} + for key, value in answers.items(): + keys = key.split(":") + if len(keys) == 2: + config[keys[0]][keys[1]] = value + if len(keys) == 3: + config[keys[0]][keys[1]][keys[2]] = value + + return config if __name__ == '__main__' : config = yaml_to_config('c24Toc12.yaml') From 55670d9f1f1b2da73fe0a08b450b525c9606bb41 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 18 May 2023 12:22:02 -0400 Subject: [PATCH 03/88] add command line option --- pre/remap_restart/remap_catchANDcn.py | 4 +- pre/remap_restart/remap_command.py | 164 +++++++++++++++ pre/remap_restart/remap_params.py | 166 ++++++--------- pre/remap_restart/remap_params.tpl | 7 +- pre/remap_restart/remap_questions.py | 289 ++++++++++---------------- pre/remap_restart/remap_restarts.py | 139 ++++--------- pre/remap_restart/remap_upper.py | 1 + pre/remap_restart/remap_utils.py | 276 +++++++++++++++--------- 8 files changed, 555 insertions(+), 491 deletions(-) create mode 100755 pre/remap_restart/remap_command.py diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 6064ea8..0f3c323 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -13,8 +13,8 @@ def get_landdir(bcsdir): k = bcsdir.find('/geometry/') if k != -1 : while bcsdir[-1] == '/': bcsdir = bcsdir[0:-1] # remove extra '/' - agrid_name = os.path.basename(bcsdir).split('_')[0] - bcsdir = bcsdir[0:k]+'/land/'+agrid_name + sub_grids = os.path.basename(bcsdir) + bcsdir = bcsdir[0:k]+'/land/'+ sub_grids return bcsdir class catchANDcn(remap_base): diff --git a/pre/remap_restart/remap_command.py b/pre/remap_restart/remap_command.py new file mode 100755 index 0000000..8b706c3 --- /dev/null +++ b/pre/remap_restart/remap_command.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# +# source install/bin/g5_modules +# +# Newer GEOS code should load a module with GEOSpyD Python3 if not run: +# module load python/GEOSpyD/Min4.10.3_py3.9 +# + +import os +import subprocess +import shlex +import ruamel.yaml +import shutil +import questionary +import glob +import argparse +from remap_utils import * +from remap_params import * + +def parse_args(program_description): + + parser = argparse.ArgumentParser(description='Remap restarts',epilog=program_description,formatter_class=argparse.RawDescriptionHelpFormatter) + + p_sub = parser.add_subparsers(help = 'Sub command help') + p_config = p_sub.add_parser( + 'config_file', + help = "Use config file as input", + ) + + p_config.add_argument('-c', '--config_file', help='YAML config file') + + p_command = p_sub.add_parser( + 'command_line', + help = "Use command line as input", + ) + p_command.add_argument('-merra2', action='store_true', default= False, help='use merra2 restarts') + p_command.add_argument('-ymdh', help='yyyymmddhh year month date hour of input and output restarts') + p_command.add_argument('-grout', help='Grid ID of the output restart, format Cxxx') + p_command.add_argument('-levsout', help='levels of output restarts') + + p_command.add_argument('-out_dir', help='directory for output restarts') + p_command.add_argument('-rst_dir', help='directory for input restarts') + + p_command.add_argument('-expid', help='restart id for input restarts') + p_command.add_argument('-newid', default="",help='restart id for output restarts') + + p_command.add_argument('-tagin', help='GCM or DAS tag associated with inputs') + p_command.add_argument('-tagout', help='GCM or DAS tag associated with outputs') + + p_command.add_argument('-wemin', help='minimum water snow water equivalent for input catch/cn') + p_command.add_argument('-wemout', help='minimum water snow water equivalent for output catch/cn') + + p_command.add_argument('-oceanin', help='ocean horizontal grid of inputs \n \ + data model: 360x180,1440x720,2880x1440,CS \n \ + coupled model: 72x36, 360x200,720x410,1440x1080') + p_command.add_argument('-oceanout', help='ocean horizontal grid of outputs \n \ + data model: 360x180,1440x720,2880x1440,CS \n \ + coupled model: 72x36, 360x200,720x410,1440x1080') + + p_command.add_argument('-ocnmdlin', default='data', help='ocean input model: data, MOM5, MOM6') + p_command.add_argument('-ocnmdlout',default='data', help='ocean output model: data, MOM5, MOM6') + p_command.add_argument('-catch_model',default='catch', help='ocean output model: catch, catchcnclm40, catchcnclm45') + + p_command.add_argument('-nobkg', action='store_true', help="Don't remap bkg files") + #p_command.add_argument('-nolbl', action='store_true', help="label final restarts with 'tagID.gridID' extension") + p_command.add_argument('-nolcv', action='store_true', help="Don't remap lcv files") + #p_command.add_argument('-bkg', action='store_true', help="Don't remap bkg files") + #p_command.add_argument('-lbl', action='store_true', help="label final restarts with 'tagID.gridID' extension") + #p_command.add_argument('-lcv', action='store_true', help="Don't remap lcv files") + p_command.add_argument('-in_altbcs', default="", help= "users' alternative boundary condition files for input") + p_command.add_argument('-out_altbcs', default="", help= "users' alternative boundary condition files for output") + p_command.add_argument('-zoom', help= "zoom for the surface input") + + p_command.add_argument('-qos', default="debug", help= "queue of slurm job") + account = get_account() + p_command.add_argument('-account', default= account, help= "account of slurm job") + p_command.add_argument('-constraint', default= 'sky', help= "machine of slurm job") + p_command.add_argument('-rs', default=3, help='flag indicating which restarts to regrid: 1 (upper air); 2 (surface) 3 (both)') + + # Parse using parse_known_args so we can pass the rest to the remap scripts + args, extra_args = parser.parse_known_args() + return args, extra_args + + +def get_answers_from_command_line(cml): + + answers = {} + answers["input:shared:MERRA-2"] = cml.merra2 + answers["input:shared:yyyymmddhh"] = cml.ymdh + answers["input:shared:model"] = cml.ocnmdlin + if not cml.merra2: + answers["input:shared:tag"] = cml.tagin + answers["input:shared:ogrid"] = cml.oceanin + answers["input:surface:catch_model"] = cml.catch_model + + answers["output:shared:agrid"] = cml.grout + answers["output:air:nlevel"] = cml.levsout + answers["output:shared:out_dir"] = os.path.abspath(cml.out_dir + '/') + answers["output:shared:expid"] = cml.newid + answers["output:shared:tag"] = cml.tagout + answers["output:shared:model"] = cml.ocnmdlout + answers["output:shared:ogrid"] = cml.oceanout + + if cml.in_altbcs.strip(): + answers["input:shared:altbcs"] = cml.in_altbcs + if cml.out_altbcs.strip(): + answers["output:shared:altbcs"] = cml.out_altbcs + + + answers["output:analysis:bkg"] = not cml.nobkg + answers["output:analysis:lcv"] = not cml.nolcv + if cml.rs == '1': + answers["output:air:remap"] = True + if cml.rs == '2': + answers["output:surface:remap"] = True + if cml.rs == '3': + answers["output:surface:remap"] = True + answers["output:air:remap"] = True + + if cml.merra2 and not cml.rst_dir: + answers['input:shared:rst_dir'] = tmp_merra2_dir(answers) + else: + answers["input:shared:rst_dir"] = os.path.abspath(cml.rst_dir + '/') + + + # zoom_default fills 'input:shared:agrid' + answers["input:surface:zoom"] = zoom_default(answers) + if cml.zoom: answers["input:surface:zoom"] = cml.zoom + if answers.get('input:shared:ogrid') == 'CS': + answers['input:shared:ogrid'] = answers['input:shared:agrid'] + if answers.get('output:shared:ogrid') == 'CS': + answers['output:shared:ogrid'] = answers['output:shared:agrid'] + + if not cml.wemin : + answers["input:surface:wemin"] = we_default(answers['input:shared:tag']) + else: + answers["input:surface:wemin"] = cml.wemin + + if not cml.wemout : + answers["output:surface:wemin"] = we_default(answers['output:shared:tag']) + else: + answers["output:surface:wemin"] = cml.wemout + + + answers["slurm:account"] = cml.account + answers["slurm:qos"] = cml.qos + answers["slurm:constraint"] = cml.constraint + + return answers + +if __name__ == "__main__": + + yaml = ruamel.yaml.YAML() + cmdl, extra_args = parse_args("test_args") + print(cmdl) + answers = get_answers_from_command_line(cmdl) + config = get_config_from_answers(answers) + with open("raw_command.yaml", "w") as f: + yaml.dump(config, f) + + params = remap_params(config) + with open("params_from_command.yaml", "w") as f: + yaml.dump(params.config, f) + diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index aa53874..fccc9b5 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -9,7 +9,7 @@ import subprocess from datetime import datetime from datetime import timedelta -from remap_utils import config_to_yaml, merra2_expid +from remap_utils import * class remap_params(object): def __init__(self, config_from_question): @@ -171,7 +171,6 @@ def init_tags(self): GITNL = ( 'GITNL', '10.19', '10.20', '10.21', '10.22', '10.23' ) D525 = ( '525', 'GEOSadas-5_25_1', 'GEOSadas-5_25_1_p5', 'GEOSadas-5_25_p7', 'GEOSadas-5_27_1', 'GEOSadas-5_29_3', 'GEOSadas-5_29_4' ) - self.newStructure = ('NL3', 'NL4', 'NL5', 'NL6', 'NL7', 'NL8', 'NL9') self.bcsTag={} for tag in F14: self.bcsTag[tag]= "Fortuna-1_4" @@ -186,7 +185,7 @@ def init_tags(self): for tag in GITOL: self.bcsTag[tag]= "Icarus_Reynolds" for tag in INL: self.bcsTag[tag]= "Icarus-NLv3_Reynolds" for tag in GITNL: self.bcsTag[tag]= "Icarus-NLv3_Reynolds" - for tag in self.newStructure: self.bcsTag[tag]= tag + for tag in NewStructureBCTag: self.bcsTag[tag]= tag for tag in D214: self.bcsTag[tag]= "Fortuna-1_4" @@ -221,11 +220,11 @@ def init_tags(self): self.tagsRank['Icarus-NLv3_Reynolds'] = 18 self.tagsRank['Icarus-NLv3_MERRA-2'] = 19 self.tagsRank['Icarus-NLv3_Ostia'] = 20 - for tag in self.newStructure: self.tagsRank[tag] = 21 + for tag in NewStructureBCTag: self.tagsRank[tag] = 21 self.bcbase={} self.bcbase['discover_ops'] = "/discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs" - self.bcbase['discover_lt'] = "/discover/nobackup/ltakacs/bcs" + self.bcbase['discover_legacy'] = "/discover/nobackup/ltakacs/bcs" self.bcbase['discover_couple'] = "/discover/nobackup/projects/gmao/ssd/aogcm/atmosphere_bcs" self.bcbase['discover_ns'] = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" @@ -245,73 +244,56 @@ def init_merra2(self): self.common_in['agrid'] = 'C180' self.common_in['ogrid'] = '1440x720' - self.common_in['bc_base']= 'discover_ops' self.common_in['tag']= 'Ganymed-4_0' self.surf_in['catch_model'] = 'catch' - def get_bcbase(self, opt): - base = '' - model = '' - - if opt.upper() == 'IN': - model = self.common_in.get('model') - base = self.common_in.get('bc_base') - if base == 'discover_ns': - return self.bcbase[base] - - if model == 'MOM6' or model == 'MOM5': - base = 'discover_couple' - - if opt.upper() == 'OUT': - model = self.common_out.get('model') - base = self.common_out.get('bc_base') - if base == 'discover_ns': - return self.bcbase[base] - if model == 'MOM6' or model == 'MOM5': - base = 'discover_couple' - assert base, 'please specify bc_base: discover_ops, discover_lt, discover_couple or an absolute path' - if base == 'discover_ops' or base == 'discover_lt' or base=='discover_couple': - return self.bcbase[base] - else: - return base - def get_bcdir(self, opt): tag = self.common_in['tag'] ogrid = self.common_in['ogrid'] model = self.common_in['model'] - bcdir = self.common_in.get('alt_bcs', None) + altbcs = self.common_in.get('altbcs', None) if opt.upper() == "OUT": tag = self.common_out['tag'] ogrid = self.common_out['ogrid'] model = self.common_out['model'] - bcdir = self.common_out.get('alt_bcs', None) - - if bcdir is None : - bc_base = self.get_bcbase(opt) - bctag = self.get_bcTag(tag,ogrid) - tagrank = self.tagsRank[bctag] - - if (tagrank >= self.tagsRank['NL3']) : - bcdir = bc_base+'/'+ bctag+'/geometry/' - elif (tagrank >= self.tagsRank['Icarus-NLv3_Reynolds']) : - bcdir = bc_base+'/Icarus-NLv3/'+bctag+'/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Icarus-NLv3/'+model+'/' - elif (tagrank >= self.tagsRank['Icarus_Reynolds']): - if bc_base == self.bcbase['discover_ops']: - bcdir = bc_base+'/Icarus_Updated/'+bctag+'/' - else: - bcdir = bc_base+'/Icarus/'+bctag+'/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Icarus/'+model+'/' - elif(tagrank >= self.tagsRank["Ganymed-4_0_Reynolds"]): - bcdir = bc_base + '/Ganymed-4_0/'+bctag+'/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Ganymed/'+model+'/' + altbcs = self.common_out.get('altbcs', None) + + bcdir = '' + base = 'discover_legacy' + if model != "data": + base = 'discover_couple' + if tag in NewStructureBCTag: + base = 'discover_ns' + if self.common_in['MERRA-2']: + base = 'discover_ops' + + bc_base = BCBase[base] + if altbcs : bc_base = altbcs + + bctag = self.get_bcTag(tag,ogrid) + tagrank = self.tagsRank[bctag] + + if (tagrank >= self.tagsRank['NL3']) : + bcdir = bc_base+'/'+ bctag+'/geometry/' + elif (tagrank >= self.tagsRank['Icarus-NLv3_Reynolds']) : + bcdir = bc_base+'/Icarus-NLv3/'+bctag+'/' + if model == 'MOM6' or model == 'MOM5': + bcdir = bc_base+'/Icarus-NLv3/'+model+'/' + elif (tagrank >= self.tagsRank['Icarus_Reynolds']): + if bc_base == self.bcbase['discover_ops']: + bcdir = bc_base+'/Icarus_Updated/'+bctag+'/' else: - bcdir = bc_base + '/' + bctag + '/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Ganymed/'+model+'/' + bcdir = bc_base+'/Icarus/'+bctag+'/' + if model == 'MOM6' or model == 'MOM5': + bcdir = bc_base+'/Icarus/'+model+'/' + elif(tagrank >= self.tagsRank["Ganymed-4_0_Reynolds"]): + bcdir = bc_base + '/Ganymed-4_0/'+bctag+'/' + if model == 'MOM6' or model == 'MOM5': + bcdir = bc_base+'/Ganymed/'+model+'/' + else: + bcdir = bc_base + '/' + bctag + '/' + if model == 'MOM6' or model == 'MOM5': + bcdir = bc_base+'/Ganymed/'+model+'/' if not os.path.exists(bcdir): exit("Cannot find bc dir " + bcdir) @@ -372,31 +354,26 @@ def get_name_with_grid( grid, names, a_o): if len(gridID) == 0 : exit("cannot find the grid subdirctory of agrid: " +agrid_+ " and ogrid " + ogrid_ + " under "+ bcdir) g = '' - if len(gridID) == 1 : g = gridID[0] - - if len(gridID) == 2 : - print(" gridIDs found", gridID) - assert omodel_ == 'MOM5' or omodel_ == 'MOM6', "found two subdirestories" - for g_ in gridID : - if omodel_ == 'MOM5': - if '_M5_' in g_ : g = g_ - if omodel_ == 'MOM6': - if '_M6_' in g_ : g = g_ - - if len(gridID) >= 3 : - print("find too many grid strings in " + bcdir) - print(" gridIDs found", gridID) - for g_ in gridID: - if g_.count('_') == 1 : - g = g_ - #WY note, found many string in the directory - print(" pick the first directory with only one '_' " + g) - break + gridID.sort(key=len) + g = gridID[0] + + # For new structure BC + for g_ in gridID : + if omodel_ == 'MOM5': + if 'M5' in g_ : g = g_ + if omodel_ == 'MOM6': + if 'M6' in g_ : g = g_ + + if len(gridID) >= 2 : + print("\n Warning! Find many GridIDs in " + bcdir) + print(" GridIDs found: ", gridID) + #WY note, found many string in the directory + print(" This GridID is chosen: " + g) return g def get_bcTag(self, tag, ogrid): bctag = self.bcsTag[tag] - if bctag in self.newStructure : return bctag + if bctag in NewStructureBCTag : return bctag if ogrid[0].upper() == "C": bctag=bctag.replace('_Reynolds','_Ostia') else: @@ -460,31 +437,8 @@ def params_for_surface(self, config_tpl): config_tpl['output']['surface']['split_saltwater'] = True config_tpl['input']['surface']['zoom']= self.surf_in['zoom'] config_tpl['input']['surface']['wemin']= self.surf_in['wemin'] - config_tpl['output']['surface']['wemin']= self.surf_out['wemout'] - - rst_dir = self.common_in['rst_dir'] + '/' - time = self.ymd + '_'+ self.hh - files = glob.glob(rst_dir +'/*catch_*'+time+'*') - if (len(files)== 0) : - files = glob.glob(rst_dir +'/*catch_*') - - if (len(files) > 0) : - config_tpl['input']['surface']['catch_model'] = 'catch' - - files = glob.glob(rst_dir +'/*catchcnclm40_*'+time+'*') - if (len(files)== 0) : - files = glob.glob(rst_dir +'/*catchcnclm40_*') - - if (len(files) > 0) : - config_tpl['input']['surface']['catch_model'] = 'catchcnclm40' - - files = glob.glob(rst_dir +'/*catchcnclm45_*'+time+'*') - if (len(files)== 0) : - files = glob.glob(rst_dir +'/*catchcnclm45_*') - - if (len(files) > 0) : - config_tpl['input']['surface']['catch_model'] = 'catchcnclm45' - + config_tpl['output']['surface']['wemin']= self.surf_out['wemin'] + config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') return config_tpl def params_for_analysis(self, config_tpl): diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index a8ca9ac..6924ed6 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -8,6 +8,9 @@ input: drymass: 1 hydrostatic: 0 shared: + #the tag info here is not critical. It is for command_line options + tag: none + altbcs: false MERRA-2: false agrid: bcs_dir: @@ -24,6 +27,9 @@ input: catch_tilefile: null output: shared: + #the tag info here is not critical. It is for command_line options + tag: none + altbcs: false agrid: bcs_dir: expid: @@ -47,7 +53,6 @@ output: bkg: true aqua: False lcv: false - slurm: account: qos: diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index d314bd8..d1817c6 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -13,73 +13,7 @@ import shutil import questionary import glob - -def fvcore_name(x): - ymdh = x['input:shared:yyyymmddhh'] - time = ymdh[0:8] + '_'+ymdh[8:10] - rst_dir = x.get('input:shared:rst_dir') - if not rst_dir : return False - files = glob.glob(rst_dir+'/*fvcore_*'+time+'*') - if len(files) ==1 : - fname = files[0] - print('\nFound ' + fname) - return fname - else: - fname = rst_dir+'/fvcore_internal_rst' - if os.path.exists(fname): - print('\nFound ' + fname) - return fname - return False - -def tmp_merra2_dir(x): - tmp_merra2 = x['output:shared:out_dir']+ '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' - return tmp_merra2 - -def data_ocean_default(resolution): - default_ = 'CS' - if resolution in ['C12','C24', 'C48'] : default_ = '360X180' - return default_ - -def we_default(tag): - default_ = '26' - if tag in ['INL','GITNL', '525'] : default_ = '13' - return default_ - -def zoom_default(x): - zoom_ = '8' - fvcore = fvcore_name(x) - if fvcore : - fvrst = os.path.dirname(os.path.realpath(__file__)) + '/fvrst.x -h ' - cmd = fvrst + fvcore - print(cmd +'\n') - p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - (output, err) = p.communicate() - p_status = p.wait() - ss = output.decode().split() - x['input:shared:agrid'] = "C"+ss[0] # save for air parameter - lat = int(ss[0]) - lon = int(ss[1]) - if (lon != lat*6) : - sys.exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') - ymdh = x.get('input:shared:yyyymmddhh') - ymdh_ = str(ss[3]) + str(ss[4])[0:2] - if (ymdh_ != ymdh) : - print("Warning: The date in fvcore is different from the date you input\n") - zoom = lat /90.0 - zoom_ = str(int(zoom)) - if zoom < 1 : zoom_ = '1' - if zoom > 8 : zoom_ = '8' - if x['input:shared:MERRA-2'] : - zoom_ = '2' - return zoom_ - -def get_account(): - cmd = 'id -gn' - p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - (accounts, err) = p.communicate() - p_status = p.wait() - accounts = accounts.decode().split() - return accounts[0] +from remap_utils import * def ask_questions(): @@ -118,24 +52,10 @@ def ask_questions(): "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_name(x), }, - { - "type": "text", - "name": "output:shared:agrid", - "message": "Enter new atmospheric grid: \n C12 C180 C1000 C270\n C24 C360 C1440 C540\n C48 C500 C2880 C1080\n C90 C720 C5760 C2160\n ", - "default": 'C360', - }, - - { - "type": "text", - "name": "output:air:nlevel", - "message": "Enter new atmospheric levels: (71 72 91 127 132 137 144 181)", - "default": "72", - }, - { "type": "select", "name": "input:shared:model", - "message": "Select input ocean model:", + "message": "Select ocean model for input restarts:", "choices": ["data", "MOM5", "MOM6"], "default": "data", "when": lambda x: not x['input:shared:MERRA-2'] @@ -144,18 +64,20 @@ def ask_questions(): { "type": "select", "name": "input:shared:ogrid", - "message": "Input Ocean grid: \n \ - Data Ocean Grids \n \ - ------------------- \n \ - 360X180 (Reynolds) \n \ - 1440X720 (MERRA-2) \n \ - 2880X1440 (OSTIA) \n \ - CS = same as atmosphere grid (OSTIA cubed-sphere) \n", - "choices": ['360X180','1440X720','2880X1440','CS'], + "message": "Select Data Ocean Grid for input restarts:", + "choices": ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'], "default": lambda x: data_ocean_default(x.get('input:shared:agrid')), "when": lambda x: x.get('input:shared:model') == 'data' and not x['input:shared:MERRA-2'], }, + { + "type": "select", + "name": "input:shared:ogrid", + "message": "Select Coupled (MOM5, MOM6) Ocean Grid for input restarts:", + "choices": ['72x36','360x200','720x410','1440x1080'], + "when": lambda x: x.get('input:shared:model') == 'MOM5' or x.get('input:shared:model')== 'MOM6' + }, + { "type": "select", "name": "output:shared:model", @@ -166,49 +88,42 @@ def ask_questions(): { "type": "select", "name": "output:shared:ogrid", - "message": "Select new ocean grid:", - "choices": ['360X180','1440X720','2880X1440','CS'], + "message": "Select Data Ocean Grid for new restarts:", + "choices": ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'], "default": lambda x: data_ocean_default(x.get('output:shared:agrid')), "when": lambda x: x['output:shared:model'] == 'data', }, - - { - "type": "select", - "name": "input:shared:ogrid", - "message": "Input ocean grid: \n \ - Coupled Ocean Grids \n \ - ------------------- \n \ - 72X36 \n \ - 360X200 \n \ - 720X410 \n \ - 1440X1080 \n ", - "choices": ['72X36','360X200','720X410','1440X1080'], - "when": lambda x: x.get('input:shared:model') == 'MOM5' or x.get('input:shared:model')== 'MOM6' - }, { "type": "select", "name": "output:shared:ogrid", - "message": "Select new ocean grid: \n \ - Coupled Ocean Grids \n \ - ------------------- \n \ - 72X36 \n \ - 360X200 \n \ - 720X410 \n \ - 1440X1080 \n ", - "choices": ['72X36','360X200','720X410','1440X1080'], + "message": "Select Couple Ocean Grid for new restarts:", + "choices": ['72x36','360x200','720x410','1440x1080'], "when": lambda x: x['output:shared:model'] != 'data', }, + { - "type": "select", - "name": "input:shared:bc_base", - "message": "Select bcs base \n \ - discover_ops: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs \n \ - discover_lt: /discover/nobackup/ltakacs/bcs \n \ - discover_couple: /discover/nobackup/projects/gmao/ssd/aogcm/atmosphere_bcs \n \ - discover_ns: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles \n \ + "type": "text", + "name": "output:shared:agrid", + "message": "Enter new atmospheric grid: \n C12 C180 C1000 C270\n C24 C360 C1440 C540\n C48 C500 C2880 C1080\n C90 C720 C5760 C2160\n ", + "default": 'C360', + }, + + { + "type": "text", + "name": "output:air:nlevel", + "message": "Enter new atmospheric levels: (71 72 91 127 132 137 144 181)", + "default": "72", + }, + + + + { + "type": "text", + "name": "input:shared:tag", + "message": "Enter GCM or DAS tag for input restarts: \n \ \n \ -Sample GCM tags for discover_ops, discover_lt, discover_couple bc base :\n \ +Sample GCM tags for legacy BCs:\n \ --------------- \n \ G40 : Ganymed-4_0 ......... Heracles-5_4_p3 \n \ ICA : Icarus .............. Jason \n \ @@ -216,73 +131,79 @@ def ask_questions(): INL : Icarus-NL ........... Jason-NL \n \ GITNL : 10.19 ............... 10.23 \n \ \n \ -Sample DAS tags for discover_ops, discover_lt, discover_couple bc base :\n \ +Sample DAS tags for legacy BCs:\n \ --------------- \n \ 5B0 : GEOSadas-5_10_0_p2 .. GEOSadas-5_11_0 \n \ 512 : GEOSadas-5_12_2 ..... GEOSadas-5_16_5\n \ 517 : GEOSadas-5_17_0 ..... GEOSadas-5_24_0_p1\n \ 525 : GEOSadas-5_25_1 ..... GEOSadas-5_29_4\n \ \n \ -Sample BC version for discover_ns ( new structure BC base): \n \ +Sample BC version for new structure BC base): \n \ -------------- \n \ NL3 : Newland version 3 \n \ NL4 : Newland version 4 \n \ NL5 : Newland version 5 \n \ -NL6 : Not generated yet \n", - "choices": ["discover_ops", "discover_lt", "discover_couple","discover_ns", "other"], - "when": lambda x: not x['input:shared:MERRA-2'], - }, - - { - "type": "text", - "name": "input:shared:tag", - "message": "Enter GCM or DAS tag for input:", +v06, v07, v08, v09: Not generated yet \n", + "default": "INL", - "when": lambda x: not x["input:shared:MERRA-2"] and not x["input:shared:bc_base"]== "discover_ns", - }, - - { - "type": "text", - "name": "input:shared:tag", - "message": "Enter BC version for input:", - "default": "NL3", - "when": lambda x: not x["input:shared:MERRA-2"] and x["input:shared:bc_base"]== "discover_ns", - }, - - { - "type": "select", - "name": "output:shared:bc_base", - "message": "Select bcs base for new restarts:", - "choices": ["discover_ops", "discover_lt", "discover_couple","discover_ns", "other"], + "when": lambda x: not x["input:shared:MERRA-2"], }, { "type": "text", "name": "output:shared:tag", "message": "Enter GCM or DAS tag for new restarts:", - "default": "INL", - "when": lambda x: not x["output:shared:bc_base"] == "discover_ns", + "default": "GITNL", + "when": lambda x: not x["input:shared:MERRA-2"], }, + # show the message if it is merra2 { "type": "text", "name": "output:shared:tag", - "message": "Enter BC version for new restarts:", - "default": "NL3", - "when": lambda x: x["output:shared:bc_base"] == "discover_ns", + "message": "Enter GCM or DAS tag for new restarts: \n \ +\n \ +Sample GCM tags for legacy BCs:\n \ +--------------- \n \ +G40 : Ganymed-4_0 ......... Heracles-5_4_p3 \n \ +ICA : Icarus .............. Jason \n \ +GITOL : 10.3 ................ 10.18 \n \ +INL : Icarus-NL ........... Jason-NL \n \ +GITNL : 10.19 ............... 10.23 \n \ +\n \ +Sample DAS tags for legacy BCs:\n \ +--------------- \n \ +5B0 : GEOSadas-5_10_0_p2 .. GEOSadas-5_11_0 \n \ +512 : GEOSadas-5_12_2 ..... GEOSadas-5_16_5\n \ +517 : GEOSadas-5_17_0 ..... GEOSadas-5_24_0_p1\n \ +525 : GEOSadas-5_25_1 ..... GEOSadas-5_29_4\n \ +\n \ +Sample BC version for new structure BC base): \n \ +-------------- \n \ +NL3 : Newland version 3 \n \ +NL4 : Newland version 4 \n \ +NL5 : Newland version 5 \n \ +v06, v07, v08, v09: Not generated yet \n", + "default": "GITNL", + "when": lambda x: x["input:shared:MERRA-2"], }, { "type": "path", - "name": "input:shared:alt_bcs", - "message": "Specify your own bcs absolute path (do not contain grid info) for restarts: \n ", - "when": lambda x: x.get("input:shared:bc_base")=="other", + "name": "input:shared:altbcs", + "message": "Enter nothing (default) or specify your own absolute bcs path for input restarts: \n\n \ +It expects the sub-structure of the path to be the same as one of the three paths that match the tag. \n \ +/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles (structure after NL3 ) \n \ +/discover/nobackup/projects/gmao/ssd/aogcm/atmosphere_bcs ( coupled model) \n \ +/discover/nobackup/projects/gmao/bcs_shared/legacy_bcs ( legacy structure)\n", + "default": "", + "when": lambda x: not x.get("input:shared:MERRA-2"), }, { "type": "path", - "name": "output:shared:alt_bcs", - "message": "Specify your own bcs path (do not contain grid info) for new restarts: \n ", - "when": lambda x: x.get("output:shared:bc_base")=="other", + "name": "output:shared:altbcs", + "message": "Enter nothing (default) or specify your own absolute bcs path for new restarts: \n ", + "default": "", }, { @@ -297,6 +218,16 @@ def ask_questions(): "message": "Would you like to remap surface?", "default": True, }, + + { + "type": "select", + "name": "input:surface:catch_model", + "message": "What is the catchment model? ", + "choices": ['catch','catchcnclm40','catchcnclm45'], + "default": 'catch', + "when": lambda x: x["output:surface:remap"] and not x["input:shared:MERRA-2"] + }, + { "type": "confirm", "name": "output:analysis:bkg", @@ -317,7 +248,7 @@ def ask_questions(): }, { "type": "text", - "name": "output:surface:wemout", + "name": "output:surface:wemin", "message": "What is value of Wemout?", "default": lambda x: we_default(x.get('output:shared:tag')) }, @@ -357,38 +288,28 @@ def ask_questions(): answers = questionary.prompt(questions) if not answers.get('input:shared:model') : answers['input:shared:model'] = 'data' + + if answers.get('input:shared:ogrid'): + answers['input:shared:ogrid'] = answers['input:shared:ogrid'].split()[0] + if answers.get('output:shared:ogrid'): + answers['output:shared:ogrid'] = answers['output:shared:ogrid'].split()[0] + if answers['input:shared:MERRA-2']: - answers['input:shared:rst_dir'] = tmp_merra2_dir(answers) + answers['input:shared:rst_dir'] = tmp_merra2_dir(answers) + answers['input:shared:altbcs'] ='' + answers['input:shared:rst_dir'] = os.path.abspath(answers['input:shared:rst_dir']) if answers.get('output:shared:ogrid') == 'CS': answers['output:shared:ogrid'] = answers['output:shared:agrid'] answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) - + return answers -def get_config_from_questionary(): - answers = ask_questions() - config = {} - config['input'] = {} - config['input']['shared'] = {} - config['input']['surface'] = {} - config['output'] = {} - config['output']['shared'] = {} - config['output']['air'] = {} - config['output']['surface'] = {} - config['output']['analysis'] = {} - config['slurm'] = {} - for key, value in answers.items(): - keys = key.split(":") - if len(keys) == 2: - config[keys[0]][keys[1]] = value - if len(keys) == 3: - config[keys[0]][keys[1]][keys[2]] = value - - return config if __name__ == "__main__": - config = get_config_from_questionary() + answers = ask_questions() + cmdl = get_command_line_from_answers(answers) + config = get_config_from_answers(answers) yaml = ruamel.yaml.YAML() with open("raw_answers.yaml", "w") as f: yaml.dump(config, f) diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index e1c9455..d32a9ad 100644 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -12,28 +12,27 @@ import ruamel.yaml import questionary from remap_utils import * -from remap_questions import get_config_from_questionary +from remap_questions import * +from remap_command import * from remap_params import * from remap_upper import * from remap_lake_landice_saltwater import * from remap_analysis import * from remap_catchANDcn import * -# Define the argument parser -def parse_args(): - program_description = textwrap.dedent(f''' +program_description = textwrap.dedent(f''' USAGE: There are three ways to use this script to remap restarts. - 1. Use an existing config file to remap: - ./remap_restarts.py config -c my_config.yaml - - 2. Use questionary to convert template remap_params.tpl to + 1. Use questionary to convert template remap_params.tpl to remap_params.yaml and then remap: ./remap_restarts.py + 2. Use an existing config file to remap: + ./remap_restarts.py config_file -c my_config.yaml + 3. Use command line to input a flattened yaml file: ./remap_restarts.py command_line -ymd 20040414 -hr 21 .... To get help, please use @@ -47,108 +46,52 @@ def parse_args(): 5. remap_catchANDcn.py uses remap_params.yaml as input for remapping 6. remap_analysis.py uses remap_params.yaml as input for remapping ''') - - parser = argparse.ArgumentParser(description='Remap restarts',epilog=program_description,formatter_class=argparse.RawDescriptionHelpFormatter) - - p_sub = parser.add_subparsers(help = 'Sub command help') - p_config = p_sub.add_parser( - 'config', - help = "Use config file as input", - ) - p_command = p_sub.add_parser( - 'command_line', - help = "Use command line as input", - ) - - p_config.add_argument('-c', '--config_file', help='YAML config file') - - p_command.add_argument('-merra2', action='store_true', help='use merra2 restarts') - p_command.add_argument('-ymd', help='yyyymmdd year mone date of input and output restarts') - p_command.add_argument('-hr', help='hh hours of input and output restarts') - p_command.add_argument('-grout', help='Grid ID of the output restart') - p_command.add_argument('-levsout', help='levels of output restarts') - - p_command.add_argument('-outdir', help='directory for output restarts') - p_command.add_argument('-d', help='directory for input restarts') - - p_command.add_argument('-expid', help='restart id for input restarts') - p_command.add_argument('-newid', help='restart id for output restarts') - - p_command.add_argument('-tagin', help='GCM or DAS tag associated with inputs') - p_command.add_argument('-tagout', help='GCM or DAS tag associated with outputs') - - p_command.add_argument('-wemin', help='minimum water snow water equivalent for input catch/cn') - p_command.add_argument('-wemout', help='minimum water snow water equivalent for output catch/cn') - - p_command.add_argument('-oceanin', default='c', help='ocean horizontal grid of inputs') - p_command.add_argument('-oceanout', default='c', help='ocean horizontal grid of outputs') - - p_command.add_argument('-ocnmdlin', default='data', help='ocean input model') - p_command.add_argument('-ocnmdlout',default='data', help='ocean output model') - - p_command.add_argument('-nobkg', action='store_true', help="Don't remap bkg files") - p_command.add_argument('-nolbl', action='store_true', help="label final restarts with 'tagID.gridID' extension") - p_command.add_argument('-nolcv', action='store_true', help="Don't remap lcv files") - p_command.add_argument('-bkg', action='store_true', help="Don't remap bkg files") - p_command.add_argument('-lbl', action='store_true', help="label final restarts with 'tagID.gridID' extension") - p_command.add_argument('-lcv', action='store_true', help="Don't remap lcv files") - p_command.add_argument('-altbcs', help= "use user's boundary condition files") - - p_command.add_argument('-rs', default=3, help='flag indicating which restarts to regrid') - - -#$GEOSBIN/regrid.pl -np -ymd ${year}${month}${day} -hr 21 -grout C${AGCM_IM} -levsout ${AGCM_LM} -outdir . -d . -expid $RSTID -tagin @EMIP_BCS_IN -oceanin e -i -nobkg -lbl -nolcv -tagout @LSMBCS -rs 3 -oceanout @OCEANOUT - - # Parse using parse_known_args so we can pass the rest to the remap scripts - # If config_file is used, then extra_args will be empty - # If flattened_yaml is used, then extra_args will be populated - args, extra_args = parser.parse_known_args() - return args, extra_args - def main(): question_flag = False config = '' - + yaml = ruamel.yaml.YAML() # Parse the command line arguments from parse_args() capturing the arguments and the rest - + cmdl, extra_args = parse_args(program_description) + answers = {} + config_yaml ='' if (len(sys.argv) > 1) : - args, extra_args = parse_args() - print(args) - if sys.argv[1] == 'config': - config = yaml_to_config(args.config_yaml) + if sys.argv[1] == 'config_file' : + config_yaml = cmdl.config_file if sys.argv[1] == 'command_line': - raw_config = get_config_from_command_line(args) - params = remap_params(raw_config) - config = params.config - question_flag = True - print(raw_config) - else: - raw_config = get_config_from_questionary() + answers = get_answers_from_command_line(cmdl) + if (len(sys.argv) == 1 or answers) : + if not answers: + answers = ask_questions() + question_flag = True + raw_config = get_config_from_answers(answers) + cmd = get_command_line_from_answers(answers) + write_cmd(answers['output:shared:out_dir'], cmd) + with open("raw_answers.yaml", "w") as f: + yaml.dump(raw_config, f) params = remap_params(raw_config) config = params.config - question_flag = True config_yaml = 'remap_params.yaml' - exit() print('\n') - print_config(config) - - questions = [ - { - "type": "confirm", - "name": "Continue", - "message": "Above is the YAML config file, would you like to continue?", - "default": True - },] - answer = questionary.prompt(questions) - - if not answer['Continue'] : - print("\nYou answered not to continue, exiting.\n") - sys.exit(0) - - if config_yaml or question_flag: write_cmd(config) - if flattened_yaml or question_flag: config_to_yaml(config, config_yaml) + if config: + print_config(config) + + questions = [ + { + "type": "confirm", + "name": "Continue", + "message": "Above is the YAML config file, would you like to continue?", + "default": True + },] + answer = questionary.prompt(questions) + + if not answer['Continue'] : + print("\nYou answered not to continue, exiting.\n") + sys.exit(0) + + # write config to yaml file + config_to_yaml(config, config_yaml) # upper air upper = upperair(params_file=config_yaml) diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 5b42f72..e44d6e9 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -91,6 +91,7 @@ def remap(self): topoin = glob.glob(in_bcsdir+'/topo_DYN_ave*.data')[0] # link topo file + cmd = '/bin/ln -s ' + topoin + ' .' print('\n'+cmd) subprocess.call(shlex.split(cmd)) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index be956c3..8b1e3c9 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -6,6 +6,87 @@ from collections import OrderedDict import shutil import questionary +import glob +import shlex + +NewStructureBCTag = ('NL3', 'NL4', 'NL5', 'v06', 'v07', 'v08', 'v09') +BCBase={} +BCBase['discover_ops'] = "/discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs" +BCBase['discover_legacy'] = "/discover/nobackup/projects/gmao/bcs_shared/legacy_bcs" +BCBase['discover_couple'] = "/discover/nobackup/projects/gmao/ssd/aogcm/atmosphere_bcs" +BCBase['discover_ns'] = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" + +def fvcore_name(x): + ymdh = x['input:shared:yyyymmddhh'] + time = ymdh[0:8] + '_'+ymdh[8:10] + rst_dir = x.get('input:shared:rst_dir') + if not rst_dir : return False + + files = glob.glob(rst_dir+'/*fvcore_*') + if len(files) == 1: + fname = files[0] + print('\nFound ' + fname) + return fname + + if len(files) > 1 : + files = glob.glob(rst_dir+'/*fvcore_*'+time+'*') + fname = files[0] + print('\nFound ' + fname) + return fname + + return False + +def tmp_merra2_dir(x): + tmp_merra2 = x['output:shared:out_dir']+ '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' + return tmp_merra2 + +def data_ocean_default(resolution): + # the default string should match the choice in remapl_question.py + default_ = 'CS (same as atmosphere OSTIA cubed-sphere grid)' + if resolution in ['C12','C24', 'C48'] : default_ = '360x180 (Reynolds)' + return default_ + +def we_default(tag): + default_ = '26' + if tag in ['INL','GITNL', '525'] : default_ = '13' + if tag in NewStructureBCTag : default_ = '13' + return default_ + +def zoom_default(x): + zoom_ = '8' + fvcore = fvcore_name(x) + if fvcore : + fvrst = os.path.dirname(os.path.realpath(__file__)) + '/fvrst.x -h ' + cmd = fvrst + fvcore + print(cmd +'\n') + p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + (output, err) = p.communicate() + p_status = p.wait() + ss = output.decode().split() + x['input:shared:agrid'] = "C"+ss[0] # save for air parameter + lat = int(ss[0]) + lon = int(ss[1]) + if (lon != lat*6) : + sys.exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') + ymdh = x.get('input:shared:yyyymmddhh') + ymdh_ = str(ss[3]) + str(ss[4])[0:2] + if (ymdh_ != ymdh) : + print("Warning: The date in fvcore is different from the date you input\n") + zoom = lat /90.0 + zoom_ = str(int(zoom)) + if zoom < 1 : zoom_ = '1' + if zoom > 8 : zoom_ = '8' + if x['input:shared:MERRA-2'] : + zoom_ = '2' + return zoom_ + +def get_account(): + cmd = 'id -gn' + p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + (accounts, err) = p.communicate() + p_status = p.wait() + accounts = accounts.decode().split() + return accounts[0] def config_to_yaml(config, yaml_file): if os.path.exists(yaml_file) : @@ -30,62 +111,16 @@ def yaml_to_config(yaml_file): config = yaml.load(stream) return config -def write_cmd(config) : - def flatten_nested(nested_dict, result=None, prefix=''): - if result is None: - result = dict() - for k, v in nested_dict.items(): - new_k = ':'.join((prefix, k)) if prefix else k - if not (isinstance(v, dict) or isinstance(v, OrderedDict)): - result.update({new_k: v}) - else: - flatten_nested(v, result, new_k) - return result - - out_dir = config['output']['shared']['out_dir'] +def write_cmd( out_dir, cmdl) : + + out_dir = os.path.realpath(out_dir) if not os.path.exists(out_dir) : os.makedirs(out_dir) bin_path = os.path.dirname(os.path.realpath(__file__)) - - cmd = '#!/usr/local/bin/csh \n' - cmd = cmd + 'set BINPATH=' + bin_path +'\n' - cmd = cmd + 'source $BINPATH/g5_modules \n' - - flat_dict = flatten_nested(config) - - k = 1 - for key, value in flat_dict.items(): - if isinstance(value, int) or isinstance(value, float) or isinstance(value, bool) or isinstance(value, type(None)): value = str(value) - if k == 1: - cmd = cmd + 'set FLAT_YAML="' + key+"="+ value+ '"\n' - else: - cmd = cmd + 'set FLAT_YAML="$FLAT_YAML '+ key+"="+ value+ '"\n' - k = k+1 - - cmd = cmd + '$BINPATH/remap_restarts.py -o $FLAT_YAML' - + cmdl = bin_path+'/'+ cmdl with open(out_dir + '/remap_restarts.CMD', 'w') as f: - f.write(cmd) + f.write(cmdl) subprocess.call(['chmod', '+x',out_dir + '/remap_restarts.CMD']) -def args_to_config(args): - # template file should be with this util file - remap_tpl = os.path.dirname(os.path.realpath(__file__)) + '/remap_params.tpl' - config = yaml_to_config(remap_tpl) - # fill in the config with args - for values in args: - [keys, value] = values.split("=") - key = keys.split(':') - if value.lower() in ['false', 'null', 'none'] : - value = False - elif value.lower() in ['true'] : - value = True - if len(key) == 2: - config[key[0]][key[1]] = value - if len(key) == 3: - config[key[0]][key[1]][key[2]] = value - - return config - def print_config( config, indent = 0 ): for k, v in config.items(): if isinstance(v, dict): @@ -117,55 +152,95 @@ def merra2_expid(config): return config -def get_config_from_command_line(cml): - - ogridID = {} - ogridID['c'] = '360X180' - ogridID['e'] = '1440x720' - ogridID['f'] = '2880x1440' - ogridID['CS'] = 'CS' - ogridID['aa'] = '72x36' - ogridID['bb'] = '360x200' - ogridID['dd'] = '720x410' - ogridID['ee'] = '1440x1080' - - answers = {} - answers["input:shared:MERRA-2"] = cml.merra2 - answers["input:shared:yyyymmddhh"] = cml.ymd + cml.hr - answers["output:shared:agrid"] = cml.grout - answers["output:air:nlevel"] = cml.levsout - answers["output:shared:out_dir"] = cml.outdir + '/' - answers["input:shared:rst_dir"] = cml.d + '/' - answers["output:shared:expid"] = cml.newid - - answers["input:shared:tag"] = cml.tagin - answers["output:shared:tag"] = cml.tagout - - answers["input:shared:model"] = cml.ocnmdlin - answers["output:shared:model"] = cml.ocnmdlout - answers["input:shared:ogrid"] = ogridID[cml.oceanin] - answers["output:shared:ogrid"] = ogridID[cml.oceanout] - - answers["output:analysis:bkg"] = cml.bkg - answers["output:analysis:lcv"] = cml.lcv - - if cml.rs == 1: - answers["output:air:remap"] = True - if cml.rs == 2: - answers["output:surface:remap"] = True - if cml.rs == 3: - answers["output:surface:remap"] = True - answers["output:air:remap"] = True - - if not answers.get('input:shared:model') : - answers['input:shared:model'] = 'data' - if answers['input:shared:MERRA-2']: - answers['input:shared:rst_dir'] = tmp_merra2_dir(answers) - if answers.get('output:shared:ogrid') == 'CS': - answers['output:shared:ogrid'] = answers['output:shared:agrid'] - answers['input:shared:rst_dir'] = os.path.abspath(answers['input:shared:rst_dir']) - answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) +def get_command_line_from_answers(answers): + + merra2 = " -merra2 " if answers["input:shared:MERRA-2"] else "" + ymdh = " -ymdh " + answers["input:shared:yyyymmddhh"] + rst_dir = " -rst_dir " + answers["input:shared:rst_dir"] + + grout = ' -grout ' + answers["output:shared:agrid"] + levsout = ' -levsout ' + answers["output:air:nlevel"] + out_dir = ' -out_dir ' + answers["output:shared:out_dir"] + newid = answers["output:shared:expid"] + + out_newid='' + if newid.strip(): + out_newid = " -newid " + newid + + in_tagin = '' + if answers.get("input:shared:tag"): + in_tagin = " -tagin " + answers["input:shared:tag"] + tagout = " -tagout " + answers["output:shared:tag"] + + ocnmdlin = '' + if answers.get("input:shared:model"): + ocnmdlin = ' -ocnmdlin ' + answers.get("input:shared:model") + ocnmdlout = ' -ocnmdlout ' + answers["output:shared:model"] + oceanin='' + if answers.get("input:shared:ogrid"): + oceanin = ' -oceanin ' + answers["input:shared:ogrid"] + oceanout = ' -oceanout ' + answers["output:shared:ogrid"] + + nobkg = '' if answers["output:analysis:bkg"] else " -nobkg " + nolcv = '' if answers["output:analysis:lcv"] else " -nolcv " + + in_altbcs = '' + out_altbcs = '' + if answers.get("input:shared:altbcs", "").strip() : + in_altbcs = " -in_altbcs " + answers["input:shared:altbcs"] + if answers.get("output:shared:altbcs", "").strip() : + out_altbcs = " -out_altbcs " + answers["output:shared:altbcs"] + + zoom = " -zoom " + answers["input:surface:zoom"] + wemin = " -wemin " + answers["input:surface:wemin"] + wemout = " -wemout " + answers["output:surface:wemin"] + catch_model ='' + if answers.get("input:surface:catch_model"): + catch_model = " -catch_model " + answers["input:surface:catch_model"] + out_rs = " -rs " + rs = 3 + if answers['output:air:remap'] and not answers['output:surface:remap']: + rs = 1 + if answers['output:surface:remap'] and not answers['output:air:remap']: + rs = 2 + out_rs = out_rs + str(rs) + + account = " -account " + answers["slurm:account"] + qos = " -qos " + answers["slurm:qos"] + constraint = " -constraint " + answers["slurm:constraint"] + + + cmdl = "remap_restarts.py command_line " + merra2 + \ + ymdh + \ + grout + \ + levsout + \ + out_newid + \ + ocnmdlin + \ + ocnmdlout + \ + oceanin + \ + oceanout + \ + in_tagin + \ + tagout + \ + rst_dir + \ + out_dir + \ + in_altbcs + \ + out_altbcs + \ + catch_model + \ + zoom + \ + wemin + \ + wemout + \ + nobkg + \ + nolcv + \ + out_rs + \ + account + \ + qos + \ + constraint + + + return cmdl + +def get_config_from_answers(answers): config = {} config['input'] = {} config['input']['shared'] = {} @@ -182,9 +257,10 @@ def get_config_from_command_line(cml): config[keys[0]][keys[1]] = value if len(keys) == 3: config[keys[0]][keys[1]][keys[2]] = value - + return config + if __name__ == '__main__' : config = yaml_to_config('c24Toc12.yaml') print_config(config) From 4926a94a8d416b2350056661c211f6b0166c1c55 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 18 May 2023 13:33:45 -0400 Subject: [PATCH 04/88] move remap tests under remap_restarts directory --- post/CMakeLists.txt | 2 -- post/tests/CMakeLists.txt | 5 ----- pre/CMakeLists.txt | 2 +- {post => pre/remap_restart}/tests/amip_c180Toc90.yaml | 0 {post => pre/remap_restart}/tests/c180Toc360.yaml | 0 {post => pre/remap_restart}/tests/c24Toc12.yaml | 0 {post => pre/remap_restart}/tests/c360Toc24.yaml | 0 {post => pre/remap_restart}/tests/f522Toc360.yaml | 0 {post => pre/remap_restart}/tests/s2sv3Toc12MOM6.yaml | 0 {post => pre/remap_restart}/tests/test_remap_cases.yaml | 0 {post => pre/remap_restart}/tests/test_remap_restarts.py | 0 11 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 post/tests/CMakeLists.txt rename {post => pre/remap_restart}/tests/amip_c180Toc90.yaml (100%) rename {post => pre/remap_restart}/tests/c180Toc360.yaml (100%) rename {post => pre/remap_restart}/tests/c24Toc12.yaml (100%) rename {post => pre/remap_restart}/tests/c360Toc24.yaml (100%) rename {post => pre/remap_restart}/tests/f522Toc360.yaml (100%) rename {post => pre/remap_restart}/tests/s2sv3Toc12MOM6.yaml (100%) rename {post => pre/remap_restart}/tests/test_remap_cases.yaml (100%) rename {post => pre/remap_restart}/tests/test_remap_restarts.py (100%) diff --git a/post/CMakeLists.txt b/post/CMakeLists.txt index bf50d8f..dca4991 100644 --- a/post/CMakeLists.txt +++ b/post/CMakeLists.txt @@ -162,5 +162,3 @@ install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/3CH.j DESTINATION post ) - -add_subdirectory(tests) diff --git a/post/tests/CMakeLists.txt b/post/tests/CMakeLists.txt deleted file mode 100644 index 80ddc86..0000000 --- a/post/tests/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ - -file(GLOB test_scripts CONFIGURE_DEPENDS ./*.py ./*.yaml) -install( - PROGRAMS ${test_scripts} - DESTINATION bin) diff --git a/pre/CMakeLists.txt b/pre/CMakeLists.txt index bbc0167..4f61326 100644 --- a/pre/CMakeLists.txt +++ b/pre/CMakeLists.txt @@ -1,7 +1,7 @@ add_subdirectory(NSIDC-OSTIA_SST-ICE_blend) add_subdirectory(prepare_ocnExtData) -file(GLOB pythonscripts CONFIGURE_DEPENDS ./remap_restart/remap*) +file(GLOB pythonscripts CONFIGURE_DEPENDS ./remap_restart/remap* ./remap_restart/tests/*.*) install( PROGRAMS ${pythonscripts} DESTINATION bin) diff --git a/post/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml similarity index 100% rename from post/tests/amip_c180Toc90.yaml rename to pre/remap_restart/tests/amip_c180Toc90.yaml diff --git a/post/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml similarity index 100% rename from post/tests/c180Toc360.yaml rename to pre/remap_restart/tests/c180Toc360.yaml diff --git a/post/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml similarity index 100% rename from post/tests/c24Toc12.yaml rename to pre/remap_restart/tests/c24Toc12.yaml diff --git a/post/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml similarity index 100% rename from post/tests/c360Toc24.yaml rename to pre/remap_restart/tests/c360Toc24.yaml diff --git a/post/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml similarity index 100% rename from post/tests/f522Toc360.yaml rename to pre/remap_restart/tests/f522Toc360.yaml diff --git a/post/tests/s2sv3Toc12MOM6.yaml b/pre/remap_restart/tests/s2sv3Toc12MOM6.yaml similarity index 100% rename from post/tests/s2sv3Toc12MOM6.yaml rename to pre/remap_restart/tests/s2sv3Toc12MOM6.yaml diff --git a/post/tests/test_remap_cases.yaml b/pre/remap_restart/tests/test_remap_cases.yaml similarity index 100% rename from post/tests/test_remap_cases.yaml rename to pre/remap_restart/tests/test_remap_cases.yaml diff --git a/post/tests/test_remap_restarts.py b/pre/remap_restart/tests/test_remap_restarts.py similarity index 100% rename from post/tests/test_remap_restarts.py rename to pre/remap_restart/tests/test_remap_restarts.py From 5ae0278146c6a1fba712f8713c028ebeea68d0e0 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Mon, 22 May 2023 10:08:07 -0400 Subject: [PATCH 05/88] update test cases --- pre/remap_restart/tests/test_remap_cases.yaml | 6 +++--- pre/remap_restart/tests/test_remap_restarts.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pre/remap_restart/tests/test_remap_cases.yaml b/pre/remap_restart/tests/test_remap_cases.yaml index 166d091..5563d26 100644 --- a/pre/remap_restart/tests/test_remap_cases.yaml +++ b/pre/remap_restart/tests/test_remap_cases.yaml @@ -13,6 +13,6 @@ c180Toc360: c360Toc24: base_line: '/discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c360Toc24/baseline/' config: 'c360Toc24.yaml' -s2sv3Toc12MOM6: - base_line: '/discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/s2sv3Toc12MOM6/baseline/' - config: 's2sv3Toc12MOM6.yaml' +#s2sv3Toc12MOM6: +# base_line: '/discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/s2sv3Toc12MOM6/baseline/' +# config: 's2sv3Toc12MOM6.yaml' diff --git a/pre/remap_restart/tests/test_remap_restarts.py b/pre/remap_restart/tests/test_remap_restarts.py index 69c6707..197e302 100755 --- a/pre/remap_restart/tests/test_remap_restarts.py +++ b/pre/remap_restart/tests/test_remap_restarts.py @@ -12,7 +12,6 @@ import glob import subprocess as sp import remap_restarts -from remap_questions import get_config_from_questionary from remap_params import * from remap_upper import * from remap_lake_landice_saltwater import * From c6d270cc58b7116577ed387d335033312d449319 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Mon, 22 May 2023 15:28:01 -0400 Subject: [PATCH 06/88] add more comments --- pre/remap_restart/remap_command.py | 2 +- pre/remap_restart/remap_params.py | 4 ++++ pre/remap_restart/remap_restarts.py | 22 ++++++++++++++-------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/pre/remap_restart/remap_command.py b/pre/remap_restart/remap_command.py index 8b706c3..50b5d20 100755 --- a/pre/remap_restart/remap_command.py +++ b/pre/remap_restart/remap_command.py @@ -59,7 +59,7 @@ def parse_args(program_description): p_command.add_argument('-ocnmdlin', default='data', help='ocean input model: data, MOM5, MOM6') p_command.add_argument('-ocnmdlout',default='data', help='ocean output model: data, MOM5, MOM6') - p_command.add_argument('-catch_model',default='catch', help='ocean output model: catch, catchcnclm40, catchcnclm45') + p_command.add_argument('-catch_model',default='catch', help='catchment model: catch, catchcnclm40, catchcnclm45') p_command.add_argument('-nobkg', action='store_true', help="Don't remap bkg files") #p_command.add_argument('-nolbl', action='store_true', help="label final restarts with 'tagID.gridID' extension") diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index fccc9b5..fb571e8 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -41,6 +41,8 @@ def __init__(self, config_from_question): config_tpl['input']['shared']['rst_dir'] = self.common_in['rst_dir']+'/' config_tpl['input']['shared']['expid'] = self.common_in.get('expid') config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] + config_tpl['input']['shared']['tag'] = self.common_in.get('tag') + config_tpl['input']['shared']['altbcs'] = self.common_in.get('altbcs') config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') config_tpl['output']['air']['remap'] = self.upper_out.get('remap') @@ -51,6 +53,8 @@ def __init__(self, config_from_question): config_tpl['output']['shared']['ogrid'] = self.common_out['ogrid'] config_tpl['output']['shared']['out_dir'] = self.common_out['out_dir'] + '/' config_tpl['output']['shared']['expid'] = self.common_out['expid'] + config_tpl['output']['shared']['tag'] = self.common_out.get('tag') + config_tpl['output']['shared']['altbcs'] = self.common_out.get('altbcs') # params for upper air config_tpl = self.params_for_air(config_tpl) diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index d32a9ad..31e5dba 100644 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -33,18 +33,24 @@ 2. Use an existing config file to remap: ./remap_restarts.py config_file -c my_config.yaml - 3. Use command line to input a flattened yaml file: - ./remap_restarts.py command_line -ymd 20040414 -hr 21 .... - To get help, please use + 3. Use command line's options as inputs: + ./remap_restarts.py command_line -ymdh 2004041421 .... + To see more command options, please use ./remap_restarts.py command_line -h + There are three help commands: + ./remap_restarts.py -h + ./remap_restarts.py config_file -h + ./remap_restarts.py command_line -h + NOTE: Each individual script can be executed independently 1. remap_questions.py generates raw_answer.yaml - 2. remap_params.py uses raw_answer.yaml and remap_params.tpl as inputs and generates remap_params.yaml - 3. remap_upper.py uses remap_params.yaml as input for remapping - 4. remap_lake_landice_saltwater.py uses remap_params.yaml as input for remapping - 5. remap_catchANDcn.py uses remap_params.yaml as input for remapping - 6. remap_analysis.py uses remap_params.yaml as input for remapping + 2. If command_line option is used, the command_line option is converted raw_answers.yaml + 3. remap_params.py uses raw_answer.yaml and remap_params.tpl as inputs and generates remap_params.yaml + 4. remap_upper.py uses remap_params.yaml as input for remapping + 5. remap_lake_landice_saltwater.py uses remap_params.yaml as input for remapping + 6. remap_catchANDcn.py uses remap_params.yaml as input for remapping + 7. remap_analysis.py uses remap_params.yaml as input for remapping ''') def main(): From c86f06fc0c93a1d614b53a9ec8599530afd9803e Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Mon, 22 May 2023 22:36:17 -0400 Subject: [PATCH 07/88] add -lbl options to add tag and resolutions information to output restart names --- pre/remap_restart/remap_analysis.py | 12 +++++++++++- pre/remap_restart/remap_catchANDcn.py | 8 +++++++- pre/remap_restart/remap_command.py | 6 ++---- pre/remap_restart/remap_lake_landice_saltwater.py | 13 ++++++++++--- pre/remap_restart/remap_params.py | 1 + pre/remap_restart/remap_params.tpl | 1 + pre/remap_restart/remap_questions.py | 6 ++++++ pre/remap_restart/remap_upper.py | 13 ++++++++++--- pre/remap_restart/remap_utils.py | 10 ++++++++++ 9 files changed, 58 insertions(+), 12 deletions(-) diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index 047186a..c7545e7 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -15,6 +15,7 @@ import fileinput import ruamel.yaml from remap_base import remap_base +from remap_utils import get_bcs_basename class analysis(remap_base): def __init__(self, **configs): @@ -61,6 +62,13 @@ def remap(self): else: expid_out = '' + in_bcsdir = config['input']['shared']['bcs_dir'] + out_bcsdir = config['output']['shared']['bcs_dir'] + label = '' + if config['output']['shared']['label']: + label = '.' + config['input']['shared']['tag'] + '.' + get_bcs_basename(in_bcsdir) + \ + '.' + config['output']['shared']['tag']+ '.' + get_bcs_basename(out_bcsdir) + aqua = config['output']['analysis']['aqua'] local_fs=[] for f in analysis_in: @@ -90,6 +98,8 @@ def remap(self): for f in local_fs: fname = os.path.basename(f) + k = fname.rfind('.') + fname = fname[0:k] + label + fname[k:] shutil.move(f, out_dir+'/'+fname) # write lcv lcv = config['output']['analysis']['lcv'] @@ -97,7 +107,7 @@ def remap(self): ymd_ = yyyymmddhh_[0:8] hh_ = yyyymmddhh_[8:10] hms_ = hh_+'0000' - rstlcvOut = out_dir+'/'+expid_out+'rst.lcv.'+ymd_+'_'+hh_+'z.bin' + rstlcvOut = out_dir+'/'+expid_out+'rst.lcv.'+ymd_+'_'+hh_+'z'+label +'.bin' cmd = bindir+'/mkdrstdate.x ' + ymd_ + ' ' + hms_ +' ' + rstlcvOut print(cmd) subprocess.call(shlex.split(cmd)) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 0f3c323..15a4b3b 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -8,6 +8,7 @@ import ruamel.yaml import shlex from remap_base import remap_base +from remap_utils import get_bcs_basename def get_landdir(bcsdir): k = bcsdir.find('/geometry/') @@ -59,7 +60,12 @@ def remap(self): out_tilefile = glob.glob(out_bcsdir+ '/*.til')[0] account = config['slurm']['account'] # even the input is binary, the output si nc4 - suffix = time+'z.nc4' + label = '' + if config['output']['shared']['label']: + label = '.' + config['input']['shared']['tag'] + '.' + get_bcs_basename(in_bcsdir) + \ + '.' + config['output']['shared']['tag']+ '.' + get_bcs_basename(out_bcsdir) + + suffix = time+'z' + label + '.nc4' if (expid) : expid = expid + '.' diff --git a/pre/remap_restart/remap_command.py b/pre/remap_restart/remap_command.py index 50b5d20..4ce6786 100755 --- a/pre/remap_restart/remap_command.py +++ b/pre/remap_restart/remap_command.py @@ -62,11 +62,8 @@ def parse_args(program_description): p_command.add_argument('-catch_model',default='catch', help='catchment model: catch, catchcnclm40, catchcnclm45') p_command.add_argument('-nobkg', action='store_true', help="Don't remap bkg files") - #p_command.add_argument('-nolbl', action='store_true', help="label final restarts with 'tagID.gridID' extension") p_command.add_argument('-nolcv', action='store_true', help="Don't remap lcv files") - #p_command.add_argument('-bkg', action='store_true', help="Don't remap bkg files") - #p_command.add_argument('-lbl', action='store_true', help="label final restarts with 'tagID.gridID' extension") - #p_command.add_argument('-lcv', action='store_true', help="Don't remap lcv files") + p_command.add_argument('-lbl', action='store_true', help="Label output restart with tag and resolution") p_command.add_argument('-in_altbcs', default="", help= "users' alternative boundary condition files for input") p_command.add_argument('-out_altbcs', default="", help= "users' alternative boundary condition files for output") p_command.add_argument('-zoom', help= "zoom for the surface input") @@ -100,6 +97,7 @@ def get_answers_from_command_line(cml): answers["output:shared:tag"] = cml.tagout answers["output:shared:model"] = cml.ocnmdlout answers["output:shared:ogrid"] = cml.oceanout + answers["output:shared:label"] = cml.lbl if cml.in_altbcs.strip(): answers["input:shared:altbcs"] = cml.in_altbcs diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index 45c1c1d..97a01f5 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -7,6 +7,7 @@ import ruamel.yaml import shlex from remap_base import remap_base +from remap_utils import get_bcs_basename class lake_landice_saltwater(remap_base): def __init__(self, **configs): @@ -43,13 +44,19 @@ def remap(self): print ("mkdir " + OutData_dir) os.makedirs(OutData_dir) - types = 'z.bin' + types = '.bin' type_str = subprocess.check_output(['file','-b', restarts_in[0]]) type_str = str(type_str) if 'Hierarchical' in type_str: - types = 'z.nc4' + types = '.nc4' yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) - suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10]+ types + + label = '' + if config['output']['shared']['label']: + label = '.' + config['input']['shared']['tag'] + '.' + get_bcs_basename(in_bcsdir) + \ + '.' + config['output']['shared']['tag']+ '.' + get_bcs_basename(out_bcsdir) + + suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + label + types saltwater = '' seaice = '' diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index fb571e8..e577211 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -55,6 +55,7 @@ def __init__(self, config_from_question): config_tpl['output']['shared']['expid'] = self.common_out['expid'] config_tpl['output']['shared']['tag'] = self.common_out.get('tag') config_tpl['output']['shared']['altbcs'] = self.common_out.get('altbcs') + config_tpl['output']['shared']['label'] = self.common_out.get('label') # params for upper air config_tpl = self.params_for_air(config_tpl) diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index 6924ed6..6f2f1fa 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -29,6 +29,7 @@ output: shared: #the tag info here is not critical. It is for command_line options tag: none + label: false altbcs: false agrid: bcs_dir: diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index d1817c6..72298c8 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -264,6 +264,12 @@ def ask_questions(): "message": "Enter new restarts expid:", "default": "", }, + { + "type": "confirm", + "name": "output:shared:label", + "message": "Would you like to add labels (tags,resoultions) to restarts' names?", + "default": False, + }, { "type": "text", diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index e44d6e9..61e3616 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -7,6 +7,7 @@ import shutil import glob from remap_base import remap_base +from remap_utils import get_bcs_basename def get_topodir(bcsdir): k = bcsdir.find('/geometry/') @@ -72,13 +73,18 @@ def remap(self): print('\nUpper air restart file names link from "_rst" to "_restart_in" \n') - types = 'z.bin' + types = '.bin' type_str = subprocess.check_output(['file','-b', restarts_in[0]]) type_str = str(type_str) if type_str.find('Hierarchical') >=0: - types = 'z.nc4' + types = '.nc4' yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) - suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10]+ types + label = '' + if config['output']['shared']['label']: + label = '.' + config['input']['shared']['tag'] + '.' + get_bcs_basename(in_bcsdir) + \ + '.' + config['output']['shared']['tag']+ '.' + get_bcs_basename(out_bcsdir) + + suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + label + types for rst in restarts_in : f = os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_restart_in' @@ -276,6 +282,7 @@ def remap(self): else: expid = '' suffix = '_rst.' + suffix + for out_rst in glob.glob("*_rst*"): filename = expid + os.path.basename(out_rst).split('_rst')[0].split('.')[-1]+suffix print('\n Move ' + out_rst + ' to ' + out_dir+"/"+filename) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 8b1e3c9..1608f1b 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -46,6 +46,13 @@ def data_ocean_default(resolution): if resolution in ['C12','C24', 'C48'] : default_ = '360x180 (Reynolds)' return default_ +def get_bcs_basename(bcs): + k = bcs.find('/geometry') + if k != -1 : + bcs = bcs[0:k] + while bcs[-1] == '/': bcs = bcs[0:-1] # remove extra '/' + return os.path.basename(bcs) + def we_default(tag): default_ = '26' if tag in ['INL','GITNL', '525'] : default_ = '13' @@ -185,6 +192,8 @@ def get_command_line_from_answers(answers): nobkg = '' if answers["output:analysis:bkg"] else " -nobkg " nolcv = '' if answers["output:analysis:lcv"] else " -nolcv " + label = ' -lbl ' if answers["output:shared:label"] else "" + in_altbcs = '' out_altbcs = '' if answers.get("input:shared:altbcs", "").strip() : @@ -230,6 +239,7 @@ def get_command_line_from_answers(answers): zoom + \ wemin + \ wemout + \ + label + \ nobkg + \ nolcv + \ out_rs + \ From 4c2c4994163459ac44bb5f83718b010869fae9af Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Tue, 23 May 2023 10:24:19 -0400 Subject: [PATCH 08/88] restrict some arguments with choices --- pre/remap_restart/remap_command.py | 50 ++++++++++++++++-------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/pre/remap_restart/remap_command.py b/pre/remap_restart/remap_command.py index 4ce6786..40c66f3 100755 --- a/pre/remap_restart/remap_command.py +++ b/pre/remap_restart/remap_command.py @@ -35,7 +35,7 @@ def parse_args(program_description): ) p_command.add_argument('-merra2', action='store_true', default= False, help='use merra2 restarts') p_command.add_argument('-ymdh', help='yyyymmddhh year month date hour of input and output restarts') - p_command.add_argument('-grout', help='Grid ID of the output restart, format Cxxx') + p_command.add_argument('-grout', help='Grid ID of the output restart, format Cxx') p_command.add_argument('-levsout', help='levels of output restarts') p_command.add_argument('-out_dir', help='directory for output restarts') @@ -50,29 +50,29 @@ def parse_args(program_description): p_command.add_argument('-wemin', help='minimum water snow water equivalent for input catch/cn') p_command.add_argument('-wemout', help='minimum water snow water equivalent for output catch/cn') - p_command.add_argument('-oceanin', help='ocean horizontal grid of inputs \n \ - data model: 360x180,1440x720,2880x1440,CS \n \ - coupled model: 72x36, 360x200,720x410,1440x1080') - p_command.add_argument('-oceanout', help='ocean horizontal grid of outputs \n \ - data model: 360x180,1440x720,2880x1440,CS \n \ - coupled model: 72x36, 360x200,720x410,1440x1080') + ocean_grids=['360x180','1440x720','2880x1440','CS', '72x36', '360x200','720x410','1440x1080'] + p_command.add_argument('-oceanin', help='ocean horizontal grid of inputs. \n \ + data model choices: 360x180,1440x720,2880x1440,CS. \n \ + coupled model choices: 72x36, 360x200,720x410,1440x1080', choices=ocean_grids) + p_command.add_argument('-oceanout', help='ocean horizontal grid of outputs. \n \ + choices are the same as option "oceanin"', choices=ocean_grids) - p_command.add_argument('-ocnmdlin', default='data', help='ocean input model: data, MOM5, MOM6') - p_command.add_argument('-ocnmdlout',default='data', help='ocean output model: data, MOM5, MOM6') - p_command.add_argument('-catch_model',default='catch', help='catchment model: catch, catchcnclm40, catchcnclm45') + p_command.add_argument('-ocnmdlin', default='data', help='ocean input model', choices=['data', 'MOM5', 'MOM6']) + p_command.add_argument('-ocnmdlout',default='data', help='ocean output model', choices=['data', 'MOM5', 'MOM6']) + p_command.add_argument('-catch_model',default='catch', help='catchment model', choices=['catch', 'catchcnclm40', 'catchcnclm45']) p_command.add_argument('-nobkg', action='store_true', help="Don't remap bkg files") p_command.add_argument('-nolcv', action='store_true', help="Don't remap lcv files") - p_command.add_argument('-lbl', action='store_true', help="Label output restart with tag and resolution") + p_command.add_argument('-lbl', action='store_true', help="Label output restarts with tags and resolutions") p_command.add_argument('-in_altbcs', default="", help= "users' alternative boundary condition files for input") p_command.add_argument('-out_altbcs', default="", help= "users' alternative boundary condition files for output") p_command.add_argument('-zoom', help= "zoom for the surface input") - p_command.add_argument('-qos', default="debug", help= "queue of slurm job") + p_command.add_argument('-qos', default = "debug", help= "queue of slurm job", choices=['debug', 'allnccs']) account = get_account() - p_command.add_argument('-account', default= account, help= "account of slurm job") + p_command.add_argument('-account', default = account, help= "account of slurm job") p_command.add_argument('-constraint', default= 'sky', help= "machine of slurm job") - p_command.add_argument('-rs', default=3, help='flag indicating which restarts to regrid: 1 (upper air); 2 (surface) 3 (both)') + p_command.add_argument('-rs', default=3, help='flag indicating which restarts to regrid: 1 (upper air); 2 (surface) 3 (both)', choices=[1,2,3]) # Parse using parse_known_args so we can pass the rest to the remap scripts args, extra_args = parser.parse_known_args() @@ -121,24 +121,26 @@ def get_answers_from_command_line(cml): answers["input:shared:rst_dir"] = os.path.abspath(cml.rst_dir + '/') - # zoom_default fills 'input:shared:agrid' - answers["input:surface:zoom"] = zoom_default(answers) - if cml.zoom: answers["input:surface:zoom"] = cml.zoom + if cml.zoom: + answers["input:surface:zoom"] = cml.zoom + else: + # zoom_default fills 'input:shared:agrid' + answers["input:surface:zoom"] = zoom_default(answers) + if answers.get('input:shared:ogrid') == 'CS': answers['input:shared:ogrid'] = answers['input:shared:agrid'] if answers.get('output:shared:ogrid') == 'CS': answers['output:shared:ogrid'] = answers['output:shared:agrid'] - if not cml.wemin : - answers["input:surface:wemin"] = we_default(answers['input:shared:tag']) - else: + if cml.wemin : answers["input:surface:wemin"] = cml.wemin - - if not cml.wemout : - answers["output:surface:wemin"] = we_default(answers['output:shared:tag']) else: - answers["output:surface:wemin"] = cml.wemout + answers["input:surface:wemin"] = we_default(answers['input:shared:tag']) + if cml.wemout : + answers["output:surface:wemin"] = cml.wemout + else: + answers["output:surface:wemin"] = we_default(answers['output:shared:tag']) answers["slurm:account"] = cml.account answers["slurm:qos"] = cml.qos From 7a2a20949ea949692d9078b8cc244f6bc519fcbd Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Tue, 23 May 2023 10:50:07 -0400 Subject: [PATCH 09/88] fix tests --- pre/remap_restart/tests/amip_c180Toc90.yaml | 1 + pre/remap_restart/tests/c180Toc360.yaml | 1 + pre/remap_restart/tests/c24Toc12.yaml | 1 + pre/remap_restart/tests/c360Toc24.yaml | 1 + pre/remap_restart/tests/f522Toc360.yaml | 1 + 5 files changed, 5 insertions(+) diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index 9e93f70..6a2c374 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -28,6 +28,7 @@ output: expid: C90CS_JM_v10.22.2_L072_C180_AMIP ogrid: C90 out_dir: $NOBACKUP/REMAP_TESTS/amip_c180Toc90/ + label: False air: # remap upper air or not remap: true diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index 1f919ef..93368c6 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -28,6 +28,7 @@ output: expid: C360CS_Jason-3_4_NL_REAMIP_MERRA2_C180 ogrid: C360 out_dir: $NOBACKUP/REMAP_TESTS/c360Toc24/ + label: False air: # remap upper air or not remap: true diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index 0b96e65..868ac12 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -28,6 +28,7 @@ output: expid: hello ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/c24Toc12/ + label: False air: # remap upper air or not remap: true diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index 08812fa..ba84b30 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -28,6 +28,7 @@ output: expid: C24c_x0046a ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/c360Toc24/ + label: False air: # remap upper air or not remap: true diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index 17deaa6..abf4908 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -27,6 +27,7 @@ output: expid: remap ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/f522Toc360/ + label: False air: nlevel: '72' remap: true From b6cb3a30e48e2bd7a0740d11629878f81dfacad9 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 25 May 2023 17:57:20 -0400 Subject: [PATCH 10/88] more straight forward questions --- pre/remap_restart/remap_catchANDcn.py | 3 +- pre/remap_restart/remap_command.py | 63 ++-- pre/remap_restart/remap_params.py | 367 +--------------------- pre/remap_restart/remap_params.tpl | 2 - pre/remap_restart/remap_questions.py | 179 ++++++----- pre/remap_restart/remap_utils.py | 425 ++++++++++++++++++++++---- 6 files changed, 509 insertions(+), 530 deletions(-) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index d3b674a..27ce66a 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -59,6 +59,8 @@ def remap(self): if not out_tilefile : out_tilefile = glob.glob(out_bcsdir+ '/*.til')[0] + out_bcsdir = get_landdir(out_bcsdir) + in_bcsdir = get_landdir(in_bcsdir) in_tilenum = 0 out_tilenum = 0 with open( in_bcsdir+'/clsm/catchment.def') as f: @@ -107,7 +109,6 @@ def remap(self): shutil.copyfile(in_rstfile,dest) in_rstfile = dest - out_bcsdir = get_landdir(out_bcsdir) log_name = out_dir+'/'+'mk_catchANDcn_log' mk_catch_j_template = """#!/bin/csh -f #SBATCH --account={account} diff --git a/pre/remap_restart/remap_command.py b/pre/remap_restart/remap_command.py index 40c66f3..0548309 100755 --- a/pre/remap_restart/remap_command.py +++ b/pre/remap_restart/remap_command.py @@ -47,8 +47,8 @@ def parse_args(program_description): p_command.add_argument('-tagin', help='GCM or DAS tag associated with inputs') p_command.add_argument('-tagout', help='GCM or DAS tag associated with outputs') - p_command.add_argument('-wemin', help='minimum water snow water equivalent for input catch/cn') - p_command.add_argument('-wemout', help='minimum water snow water equivalent for output catch/cn') + p_command.add_argument('-in_wemin', help='minimum water snow water equivalent for input catch/cn') + p_command.add_argument('-out_wemin', help='minimum water snow water equivalent for output catch/cn') ocean_grids=['360x180','1440x720','2880x1440','CS', '72x36', '360x200','720x410','1440x1080'] p_command.add_argument('-oceanin', help='ocean horizontal grid of inputs. \n \ @@ -64,15 +64,15 @@ def parse_args(program_description): p_command.add_argument('-nobkg', action='store_true', help="Don't remap bkg files") p_command.add_argument('-nolcv', action='store_true', help="Don't remap lcv files") p_command.add_argument('-lbl', action='store_true', help="Label output restarts with tags and resolutions") - p_command.add_argument('-in_altbcs', default="", help= "users' alternative boundary condition files for input") - p_command.add_argument('-out_altbcs', default="", help= "users' alternative boundary condition files for output") + p_command.add_argument('-in_bcsdir', default="", help= "users' alternative boundary condition files for input. If not specified (default), it will be deduced from tag and resolution information") + p_command.add_argument('-out_bcsdir', default="", help= "users' alternative boundary condition files for output. If not specified (default), it will be deduced from tag and resolution information") p_command.add_argument('-zoom', help= "zoom for the surface input") p_command.add_argument('-qos', default = "debug", help= "queue of slurm job", choices=['debug', 'allnccs']) account = get_account() p_command.add_argument('-account', default = account, help= "account of slurm job") p_command.add_argument('-constraint', default= 'sky', help= "machine of slurm job") - p_command.add_argument('-rs', default=3, help='flag indicating which restarts to regrid: 1 (upper air); 2 (surface) 3 (both)', choices=[1,2,3]) + p_command.add_argument('-rs', default= '3', help='flag indicating which restarts to regrid: 1 (upper air); 2 (surface) 3 (both)', choices=['1','2','3']) # Parse using parse_known_args so we can pass the rest to the remap scripts args, extra_args = parser.parse_known_args() @@ -85,25 +85,39 @@ def get_answers_from_command_line(cml): answers["input:shared:MERRA-2"] = cml.merra2 answers["input:shared:yyyymmddhh"] = cml.ymdh answers["input:shared:model"] = cml.ocnmdlin - if not cml.merra2: + answers["output:shared:out_dir"] = os.path.abspath(cml.out_dir + '/') + if cml.merra2: + init_merra2(answers) + else: answers["input:shared:tag"] = cml.tagin - answers["input:shared:ogrid"] = cml.oceanin - answers["input:surface:catch_model"] = cml.catch_model - + answers["input:surface:catch_model"] = cml.catch_model + answers["input:shared:rst_dir"] = os.path.abspath(cml.rst_dir + '/') + fvcore_name(answers) + ogrid = cml.oceanin + if ogrid == "CS": + ogrid = answers["input:shared:agrid"] + answers["input:shared:ogrid"] = ogrid + answers["output:shared:agrid"] = cml.grout answers["output:air:nlevel"] = cml.levsout - answers["output:shared:out_dir"] = os.path.abspath(cml.out_dir + '/') answers["output:shared:expid"] = cml.newid answers["output:shared:tag"] = cml.tagout answers["output:shared:model"] = cml.ocnmdlout - answers["output:shared:ogrid"] = cml.oceanout answers["output:shared:label"] = cml.lbl + ogrid = cml.oceanout + if ogrid == "CS": + ogrid = answers["output:shared:agrid"] + answers["output:shared:ogrid"] = ogrid - if cml.in_altbcs.strip(): - answers["input:shared:altbcs"] = cml.in_altbcs - if cml.out_altbcs.strip(): - answers["output:shared:altbcs"] = cml.out_altbcs + if cml.in_bcsdir.strip(): + answers["input:shared:bcs_dir"] = cml.in_bcsdir + else: + answers["input:shared:bcs_dir"] = get_bcsdir(answers, "IN") + if cml.out_bcsdir.strip(): + answers["output:shared:bcs_dir"] = cml.out_bcsdir + else: + answers["output:shared:bcs_dir"] = get_bcsdir(answers, "OUT") answers["output:analysis:bkg"] = not cml.nobkg answers["output:analysis:lcv"] = not cml.nolcv @@ -115,30 +129,19 @@ def get_answers_from_command_line(cml): answers["output:surface:remap"] = True answers["output:air:remap"] = True - if cml.merra2 and not cml.rst_dir: - answers['input:shared:rst_dir'] = tmp_merra2_dir(answers) - else: - answers["input:shared:rst_dir"] = os.path.abspath(cml.rst_dir + '/') - - if cml.zoom: answers["input:surface:zoom"] = cml.zoom else: # zoom_default fills 'input:shared:agrid' answers["input:surface:zoom"] = zoom_default(answers) - if answers.get('input:shared:ogrid') == 'CS': - answers['input:shared:ogrid'] = answers['input:shared:agrid'] - if answers.get('output:shared:ogrid') == 'CS': - answers['output:shared:ogrid'] = answers['output:shared:agrid'] - - if cml.wemin : - answers["input:surface:wemin"] = cml.wemin + if cml.in_wemin : + answers["input:surface:wemin"] = cml.in_wemin else: answers["input:surface:wemin"] = we_default(answers['input:shared:tag']) - if cml.wemout : - answers["output:surface:wemin"] = cml.wemout + if cml.out_wemin : + answers["output:surface:wemin"] = cml.out_wemin else: answers["output:surface:wemin"] = we_default(answers['output:shared:tag']) diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index 494eaa8..c2d8956 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -21,11 +21,6 @@ def __init__(self, config_from_question): self.surf_out = config_from_question['output']['surface'] self.ana_out = config_from_question['output']['analysis'] - self.init_time() - self.init_tags() - self.init_merra2() - - # load input yaml yaml = ruamel.yaml.YAML() stream = '' @@ -43,10 +38,10 @@ def __init__(self, config_from_question): config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] config_tpl['input']['shared']['tag'] = self.common_in.get('tag') config_tpl['input']['shared']['altbcs'] = self.common_in.get('altbcs') - - config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') - config_tpl['output']['air']['remap'] = self.upper_out.get('remap') config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') + + config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') + config_tpl['output']['air']['remap'] = self.upper_out.get('remap') config_tpl['output']['surface']['remap_water'] = self.surf_out.get('remap') config_tpl['output']['surface']['remap_catch'] = self.surf_out.get('remap') config_tpl['output']['shared']['agrid'] = self.common_out['agrid'] @@ -57,360 +52,24 @@ def __init__(self, config_from_question): config_tpl['output']['shared']['altbcs'] = self.common_out.get('altbcs') config_tpl['output']['shared']['label'] = self.common_out.get('label') + config_tpl['input']['shared']['bcs_dir'] = self.common_in['bcs_dir'] + config_tpl['output']['shared']['bcs_dir'] = self.common_out['bcs_dir'] # params for upper air config_tpl = self.params_for_air(config_tpl) config_tpl = self.params_for_surface(config_tpl) config_tpl = self.params_for_analysis(config_tpl) config_tpl = self.options_for_slurm(config_tpl) - # get bc directory and tile file - in_bcsdir = self.get_bcdir("IN") - out_bcsdir = self.get_bcdir("OUT") - config_tpl['input']['shared']['bcs_dir'] = in_bcsdir+ '/' - config_tpl['output']['shared']['bcs_dir'] = out_bcsdir + '/' - self.config = config_tpl - def init_tags(self): - # copy and paste from remap.pl - # minor change. Add "D" to the number for each group - # BCS Tag: Fortuna-1_4 - F14 = ( 'F14', 'Fortuna-1_4', 'Fortuna-1_4_p1' ) - D214 = ( 'D214', 'GEOSdas-2_1_4', 'GEOSdas-2_1_4-m1', - 'GEOSdas-2_1_4-m2', 'GEOSdas-2_1_4-m3', 'GEOSdas-2_1_4-m4' ) - D540 = ( 'D540', 'GEOSadas-5_4_0', 'GEOSadas-5_4_0_p1', - 'GEOSadas-5_4_0_p2', 'GEOSadas-5_4_0_p3', 'GEOSadas-5_4_0_p4', - 'GEOSadas-5_4_1', 'GEOSadas-5_4_1_p1', 'GEOSadas-5_4_2', - 'GEOSadas-5_4_3', 'GEOSadas-5_4_4', 'GEOSadas-5_5_0', - 'GEOSadas-5_5_1', 'GEOSadas-5_5_2', 'GEOSadas-5_5_3' ) - - # BCS Tag: Fortuna-2_0 - #--------------------- - F20 = ( 'F20', 'Fortuna-2_0') - - # BCS Tag: Fortuna-2_1 - #--------------------- - F21 = ( 'F21', 'Fortuna-2_1', 'Fortuna-2_1_p1', - 'Fortuna-2_1_p2', 'Fortuna-2_1_p3', 'Fortuna-2_2', - 'Fortuna-2_2_p1', 'Fortuna-2_2_p2', 'Fortuna-2_3', - 'Fortuna-2_3_p1', 'Fortuna-2_4', 'Fortuna-2_4_p1', - 'Fortuna-2_4_p2', 'Fortuna-2_5', 'Fortuna-2_5_BETA0', - 'Fortuna-2_5_p1', 'Fortuna-2_5_p2', 'Fortuna-2_5_p3', - 'Fortuna-2_5_p4', 'Fortuna-2_5_p5', 'Fortuna-2_5_p6', - 'Fortuna-2_5_pp2' ) - D561 = ( 'D561', 'GEOSadas-5_6_1', 'GEOSadas-5_6_1_p1', - 'GEOSadas-5_6_1_p2', 'GEOSadas-5_6_1_p3', 'GEOSadas-5_6_1_p4', - 'GEOSadas-5_6_2', 'GEOSadas-5_6_2_p1', 'GEOSadas-5_6_2_p2', - 'GEOSadas-5_6_2_p3', 'GEOSadas-5_6_2_p4', 'GEOSadas-5_6_2_p5', - 'GEOSadas-5_6_2_p6', 'GEOSadas-5_7_1', 'GEOSadas-5_7_1_p1', - 'GEOSadas-5_7_1_p2', 'GEOSadas-5_7_2', 'GEOSadas-5_7_2_p1', - 'GEOSadas-5_7_2_p2', 'GEOSadas-5_7_2_p2_m1','GEOSadas-5_7_2_p3', - 'GEOSadas-5_7_2_p3_m1', 'GEOSadas-5_7_2_p3_m2','GEOSadas-5_7_2_p4', - 'GEOSadas-5_7_2_p5', 'GEOSadas-5_7_2_p5_m1','GEOSadas-5_7_3', - 'GEOSadas-5_7_3_p1', 'GEOSadas-5_7_3_p2', 'GEOSadas-5_7_3_p2' ) - - # BCS Tag: Ganymed-1_0 - #--------------------- - G10 = ( 'G10', 'Ganymed-1_0', 'Ganymed-1_0_BETA', - 'Ganymed-1_0_BETA1', 'Ganymed-1_0_BETA2', 'Ganymed-1_0_BETA3', - 'Ganymed-1_0_BETA4' ) - - D580 = ( 'D580', 'GEOSadas-5_8_0', 'GEOSadas-5_9_0', - 'GEOSadas-5_9_1' ) - - # BCS Tags: Ganymed-1_0_M and Ganymed-1_0_D - #------------------------------------------ - G10p = ( 'G10p', 'Ganymed-1_0_p1', 'Ganymed-1_0_p2', - 'Ganymed-1_0_p3', 'Ganymed-1_0_p4', 'Ganymed-1_0_p5', - 'Ganymed-1_0_p6' ) - - D591p= ( 'D591p', 'GEOSadas-5_9_1_p1', 'GEOSadas-5_9_1_p2', - 'GEOSadas-5_9_1_p3', 'GEOSadas-5_9_1_p4', 'GEOSadas-5_9_1_p5', - 'GEOSadas-5_9_1_p6', 'GEOSadas-5_9_1_p7', 'GEOSadas-5_9_1_p8', - 'GEOSadas-5_9_1_p9' ) - - # BCS Tags: Ganymed-1_0_M and Ganymed-1_0_D w/ new landice rst - #------------------------------------------------------------------------ - G20 = ( 'G20', 'Ganymed-2_0', 'Ganymed-2_1', - 'Ganymed-2_1_p1', 'Ganymed-2_1_p2', 'Ganymed-2_1_p3', - 'Ganymed-2_1_p4', 'Ganymed-2_1_p5', 'Ganymed-2_1_p6' ) - D5A0 = ( 'D5A0', 'GEOSadas-5_10_0', 'GEOSadas-5_10_0_p1' ) - - - # BCS Tags: Ganymed-1_0_Reynolds and Ganymed-1_0_Ostia - #----------------------------------------------------- - G30 = ( 'G30', 'Ganymed-3_0', 'Ganymed-3_0_p1' ) - D5B0 = ( '5B0', 'GEOSadas-5_10_0_p2', 'GEOSadas-5_11_0' ) - - # BCS Tags: Ganymed-4_0_Reynolds, Ganymed-4_0_MERRA-2, and Ganymed-4_0_Ostia - #--------------------------------------------------------------------------- - G40 = ( 'G40', 'Ganymed-4_0', 'Ganymed-4_0_p1', - 'Ganymed-4_1', 'Heracles-1_0', 'Heracles-1_1', - 'Heracles-2_0', 'Heracles-2_1', 'Heracles-3_0', - 'Heracles-4_0', 'Heracles-5_4_p3' ) - D512 = ( '512', 'GEOSadas-5_12_2', 'GEOSadas-5_12_4', - 'GEOSadas-5_12_4_p1', 'GEOSadas-5_12_4_p2', 'GEOSadas-5_12_4_p3', - 'GEOSadas-5_12_5', 'GEOSadas-5_13_0_p1', 'GEOSadas-5_13_0_p2', - 'GEOSadas-5_13_1', 'GEOSadas-5_16_5' ) - - # BCS Tags: Icarus (New Land Parameters, New Topography) - #--------------------------------------------------------------------------- - ICA = ( 'ICA', 'Icarus', 'Jason' ) - D517 = ( '517', 'GEOSadas-5_17_0', 'GEOSadas-5_17_1', 'GEOSadas-5_18_0', - 'GEOSadas-5_18_1', 'GEOSadas-5_18_2', 'GEOSadas-5_18_3', - 'GEOSadas-5_18_3_p1', 'GEOSadas-5_19_0', 'GEOSadas-5_20_0', - 'GEOSadas-5_20_0_p1', 'GEOSadas-5_20_0_p2', 'GEOSadas-5_21_0', - 'GEOSadas-5_21_2', 'GEOSadas-5_21_3_p1', 'GEOSadas-5_22_0', - 'GEOSadas-5_22_0_p1', 'GEOSadas-5_22_0_p2', 'GEOSadas-5_23_0', - 'GEOSadas-5_23_0_p1', 'GEOSadas-5_24_0', 'GEOSadas-5_24_0_p1' ) - GITOL = ( 'GITOL', '10.3', '10.4', '10.5', - '10.6', '10.7', '10.8', - '10.9', '10.10', '10.11', - '10.12', '10.13', '10.14', - '10.15', '10.16', '10.17', - '10.18' ) - - # BCS Tags: Icarus-NLv3 (New Land Parameters) - #--------------------------------------------------------------------------- - INL = ( 'INL', 'Icarus-NL', 'Icarus-NLv3', 'Jason-NL' ) - GITNL = ( 'GITNL', '10.19', '10.20', '10.21', '10.22', '10.23' ) - D525 = ( '525', 'GEOSadas-5_25_1', 'GEOSadas-5_25_1_p5', 'GEOSadas-5_25_p7', - 'GEOSadas-5_27_1', 'GEOSadas-5_29_3', 'GEOSadas-5_29_4' ) - - self.bcsTag={} - for tag in F14: self.bcsTag[tag]= "Fortuna-1_4" - for tag in F20: self.bcsTag[tag]= "Fortuna-2_0" - for tag in F21: self.bcsTag[tag]= "Fortuna-2_1" - for tag in G10: self.bcsTag[tag]= "Ganymed-1_0" - for tag in G10p: self.bcsTag[tag]= "Ganymed-1_0_M" - for tag in G20: self.bcsTag[tag]= "Ganymed-1_0_M" - for tag in G30: self.bcsTag[tag]= "Ganymed-1_0_Reynolds" - for tag in G40: self.bcsTag[tag]= "Ganymed-4_0_Reynolds" - for tag in ICA: self.bcsTag[tag]= "Icarus_Reynolds" - for tag in GITOL: self.bcsTag[tag]= "Icarus_Reynolds" - for tag in INL: self.bcsTag[tag]= "Icarus-NLv3_Reynolds" - for tag in GITNL: self.bcsTag[tag]= "Icarus-NLv3_Reynolds" - for tag in NewStructureBCTag: self.bcsTag[tag]= tag - - - for tag in D214: self.bcsTag[tag]= "Fortuna-1_4" - for tag in D540: self.bcsTag[tag]= "Fortuna-1_4" - for tag in D561: self.bcsTag[tag]= "Fortuna-2_1" - for tag in D580: self.bcsTag[tag]= "Ganymed-1_0" - for tag in D591p: self.bcsTag[tag]= "Ganymed-1_0_M" - for tag in D5A0: self.bcsTag[tag]= "Ganymed-1_0_M" - for tag in D5B0: self.bcsTag[tag]= "Ganymed-1_0_Reynolds" - for tag in D512: self.bcsTag[tag]= "Ganymed-4_0_Reynolds" - for tag in D517: self.bcsTag[tag]= "Icarus_Reynolds" - for tag in D525: self.bcsTag[tag]= "Icarus-NLv3_Reynolds" - - self.tagsRank ={} - self.tagsRank['Fortuna-1_4'] = 1 - self.tagsRank['Fortuna-1_5'] = 2 - self.tagsRank['Fortuna-2_0'] = 3 - self.tagsRank['Fortuna-2_1'] = 4 - self.tagsRank['Ganymed-1_0'] = 5 - self.tagsRank['Ganymed-1_0_m1'] = 6 - self.tagsRank['Ganymed-1_0_m2'] = 7 - self.tagsRank['Ganymed-1_0_M'] = 8 - self.tagsRank['Ganymed-1_0_D'] = 9 - self.tagsRank['Ganymed-1_0_Reynolds'] = 10 - self.tagsRank['Ganymed-1_0_Ostia'] = 11 - self.tagsRank['Ganymed-4_0_Reynolds'] = 12 - self.tagsRank['Ganymed-4_0_Ostia'] = 13 - self.tagsRank['Ganymed-4_0_MERRA-2'] = 14 - self.tagsRank['Icarus_Reynolds'] = 15 - self.tagsRank['Icarus_MERRA-2'] = 16 - self.tagsRank['Icarus_Ostia'] = 17 - self.tagsRank['Icarus-NLv3_Reynolds'] = 18 - self.tagsRank['Icarus-NLv3_MERRA-2'] = 19 - self.tagsRank['Icarus-NLv3_Ostia'] = 20 - for tag in NewStructureBCTag: self.tagsRank[tag] = 21 - - def init_time(self): - ymdh = self.common_in.get('yyyymmddhh') - self.yyyymm = ymdh[0:6] - self.yyyy = ymdh[0:4] - self.mm = ymdh[4:6] - self.dd = ymdh[6:8] - self.hh = ymdh[8:10] - self.ymd = ymdh[0:8] - - def init_merra2(self): - if not self.common_in['MERRA-2']: - return - self.common_in = merra2_expid(self.common_in) - - self.common_in['agrid'] = 'C180' - self.common_in['ogrid'] = '1440x720' - self.common_in['tag']= 'Ganymed-4_0' - self.surf_in['catch_model'] = 'catch' - - def get_bcdir(self, opt): - tag = self.common_in['tag'] - ogrid = self.common_in['ogrid'] - model = self.common_in['model'] - altbcs = self.common_in.get('altbcs', None) - if opt.upper() == "OUT": - tag = self.common_out['tag'] - ogrid = self.common_out['ogrid'] - model = self.common_out['model'] - altbcs = self.common_out.get('altbcs', None) - - bcdir = '' - base = 'discover_legacy' - if model != "data": - base = 'discover_couple' - if tag in NewStructureBCTag: - base = 'discover_ns' - if self.common_in['MERRA-2']: - base = 'discover_ops' - - bc_base = BCBase[base] - if altbcs : bc_base = altbcs - - bctag = self.get_bcTag(tag,ogrid) - tagrank = self.tagsRank[bctag] - - if (tagrank >= self.tagsRank['NL3']) : - bcdir = bc_base+'/'+ bctag+'/geometry/' - elif (tagrank >= self.tagsRank['Icarus-NLv3_Reynolds']) : - bcdir = bc_base+'/Icarus-NLv3/'+bctag+'/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Icarus-NLv3/'+model+'/' - elif (tagrank >= self.tagsRank['Icarus_Reynolds']): - if bc_base == BCBase['discover_ops']: - bcdir = bc_base+'/Icarus_Updated/'+bctag+'/' - else: - bcdir = bc_base+'/Icarus/'+bctag+'/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Icarus/'+model+'/' - elif(tagrank >= self.tagsRank["Ganymed-4_0_Reynolds"]): - bcdir = bc_base + '/Ganymed-4_0/'+bctag+'/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Ganymed/'+model+'/' - else: - bcdir = bc_base + '/' + bctag + '/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Ganymed/'+model+'/' - - if not os.path.exists(bcdir): - exit("Cannot find bc dir " + bcdir) - - gridStr = self.get_grid_subdir(bcdir,opt) - bcdir = bcdir + '/' + gridStr - - return bcdir - - def get_grid_subdir(self, bcdir, opt): - - def get_name_with_grid( grid, names, a_o): - if not grid : - return names - namex = [] - if (grid[0].upper() == 'C'): - n = int(grid[1:]) - s1 ='{:04d}x6C'.format(n) - j=n*6 - s2 =str(n) - s3 =str(j) - # first try - for aoname in names: - name = '' - if(a_o == 'a'): - name = aoname.split('_')[0] - else: - name = aoname.split('_')[-1] - if (name.find(s1) != -1 or (name.find(s2) != -1 and name.find(s3) != -1 )): - namex.append(aoname) - else: - xy = grid.upper().split('X') - s2 = xy[0] - s3 = xy[1] - for aoname in names: - name = '' - if(a_o == 'a'): - name = aoname.split('_')[0] - else: - name = aoname.split('_')[-1] - if (name.find(s2) != -1 and name.find(s3) != -1): namex.append(aoname) - return namex - #v3.5 - #dirnames = [ f.name for f in os.scandir(bcdir) if f.is_dir()] - #v2.7 - dirnames = [f for f in os.listdir(bcdir) if os.path.isdir(os.path.join(bcdir,f))] - - agrid_ = self.common_in['agrid'] - ogrid_ = self.common_in['ogrid'] - omodel_ = self.common_in.get('model') - if opt.upper() == "OUT" : - agrid_ = self.common_out['agrid'] - ogrid_ = self.common_out['ogrid'] - omodel_ = self.common_out.get('model') - - anames = get_name_with_grid(agrid_, dirnames, 'a') - gridID = get_name_with_grid(ogrid_, anames, 'o') - if len(gridID) == 0 : - exit("cannot find the grid subdirctory of agrid: " +agrid_+ " and ogrid " + ogrid_ + " under "+ bcdir) - g = '' - gridID.sort(key=len) - g = gridID[0] - - # For new structure BC - for g_ in gridID : - if omodel_ == 'MOM5': - if 'M5' in g_ : g = g_ - if omodel_ == 'MOM6': - if 'M6' in g_ : g = g_ - - if len(gridID) >= 2 : - print("\n Warning! Find many GridIDs in " + bcdir) - print(" GridIDs found: ", gridID) - #WY note, found many string in the directory - print(" This GridID is chosen: " + g) - return g - - def get_bcTag(self, tag, ogrid): - bctag = self.bcsTag[tag] - if bctag in NewStructureBCTag : return bctag - if ogrid[0].upper() == "C": - bctag=bctag.replace('_Reynolds','_Ostia') - else: - xy = ogrid.upper().split('X') - x = int(xy[0]) - if x == 1440: bctag=bctag.replace('_Reynolds','_MERRA-2') - if x == 2880: - bctag=bctag.replace('_Reynolds','_Ostia') - bctag=bctag.replace('_M','_D') - return bctag - def params_for_air(self, config_tpl): if self.common_in['MERRA-2']: return config_tpl - # verify agrid - rst_dir = self.common_in['rst_dir'] + '/' - time = self.ymd + '_'+ self.hh - files = glob.glob(rst_dir +'/*fvcore_*'+time+'*') - if len(files) == 0 : - fname_ = rst_dir +'/fvcore_internal_rst' - if os.path.exists(fname_) : - files.append(fname_) - - # get expid - if (len(files) >0) : - fname = os.path.basename(files[0]) - expid = fname.split('fvcore')[0] - config_tpl['input']['shared']['expid'] = expid[0:-1] #remove the last '.' - - agrid_ = self.common_in['agrid'] - if self.common_in['ogrid'] == 'CS' : - config_tpl['input']['shared']['ogrid'] = agrid_ - self.common_in['ogrid'] = agrid_ ogrid = config_tpl['input']['shared']['ogrid'] tagout = self.common_out['tag'] - bctag = self.get_bcTag(tagout, ogrid) - tagrank = self.tagsRank[bctag] + bctag = get_bcTag(tagout, ogrid) + tagrank = TagsRank[bctag] if ( not config_tpl['input']['air']['drymass']) : config_tpl['input']['air']['drymass'] = 0 if tagrank >=12 : @@ -428,11 +87,11 @@ def params_for_surface(self, config_tpl): config_tpl['output']['surface']['surflay'] = 20. tagout = self.common_out['tag'] ogrid = self.common_out['ogrid'] - bctag = self.get_bcTag(tagout, ogrid) - tagrank = self.tagsRank[bctag] + bctag = get_bcTag(tagout, ogrid) + tagrank = TagsRank[bctag] if tagrank >=12 : config_tpl['output']['surface']['surflay'] = 50. - if tagrank >= self.tagsRank["Icarus_Reynolds"]: + if tagrank >= TagsRank["Icarus_Reynolds"]: config_tpl['output']['surface']['split_saltwater'] = True config_tpl['input']['surface']['zoom']= self.surf_in['zoom'] config_tpl['input']['surface']['wemin']= self.surf_in['wemin'] @@ -446,9 +105,9 @@ def params_for_analysis(self, config_tpl): ogrid = self.common_out['ogrid'] tagout = self.common_out['tag'] - bctag = self.get_bcTag(tagout, ogrid) - tagrank = self.tagsRank[bctag] - if tagrank >= self.tagsRank["Ganymed-4_0_Reynolds"] : + bctag = get_bcTag(tagout, ogrid) + tagrank = TagsRank[bctag] + if tagrank >= TagsRank["Ganymed-4_0_Reynolds"] : config_tpl['output']['analysis']['aqua'] = True return config_tpl diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index 6f2f1fa..cecf3d3 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -10,7 +10,6 @@ input: shared: #the tag info here is not critical. It is for command_line options tag: none - altbcs: false MERRA-2: false agrid: bcs_dir: @@ -30,7 +29,6 @@ output: #the tag info here is not critical. It is for command_line options tag: none label: false - altbcs: false agrid: bcs_dir: expid: diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 31d1ec8..188bac4 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -15,6 +15,26 @@ import glob from remap_utils import * +def remove_ogrid_comment(x, opt): + ogrid = '' + if opt == "IN": + ogrid = x.get('input:shared:ogrid') + else: + ogrid = x.get('output:shared:ogrid') + if not ogrid: return False + + ogrid = ogrid.split()[0] + if opt == "IN": + if ogrid == 'CS': + ogrid = x['input:shared:agrid'] + x['input:shared:ogrid'] = ogrid + else: + if ogrid == 'CS': + ogrid = x['output:shared:agrid'] + x['output:shared:ogrid'] = ogrid + + return False + def ask_questions(): questions = [ @@ -42,6 +62,13 @@ def ask_questions(): "name": "output:shared:out_dir", "message": "Enter the directory for new restarts:\n" }, + { + "type": "path", + "name": "output:shared:out_dir", + "message": "init merra2 \n", + # always return false, no show but change x + "when": lambda x: init_merra2(x), + }, { "type": "text", @@ -55,7 +82,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:model", - "message": "Select ocean model for input restarts:", + "message": "Select ocean model that matches input restarts:", "choices": ["data", "MOM5", "MOM6"], "default": "data", "when": lambda x: not x['input:shared:MERRA-2'] @@ -64,20 +91,36 @@ def ask_questions(): { "type": "select", "name": "input:shared:ogrid", - "message": "Select Data Ocean Grid for input restarts:", + "message": "Select data ocean grid that matches input restarts:", "choices": ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'], "default": lambda x: data_ocean_default(x.get('input:shared:agrid')), "when": lambda x: x.get('input:shared:model') == 'data' and not x['input:shared:MERRA-2'], }, + { + "type": "text", + # remove the comments + "name": "input:shared:ogrid", + "message": "remove the comment of ogrid", + # always return false, so it never shows + "when": lambda x: remove_ogrid_comment(x, 'IN') + }, + { "type": "select", "name": "input:shared:ogrid", - "message": "Select Coupled (MOM5, MOM6) Ocean Grid for input restarts:", + "message": "Select coupled (MOM5, MOM6) ocean grid that matches input restarts:", "choices": ['72x36','360x200','720x410','1440x1080'], "when": lambda x: x.get('input:shared:model') == 'MOM5' or x.get('input:shared:model')== 'MOM6' }, + { + "type": "text", + "name": "output:shared:agrid", + "message": "Enter new atmospheric grid: \n C12 C180 C1000 C270\n C24 C360 C1440 C540\n C48 C500 C2880 C1080\n C90 C720 C5760 C2160 C1536\n ", + "default": 'C360', + }, + { "type": "select", "name": "output:shared:model", @@ -88,27 +131,27 @@ def ask_questions(): { "type": "select", "name": "output:shared:ogrid", - "message": "Select Data Ocean Grid for new restarts:", + "message": "Select data ocean grid for new restarts:", "choices": ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'], "default": lambda x: data_ocean_default(x.get('output:shared:agrid')), "when": lambda x: x['output:shared:model'] == 'data', }, + { + "type": "text", + # remove the comments + "name": "output:shared:ogrid", + "message": "remove the comment of ogrid", + # always return false, so it never shows + "when": lambda x: remove_ogrid_comment(x, 'OUT') + }, { "type": "select", "name": "output:shared:ogrid", - "message": "Select Couple Ocean Grid for new restarts:", + "message": "Select couple ocean grid for new restarts:", "choices": ['72x36','360x200','720x410','1440x1080'], "when": lambda x: x['output:shared:model'] != 'data', }, - - { - "type": "text", - "name": "output:shared:agrid", - "message": "Enter new atmospheric grid: \n C12 C180 C1000 C270\n C24 C360 C1440 C540\n C48 C500 C2880 C1080\n C90 C720 C5760 C2160 C1536\n ", - "default": 'C360', - }, - { "type": "text", "name": "output:air:nlevel", @@ -116,33 +159,27 @@ def ask_questions(): "default": "72", }, - - { "type": "text", "name": "input:shared:tag", - "message": "Enter GCM or DAS tag for input restarts: \n \ + "message": "Enter BC version (GCM or DAS tag) that matches input restarts: \n \ \n \ -Sample GCM tags for legacy BCs:\n \ +BC version: GCM or DAS tag \n \ --------------- \n \ -G40 : Ganymed-4_0 ......... Heracles-5_4_p3 \n \ -ICA : Icarus .............. Jason \n \ -GITOL : 10.3 ................ 10.18 \n \ -INL : Icarus-NL ........... Jason-NL \n \ -GITNL : 10.19 ............... 10.23 \n \ +G40 : Ganymed-4_0 ......... Heracles-5_4_p3 \n \ +ICA : Icarus .............. Jason \n \ +GITOL : 10.3 ................ 10.18 \n \ +INL : Icarus-NL ........... Jason-NL \n \ +GITNL : 10.19 ............... 10.23 \n \ \n \ -Sample DAS tags for legacy BCs:\n \ ---------------- \n \ -5B0 : GEOSadas-5_10_0_p2 .. GEOSadas-5_11_0 \n \ -512 : GEOSadas-5_12_2 ..... GEOSadas-5_16_5\n \ -517 : GEOSadas-5_17_0 ..... GEOSadas-5_24_0_p1\n \ -525 : GEOSadas-5_25_1 ..... GEOSadas-5_29_4\n \ +5B0 : GEOSadas-5_10_0_p2 .. GEOSadas-5_11_0 \n \ +512 : GEOSadas-5_12_2 ..... GEOSadas-5_16_5\n \ +517 : GEOSadas-5_17_0 ..... GEOSadas-5_24_0_p1\n \ +525 : GEOSadas-5_25_1 ..... GEOSadas-5_29_4\n \ \n \ -Sample BC version for new structure BC base): \n \ --------------- \n \ -NL3 : Newland version 3 \n \ -NL4 : Newland version 4 \n \ -NL5 : Newland version 5 \n \ +NL3 : Newland version 3 \n \ +NL4 : Newland version 4 \n \ +NL5 : Newland version 5 \n \ v06, v07, v08, v09: Not generated yet \n", "default": "INL", @@ -160,50 +197,41 @@ def ask_questions(): { "type": "text", "name": "output:shared:tag", - "message": "Enter GCM or DAS tag for new restarts: \n \ + "message": " Enter BC version (GCM or DAS tag) for new restarts: \n \ \n \ -Sample GCM tags for legacy BCs:\n \ +BC version: GCM or DAS tag \n \ --------------- \n \ -G40 : Ganymed-4_0 ......... Heracles-5_4_p3 \n \ -ICA : Icarus .............. Jason \n \ -GITOL : 10.3 ................ 10.18 \n \ -INL : Icarus-NL ........... Jason-NL \n \ -GITNL : 10.19 ............... 10.23 \n \ +G40 : Ganymed-4_0 ......... Heracles-5_4_p3 \n \ +ICA : Icarus .............. Jason \n \ +GITOL : 10.3 ................ 10.18 \n \ +INL : Icarus-NL ........... Jason-NL \n \ +GITNL : 10.19 ............... 10.23 \n \ \n \ -Sample DAS tags for legacy BCs:\n \ ---------------- \n \ -5B0 : GEOSadas-5_10_0_p2 .. GEOSadas-5_11_0 \n \ -512 : GEOSadas-5_12_2 ..... GEOSadas-5_16_5\n \ -517 : GEOSadas-5_17_0 ..... GEOSadas-5_24_0_p1\n \ -525 : GEOSadas-5_25_1 ..... GEOSadas-5_29_4\n \ +5B0 : GEOSadas-5_10_0_p2 .. GEOSadas-5_11_0 \n \ +512 : GEOSadas-5_12_2 ..... GEOSadas-5_16_5\n \ +517 : GEOSadas-5_17_0 ..... GEOSadas-5_24_0_p1\n \ +525 : GEOSadas-5_25_1 ..... GEOSadas-5_29_4\n \ \n \ -Sample BC version for new structure BC base): \n \ --------------- \n \ -NL3 : Newland version 3 \n \ -NL4 : Newland version 4 \n \ -NL5 : Newland version 5 \n \ +NL3 : Newland version 3 \n \ +NL4 : Newland version 4 \n \ +NL5 : Newland version 5 \n \ v06, v07, v08, v09: Not generated yet \n", + "default": "GITNL", "when": lambda x: x["input:shared:MERRA-2"], }, { "type": "path", - "name": "input:shared:altbcs", - "message": "Enter nothing (default) or specify your own absolute bcs path for input restarts: \n\n \ -It expects the sub-structure of the path to be the same as one of the three paths that match the tag. \n \ -/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles (structure after NL3 ) \n \ -/discover/nobackup/projects/gmao/ssd/aogcm/atmosphere_bcs ( coupled model) \n \ -/discover/nobackup/projects/gmao/bcs_shared/legacy_bcs ( legacy structure)\n", - "default": "", - "when": lambda x: not x.get("input:shared:MERRA-2"), + "name": "input:shared:bcs_dir", + "message": "Is this BCS matching input restarts? If no, enter your own absolute path: \n", + "default": lambda x: get_bcsdir(x, "IN"), }, - { "type": "path", - "name": "output:shared:altbcs", - "message": "Enter nothing (default) or specify your own absolute bcs path for new restarts: \n ", - "default": "", + "name": "output:shared:bcs_dir", + "message": "Is this BCS for new restarts? If no, enter your own absolute path: \n", + "default": lambda x: get_bcsdir(x, "OUT"), }, { @@ -212,6 +240,7 @@ def ask_questions(): "message": "Would you like to remap upper air?", "default": True, }, + { "type": "confirm", "name": "output:surface:remap", @@ -243,19 +272,19 @@ def ask_questions(): { "type": "text", "name": "input:surface:wemin", - "message": "What is value of Wemin?", + "message": "What is value of wemin (minimum snow water equivalent parameter) for surface inputs?", "default": lambda x: we_default(x.get('input:shared:tag')) }, { "type": "text", "name": "output:surface:wemin", - "message": "What is value of Wemout?", + "message": "What is value of wemin (minimum snow water equivalent parameter) for new surface restarts?", "default": lambda x: we_default(x.get('output:shared:tag')) }, { "type": "text", "name": "input:surface:zoom", - "message": "What is value of zoom [1-8]?", + "message": "What is value of zoom (paremeter of radius search, smaller value means larger radius) for surface inputs [1-8]?", "default": lambda x: zoom_default(x) }, { @@ -292,26 +321,12 @@ def ask_questions(): }, ] answers = questionary.prompt(questions) - if not answers.get('input:shared:model') : - answers['input:shared:model'] = 'data' - if answers.get('input:shared:ogrid'): - answers['input:shared:ogrid'] = answers['input:shared:ogrid'].split()[0] - if answers.get('output:shared:ogrid'): - answers['output:shared:ogrid'] = answers['output:shared:ogrid'].split()[0] - - if answers['input:shared:MERRA-2']: - answers['input:shared:rst_dir'] = tmp_merra2_dir(answers) - answers['input:shared:altbcs'] ='' - - answers['input:shared:rst_dir'] = os.path.abspath(answers['input:shared:rst_dir']) - if answers.get('output:shared:ogrid') == 'CS': - answers['output:shared:ogrid'] = answers['output:shared:agrid'] - answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) + answers['input:shared:rst_dir'] = os.path.abspath(answers['input:shared:rst_dir']) + answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) return answers - if __name__ == "__main__": answers = ask_questions() cmdl = get_command_line_from_answers(answers) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 1608f1b..fa3a5c3 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -9,6 +9,8 @@ import glob import shlex +# define some constants for bcs + NewStructureBCTag = ('NL3', 'NL4', 'NL5', 'v06', 'v07', 'v08', 'v09') BCBase={} BCBase['discover_ops'] = "/discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs" @@ -16,29 +18,235 @@ BCBase['discover_couple'] = "/discover/nobackup/projects/gmao/ssd/aogcm/atmosphere_bcs" BCBase['discover_ns'] = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" +BCSTag = {} +TagsRank = {} +tag_initialized = False + +def init_tags(): + # copy and paste from rigrid.pl + # minor change. Add "D" to the number for each group + # BCS Tag: Fortuna-1_4 + global tag_initialized + if tag_initialized : return + F14 = ( 'F14', 'Fortuna-1_4', 'Fortuna-1_4_p1' ) + D214 = ( 'D214', 'GEOSdas-2_1_4', 'GEOSdas-2_1_4-m1', + 'GEOSdas-2_1_4-m2', 'GEOSdas-2_1_4-m3', 'GEOSdas-2_1_4-m4' ) + D540 = ( 'D540', 'GEOSadas-5_4_0', 'GEOSadas-5_4_0_p1', + 'GEOSadas-5_4_0_p2', 'GEOSadas-5_4_0_p3', 'GEOSadas-5_4_0_p4', + 'GEOSadas-5_4_1', 'GEOSadas-5_4_1_p1', 'GEOSadas-5_4_2', + 'GEOSadas-5_4_3', 'GEOSadas-5_4_4', 'GEOSadas-5_5_0', + 'GEOSadas-5_5_1', 'GEOSadas-5_5_2', 'GEOSadas-5_5_3' ) + + # BCS Tag: Fortuna-2_0 + #--------------------- + F20 = ( 'F20', 'Fortuna-2_0') + + # BCS Tag: Fortuna-2_1 + #--------------------- + F21 = ( 'F21', 'Fortuna-2_1', 'Fortuna-2_1_p1', + 'Fortuna-2_1_p2', 'Fortuna-2_1_p3', 'Fortuna-2_2', + 'Fortuna-2_2_p1', 'Fortuna-2_2_p2', 'Fortuna-2_3', + 'Fortuna-2_3_p1', 'Fortuna-2_4', 'Fortuna-2_4_p1', + 'Fortuna-2_4_p2', 'Fortuna-2_5', 'Fortuna-2_5_BETA0', + 'Fortuna-2_5_p1', 'Fortuna-2_5_p2', 'Fortuna-2_5_p3', + 'Fortuna-2_5_p4', 'Fortuna-2_5_p5', 'Fortuna-2_5_p6', + 'Fortuna-2_5_pp2' ) + D561 = ( 'D561', 'GEOSadas-5_6_1', 'GEOSadas-5_6_1_p1', + 'GEOSadas-5_6_1_p2', 'GEOSadas-5_6_1_p3', 'GEOSadas-5_6_1_p4', + 'GEOSadas-5_6_2', 'GEOSadas-5_6_2_p1', 'GEOSadas-5_6_2_p2', + 'GEOSadas-5_6_2_p3', 'GEOSadas-5_6_2_p4', 'GEOSadas-5_6_2_p5', + 'GEOSadas-5_6_2_p6', 'GEOSadas-5_7_1', 'GEOSadas-5_7_1_p1', + 'GEOSadas-5_7_1_p2', 'GEOSadas-5_7_2', 'GEOSadas-5_7_2_p1', + 'GEOSadas-5_7_2_p2', 'GEOSadas-5_7_2_p2_m1','GEOSadas-5_7_2_p3', + 'GEOSadas-5_7_2_p3_m1', 'GEOSadas-5_7_2_p3_m2','GEOSadas-5_7_2_p4', + 'GEOSadas-5_7_2_p5', 'GEOSadas-5_7_2_p5_m1','GEOSadas-5_7_3', + 'GEOSadas-5_7_3_p1', 'GEOSadas-5_7_3_p2', 'GEOSadas-5_7_3_p2' ) + + # BCS Tag: Ganymed-1_0 + #--------------------- + G10 = ( 'G10', 'Ganymed-1_0', 'Ganymed-1_0_BETA', + 'Ganymed-1_0_BETA1', 'Ganymed-1_0_BETA2', 'Ganymed-1_0_BETA3', + 'Ganymed-1_0_BETA4' ) + + D580 = ( 'D580', 'GEOSadas-5_8_0', 'GEOSadas-5_9_0', + 'GEOSadas-5_9_1' ) + + # BCS Tags: Ganymed-1_0_M and Ganymed-1_0_D + #------------------------------------------ + G10p = ( 'G10p', 'Ganymed-1_0_p1', 'Ganymed-1_0_p2', + 'Ganymed-1_0_p3', 'Ganymed-1_0_p4', 'Ganymed-1_0_p5', + 'Ganymed-1_0_p6' ) + + D591p= ( 'D591p', 'GEOSadas-5_9_1_p1', 'GEOSadas-5_9_1_p2', + 'GEOSadas-5_9_1_p3', 'GEOSadas-5_9_1_p4', 'GEOSadas-5_9_1_p5', + 'GEOSadas-5_9_1_p6', 'GEOSadas-5_9_1_p7', 'GEOSadas-5_9_1_p8', + 'GEOSadas-5_9_1_p9' ) + + # BCS Tags: Ganymed-1_0_M and Ganymed-1_0_D w/ new landice rst + #------------------------------------------------------------------------ + G20 = ( 'G20', 'Ganymed-2_0', 'Ganymed-2_1', + 'Ganymed-2_1_p1', 'Ganymed-2_1_p2', 'Ganymed-2_1_p3', + 'Ganymed-2_1_p4', 'Ganymed-2_1_p5', 'Ganymed-2_1_p6' ) + D5A0 = ( 'D5A0', 'GEOSadas-5_10_0', 'GEOSadas-5_10_0_p1' ) + + + # BCS Tags: Ganymed-1_0_Reynolds and Ganymed-1_0_Ostia + #----------------------------------------------------- + G30 = ( 'G30', 'Ganymed-3_0', 'Ganymed-3_0_p1' ) + D5B0 = ( '5B0', 'GEOSadas-5_10_0_p2', 'GEOSadas-5_11_0' ) + + # BCS Tags: Ganymed-4_0_Reynolds, Ganymed-4_0_MERRA-2, and Ganymed-4_0_Ostia + #--------------------------------------------------------------------------- + G40 = ( 'G40', 'Ganymed-4_0', 'Ganymed-4_0_p1', + 'Ganymed-4_1', 'Heracles-1_0', 'Heracles-1_1', + 'Heracles-2_0', 'Heracles-2_1', 'Heracles-3_0', + 'Heracles-4_0', 'Heracles-5_4_p3' ) + D512 = ( '512', 'GEOSadas-5_12_2', 'GEOSadas-5_12_4', + 'GEOSadas-5_12_4_p1', 'GEOSadas-5_12_4_p2', 'GEOSadas-5_12_4_p3', + 'GEOSadas-5_12_5', 'GEOSadas-5_13_0_p1', 'GEOSadas-5_13_0_p2', + 'GEOSadas-5_13_1', 'GEOSadas-5_16_5' ) + + # BCS Tags: Icarus (New Land Parameters, New Topography) + #--------------------------------------------------------------------------- + ICA = ( 'ICA', 'Icarus', 'Jason' ) + D517 = ( '517', 'GEOSadas-5_17_0', 'GEOSadas-5_17_1', 'GEOSadas-5_18_0', + 'GEOSadas-5_18_1', 'GEOSadas-5_18_2', 'GEOSadas-5_18_3', + 'GEOSadas-5_18_3_p1', 'GEOSadas-5_19_0', 'GEOSadas-5_20_0', + 'GEOSadas-5_20_0_p1', 'GEOSadas-5_20_0_p2', 'GEOSadas-5_21_0', + 'GEOSadas-5_21_2', 'GEOSadas-5_21_3_p1', 'GEOSadas-5_22_0', + 'GEOSadas-5_22_0_p1', 'GEOSadas-5_22_0_p2', 'GEOSadas-5_23_0', + 'GEOSadas-5_23_0_p1', 'GEOSadas-5_24_0', 'GEOSadas-5_24_0_p1' ) + GITOL = ( 'GITOL', '10.3', '10.4', '10.5', + '10.6', '10.7', '10.8', + '10.9', '10.10', '10.11', + '10.12', '10.13', '10.14', + '10.15', '10.16', '10.17', + '10.18' ) + + # BCS Tags: Icarus-NLv3 (New Land Parameters) + #--------------------------------------------------------------------------- + INL = ( 'INL', 'Icarus-NL', 'Icarus-NLv3', 'Jason-NL' ) + GITNL = ( 'GITNL', '10.19', '10.20', '10.21', '10.22', '10.23' ) + D525 = ( '525', 'GEOSadas-5_25_1', 'GEOSadas-5_25_1_p5', 'GEOSadas-5_25_p7', + 'GEOSadas-5_27_1', 'GEOSadas-5_29_3', 'GEOSadas-5_29_4' ) + + for tag in F14: BCSTag[tag]= "Fortuna-1_4" + for tag in F20: BCSTag[tag]= "Fortuna-2_0" + for tag in F21: BCSTag[tag]= "Fortuna-2_1" + for tag in G10: BCSTag[tag]= "Ganymed-1_0" + for tag in G10p: BCSTag[tag]= "Ganymed-1_0_M" + for tag in G20: BCSTag[tag]= "Ganymed-1_0_M" + for tag in G30: BCSTag[tag]= "Ganymed-1_0_Reynolds" + for tag in G40: BCSTag[tag]= "Ganymed-4_0_Reynolds" + for tag in ICA: BCSTag[tag]= "Icarus_Reynolds" + for tag in GITOL: BCSTag[tag]= "Icarus_Reynolds" + for tag in INL: BCSTag[tag]= "Icarus-NLv3_Reynolds" + for tag in GITNL: BCSTag[tag]= "Icarus-NLv3_Reynolds" + for tag in NewStructureBCTag: BCSTag[tag]= tag + + + for tag in D214: BCSTag[tag]= "Fortuna-1_4" + for tag in D540: BCSTag[tag]= "Fortuna-1_4" + for tag in D561: BCSTag[tag]= "Fortuna-2_1" + for tag in D580: BCSTag[tag]= "Ganymed-1_0" + for tag in D591p: BCSTag[tag]= "Ganymed-1_0_M" + for tag in D5A0: BCSTag[tag]= "Ganymed-1_0_M" + for tag in D5B0: BCSTag[tag]= "Ganymed-1_0_Reynolds" + for tag in D512: BCSTag[tag]= "Ganymed-4_0_Reynolds" + for tag in D517: BCSTag[tag]= "Icarus_Reynolds" + for tag in D525: BCSTag[tag]= "Icarus-NLv3_Reynolds" + + TagsRank['Fortuna-1_4'] = 1 + TagsRank['Fortuna-1_5'] = 2 + TagsRank['Fortuna-2_0'] = 3 + TagsRank['Fortuna-2_1'] = 4 + TagsRank['Ganymed-1_0'] = 5 + TagsRank['Ganymed-1_0_m1'] = 6 + TagsRank['Ganymed-1_0_m2'] = 7 + TagsRank['Ganymed-1_0_M'] = 8 + TagsRank['Ganymed-1_0_D'] = 9 + TagsRank['Ganymed-1_0_Reynolds'] = 10 + TagsRank['Ganymed-1_0_Ostia'] = 11 + TagsRank['Ganymed-4_0_Reynolds'] = 12 + TagsRank['Ganymed-4_0_Ostia'] = 13 + TagsRank['Ganymed-4_0_MERRA-2'] = 14 + TagsRank['Icarus_Reynolds'] = 15 + TagsRank['Icarus_MERRA-2'] = 16 + TagsRank['Icarus_Ostia'] = 17 + TagsRank['Icarus-NLv3_Reynolds'] = 18 + TagsRank['Icarus-NLv3_MERRA-2'] = 19 + TagsRank['Icarus-NLv3_Ostia'] = 20 + for tag in NewStructureBCTag: TagsRank[tag] = 21 + tag_initialized = True + +def init_merra2(x): + if not x.get('input:shared:MERRA-2') : return False + + yyyymm = int(x.get('input:shared:yyyymmddhh')[0:6]) + if yyyymm < 197901 : + exit("Error. MERRA-2 data < 1979 not available\n") + elif (yyyymm < 199201): + expid = "d5124_m2_jan79" + elif (yyyymm < 200106): + expid = "d5124_m2_jan91" + elif (yyyymm < 201101): + expid = "d5124_m2_jan00" + elif (yyyymm < 202106): + expid = "d5124_m2_jan10" + # There was a rewind in MERRA2 from Jun 2021 to Sept 2021 + elif (yyyymm < 202110): + expid = "d5124_m2_jun21" + else: + expid = "d5124_m2_jan10" + x['input:shared:expid'] = expid + x['input:shared:model'] = 'data' + x['input:shared:agrid'] = 'C180' + x['input:shared:ogrid'] = '1440x720' + x['input:shared:tag'] = 'Ganymed-4_0' + x['input:surface:catch_model'] = 'catch' + x['input:shared:rst_dir'] = x['output:shared:out_dir'] + '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' + + return False + def fvcore_name(x): ymdh = x['input:shared:yyyymmddhh'] time = ymdh[0:8] + '_'+ymdh[8:10] rst_dir = x.get('input:shared:rst_dir') if not rst_dir : return False + x['input:shared:rst_dir'] = rst_dir.strip() # remove extra space + files = glob.glob(rst_dir+'/*fvcore_*') + + if len (files) == 0 : return False + if len(files) == 1: fname = files[0] - print('\nFound ' + fname) - return fname if len(files) > 1 : files = glob.glob(rst_dir+'/*fvcore_*'+time+'*') fname = files[0] - print('\nFound ' + fname) - return fname - - return False - -def tmp_merra2_dir(x): - tmp_merra2 = x['output:shared:out_dir']+ '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' - return tmp_merra2 + fvrst = os.path.dirname(os.path.realpath(__file__)) + '/fvrst.x -h ' + cmd = fvrst + fname + #print(cmd +'\n') + p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + (output, err) = p.communicate() + p_status = p.wait() + ss = output.decode().split() + lat = int(ss[0]) + lon = int(ss[1]) + if (lon != lat*6) : + sys.exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') + ymdh = x.get('input:shared:yyyymmddhh') + ymdh_ = str(ss[3]) + str(ss[4])[0:2] + if (ymdh_ != ymdh) : + print("Warning: The date in fvcore is " + ymdh_ ) + x['input:shared:agrid'] = "C"+ss[0] # save for air parameter + print("The input fvcore restart has air grid: " + "C" + ss[0] + '\n') + expid = os.path.basename(fname).split('fvcore')[0] + expid = expid[0:-1] + x['input:shared:expid'] = expid + return fname def data_ocean_default(resolution): # the default string should match the choice in remapl_question.py @@ -61,24 +269,9 @@ def we_default(tag): def zoom_default(x): zoom_ = '8' - fvcore = fvcore_name(x) - if fvcore : - fvrst = os.path.dirname(os.path.realpath(__file__)) + '/fvrst.x -h ' - cmd = fvrst + fvcore - print(cmd +'\n') - p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - (output, err) = p.communicate() - p_status = p.wait() - ss = output.decode().split() - x['input:shared:agrid'] = "C"+ss[0] # save for air parameter - lat = int(ss[0]) - lon = int(ss[1]) - if (lon != lat*6) : - sys.exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') - ymdh = x.get('input:shared:yyyymmddhh') - ymdh_ = str(ss[3]) + str(ss[4])[0:2] - if (ymdh_ != ymdh) : - print("Warning: The date in fvcore is different from the date you input\n") + cxx = x.get('input:shared:agrid') + if cxx : + lat = int(cxx[1:]) zoom = lat /90.0 zoom_ = str(int(zoom)) if zoom < 1 : zoom_ = '1' @@ -136,29 +329,6 @@ def print_config( config, indent = 0 ): else: print(" " * indent, f"{k}: {v}") -def merra2_expid(config): - if not config['MERRA-2']: - return config - yyyymm = int(config.get('yyyymmddhh')[0:6]) - if yyyymm < 197901 : - exit("Error. MERRA-2 data < 1979 not available\n") - elif (yyyymm < 199201): - expid = "d5124_m2_jan79" - elif (yyyymm < 200106): - expid = "d5124_m2_jan91" - elif (yyyymm < 201101): - expid = "d5124_m2_jan00" - elif (yyyymm < 202106): - expid = "d5124_m2_jan10" - # There was a rewind in MERRA2 from Jun 2021 to Sept 2021 - elif (yyyymm < 202110): - expid = "d5124_m2_jun21" - else: - expid = "d5124_m2_jan10" - config['expid'] = expid - - return config - def get_command_line_from_answers(answers): merra2 = " -merra2 " if answers["input:shared:MERRA-2"] else "" @@ -194,19 +364,23 @@ def get_command_line_from_answers(answers): label = ' -lbl ' if answers["output:shared:label"] else "" - in_altbcs = '' - out_altbcs = '' - if answers.get("input:shared:altbcs", "").strip() : - in_altbcs = " -in_altbcs " + answers["input:shared:altbcs"] - if answers.get("output:shared:altbcs", "").strip() : - out_altbcs = " -out_altbcs " + answers["output:shared:altbcs"] + in_bcsdir = '' + default_bc = get_bcsdir(answers, "IN") + if not os.path.samefile(default_bc, answers.get("input:shared:bcs_dir").strip()) : + in_bcsdir = ' -in_bcsdir ' + answers.get("input:shared:bcs_dir") + + out_bcsdir = '' + default_bc = get_bcsdir(answers, "OUT") + if not os.path.samefile(default_bc, answers.get("output:shared:bcs_dir").strip()) : + out_bcsdir = ' -out_bcsdir ' + answers.get("output:shared:bcs_dir") zoom = " -zoom " + answers["input:surface:zoom"] - wemin = " -wemin " + answers["input:surface:wemin"] - wemout = " -wemout " + answers["output:surface:wemin"] + wemin = " -in_wemin " + answers["input:surface:wemin"] + wemout = " -out_wemin " + answers["output:surface:wemin"] catch_model ='' if answers.get("input:surface:catch_model"): catch_model = " -catch_model " + answers["input:surface:catch_model"] + out_rs = " -rs " rs = 3 if answers['output:air:remap'] and not answers['output:surface:remap']: @@ -233,8 +407,8 @@ def get_command_line_from_answers(answers): tagout + \ rst_dir + \ out_dir + \ - in_altbcs + \ - out_altbcs + \ + in_bcsdir + \ + out_bcsdir + \ catch_model + \ zoom + \ wemin + \ @@ -270,6 +444,135 @@ def get_config_from_answers(answers): return config +def get_bcTag(tag, ogrid): + bctag = BCSTag[tag] + if bctag in NewStructureBCTag : return bctag + if ogrid[0].upper() == "C": + bctag=bctag.replace('_Reynolds','_Ostia') + else: + xy = ogrid.upper().split('X') + x = int(xy[0]) + if x == 1440: bctag=bctag.replace('_Reynolds','_MERRA-2') + if x == 2880: + bctag=bctag.replace('_Reynolds','_Ostia') + bctag=bctag.replace('_M','_D') + return bctag + +def get_grid_subdir(bcdir, agrid, ogrid, model): + def get_name_with_grid( grid, names, a_o): + if not grid : + return names + namex = [] + if (grid[0].upper() == 'C'): + n = int(grid[1:]) + s1 ='{:04d}x6C'.format(n) + j=n*6 + s2 =str(n) + s3 =str(j) + # first try + for aoname in names: + name = '' + if(a_o == 'a'): + name = aoname.split('_')[0] + else: + name = aoname.split('_')[-1] + if (name.find(s1) != -1 or (name.find(s2) != -1 and name.find(s3) != -1 )): + namex.append(aoname) + else: + xy = grid.upper().split('X') + s2 = xy[0] + s3 = xy[1] + for aoname in names: + name = '' + if(a_o == 'a'): + name = aoname.split('_')[0] + else: + name = aoname.split('_')[-1] + if (name.find(s2) != -1 and name.find(s3) != -1): namex.append(aoname) + return namex + #v3.5 + #dirnames = [ f.name for f in os.scandir(bcdir) if f.is_dir()] + #v2.7 + dirnames = [f for f in os.listdir(bcdir) if os.path.isdir(os.path.join(bcdir,f))] + anames = get_name_with_grid(agrid, dirnames, 'a') + gridID = get_name_with_grid(ogrid, anames, 'o') + if len(gridID) == 0 : + exit("cannot find the grid subdirctory of agrid: " +agrid+ " and ogrid " + ogrid + " under "+ bcdir) + g = '' + gridID.sort(key=len) + g = gridID[0] + + # For new structure BC + for g_ in gridID : + if model == 'MOM5': + if 'M5' in g_ : g = g_ + if model == 'MOM6': + if 'M6' in g_ : g = g_ + + if len(gridID) >= 2 : + print("\n Warning! Find many GridIDs in " + bcdir) + print(" GridIDs found: ", gridID) + #WY note, found many string in the directory + print(" This GridID is chosen: " + g) + return g + +def get_bcsdir(x, opt): + init_tags() + tag = x.get('input:shared:tag') + agrid = x.get('input:shared:agrid') + ogrid = x.get('input:shared:ogrid') + model = x.get('input:shared:model') + if opt.upper() == "OUT": + tag = x.get('output:shared:tag') + agrid = x.get('output:shared:agrid') + ogrid = x.get('output:shared:ogrid') + model = x.get('output:shared:model') + + bcdir = '' + base = 'discover_legacy' + if model != "data": + base = 'discover_couple' + if tag in NewStructureBCTag: + base = 'discover_ns' + if x.get('input:shared:MERRA-2'): + base = 'discover_ops' + + bc_base = BCBase[base] + + bctag = get_bcTag(tag,ogrid) + tagrank = TagsRank[bctag] + + if (tagrank >= TagsRank['NL3']) : + bcdir = bc_base+'/'+ bctag+'/geometry/' + elif (tagrank >= TagsRank['Icarus-NLv3_Reynolds']) : + bcdir = bc_base+'/Icarus-NLv3/'+bctag+'/' + if model == 'MOM6' or model == 'MOM5': + bcdir = bc_base+'/Icarus-NLv3/'+model+'/' + elif (tagrank >= TagsRank['Icarus_Reynolds']): + if bc_base == BCBase['discover_ops']: + bcdir = bc_base+'/Icarus_Updated/'+bctag+'/' + else: + bcdir = bc_base+'/Icarus/'+bctag+'/' + if model == 'MOM6' or model == 'MOM5': + bcdir = bc_base+'/Icarus/'+model+'/' + elif(tagrank >= TagsRank["Ganymed-4_0_Reynolds"]): + bcdir = bc_base + '/Ganymed-4_0/'+bctag+'/' + if model == 'MOM6' or model == 'MOM5': + bcdir = bc_base+'/Ganymed/'+model+'/' + else: + bcdir = bc_base + '/' + bctag + '/' + if model == 'MOM6' or model == 'MOM5': + bcdir = bc_base+'/Ganymed/'+model+'/' + + if not os.path.exists(bcdir): + exit("Cannot find bc dir " + bcdir) + + + gridStr = get_grid_subdir(bcdir,agrid, ogrid, model) + bcdir = bcdir + gridStr + + return bcdir + if __name__ == '__main__' : config = yaml_to_config('c24Toc12.yaml') From b4f5c5f290095a6b8471eba56579c7696788c769 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Fri, 26 May 2023 11:36:21 -0400 Subject: [PATCH 11/88] convert merra2 binary files to nc4 before regrid --- pre/remap_restart/remap_bin2nc.py | 172 ++++++++++++++++++ pre/remap_restart/remap_fv.yaml | 67 +++++++ pre/remap_restart/remap_gocart.yaml | 49 +++++ pre/remap_restart/remap_lake.yaml | 42 +++++ .../remap_lake_landice_saltwater.py | 11 +- pre/remap_restart/remap_landice.yaml | 68 +++++++ pre/remap_restart/remap_moist.yaml | 56 ++++++ pre/remap_restart/remap_params.py | 2 - pre/remap_restart/remap_pchem.yaml | 49 +++++ pre/remap_restart/remap_salt.yaml | 81 +++++++++ pre/remap_restart/remap_upper.py | 17 +- 11 files changed, 605 insertions(+), 9 deletions(-) create mode 100755 pre/remap_restart/remap_bin2nc.py create mode 100644 pre/remap_restart/remap_fv.yaml create mode 100644 pre/remap_restart/remap_gocart.yaml create mode 100644 pre/remap_restart/remap_lake.yaml create mode 100644 pre/remap_restart/remap_landice.yaml create mode 100644 pre/remap_restart/remap_moist.yaml create mode 100644 pre/remap_restart/remap_pchem.yaml create mode 100644 pre/remap_restart/remap_salt.yaml diff --git a/pre/remap_restart/remap_bin2nc.py b/pre/remap_restart/remap_bin2nc.py new file mode 100755 index 0000000..7107b50 --- /dev/null +++ b/pre/remap_restart/remap_bin2nc.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python + +from yaml import safe_load, load, dump +from netCDF4 import Dataset +import numpy as np +from scipy.io import FortranFile + +def writeCVars(coordVars,dimensions): + + for v in coordVars: + if v=='time': + coordVars[v][:]=0 + elif (v=='lat') | (v=='lon') | (v=='lev') | (v=='edges'): + dsize=dimensions[v] + coords=np.arange(1,dsize+1) + coordVars[v][:]=coords + +def writeVar(v, vars, dimensions, bintype, binf): + var=vars[v['short_name']] + dims = v['dimension'] + # is tile? + if 'tile' in dims: + if ('subtile' in dims) & (len(dims)==2): + rec=binf.read_reals(dtype=bintype) + tilesize=dimensions.get('tile') + for i in range(dimensions.get('subtile')): + var[i,:]=rec[i*tilesize:(i+1)*tilesize] + elif (len(dims)==2) & ( ('unknown_dim1' in dims) | ('unknown_dim2' in dims) ): + if 'unknown_dim1' in dims: + dsize=dimensions.get('unknown_dim1') + if 'unknown_dim2' in dims: + dsize=dimensions.get('unknown_dim2') + for i in range(dsize): + rec = binf.read_reals(dtype=bintype) + var[i,:]=rec + elif (len(dims)==3) & ('unknown_dim1' in dims) & ('unknown_dim2' in dims): + for j in range(dimensions.get('unknown_dim2')): + for i in range(dimensions.get('unknown_dim1')): + rec = binf.read_reals(dtype=bintype) + var[j,i,:]=rec + elif (len(dims)==1): + rec = binf.read_reals(dtype=bintype) + var[:]=rec + + # is gridded + elif 'lat' in dims: + if 'lev' in dims: + for i in range(dimensions.get('lev')): + rec = binf.read_reals(dtype=bintype) + var[i,:,:]=rec + elif 'edges' in dims: + for i in range(dimensions.get('edges')): + rec = binf.read_reals(dtype=bintype) + var[i,:,:]=rec + else: + rec = binf.read_reals(dtype=bintype) + var[:,:]=rec + # is just lev + else: + if 'edges' in dims: + rec = binf.read_reals(dtype=bintype) + var[:]=rec + +def defineDimVars(fid,dims): + coordVars = {} + for d in dims: + if d=="lon": + newVar=fid.createVariable(d,'f8',d) + setattr(newVar,'units','degrees_east') + setattr(newVar,'long_name','Longitude') + coordVars.update([(d,newVar)]) + if d=="lat": + newVar=fid.createVariable(d,'f8',d) + setattr(newVar,'units','degrees_north') + setattr(newVar,'long_name','Latitude') + coordVars.update([(d,newVar)]) + if d=="lev": + newVar=fid.createVariable(d,'f8',d) + setattr(newVar,'units','layer') + setattr(newVar,'long_name','sigma_at_layer_midpoints') + setattr(newVar,'standard_name','atmosphere_hybrid_sigma_pressure_coordinate') + setattr(newVar,'coordinate','eta') + setattr(newVar,'positive','down') + setattr(newVar,'formulaTerms','ap: ak b: bk ps: ps p0: p00') + coordVars.update([(d,newVar)]) + if d=="edges": + newVar=fid.createVariable(d,'f8',d) + setattr(newVar,'units','level') + setattr(newVar,'long_name','sigma_at_layer edges') + setattr(newVar,'standard_name','atmosphere_hybrid_sigma_pressure_coordinate') + setattr(newVar,'coordinate','eta') + setattr(newVar,'positive','down') + setattr(newVar,'formulaTerms','ap: ak b: bk ps: ps p0: p00') + coordVars.update([(d,newVar)]) + if d=="time": + newVar=fid.createVariable(d,'f8',d) + setattr(newVar,'units','minutes since ') + setattr(newVar,'long_name','time') + coordVars.update([(d,newVar)]) + + return coordVars + +def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=False): + file = yamlfile + hdr = hasHeader + + bintype=np.float32 + if (isDouble): + bintype=np.float64 + + + f=open(file,'r') + + data=safe_load(f) + + dimensions = data["dimensions"] + variables = data["variables"] + + if debug: + print(dump(variables)) + print(dump(dimensions)) + + ncfid = Dataset(ncfile,mode='w',format='NETCDF4') + + dimids = {} + for d in dimensions: + dsize = dimensions.get(d) + newDim = ncfid.createDimension(d,dsize) + dimids.update([(d,newDim)]) + + cVars = defineDimVars(ncfid,dimids) + + vars = {} + for v in variables: + sname = v['short_name'] + lname = v['long_name'] + units = v['units'] + dims = v['dimension'] + newVar = ncfid.createVariable(sname,'f4',dims) + setattr(newVar,'long_name',lname) + setattr(newVar,'units',units) + vars.update([(sname,newVar)]) + + writeCVars(cVars,dimensions) + + binf =FortranFile(binfile,'r') + if hdr: + if debug: + print("reading header") + rec = binf.read_ints(dtype=np.int32) + if debug: + print(rec) + rec = binf.read_ints(dtype=np.int32) + if debug: + print(rec) + + for v in variables: + writeVar(v,vars,dimensions,bintype, binf) + + ncfid.close() + +if __name__ == "__main__": + + bin2nc('fvcore_internal_rst', 'fvcore_internal_rst.nc4', 'remap_fv.yaml', isDouble=True, hasHeader=True) + bin2nc('moist_internal_rst', 'moist_internal_rst.nc4', 'remap_moist.yaml') + bin2nc('pchem_internal_rst', 'pchem_internal_rst.nc4', 'remap_pchem.yaml') + bin2nc('lake_internal_rst', 'lake_internal_rst.nc4', 'remap_lake.yaml') + bin2nc('landice_internal_rst', 'landice_internal_rst.nc4', 'remap_landice.yaml') + bin2nc('saltwater_internal_rst', 'saltwater_internal_rst.nc4', 'remap_saltwater.yaml') + + + diff --git a/pre/remap_restart/remap_fv.yaml b/pre/remap_restart/remap_fv.yaml new file mode 100644 index 0000000..8a2c699 --- /dev/null +++ b/pre/remap_restart/remap_fv.yaml @@ -0,0 +1,67 @@ +variables: + - short_name: AK + long_name: hybrid_sigma_pressure_a + units: 'Pa' + dimension: + - edges + - short_name: BK + long_name: hybrid_sigma_pressure_b + units: '1' + dimension: + - edges + - short_name: U + long_name: eastward_wind + units: 'm s-1' + dimension: + - lev + - lat + - lon + - short_name: V + long_name: northward_wind + units: 'm s-1' + dimension: + - lev + - lat + - lon + - short_name: PT + long_name: scaled_potential_temperature + units: 'K Pa$^{-\kappa}$' + dimension: + - lev + - lat + - lon + - short_name: PE + long_name: air_pressure + units: 'Pa' + dimension: + - edges + - lat + - lon + - short_name: PKZ + long_name: pressure_to_kappa + units: 'Pa$^\kappa$' + dimension: + - lev + - lat + - lon + - short_name: DZ + long_name: height_thickness + units: 'm' + dimension: + - lev + - lat + - lon + - short_name: W + long_name: vertical_velocity + units: 'm s-1' + dimension: + - lev + - lat + - lon +dimensions: + lon: 180 + lat: 1080 + lev: 72 + edges: 73 + time: 1 + diff --git a/pre/remap_restart/remap_gocart.yaml b/pre/remap_restart/remap_gocart.yaml new file mode 100644 index 0000000..4faf5de --- /dev/null +++ b/pre/remap_restart/remap_gocart.yaml @@ -0,0 +1,49 @@ +variables: + - short_name: OX + long_name: odd_oxygen_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon + - short_name: N20 + long_name: nitrous_oxide_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon + - short_name: CFC11 + long_name: CFC11_(CCl3F)_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon + - short_name: CFC12 + long_name: CFC12_(CCl2F2)_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon + - short_name: HCFC22 + long_name: HCFC22_(CHClF2)_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon + - short_name: CH4 + long_name: methane_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon +dimensions: + lon: 180 + lat: 1080 + lev: 72 + time: 1 + diff --git a/pre/remap_restart/remap_lake.yaml b/pre/remap_restart/remap_lake.yaml new file mode 100644 index 0000000..65dcef1 --- /dev/null +++ b/pre/remap_restart/remap_lake.yaml @@ -0,0 +1,42 @@ +variables: + - short_name: TS + long_name: surface_skin_temperature + units: 'K' + dimension: + - subtile + - tile + - short_name: QS + long_name: surface_specific_humidity + units: 'kg kg-1' + dimension: + - subtile + - tile + - short_name: FR + long_name: ice_fraction + units: '1' + dimension: + - subtile + - tile + - short_name: CH + long_name: surface_heat_exchange_coefficient + units: 'kg m-2 s-1' + dimension: + - subtile + - tile + - short_name: CM + long_name: surface_momentum_exchange_coefficient + units: 'kg m-2 s-1' + dimension: + - subtile + - tile + - short_name: CQ + long_name: surface_moisture_exchange_coefficient + units: 'kg m-2 s-1' + dimension: + - subtile + - tile +dimensions: + tile: 14910 + subtile: 2 + time: 1 + diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index 97a01f5..d0f1b5f 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -8,6 +8,7 @@ import shlex from remap_base import remap_base from remap_utils import get_bcs_basename +from remap_bin2nc import bin2nc class lake_landice_saltwater(remap_base): def __init__(self, **configs): @@ -194,12 +195,18 @@ def copy_merra2(self): surfin = [ merra_2_rst_dir + expid+'.lake_internal_rst.' + suffix, merra_2_rst_dir + expid+'.landice_internal_rst.' + suffix, merra_2_rst_dir + expid+'.saltwater_internal_rst.'+ suffix] - - for f in surfin : + bin2nc_yaml = ['remap_lake.yaml', 'remap_landice.yaml','remap_salt.yaml'] + bin_path = os.path.dirname(os.path.realpath(__file__)) + for (f,yf) in zip(surfin, bin2nc_yaml): fname = os.path.basename(f) dest = rst_dir + '/'+fname print("Copy file "+f +" to " + rst_dir) shutil.copy(f, dest) + ncdest = dest.replace('z.bin', 'z.nc4') + yaml_file = bin_path + '/'+yf + print('Convert bin to nc4:' + dest + ' to \n' + ncdest + '\n') + bin2nc(dest, ncdest, yaml_file) + os.remove(dest) if __name__ == '__main__' : lls = lake_landice_saltwater(params_file='remap_params.yaml') diff --git a/pre/remap_restart/remap_landice.yaml b/pre/remap_restart/remap_landice.yaml new file mode 100644 index 0000000..159f4b3 --- /dev/null +++ b/pre/remap_restart/remap_landice.yaml @@ -0,0 +1,68 @@ +variables: + - short_name: TS + long_name: surface_skin_temperature + units: 'K' + dimension: + - unknown_dim1 + - tile + - short_name: QS + long_name: surface_specific_humidity + units: 'kg kg-1' + dimension: + - unknown_dim1 + - tile + - short_name: FR + long_name: ice_fraction + units: '1' + dimension: + - unknown_dim1 + - tile + - short_name: CH + long_name: surface_heat_exchange_coefficient + units: 'kg m-2 s-1' + dimension: + - unknown_dim1 + - tile + - short_name: CM + long_name: surface_momentum_exchange_coefficient + units: 'kg m-2 s-1' + dimension: + - unknown_dim1 + - tile + - short_name: CQ + long_name: surface_moisture_exchange_coefficient + units: 'kg m-2 s-1' + dimension: + - unknown_dim1 + - tile + - short_name: WESN + long_name: snow_layer_mass + units: 'kg m-2' + dimension: + - unknown_dim2 + - tile + - short_name: HTSN + long_name: snow_layer_heat_content + units: 'J m-2' + dimension: + - unknown_dim2 + - tile + - short_name: SNDZ + long_name: snow_layer_heat_depth + units: 'm' + dimension: + - unknown_dim2 + - tile + - short_name: TICE + long_name: ice_layer_temperature + units: 'k' + dimension: + - unknown_dim2 + - unknown_dim1 + - tile +dimensions: + tile: 5997 + unknown_dim1: 2 + unknown_dim2: 15 + time: 1 + diff --git a/pre/remap_restart/remap_moist.yaml b/pre/remap_restart/remap_moist.yaml new file mode 100644 index 0000000..7085f38 --- /dev/null +++ b/pre/remap_restart/remap_moist.yaml @@ -0,0 +1,56 @@ +variables: + - short_name: Q + long_name: specific_humidity + units: 'kg kg-1' + dimension: + - lev + - lat + - lon + - short_name: QLLS + long_name: mass_fraction_of_large_scale_cloud_liquid_water + units: 'kg kg-1' + dimension: + - lev + - lat + - lon + - short_name: QLCN + long_name: mass_fraction_of_convective_cloud_liquid_water + units: 'kg kg-1' + dimension: + - lev + - lat + - lon + - short_name: CLLS + long_name: large_scale_cloud_area_fraction + units: '1' + dimension: + - lev + - lat + - lon + - short_name: CLCN + long_name: convective_cloud_area_fraction + units: '1' + dimension: + - lev + - lat + - lon + - short_name: QILS + long_name: mass_fraction_of_large_scale_cloud_ice_water + units: '1' + dimension: + - lev + - lat + - lon + - short_name: QICN + long_name: mass_fraction_of_convective_cloud_ice_water + units: '1' + dimension: + - lev + - lat + - lon +dimensions: + lon: 180 + lat: 1080 + lev: 72 + time: 1 + diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index c2d8956..1b6da5c 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -37,7 +37,6 @@ def __init__(self, config_from_question): config_tpl['input']['shared']['expid'] = self.common_in.get('expid') config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] config_tpl['input']['shared']['tag'] = self.common_in.get('tag') - config_tpl['input']['shared']['altbcs'] = self.common_in.get('altbcs') config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') @@ -49,7 +48,6 @@ def __init__(self, config_from_question): config_tpl['output']['shared']['out_dir'] = self.common_out['out_dir'] + '/' config_tpl['output']['shared']['expid'] = self.common_out['expid'] config_tpl['output']['shared']['tag'] = self.common_out.get('tag') - config_tpl['output']['shared']['altbcs'] = self.common_out.get('altbcs') config_tpl['output']['shared']['label'] = self.common_out.get('label') config_tpl['input']['shared']['bcs_dir'] = self.common_in['bcs_dir'] diff --git a/pre/remap_restart/remap_pchem.yaml b/pre/remap_restart/remap_pchem.yaml new file mode 100644 index 0000000..4faf5de --- /dev/null +++ b/pre/remap_restart/remap_pchem.yaml @@ -0,0 +1,49 @@ +variables: + - short_name: OX + long_name: odd_oxygen_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon + - short_name: N20 + long_name: nitrous_oxide_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon + - short_name: CFC11 + long_name: CFC11_(CCl3F)_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon + - short_name: CFC12 + long_name: CFC12_(CCl2F2)_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon + - short_name: HCFC22 + long_name: HCFC22_(CHClF2)_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon + - short_name: CH4 + long_name: methane_volume_mixing_ratio + units: 'mol mol-1' + dimension: + - lev + - lat + - lon +dimensions: + lon: 180 + lat: 1080 + lev: 72 + time: 1 + diff --git a/pre/remap_restart/remap_salt.yaml b/pre/remap_restart/remap_salt.yaml new file mode 100644 index 0000000..af3f8af --- /dev/null +++ b/pre/remap_restart/remap_salt.yaml @@ -0,0 +1,81 @@ +variables: + - short_name: 'HSKINW' + long_name: 'water_skin_layer_mass' + units: 'kg m-2' + dimension: + - tile + - short_name: 'TSKINW' + long_name: 'water_skin_temperature' + units: 'K' + dimension: + - tile + - short_name: 'SSKINW' + long_name: 'water_skin_salinity' + units: 'psu' + dimension: + - tile + - short_name: 'HSKINI' + long_name: 'ice_skin_layer_mass' + units: 'kg m-2' + dimension: + - tile + - short_name: 'TSKINI' + long_name: 'ice_skin_temperature' + units: 'K' + dimension: + - tile + - short_name: 'SSKINI' + long_name: 'ice_skin_salinity' + units: 'psu' + dimension: + - tile + - short_name: 'QS' + long_name: 'surface_specific_humidity' + units: 'kg kg-1' + dimension: + - subtile + - tile + - short_name: 'CH' + long_name: 'surface_heat_exchange_coefficient' + units: 'kg m-2 s-1' + dimension: + - subtile + - tile + - short_name: 'CM' + long_name: 'surface_momentum_exchange_coefficient' + units: 'kg m-2 s-1' + dimension: + - subtile + - tile + - short_name: 'CQ' + long_name: 'surface_moisture_exchange_coefficient' + units: 'kg m-2 s-1' + dimension: + - subtile + - tile + - short_name: 'Z0' + long_name: 'aerodynamic_roughness' + units: 'm' + dimension: + - subtile + - tile + - short_name: 'WW' + long_name: 'vertical_velocity_scale_squared' + units: 'm+2 s-2' + dimension: + - subtile + - tile + - short_name: 'TWMTS' + long_name: 'departure_of_skin_temperature_from_mean_interface_temperature' + units: 'K' + dimension: + - tile + - short_name: 'DTWARM' + long_name: 'departure_of_departure_of_TDEL_from_TS_FOUND' + units: 'K' + dimension: + - tile +dimensions: + tile: 1385431 + subtile: 2 + time: 1 diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index a066238..2c3b475 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -8,6 +8,7 @@ import glob from remap_base import remap_base from remap_utils import get_bcs_basename +from remap_bin2nc import bin2nc def get_topodir(bcsdir): k = bcsdir.find('/geometry/') @@ -72,7 +73,6 @@ def remap(self): os.chdir(tmpdir) print('\nUpper air restart file names link from "_rst" to "_restart_in" \n') - types = '.bin' type_str = subprocess.check_output(['file','-b', restarts_in[0]]) type_str = str(type_str) @@ -85,7 +85,6 @@ def remap(self): '.' + config['output']['shared']['tag']+ '.' + get_bcs_basename(out_bcsdir) suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + label + types - for rst in restarts_in : f = os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_restart_in' cmd = '/bin/ln -s ' + rst + ' ' + f @@ -339,15 +338,23 @@ def copy_merra2(self): upperin =[merra_2_rst_dir + expid+'.fvcore_internal_rst.' + suffix, merra_2_rst_dir + expid+'.moist_internal_rst.' + suffix, - merra_2_rst_dir + expid+'.agcm_import_rst.' + suffix, merra_2_rst_dir + expid+'.gocart_internal_rst.' + suffix, merra_2_rst_dir + expid+'.pchem_internal_rst.' + suffix ] - - for f in upperin : + bin2nc_yaml = ['remap_fv.yaml', 'remap_moist.yaml', 'remap_gocart.yaml', 'remap_pchem.yaml'] + bin_path = os.path.dirname(os.path.realpath(__file__)) + for (f, yf) in zip(upperin,bin2nc_yaml) : fname = os.path.basename(f) dest = rst_dir + '/'+fname print("Copy file "+f +" to " + rst_dir) shutil.copy(f, dest) + ncdest = dest.replace('z.bin', 'z.nc4') + yaml_file = bin_path + '/'+yf + print('Convert bin to nc4:' + dest + ' to \n' + ncdest + '\n') + if '_fv' in yf: + bin2nc(dest, ncdest, yaml_file, isDouble=True, hasHeader=True) + else: + bin2nc(dest, ncdest, yaml_file) + os.remove(dest) if __name__ == '__main__' : air = upperair(params_file='remap_params.yaml') From c1f6c2c87f11653532c360e8951b5c6a85013020 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Fri, 26 May 2023 14:27:47 -0400 Subject: [PATCH 12/88] added catch.yaml for completeness. --- pre/remap_restart/remap_bin2nc.py | 25 ++- pre/remap_restart/remap_catch.yaml | 297 +++++++++++++++++++++++++++++ 2 files changed, 308 insertions(+), 14 deletions(-) create mode 100644 pre/remap_restart/remap_catch.yaml diff --git a/pre/remap_restart/remap_bin2nc.py b/pre/remap_restart/remap_bin2nc.py index 7107b50..a5d87a9 100755 --- a/pre/remap_restart/remap_bin2nc.py +++ b/pre/remap_restart/remap_bin2nc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from yaml import safe_load, load, dump from netCDF4 import Dataset @@ -101,15 +101,13 @@ def defineDimVars(fid,dims): return coordVars def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=False): - file = yamlfile - hdr = hasHeader bintype=np.float32 if (isDouble): bintype=np.float64 - f=open(file,'r') + f=open(yamlfile,'r') data=safe_load(f) @@ -144,7 +142,7 @@ def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=Fal writeCVars(cVars,dimensions) binf =FortranFile(binfile,'r') - if hdr: + if hasHeader: if debug: print("reading header") rec = binf.read_ints(dtype=np.int32) @@ -161,12 +159,11 @@ def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=Fal if __name__ == "__main__": - bin2nc('fvcore_internal_rst', 'fvcore_internal_rst.nc4', 'remap_fv.yaml', isDouble=True, hasHeader=True) - bin2nc('moist_internal_rst', 'moist_internal_rst.nc4', 'remap_moist.yaml') - bin2nc('pchem_internal_rst', 'pchem_internal_rst.nc4', 'remap_pchem.yaml') - bin2nc('lake_internal_rst', 'lake_internal_rst.nc4', 'remap_lake.yaml') - bin2nc('landice_internal_rst', 'landice_internal_rst.nc4', 'remap_landice.yaml') - bin2nc('saltwater_internal_rst', 'saltwater_internal_rst.nc4', 'remap_saltwater.yaml') - - - + bin2nc('fvcore_internal_rst', 'fvcore_internal_rst.nc4', 'remap_fv.yaml', isDouble=True, hasHeader=True) + bin2nc('moist_internal_rst', 'moist_internal_rst.nc4', 'remap_moist.yaml') + bin2nc('pchem_internal_rst', 'pchem_internal_rst.nc4', 'remap_pchem.yaml') + bin2nc('gocart_internal_rst', 'gocart_internal_rst.nc4', 'remap_gocart.yaml') + bin2nc('lake_internal_rst', 'lake_internal_rst.nc4', 'remap_lake.yaml') + bin2nc('landice_internal_rst', 'landice_internal_rst.nc4', 'remap_landice.yaml') + bin2nc('saltwater_internal_rst','saltwater_internal_rst.nc4', 'remap_saltwater.yaml') + bin2nc('catch_internal_rst', 'catch_internal_rst.nc4', 'remap_catch.yaml') diff --git a/pre/remap_restart/remap_catch.yaml b/pre/remap_restart/remap_catch.yaml new file mode 100644 index 0000000..26d7fbf --- /dev/null +++ b/pre/remap_restart/remap_catch.yaml @@ -0,0 +1,297 @@ +variables: + - short_name: 'BF1' + long_name: 'topo_baseflow_param_1' + units: 'kg m-4' + dimension: + - tile + - short_name: 'BF2' + long_name: 'topo_baseflow_param_2' + units: 'm' + dimension: + - tile + - short_name: 'BF3' + long_name: 'topo_baseflow_param_3' + units: 'log(m)' + dimension: + - tile + - short_name: 'VGWMAX' + long_name: 'max_rootzone_water_content' + units: 'kg m-2' + dimension: + - tile + - short_name: 'CDCR1' + long_name: 'moisture_threshold' + units: 'kg m-2' + dimension: + - tile + - short_name: 'CDCR2' + long_name: 'max_water_content' + units: 'kg m-2' + dimension: + - tile + - short_name: 'PSIS' + long_name: 'saturated_matric_potential' + units: 'm' + dimension: + - tile + - short_name: 'BEE' + long_name: 'clapp_hornberger_b' + units: '1' + dimension: + - tile + - short_name: 'POROS' + long_name: 'soil_porosity' + units: '1' + dimension: + - tile + - short_name: 'WPWET' + long_name: 'wetness_at_wilting_point' + units: '1' + dimension: + - tile + - short_name: 'COND' + long_name: 'sfc_sat_hydraulic_conduct' + units: 'm s-1' + dimension: + - tile + - short_name: 'GNU' + long_name: 'vertical_transmissivity' + units: 'm-1' + dimension: + - tile + - short_name: 'ARS1' + long_name: 'wetness_param_1' + units: 'm+2 kg-1' + dimension: + - tile + - short_name: 'ARS2' + long_name: 'wetness_param_2' + units: 'm+2 kg-1' + dimension: + - tile + - short_name: 'ARS3' + long_name: 'wetness_param_3' + units: 'm+4 kg-2' + dimension: + - tile + - short_name: 'ARA1' + long_name: 'shape_param_1' + units: 'm+2 kg-1' + dimension: + - tile + - short_name: 'ARA2' + long_name: 'shape_param_2' + units: '1' + dimension: + - tile + - short_name: 'ARA3' + long_name: 'shape_param_3' + units: 'm+2 kg-1' + dimension: + - tile + - short_name: 'ARA4' + long_name: 'shape_param_4' + units: '1' + dimension: + - tile + - short_name: 'ARW1' + long_name: 'min_theta_param_1' + units: 'm+2 kg-1' + dimension: + - tile + - short_name: 'ARW2' + long_name: 'min_theta_param_2' + units: 'm+2 kg-1' + dimension: + - tile + - short_name: 'ARW3' + long_name: 'min_theta_param_3' + units: 'm+4 kg-2' + dimension: + - tile + - short_name: 'ARW4' + long_name: 'min_theta_param_4' + units: '1' + dimension: + - tile + - short_name: 'TSA1' + long_name: 'water_transfer_param_1' + units: '1' + dimension: + - tile + - short_name: 'TSA2' + long_name: 'water_transfer_param_2' + units: '1' + dimension: + - tile + - short_name: 'TSB1' + long_name: 'water_transfer_param_3' + units: '1' + dimension: + - tile + - short_name: 'TSB2' + long_name: 'water_transfer_param_4' + units: '1' + dimension: + - tile + - short_name: 'ATAU' + long_name: 'water_transfer_param_5' + units: '1' + dimension: + - tile + - short_name: 'BTAU' + long_name: 'water_transfer_param_6' + units: '1' + dimension: + - tile + - short_name: 'OLD_ITY' + long_name: 'Placeholder. Used to be vegetation_type.' + units: '1' + dimension: + - tile + - short_name: 'TC' + long_name: 'canopy_temperature' + units: 'K' + dimension: + - subtile + - tile + - short_name: 'QC' + long_name: 'canopy_specific_humidity' + units: 'kg kg-1' + dimension: + - subtile + - tile + - short_name: 'CAPAC' + long_name: 'interception_reservoir_capac' + units: 'kg m-2' + dimension: + - tile + - short_name: 'CATDEF' + long_name: 'catchment_deficit' + units: 'kg m-2' + dimension: + - tile + - short_name: 'RZEXC' + long_name: 'root_zone_excess' + units: 'kg m-2' + dimension: + - tile + - short_name: 'SRFEXC' + long_name: 'surface_excess' + units: 'kg m-2' + dimension: + - tile + - short_name: 'GHTCNT1' + long_name: 'soil_heat_content_layer_1' + units: 'J m-2' + dimension: + - tile + - short_name: 'GHTCNT2' + long_name: 'soil_heat_content_layer_2' + units: 'J_m-2' + dimension: + - tile + - short_name: 'GHTCNT3' + long_name: 'soil_heat_content_layer_3' + units: 'J m-2' + dimension: + - tile + - short_name: 'GHTCNT4' + long_name: 'soil_heat_content_layer_4' + units: 'J m-2' + dimension: + - tile + - short_name: 'GHTCNT5' + long_name: 'soil_heat_content_layer_5' + units: 'J m-2' + dimension: + - tile + - short_name: 'GHTCNT6' + long_name: 'soil_heat_content_layer_6' + units: 'J m-2' + dimension: + - tile + - short_name: 'TSURF' + long_name: 'mean_catchment_temp_incl_snw' + units: 'K' + dimension: + - tile + - short_name: 'WESNN1' + long_name: 'snow_mass_layer_1' + units: 'kg m-2' + dimension: + - tile + - short_name: 'WESNN2' + long_name: 'snow_mass_layer_2' + units: 'kg m-2' + dimension: + - tile + - short_name: 'WESNN3' + long_name: 'snow_mass_layer_3' + units: 'kg m-2' + dimension: + - tile + - short_name: 'HTSNNN1' + long_name: 'heat_content_snow_layer_1' + units: 'J m-2' + dimension: + - tile + - short_name: 'HTSNNN2' + long_name: 'heat_content_snow_layer_2' + units: 'J m-2' + dimension: + - tile + - short_name: 'HTSNNN3' + long_name: 'heat_content_snow_layer_3' + units: 'J m-2' + dimension: + - tile + - short_name: 'SNDZN1' + long_name: 'snow_depth_layer_1' + units: 'm' + dimension: + - tile + - short_name: 'SNDZN2' + long_name: 'snow_depth_layer_2' + units: 'm' + dimension: + - tile + - short_name: 'SNDZN3' + long_name: 'snow_depth_layer_3' + units: 'm' + dimension: + - tile + - short_name: 'CH' + long_name: 'surface_heat_exchange_coefficient' + units: 'kg m-2 s-1' + dimension: + - subtile + - tile + - short_name: 'CM' + long_name: 'surface_momentum_exchange_coefficient' + units: 'kg m-2 s-1' + dimension: + - subtile + - tile + - short_name: 'CQ' + long_name: 'surface_moisture_exchange_coffiecient' + units: 'kg m-2 s-1' + dimension: + - subtile + - tile + - short_name: 'FR' + long_name: 'subtile_fractions' + units: '1' + dimension: + - subtile + - tile + - short_name: 'WW' + long_name: 'vertical_velocity_scale_squared' + units: 'm+2 s-2' + dimension: + - subtile + - tile +dimensions: + tile: 195136 + subtile: 4 + time: 1 From 73b03a5e9f0d660789175c42cc39e7e04e40e65d Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Tue, 30 May 2023 11:20:19 -0400 Subject: [PATCH 13/88] fix bcs for merra2 input --- pre/remap_restart/remap_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index fa3a5c3..1897e31 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -534,7 +534,7 @@ def get_bcsdir(x, opt): base = 'discover_couple' if tag in NewStructureBCTag: base = 'discover_ns' - if x.get('input:shared:MERRA-2'): + if x.get('input:shared:MERRA-2') and opt.upper() == "IN": base = 'discover_ops' bc_base = BCBase[base] From 2b074926e7fd9b0703d654ca5b7dc73350f533bd Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 31 May 2023 09:05:52 -0400 Subject: [PATCH 14/88] convert to CS for cubed-sphere ocean grid --- pre/remap_restart/remap_utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 1897e31..ec56d4d 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -355,9 +355,16 @@ def get_command_line_from_answers(answers): ocnmdlout = ' -ocnmdlout ' + answers["output:shared:model"] oceanin='' - if answers.get("input:shared:ogrid"): - oceanin = ' -oceanin ' + answers["input:shared:ogrid"] - oceanout = ' -oceanout ' + answers["output:shared:ogrid"] + ogrid = answers.get("input:shared:ogrid") + if ogrid : + if ogrid[0] == 'C' + ogrid = "CS" + oceanin = ' -oceanin ' + ogrid + + ogrid = answers.get("output:shared:ogrid") + if ogrid[0] == 'C': + ogrid = "CS" + oceanout = ' -oceanout ' + ogrid nobkg = '' if answers["output:analysis:bkg"] else " -nobkg " nolcv = '' if answers["output:analysis:lcv"] else " -nolcv " From 494d3f94dc85fcd8f040af77fe25fff0c3cecd9a Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 31 May 2023 09:29:02 -0400 Subject: [PATCH 15/88] fixed typo --- pre/remap_restart/remap_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index ec56d4d..c7d8d0a 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -357,7 +357,7 @@ def get_command_line_from_answers(answers): oceanin='' ogrid = answers.get("input:shared:ogrid") if ogrid : - if ogrid[0] == 'C' + if ogrid[0] == 'C': ogrid = "CS" oceanin = ' -oceanin ' + ogrid From 6909574e4c3edc1417a166ff9b054e332590cb4d Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Mon, 5 Jun 2023 15:42:28 -0400 Subject: [PATCH 16/88] add no-prompt argument --- pre/remap_restart/remap_command.py | 1 + pre/remap_restart/remap_restarts.py | 17 ++++++++++------- pre/remap_restart/remap_utils.py | 4 ++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pre/remap_restart/remap_command.py b/pre/remap_restart/remap_command.py index 0548309..7ab0ed0 100755 --- a/pre/remap_restart/remap_command.py +++ b/pre/remap_restart/remap_command.py @@ -63,6 +63,7 @@ def parse_args(program_description): p_command.add_argument('-nobkg', action='store_true', help="Don't remap bkg files") p_command.add_argument('-nolcv', action='store_true', help="Don't remap lcv files") + p_command.add_argument('-np', action='store_true', help="No prompt. Overwrite config files without prompting questions") p_command.add_argument('-lbl', action='store_true', help="Label output restarts with tags and resolutions") p_command.add_argument('-in_bcsdir', default="", help= "users' alternative boundary condition files for input. If not specified (default), it will be deduced from tag and resolution information") p_command.add_argument('-out_bcsdir', default="", help= "users' alternative boundary condition files for output. If not specified (default), it will be deduced from tag and resolution information") diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index 31e5dba..fcf2cfc 100644 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -61,11 +61,13 @@ def main(): cmdl, extra_args = parse_args(program_description) answers = {} config_yaml ='' + noprompt = False if (len(sys.argv) > 1) : if sys.argv[1] == 'config_file' : config_yaml = cmdl.config_file if sys.argv[1] == 'command_line': answers = get_answers_from_command_line(cmdl) + noprompt = cmdl.np if (len(sys.argv) == 1 or answers) : if not answers: answers = ask_questions() @@ -83,21 +85,22 @@ def main(): if config: print_config(config) - questions = [ + if not noprompt : + questions = [ { "type": "confirm", "name": "Continue", "message": "Above is the YAML config file, would you like to continue?", "default": True },] - answer = questionary.prompt(questions) + answer = questionary.prompt(questions) - if not answer['Continue'] : - print("\nYou answered not to continue, exiting.\n") - sys.exit(0) + if not answer['Continue'] : + print("\nYou answered not to continue, exiting.\n") + sys.exit(0) - # write config to yaml file - config_to_yaml(config, config_yaml) + # write config to yaml file + config_to_yaml(config, config_yaml,noprompt = noprompt) # upper air upper = upperair(params_file=config_yaml) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index c7d8d0a..4073b4a 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -288,8 +288,8 @@ def get_account(): accounts = accounts.decode().split() return accounts[0] -def config_to_yaml(config, yaml_file): - if os.path.exists(yaml_file) : +def config_to_yaml(config, yaml_file, noprompt = False): + if os.path.exists(yaml_file) and not noprompt : overwrite = questionary.confirm("Do you want to overwrite " + yaml_file + "?" , default=False).ask() if not overwrite : while True: From 1772058d7f98d59c9e1e9760dccdd36f2907cd93 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Mon, 26 Jun 2023 10:47:32 -0400 Subject: [PATCH 17/88] change tag to bc version --- pre/remap_restart/remap_questions.py | 40 ++--- pre/remap_restart/remap_utils.py | 237 +++++++++++++-------------- 2 files changed, 128 insertions(+), 149 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 188bac4..a939fd7 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -162,62 +162,50 @@ def ask_questions(): { "type": "text", "name": "input:shared:tag", - "message": "Enter BC version (GCM or DAS tag) that matches input restarts: \n \ + "message": "Enter BC version that matches input restarts: \n \ \n \ -BC version: GCM or DAS tag \n \ +BC version \n \ --------------- \n \ -G40 : Ganymed-4_0 ......... Heracles-5_4_p3 \n \ -ICA : Icarus .............. Jason \n \ -GITOL : 10.3 ................ 10.18 \n \ -INL : Icarus-NL ........... Jason-NL \n \ -GITNL : 10.19 ............... 10.23 \n \ +ICA : Icarus \n \ +NLv3 : New land version 3 \n \ \n \ -5B0 : GEOSadas-5_10_0_p2 .. GEOSadas-5_11_0 \n \ -512 : GEOSadas-5_12_2 ..... GEOSadas-5_16_5\n \ -517 : GEOSadas-5_17_0 ..... GEOSadas-5_24_0_p1\n \ -525 : GEOSadas-5_25_1 ..... GEOSadas-5_29_4\n \ +New directory structures: \n \ \n \ NL3 : Newland version 3 \n \ NL4 : Newland version 4 \n \ NL5 : Newland version 5 \n \ v06, v07, v08, v09: Not generated yet \n", - "default": "INL", + "default": "ICA", "when": lambda x: not x["input:shared:MERRA-2"], }, { "type": "text", "name": "output:shared:tag", - "message": "Enter GCM or DAS tag for new restarts:", - "default": "GITNL", + "message": "Enter BC version for new restarts:", + "default": "NLv3", "when": lambda x: not x["input:shared:MERRA-2"], }, # show the message if it is merra2 { "type": "text", "name": "output:shared:tag", - "message": " Enter BC version (GCM or DAS tag) for new restarts: \n \ + "message": " Enter BC version for new restarts: \n \ \n \ -BC version: GCM or DAS tag \n \ +BC version \n \ --------------- \n \ -G40 : Ganymed-4_0 ......... Heracles-5_4_p3 \n \ -ICA : Icarus .............. Jason \n \ -GITOL : 10.3 ................ 10.18 \n \ -INL : Icarus-NL ........... Jason-NL \n \ -GITNL : 10.19 ............... 10.23 \n \ +ICA : Icarus \n \ +NLv3 : New land version 3 \n \ \n \ -5B0 : GEOSadas-5_10_0_p2 .. GEOSadas-5_11_0 \n \ -512 : GEOSadas-5_12_2 ..... GEOSadas-5_16_5\n \ -517 : GEOSadas-5_17_0 ..... GEOSadas-5_24_0_p1\n \ -525 : GEOSadas-5_25_1 ..... GEOSadas-5_29_4\n \ +New directory structures: \n \ \n \ NL3 : Newland version 3 \n \ NL4 : Newland version 4 \n \ NL5 : Newland version 5 \n \ v06, v07, v08, v09: Not generated yet \n", - "default": "GITNL", + "default": "NLv3", "when": lambda x: x["input:shared:MERRA-2"], }, diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 4073b4a..7d7507f 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -28,145 +28,136 @@ def init_tags(): # BCS Tag: Fortuna-1_4 global tag_initialized if tag_initialized : return - F14 = ( 'F14', 'Fortuna-1_4', 'Fortuna-1_4_p1' ) - D214 = ( 'D214', 'GEOSdas-2_1_4', 'GEOSdas-2_1_4-m1', - 'GEOSdas-2_1_4-m2', 'GEOSdas-2_1_4-m3', 'GEOSdas-2_1_4-m4' ) - D540 = ( 'D540', 'GEOSadas-5_4_0', 'GEOSadas-5_4_0_p1', - 'GEOSadas-5_4_0_p2', 'GEOSadas-5_4_0_p3', 'GEOSadas-5_4_0_p4', - 'GEOSadas-5_4_1', 'GEOSadas-5_4_1_p1', 'GEOSadas-5_4_2', - 'GEOSadas-5_4_3', 'GEOSadas-5_4_4', 'GEOSadas-5_5_0', - 'GEOSadas-5_5_1', 'GEOSadas-5_5_2', 'GEOSadas-5_5_3' ) +# F14 = ( 'F14', 'Fortuna-1_4', 'Fortuna-1_4_p1' ) +# D214 = ( 'D214', 'GEOSdas-2_1_4', 'GEOSdas-2_1_4-m1', +# 'GEOSdas-2_1_4-m2', 'GEOSdas-2_1_4-m3', 'GEOSdas-2_1_4-m4' ) +# D540 = ( 'D540', 'GEOSadas-5_4_0', 'GEOSadas-5_4_0_p1', +# 'GEOSadas-5_4_0_p2', 'GEOSadas-5_4_0_p3', 'GEOSadas-5_4_0_p4', +# 'GEOSadas-5_4_1', 'GEOSadas-5_4_1_p1', 'GEOSadas-5_4_2', +# 'GEOSadas-5_4_3', 'GEOSadas-5_4_4', 'GEOSadas-5_5_0', +# 'GEOSadas-5_5_1', 'GEOSadas-5_5_2', 'GEOSadas-5_5_3' ) # BCS Tag: Fortuna-2_0 #--------------------- - F20 = ( 'F20', 'Fortuna-2_0') +# F20 = ( 'F20', 'Fortuna-2_0') # BCS Tag: Fortuna-2_1 #--------------------- - F21 = ( 'F21', 'Fortuna-2_1', 'Fortuna-2_1_p1', - 'Fortuna-2_1_p2', 'Fortuna-2_1_p3', 'Fortuna-2_2', - 'Fortuna-2_2_p1', 'Fortuna-2_2_p2', 'Fortuna-2_3', - 'Fortuna-2_3_p1', 'Fortuna-2_4', 'Fortuna-2_4_p1', - 'Fortuna-2_4_p2', 'Fortuna-2_5', 'Fortuna-2_5_BETA0', - 'Fortuna-2_5_p1', 'Fortuna-2_5_p2', 'Fortuna-2_5_p3', - 'Fortuna-2_5_p4', 'Fortuna-2_5_p5', 'Fortuna-2_5_p6', - 'Fortuna-2_5_pp2' ) - D561 = ( 'D561', 'GEOSadas-5_6_1', 'GEOSadas-5_6_1_p1', - 'GEOSadas-5_6_1_p2', 'GEOSadas-5_6_1_p3', 'GEOSadas-5_6_1_p4', - 'GEOSadas-5_6_2', 'GEOSadas-5_6_2_p1', 'GEOSadas-5_6_2_p2', - 'GEOSadas-5_6_2_p3', 'GEOSadas-5_6_2_p4', 'GEOSadas-5_6_2_p5', - 'GEOSadas-5_6_2_p6', 'GEOSadas-5_7_1', 'GEOSadas-5_7_1_p1', - 'GEOSadas-5_7_1_p2', 'GEOSadas-5_7_2', 'GEOSadas-5_7_2_p1', - 'GEOSadas-5_7_2_p2', 'GEOSadas-5_7_2_p2_m1','GEOSadas-5_7_2_p3', - 'GEOSadas-5_7_2_p3_m1', 'GEOSadas-5_7_2_p3_m2','GEOSadas-5_7_2_p4', - 'GEOSadas-5_7_2_p5', 'GEOSadas-5_7_2_p5_m1','GEOSadas-5_7_3', - 'GEOSadas-5_7_3_p1', 'GEOSadas-5_7_3_p2', 'GEOSadas-5_7_3_p2' ) +# F21 = ( 'F21', 'Fortuna-2_1', 'Fortuna-2_1_p1', +# 'Fortuna-2_1_p2', 'Fortuna-2_1_p3', 'Fortuna-2_2', +# 'Fortuna-2_2_p1', 'Fortuna-2_2_p2', 'Fortuna-2_3', +# 'Fortuna-2_3_p1', 'Fortuna-2_4', 'Fortuna-2_4_p1', +# 'Fortuna-2_4_p2', 'Fortuna-2_5', 'Fortuna-2_5_BETA0', +# 'Fortuna-2_5_p1', 'Fortuna-2_5_p2', 'Fortuna-2_5_p3', +# 'Fortuna-2_5_p4', 'Fortuna-2_5_p5', 'Fortuna-2_5_p6', +# 'Fortuna-2_5_pp2' ) +# D561 = ( 'D561', 'GEOSadas-5_6_1', 'GEOSadas-5_6_1_p1', +# 'GEOSadas-5_6_1_p2', 'GEOSadas-5_6_1_p3', 'GEOSadas-5_6_1_p4', +# 'GEOSadas-5_6_2', 'GEOSadas-5_6_2_p1', 'GEOSadas-5_6_2_p2', +# 'GEOSadas-5_6_2_p3', 'GEOSadas-5_6_2_p4', 'GEOSadas-5_6_2_p5', +# 'GEOSadas-5_6_2_p6', 'GEOSadas-5_7_1', 'GEOSadas-5_7_1_p1', +# 'GEOSadas-5_7_1_p2', 'GEOSadas-5_7_2', 'GEOSadas-5_7_2_p1', +# 'GEOSadas-5_7_2_p2', 'GEOSadas-5_7_2_p2_m1','GEOSadas-5_7_2_p3', +# 'GEOSadas-5_7_2_p3_m1', 'GEOSadas-5_7_2_p3_m2','GEOSadas-5_7_2_p4', +# 'GEOSadas-5_7_2_p5', 'GEOSadas-5_7_2_p5_m1','GEOSadas-5_7_3', +# 'GEOSadas-5_7_3_p1', 'GEOSadas-5_7_3_p2', 'GEOSadas-5_7_3_p2' ) # BCS Tag: Ganymed-1_0 - #--------------------- - G10 = ( 'G10', 'Ganymed-1_0', 'Ganymed-1_0_BETA', - 'Ganymed-1_0_BETA1', 'Ganymed-1_0_BETA2', 'Ganymed-1_0_BETA3', - 'Ganymed-1_0_BETA4' ) - - D580 = ( 'D580', 'GEOSadas-5_8_0', 'GEOSadas-5_9_0', - 'GEOSadas-5_9_1' ) - - # BCS Tags: Ganymed-1_0_M and Ganymed-1_0_D - #------------------------------------------ - G10p = ( 'G10p', 'Ganymed-1_0_p1', 'Ganymed-1_0_p2', - 'Ganymed-1_0_p3', 'Ganymed-1_0_p4', 'Ganymed-1_0_p5', - 'Ganymed-1_0_p6' ) - - D591p= ( 'D591p', 'GEOSadas-5_9_1_p1', 'GEOSadas-5_9_1_p2', - 'GEOSadas-5_9_1_p3', 'GEOSadas-5_9_1_p4', 'GEOSadas-5_9_1_p5', - 'GEOSadas-5_9_1_p6', 'GEOSadas-5_9_1_p7', 'GEOSadas-5_9_1_p8', - 'GEOSadas-5_9_1_p9' ) - - # BCS Tags: Ganymed-1_0_M and Ganymed-1_0_D w/ new landice rst - #------------------------------------------------------------------------ - G20 = ( 'G20', 'Ganymed-2_0', 'Ganymed-2_1', - 'Ganymed-2_1_p1', 'Ganymed-2_1_p2', 'Ganymed-2_1_p3', - 'Ganymed-2_1_p4', 'Ganymed-2_1_p5', 'Ganymed-2_1_p6' ) - D5A0 = ( 'D5A0', 'GEOSadas-5_10_0', 'GEOSadas-5_10_0_p1' ) - - - # BCS Tags: Ganymed-1_0_Reynolds and Ganymed-1_0_Ostia - #----------------------------------------------------- - G30 = ( 'G30', 'Ganymed-3_0', 'Ganymed-3_0_p1' ) - D5B0 = ( '5B0', 'GEOSadas-5_10_0_p2', 'GEOSadas-5_11_0' ) - - # BCS Tags: Ganymed-4_0_Reynolds, Ganymed-4_0_MERRA-2, and Ganymed-4_0_Ostia - #--------------------------------------------------------------------------- + #--------------------- +# G10 = ( 'G10', 'Ganymed-1_0', 'Ganymed-1_0_BETA', +# 'Ganymed-1_0_BETA1', 'Ganymed-1_0_BETA2', 'Ganymed-1_0_BETA3', +# 'Ganymed-1_0_BETA4' ) +# +# D580 = ( 'D580', 'GEOSadas-5_8_0', 'GEOSadas-5_9_0', +# 'GEOSadas-5_9_1' ) +# +# # BCS Tags: Ganymed-1_0_M and Ganymed-1_0_D +# #------------------------------------------ +# G10p = ( 'G10p', 'Ganymed-1_0_p1', 'Ganymed-1_0_p2', +# 'Ganymed-1_0_p3', 'Ganymed-1_0_p4', 'Ganymed-1_0_p5', +# 'Ganymed-1_0_p6' ) +# +# D591p= ( 'D591p', 'GEOSadas-5_9_1_p1', 'GEOSadas-5_9_1_p2', +# 'GEOSadas-5_9_1_p3', 'GEOSadas-5_9_1_p4', 'GEOSadas-5_9_1_p5', +# 'GEOSadas-5_9_1_p6', 'GEOSadas-5_9_1_p7', 'GEOSadas-5_9_1_p8', +# 'GEOSadas-5_9_1_p9' ) +# +# # BCS Tags: Ganymed-1_0_M and Ganymed-1_0_D w/ new landice rst +# #------------------------------------------------------------------------ +# G20 = ( 'G20', 'Ganymed-2_0', 'Ganymed-2_1', +# 'Ganymed-2_1_p1', 'Ganymed-2_1_p2', 'Ganymed-2_1_p3', +# 'Ganymed-2_1_p4', 'Ganymed-2_1_p5', 'Ganymed-2_1_p6' ) +# D5A0 = ( 'D5A0', 'GEOSadas-5_10_0', 'GEOSadas-5_10_0_p1' ) +# +# +# BCS Tags: Ganymed-1_0_Reynolds and Ganymed-1_0_Ostia +#----------------------------------------------------- +# G30 = ( 'G30', 'Ganymed-3_0', 'Ganymed-3_0_p1' ) +# D5B0 = ( '5B0', 'GEOSadas-5_10_0_p2', 'GEOSadas-5_11_0' ) +# +# # BCS Tags: Ganymed-4_0_Reynolds, Ganymed-4_0_MERRA-2, and Ganymed-4_0_Ostia +# #--------------------------------------------------------------------------- G40 = ( 'G40', 'Ganymed-4_0', 'Ganymed-4_0_p1', 'Ganymed-4_1', 'Heracles-1_0', 'Heracles-1_1', 'Heracles-2_0', 'Heracles-2_1', 'Heracles-3_0', 'Heracles-4_0', 'Heracles-5_4_p3' ) - D512 = ( '512', 'GEOSadas-5_12_2', 'GEOSadas-5_12_4', - 'GEOSadas-5_12_4_p1', 'GEOSadas-5_12_4_p2', 'GEOSadas-5_12_4_p3', - 'GEOSadas-5_12_5', 'GEOSadas-5_13_0_p1', 'GEOSadas-5_13_0_p2', - 'GEOSadas-5_13_1', 'GEOSadas-5_16_5' ) - - # BCS Tags: Icarus (New Land Parameters, New Topography) - #--------------------------------------------------------------------------- +# D512 = ( '512', 'GEOSadas-5_12_2', 'GEOSadas-5_12_4', +# 'GEOSadas-5_12_4_p1', 'GEOSadas-5_12_4_p2', 'GEOSadas-5_12_4_p3', +# 'GEOSadas-5_12_5', 'GEOSadas-5_13_0_p1', 'GEOSadas-5_13_0_p2', +# 'GEOSadas-5_13_1', 'GEOSadas-5_16_5' ) +# +# # BCS Tags: Icarus (New Land Parameters, New Topography) +# #--------------------------------------------------------------------------- ICA = ( 'ICA', 'Icarus', 'Jason' ) - D517 = ( '517', 'GEOSadas-5_17_0', 'GEOSadas-5_17_1', 'GEOSadas-5_18_0', - 'GEOSadas-5_18_1', 'GEOSadas-5_18_2', 'GEOSadas-5_18_3', - 'GEOSadas-5_18_3_p1', 'GEOSadas-5_19_0', 'GEOSadas-5_20_0', - 'GEOSadas-5_20_0_p1', 'GEOSadas-5_20_0_p2', 'GEOSadas-5_21_0', - 'GEOSadas-5_21_2', 'GEOSadas-5_21_3_p1', 'GEOSadas-5_22_0', - 'GEOSadas-5_22_0_p1', 'GEOSadas-5_22_0_p2', 'GEOSadas-5_23_0', - 'GEOSadas-5_23_0_p1', 'GEOSadas-5_24_0', 'GEOSadas-5_24_0_p1' ) - GITOL = ( 'GITOL', '10.3', '10.4', '10.5', - '10.6', '10.7', '10.8', - '10.9', '10.10', '10.11', - '10.12', '10.13', '10.14', - '10.15', '10.16', '10.17', - '10.18' ) + NLv3 = ( 'NLv3', 'NLV3') +# D517 = ( '517', 'GEOSadas-5_17_0', 'GEOSadas-5_17_1', 'GEOSadas-5_18_0', +# 'GEOSadas-5_18_1', 'GEOSadas-5_18_2', 'GEOSadas-5_18_3', +# 'GEOSadas-5_18_3_p1', 'GEOSadas-5_19_0', 'GEOSadas-5_20_0', +# 'GEOSadas-5_20_0_p1', 'GEOSadas-5_20_0_p2', 'GEOSadas-5_21_0', +# 'GEOSadas-5_21_2', 'GEOSadas-5_21_3_p1', 'GEOSadas-5_22_0', +# 'GEOSadas-5_22_0_p1', 'GEOSadas-5_22_0_p2', 'GEOSadas-5_23_0', +# 'GEOSadas-5_23_0_p1', 'GEOSadas-5_24_0', 'GEOSadas-5_24_0_p1' ) +# GITOL = ( 'GITOL', '10.3', '10.4', '10.5', +# '10.6', '10.7', '10.8', +# '10.9', '10.10', '10.11', +# '10.12', '10.13', '10.14', +# '10.15', '10.16', '10.17', +# '10.18' ) # BCS Tags: Icarus-NLv3 (New Land Parameters) #--------------------------------------------------------------------------- - INL = ( 'INL', 'Icarus-NL', 'Icarus-NLv3', 'Jason-NL' ) - GITNL = ( 'GITNL', '10.19', '10.20', '10.21', '10.22', '10.23' ) - D525 = ( '525', 'GEOSadas-5_25_1', 'GEOSadas-5_25_1_p5', 'GEOSadas-5_25_p7', - 'GEOSadas-5_27_1', 'GEOSadas-5_29_3', 'GEOSadas-5_29_4' ) - - for tag in F14: BCSTag[tag]= "Fortuna-1_4" - for tag in F20: BCSTag[tag]= "Fortuna-2_0" - for tag in F21: BCSTag[tag]= "Fortuna-2_1" - for tag in G10: BCSTag[tag]= "Ganymed-1_0" - for tag in G10p: BCSTag[tag]= "Ganymed-1_0_M" - for tag in G20: BCSTag[tag]= "Ganymed-1_0_M" - for tag in G30: BCSTag[tag]= "Ganymed-1_0_Reynolds" +# INL = ( 'INL', 'Icarus-NL', 'Icarus-NLv3', 'Jason-NL' ) +# GITNL = ( 'GITNL', '10.19', '10.20', '10.21', '10.22', '10.23' ) +# D525 = ( '525', 'GEOSadas-5_25_1', 'GEOSadas-5_25_1_p5', 'GEOSadas-5_25_p7', +# 'GEOSadas-5_27_1', 'GEOSadas-5_29_3', 'GEOSadas-5_29_4' ) + +# for tag in F14: BCSTag[tag]= "Fortuna-1_4" +# for tag in F20: BCSTag[tag]= "Fortuna-2_0" +# for tag in F21: BCSTag[tag]= "Fortuna-2_1" +# for tag in G10: BCSTag[tag]= "Ganymed-1_0" +# for tag in G10p: BCSTag[tag]= "Ganymed-1_0_M" +# for tag in G20: BCSTag[tag]= "Ganymed-1_0_M" +# for tag in G30: BCSTag[tag]= "Ganymed-1_0_Reynolds" for tag in G40: BCSTag[tag]= "Ganymed-4_0_Reynolds" for tag in ICA: BCSTag[tag]= "Icarus_Reynolds" - for tag in GITOL: BCSTag[tag]= "Icarus_Reynolds" - for tag in INL: BCSTag[tag]= "Icarus-NLv3_Reynolds" - for tag in GITNL: BCSTag[tag]= "Icarus-NLv3_Reynolds" - for tag in NewStructureBCTag: BCSTag[tag]= tag - - - for tag in D214: BCSTag[tag]= "Fortuna-1_4" - for tag in D540: BCSTag[tag]= "Fortuna-1_4" - for tag in D561: BCSTag[tag]= "Fortuna-2_1" - for tag in D580: BCSTag[tag]= "Ganymed-1_0" - for tag in D591p: BCSTag[tag]= "Ganymed-1_0_M" - for tag in D5A0: BCSTag[tag]= "Ganymed-1_0_M" - for tag in D5B0: BCSTag[tag]= "Ganymed-1_0_Reynolds" - for tag in D512: BCSTag[tag]= "Ganymed-4_0_Reynolds" - for tag in D517: BCSTag[tag]= "Icarus_Reynolds" - for tag in D525: BCSTag[tag]= "Icarus-NLv3_Reynolds" - - TagsRank['Fortuna-1_4'] = 1 - TagsRank['Fortuna-1_5'] = 2 - TagsRank['Fortuna-2_0'] = 3 - TagsRank['Fortuna-2_1'] = 4 - TagsRank['Ganymed-1_0'] = 5 - TagsRank['Ganymed-1_0_m1'] = 6 - TagsRank['Ganymed-1_0_m2'] = 7 - TagsRank['Ganymed-1_0_M'] = 8 - TagsRank['Ganymed-1_0_D'] = 9 - TagsRank['Ganymed-1_0_Reynolds'] = 10 - TagsRank['Ganymed-1_0_Ostia'] = 11 + for tag in NLv3: BCSTag[tag]= "Icarus-NLv3_Reynolds" +# for tag in GITOL: BCSTag[tag]= "Icarus_Reynolds" +# for tag in INL: BCSTag[tag]= "Icarus-NLv3_Reynolds" +# for tag in GITNL: BCSTag[tag]= "Icarus-NLv3_Reynolds" +# for tag in NewStructureBCTag: BCSTag[tag]= tag +# + +# for tag in D214: BCSTag[tag]= "Fortuna-1_4" +# for tag in D540: BCSTag[tag]= "Fortuna-1_4" +# for tag in D561: BCSTag[tag]= "Fortuna-2_1" +# for tag in D580: BCSTag[tag]= "Ganymed-1_0" +# for tag in D591p: BCSTag[tag]= "Ganymed-1_0_M" +# for tag in D5A0: BCSTag[tag]= "Ganymed-1_0_M" +# for tag in D5B0: BCSTag[tag]= "Ganymed-1_0_Reynolds" +# for tag in D512: BCSTag[tag]= "Ganymed-4_0_Reynolds" +# for tag in D517: BCSTag[tag]= "Icarus_Reynolds" +# for tag in D525: BCSTag[tag]= "Icarus-NLv3_Reynolds" +# TagsRank['Ganymed-4_0_Reynolds'] = 12 TagsRank['Ganymed-4_0_Ostia'] = 13 TagsRank['Ganymed-4_0_MERRA-2'] = 14 @@ -263,8 +254,8 @@ def get_bcs_basename(bcs): def we_default(tag): default_ = '26' - if tag in ['INL','GITNL', '525'] : default_ = '13' - if tag in NewStructureBCTag : default_ = '13' + if tag =='NLv3' : default_ = '13' + if tag in NewStructureBCTag : default_ = '13' return default_ def zoom_default(x): From 6766dd0c72dfcf16ca781c06fc0f61b4a704a3b0 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Mon, 26 Jun 2023 12:21:16 -0400 Subject: [PATCH 18/88] constraint for MERRA-2 hours (21) --- pre/remap_restart/remap_questions.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index a939fd7..bccb9a3 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -55,8 +55,15 @@ def ask_questions(): "name": "input:shared:yyyymmddhh", "message": "From what restart date/time would you like to remap? (must be 10 digits: yyyymmddhh)", "validate": lambda text: len(text)==10 , + "when": lambda x: not x['input:shared:MERRA-2'], + }, + { + "type": "text", + "name": "input:shared:yyyymmddhh", + "message": "From what restart date would you like to remap? (must be 8 digits: yyyymmdd, hour=21z)", + "validate": lambda text: len(text)==8 , + "when": lambda x: x['input:shared:MERRA-2'], }, - { "type": "path", "name": "output:shared:out_dir", @@ -309,7 +316,8 @@ def ask_questions(): }, ] answers = questionary.prompt(questions) - + if answers['input:shared:MERRA-2'] : + answers['input:shared:yyyymmddhh'] = answers['input:shared:yyyymmddhh']+ '21' answers['input:shared:rst_dir'] = os.path.abspath(answers['input:shared:rst_dir']) answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) From 293dca5cd0e4f29999634755ef7cdf7b8daf09e3 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 28 Jun 2023 16:51:24 -0400 Subject: [PATCH 19/88] change message format --- pre/remap_restart/remap_questions.py | 58 ++++++++++++++++------------ 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index bccb9a3..cb0e59f 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -80,7 +80,11 @@ def ask_questions(): { "type": "text", "name": "input:shared:agrid", - "message": "Enter input atmospheric grid: \n C12 C180 C1000 C270\n C24 C360 C1440 C540\n C48 C500 C2880 C1080\n C90 C720 C5760 C2160 C1536\n ", + "message": "Enter input atmospheric grid: \n \ + C12 C180 C1000 C270\n \ + C24 C360 C1440 C540\n \ + C48 C500 C2880 C1080\n \ + C90 C720 C5760 C2160 C1536\n ", "default": 'C360', # if it is merra-2 or has_fvcore, agrid is deduced "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_name(x), @@ -124,7 +128,11 @@ def ask_questions(): { "type": "text", "name": "output:shared:agrid", - "message": "Enter new atmospheric grid: \n C12 C180 C1000 C270\n C24 C360 C1440 C540\n C48 C500 C2880 C1080\n C90 C720 C5760 C2160 C1536\n ", + "message": "Enter new atmospheric grid: \n \ + C12 C180 C1000 C270\n \ + C24 C360 C1440 C540\n \ + C48 C500 C2880 C1080\n \ + C90 C720 C5760 C2160 C1536\n ", "default": 'C360', }, @@ -169,21 +177,20 @@ def ask_questions(): { "type": "text", "name": "input:shared:tag", - "message": "Enter BC version that matches input restarts: \n \ + "message": "Enter BC version that matches input restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) \n \ \n \ -BC version \n \ ---------------- \n \ -ICA : Icarus \n \ -NLv3 : New land version 3 \n \ + BC version \n \ + --------------- \n \ + ICA : Icarus \n \ + NLv3 : New land version 3 \n \ \n \ -New directory structures: \n \ + New directory structures: \n \ \n \ -NL3 : Newland version 3 \n \ -NL4 : Newland version 4 \n \ -NL5 : Newland version 5 \n \ -v06, v07, v08, v09: Not generated yet \n", - - "default": "ICA", + NL3 : Newland version 3 \n \ + NL4 : Newland version 4 \n \ + NL5 : Newland version 5 \n \ + v06, v07, v08, v09: Not generated yet \n", + "validate": lambda text : text in ["ICA", "NLv3", "NL3", "NL4", "NL5", "v06", "v07"], "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -191,6 +198,7 @@ def ask_questions(): "type": "text", "name": "output:shared:tag", "message": "Enter BC version for new restarts:", + "validate": lambda text : text in ["ICA", "NLv3", "NL3", "NL4", "NL5", "v06", "v07"], "default": "NLv3", "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -198,20 +206,20 @@ def ask_questions(): { "type": "text", "name": "output:shared:tag", - "message": " Enter BC version for new restarts: \n \ + "message": " Enter BC version for new restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) \n \ \n \ -BC version \n \ ---------------- \n \ -ICA : Icarus \n \ -NLv3 : New land version 3 \n \ + BC version \n \ + --------------- \n \ + ICA : Icarus \n \ + NLv3 : New land version 3 \n \ \n \ -New directory structures: \n \ + New directory structures: \n \ \n \ -NL3 : Newland version 3 \n \ -NL4 : Newland version 4 \n \ -NL5 : Newland version 5 \n \ -v06, v07, v08, v09: Not generated yet \n", - + NL3 : Newland version 3 \n \ + NL4 : Newland version 4 \n \ + NL5 : Newland version 5 \n \ + v06, v07, v08, v09: Not generated yet \n", + "validate": lambda text : text in ["ICA", "NLv3", "NL3", "NL4", "NL5", "v06", "v07"], "default": "NLv3", "when": lambda x: x["input:shared:MERRA-2"], }, From cbf6da28b12888f5fdd0a100b92ebabfd36a3ffd Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 29 Jun 2023 09:01:31 -0400 Subject: [PATCH 20/88] change name 'tag' to 'bc_version' --- pre/remap_restart/remap_analysis.py | 4 +- pre/remap_restart/remap_catchANDcn.py | 4 +- pre/remap_restart/remap_command.py | 18 +- .../remap_lake_landice_saltwater.py | 4 +- pre/remap_restart/remap_params.py | 30 ++-- pre/remap_restart/remap_params.tpl | 8 +- pre/remap_restart/remap_questions.py | 12 +- pre/remap_restart/remap_upper.py | 4 +- pre/remap_restart/remap_utils.py | 154 +++++++++--------- 9 files changed, 119 insertions(+), 119 deletions(-) diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index c47191a..32834c4 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -66,8 +66,8 @@ def remap(self): out_bcsdir = config['output']['shared']['bcs_dir'] label = '' if config['output']['shared']['label']: - label = '.' + config['input']['shared']['tag'] + '.' + get_bcs_basename(in_bcsdir) + \ - '.' + config['output']['shared']['tag']+ '.' + get_bcs_basename(out_bcsdir) + label = '.' + config['input']['shared']['bc_version'] + '.' + get_bcs_basename(in_bcsdir) + \ + '.' + config['output']['shared']['bc_version']+ '.' + get_bcs_basename(out_bcsdir) aqua = config['output']['analysis']['aqua'] local_fs=[] diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 1b33559..abb2bff 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -97,8 +97,8 @@ def remap(self): # even the input is binary, the output would be nc4 label = '' if config['output']['shared']['label']: - label = '.' + config['input']['shared']['tag'] + '.' + get_bcs_basename(in_bcsdir) + \ - '.' + config['output']['shared']['tag']+ '.' + get_bcs_basename(out_bcsdir) + label = '.' + config['input']['shared']['bc_version'] + '.' + get_bcs_basename(in_bcsdir) + \ + '.' + config['output']['shared']['bc_version']+ '.' + get_bcs_basename(out_bcsdir) suffix = time+'z' + label + '.nc4' diff --git a/pre/remap_restart/remap_command.py b/pre/remap_restart/remap_command.py index 7ab0ed0..a8d7e50 100755 --- a/pre/remap_restart/remap_command.py +++ b/pre/remap_restart/remap_command.py @@ -44,8 +44,8 @@ def parse_args(program_description): p_command.add_argument('-expid', help='restart id for input restarts') p_command.add_argument('-newid', default="",help='restart id for output restarts') - p_command.add_argument('-tagin', help='GCM or DAS tag associated with inputs') - p_command.add_argument('-tagout', help='GCM or DAS tag associated with outputs') + p_command.add_argument('-bcvin', help='bc_version associated with inputs') + p_command.add_argument('-bcvout', help='bc_version associated with outputs') p_command.add_argument('-in_wemin', help='minimum water snow water equivalent for input catch/cn') p_command.add_argument('-out_wemin', help='minimum water snow water equivalent for output catch/cn') @@ -64,9 +64,9 @@ def parse_args(program_description): p_command.add_argument('-nobkg', action='store_true', help="Don't remap bkg files") p_command.add_argument('-nolcv', action='store_true', help="Don't remap lcv files") p_command.add_argument('-np', action='store_true', help="No prompt. Overwrite config files without prompting questions") - p_command.add_argument('-lbl', action='store_true', help="Label output restarts with tags and resolutions") - p_command.add_argument('-in_bcsdir', default="", help= "users' alternative boundary condition files for input. If not specified (default), it will be deduced from tag and resolution information") - p_command.add_argument('-out_bcsdir', default="", help= "users' alternative boundary condition files for output. If not specified (default), it will be deduced from tag and resolution information") + p_command.add_argument('-lbl', action='store_true', help="Label output restarts with bc_versions and resolutions") + p_command.add_argument('-in_bcsdir', default="", help= "users' alternative boundary condition files for input. If not specified (default), it will be deduced from bc_version and resolution information") + p_command.add_argument('-out_bcsdir', default="", help= "users' alternative boundary condition files for output. If not specified (default), it will be deduced from bc_version and resolution information") p_command.add_argument('-zoom', help= "zoom for the surface input") p_command.add_argument('-qos', default = "debug", help= "queue of slurm job", choices=['debug', 'allnccs']) @@ -90,7 +90,7 @@ def get_answers_from_command_line(cml): if cml.merra2: init_merra2(answers) else: - answers["input:shared:tag"] = cml.tagin + answers["input:shared:bc_version"] = cml.bcvin answers["input:surface:catch_model"] = cml.catch_model answers["input:shared:rst_dir"] = os.path.abspath(cml.rst_dir + '/') fvcore_name(answers) @@ -102,7 +102,7 @@ def get_answers_from_command_line(cml): answers["output:shared:agrid"] = cml.grout answers["output:air:nlevel"] = cml.levsout answers["output:shared:expid"] = cml.newid - answers["output:shared:tag"] = cml.tagout + answers["output:shared:bc_version"] = cml.bcvout answers["output:shared:model"] = cml.ocnmdlout answers["output:shared:label"] = cml.lbl ogrid = cml.oceanout @@ -139,12 +139,12 @@ def get_answers_from_command_line(cml): if cml.in_wemin : answers["input:surface:wemin"] = cml.in_wemin else: - answers["input:surface:wemin"] = we_default(answers['input:shared:tag']) + answers["input:surface:wemin"] = we_default(answers['input:shared:bc_version']) if cml.out_wemin : answers["output:surface:wemin"] = cml.out_wemin else: - answers["output:surface:wemin"] = we_default(answers['output:shared:tag']) + answers["output:surface:wemin"] = we_default(answers['output:shared:bc_version']) answers["slurm:account"] = cml.account answers["slurm:qos"] = cml.qos diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index d0f1b5f..5faf113 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -54,8 +54,8 @@ def remap(self): label = '' if config['output']['shared']['label']: - label = '.' + config['input']['shared']['tag'] + '.' + get_bcs_basename(in_bcsdir) + \ - '.' + config['output']['shared']['tag']+ '.' + get_bcs_basename(out_bcsdir) + label = '.' + config['input']['shared']['bc_version'] + '.' + get_bcs_basename(in_bcsdir) + \ + '.' + config['output']['shared']['bc_version']+ '.' + get_bcs_basename(out_bcsdir) suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + label + types diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index 1b6da5c..2253130 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -36,7 +36,7 @@ def __init__(self, config_from_question): config_tpl['input']['shared']['rst_dir'] = self.common_in['rst_dir']+'/' config_tpl['input']['shared']['expid'] = self.common_in.get('expid') config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] - config_tpl['input']['shared']['tag'] = self.common_in.get('tag') + config_tpl['input']['shared']['bc_version'] = self.common_in.get('bc_version') config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') @@ -47,7 +47,7 @@ def __init__(self, config_from_question): config_tpl['output']['shared']['ogrid'] = self.common_out['ogrid'] config_tpl['output']['shared']['out_dir'] = self.common_out['out_dir'] + '/' config_tpl['output']['shared']['expid'] = self.common_out['expid'] - config_tpl['output']['shared']['tag'] = self.common_out.get('tag') + config_tpl['output']['shared']['bc_version'] = self.common_out.get('bc_version') config_tpl['output']['shared']['label'] = self.common_out.get('label') config_tpl['input']['shared']['bcs_dir'] = self.common_in['bcs_dir'] @@ -65,12 +65,12 @@ def params_for_air(self, config_tpl): return config_tpl ogrid = config_tpl['input']['shared']['ogrid'] - tagout = self.common_out['tag'] - bctag = get_bcTag(tagout, ogrid) - tagrank = TagsRank[bctag] + bcvout = self.common_out['bc_version'] + bcVersion = get_bcVersion(bcvout, ogrid) + bc_versionrank = BcsRank[bcVersion] if ( not config_tpl['input']['air']['drymass']) : config_tpl['input']['air']['drymass'] = 0 - if tagrank >=12 : + if bc_versionrank >=12 : config_tpl['input']['air']['drymass'] = 1 return config_tpl @@ -83,13 +83,13 @@ def options_for_slurm(self, config_tpl): def params_for_surface(self, config_tpl): config_tpl['output']['surface']['surflay'] = 20. - tagout = self.common_out['tag'] + bcvout = self.common_out['bc_version'] ogrid = self.common_out['ogrid'] - bctag = get_bcTag(tagout, ogrid) - tagrank = TagsRank[bctag] - if tagrank >=12 : + bcVersion = get_bcVersion(bcvout, ogrid) + bc_versionrank = BcsRank[bcVersion] + if bc_versionrank >=12 : config_tpl['output']['surface']['surflay'] = 50. - if tagrank >= TagsRank["Icarus_Reynolds"]: + if bc_versionrank >= BcsRank["Icarus_Reynolds"]: config_tpl['output']['surface']['split_saltwater'] = True config_tpl['input']['surface']['zoom']= self.surf_in['zoom'] config_tpl['input']['surface']['wemin']= self.surf_in['wemin'] @@ -102,10 +102,10 @@ def params_for_analysis(self, config_tpl): config_tpl['output']['analysis']['bkg'] = self.ana_out.get('bkg') ogrid = self.common_out['ogrid'] - tagout = self.common_out['tag'] - bctag = get_bcTag(tagout, ogrid) - tagrank = TagsRank[bctag] - if tagrank >= TagsRank["Ganymed-4_0_Reynolds"] : + bcvout = self.common_out['bc_version'] + bcVersion = get_bcVersion(bcvout, ogrid) + bc_versionrank = BcsRank[bcVersion] + if bc_versionrank >= BcsRank["Ganymed-4_0_Reynolds"] : config_tpl['output']['analysis']['aqua'] = True return config_tpl diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index cecf3d3..cbabcda 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -8,8 +8,8 @@ input: drymass: 1 hydrostatic: 0 shared: - #the tag info here is not critical. It is for command_line options - tag: none + #the bc_version info here is not critical. It is used in command_line options + bc_version: none MERRA-2: false agrid: bcs_dir: @@ -26,8 +26,8 @@ input: catch_tilefile: null output: shared: - #the tag info here is not critical. It is for command_line options - tag: none + #the bc_version info here is not critical. It is used in command_line options + bc_version: none label: false agrid: bcs_dir: diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index cb0e59f..031d477 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -176,7 +176,7 @@ def ask_questions(): { "type": "text", - "name": "input:shared:tag", + "name": "input:shared:bc_version", "message": "Enter BC version that matches input restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) \n \ \n \ BC version \n \ @@ -196,7 +196,7 @@ def ask_questions(): { "type": "text", - "name": "output:shared:tag", + "name": "output:shared:bc_version", "message": "Enter BC version for new restarts:", "validate": lambda text : text in ["ICA", "NLv3", "NL3", "NL4", "NL5", "v06", "v07"], "default": "NLv3", @@ -205,7 +205,7 @@ def ask_questions(): # show the message if it is merra2 { "type": "text", - "name": "output:shared:tag", + "name": "output:shared:bc_version", "message": " Enter BC version for new restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) \n \ \n \ BC version \n \ @@ -276,13 +276,13 @@ def ask_questions(): "type": "text", "name": "input:surface:wemin", "message": "What is value of wemin (minimum snow water equivalent parameter) for surface inputs?", - "default": lambda x: we_default(x.get('input:shared:tag')) + "default": lambda x: we_default(x.get('input:shared:bc_version')) }, { "type": "text", "name": "output:surface:wemin", "message": "What is value of wemin (minimum snow water equivalent parameter) for new surface restarts?", - "default": lambda x: we_default(x.get('output:shared:tag')) + "default": lambda x: we_default(x.get('output:shared:bc_version')) }, { "type": "text", @@ -299,7 +299,7 @@ def ask_questions(): { "type": "confirm", "name": "output:shared:label", - "message": "Would you like to add labels (tags,resoultions) to restarts' names?", + "message": "Would you like to add labels (bc_versions,resoultions) to restarts' names?", "default": False, }, diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 2c3b475..4daa954 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -81,8 +81,8 @@ def remap(self): yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) label = '' if config['output']['shared']['label']: - label = '.' + config['input']['shared']['tag'] + '.' + get_bcs_basename(in_bcsdir) + \ - '.' + config['output']['shared']['tag']+ '.' + get_bcs_basename(out_bcsdir) + label = '.' + config['input']['shared']['bc_version'] + '.' + get_bcs_basename(in_bcsdir) + \ + '.' + config['output']['shared']['bc_version']+ '.' + get_bcs_basename(out_bcsdir) suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + label + types for rst in restarts_in : diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 785432e..308b124 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -11,23 +11,23 @@ # define some constants for bcs -NewStructureBCTag = ('NL3', 'NL4', 'NL5', 'v06', 'v07', 'v08', 'v09') +NewStructureBCList = ['NL3', 'NL4', 'NL5', 'v06', 'v07', 'v08', 'v09'] BCBase={} BCBase['discover_ops'] = "/discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs" BCBase['discover_legacy'] = "/discover/nobackup/projects/gmao/bcs_shared/legacy_bcs" BCBase['discover_couple'] = "/discover/nobackup/projects/gmao/ssd/aogcm/atmosphere_bcs" BCBase['discover_ns'] = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" -BCSTag = {} -TagsRank = {} -tag_initialized = False +BcsVersionMap = {} +BcsRank = {} +bc_version_initialized = False -def init_tags(): +def init_bc_versions(): # copy and paste from rigrid.pl # minor change. Add "D" to the number for each group # BCS Tag: Fortuna-1_4 - global tag_initialized - if tag_initialized : return + global bc_version_initialized + if bc_version_initialized : return # F14 = ( 'F14', 'Fortuna-1_4', 'Fortuna-1_4_p1' ) # D214 = ( 'D214', 'GEOSdas-2_1_4', 'GEOSdas-2_1_4-m1', # 'GEOSdas-2_1_4-m2', 'GEOSdas-2_1_4-m3', 'GEOSdas-2_1_4-m4' ) @@ -109,7 +109,7 @@ def init_tags(): # # BCS Tags: Icarus (New Land Parameters, New Topography) # #--------------------------------------------------------------------------- ICA = ( 'ICA', 'Icarus', 'Jason' ) - NLv3 = ( 'NLv3', 'NLV3') + NLv3 = ( 'NLv3',) # D517 = ( '517', 'GEOSadas-5_17_0', 'GEOSadas-5_17_1', 'GEOSadas-5_18_0', # 'GEOSadas-5_18_1', 'GEOSadas-5_18_2', 'GEOSadas-5_18_3', # 'GEOSadas-5_18_3_p1', 'GEOSadas-5_19_0', 'GEOSadas-5_20_0', @@ -131,44 +131,44 @@ def init_tags(): # D525 = ( '525', 'GEOSadas-5_25_1', 'GEOSadas-5_25_1_p5', 'GEOSadas-5_25_p7', # 'GEOSadas-5_27_1', 'GEOSadas-5_29_3', 'GEOSadas-5_29_4' ) -# for tag in F14: BCSTag[tag]= "Fortuna-1_4" -# for tag in F20: BCSTag[tag]= "Fortuna-2_0" -# for tag in F21: BCSTag[tag]= "Fortuna-2_1" -# for tag in G10: BCSTag[tag]= "Ganymed-1_0" -# for tag in G10p: BCSTag[tag]= "Ganymed-1_0_M" -# for tag in G20: BCSTag[tag]= "Ganymed-1_0_M" -# for tag in G30: BCSTag[tag]= "Ganymed-1_0_Reynolds" - for tag in G40: BCSTag[tag]= "Ganymed-4_0_Reynolds" - for tag in ICA: BCSTag[tag]= "Icarus_Reynolds" - for tag in NLv3: BCSTag[tag]= "Icarus-NLv3_Reynolds" -# for tag in GITOL: BCSTag[tag]= "Icarus_Reynolds" -# for tag in INL: BCSTag[tag]= "Icarus-NLv3_Reynolds" -# for tag in GITNL: BCSTag[tag]= "Icarus-NLv3_Reynolds" -# for tag in NewStructureBCTag: BCSTag[tag]= tag +# for bc_version in F14: BcsVersionMap[bc_version]= "Fortuna-1_4" +# for bc_version in F20: BcsVersionMap[bc_version]= "Fortuna-2_0" +# for bc_version in F21: BcsVersionMap[bc_version]= "Fortuna-2_1" +# for bc_version in G10: BcsVersionMap[bc_version]= "Ganymed-1_0" +# for bc_version in G10p: BcsVersionMap[bc_version]= "Ganymed-1_0_M" +# for bc_version in G20: BcsVersionMap[bc_version]= "Ganymed-1_0_M" +# for bc_version in G30: BcsVersionMap[bc_version]= "Ganymed-1_0_Reynolds" + for bc_version in G40: BcsVersionMap[bc_version]= "Ganymed-4_0_Reynolds" + for bc_version in ICA: BcsVersionMap[bc_version]= "Icarus_Reynolds" + for bc_version in NLv3: BcsVersionMap[bc_version]= "Icarus-NLv3_Reynolds" +# for bc_version in GITOL: BcsVersionMap[bc_version]= "Icarus_Reynolds" +# for bc_version in INL: BcsVersionMap[bc_version]= "Icarus-NLv3_Reynolds" +# for bc_version in GITNL: BcsVersionMap[bc_version]= "Icarus-NLv3_Reynolds" +# for bc_version in NewStructureBCList: BcsVersionMap[bc_version]= bc_version # -# for tag in D214: BCSTag[tag]= "Fortuna-1_4" -# for tag in D540: BCSTag[tag]= "Fortuna-1_4" -# for tag in D561: BCSTag[tag]= "Fortuna-2_1" -# for tag in D580: BCSTag[tag]= "Ganymed-1_0" -# for tag in D591p: BCSTag[tag]= "Ganymed-1_0_M" -# for tag in D5A0: BCSTag[tag]= "Ganymed-1_0_M" -# for tag in D5B0: BCSTag[tag]= "Ganymed-1_0_Reynolds" -# for tag in D512: BCSTag[tag]= "Ganymed-4_0_Reynolds" -# for tag in D517: BCSTag[tag]= "Icarus_Reynolds" -# for tag in D525: BCSTag[tag]= "Icarus-NLv3_Reynolds" +# for bc_version in D214: BcsVersionMap[bc_version]= "Fortuna-1_4" +# for bc_version in D540: BcsVersionMap[bc_version]= "Fortuna-1_4" +# for bc_version in D561: BcsVersionMap[bc_version]= "Fortuna-2_1" +# for bc_version in D580: BcsVersionMap[bc_version]= "Ganymed-1_0" +# for bc_version in D591p: BcsVersionMap[bc_version]= "Ganymed-1_0_M" +# for bc_version in D5A0: BcsVersionMap[bc_version]= "Ganymed-1_0_M" +# for bc_version in D5B0: BcsVersionMap[bc_version]= "Ganymed-1_0_Reynolds" +# for bc_version in D512: BcsVersionMap[bc_version]= "Ganymed-4_0_Reynolds" +# for bc_version in D517: BcsVersionMap[bc_version]= "Icarus_Reynolds" +# for bc_version in D525: BcsVersionMap[bc_version]= "Icarus-NLv3_Reynolds" # - TagsRank['Ganymed-4_0_Reynolds'] = 12 - TagsRank['Ganymed-4_0_Ostia'] = 13 - TagsRank['Ganymed-4_0_MERRA-2'] = 14 - TagsRank['Icarus_Reynolds'] = 15 - TagsRank['Icarus_MERRA-2'] = 16 - TagsRank['Icarus_Ostia'] = 17 - TagsRank['Icarus-NLv3_Reynolds'] = 18 - TagsRank['Icarus-NLv3_MERRA-2'] = 19 - TagsRank['Icarus-NLv3_Ostia'] = 20 - for tag in NewStructureBCTag: TagsRank[tag] = 21 - tag_initialized = True + BcsRank['Ganymed-4_0_Reynolds'] = 12 + BcsRank['Ganymed-4_0_Ostia'] = 13 + BcsRank['Ganymed-4_0_MERRA-2'] = 14 + BcsRank['Icarus_Reynolds'] = 15 + BcsRank['Icarus_MERRA-2'] = 16 + BcsRank['Icarus_Ostia'] = 17 + BcsRank['Icarus-NLv3_Reynolds'] = 18 + BcsRank['Icarus-NLv3_MERRA-2'] = 19 + BcsRank['Icarus-NLv3_Ostia'] = 20 + for bc_version in NewStructureBCList: BcsRank[bc_version] = 21 + bc_version_initialized = True def init_merra2(x): if not x.get('input:shared:MERRA-2') : return False @@ -193,7 +193,7 @@ def init_merra2(x): x['input:shared:model'] = 'data' x['input:shared:agrid'] = 'C180' x['input:shared:ogrid'] = '1440x720' - x['input:shared:tag'] = 'Ganymed-4_0' + x['input:shared:bc_version'] = 'Ganymed-4_0' x['input:surface:catch_model'] = 'catch' x['input:shared:rst_dir'] = x['output:shared:out_dir'] + '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' @@ -253,10 +253,10 @@ def get_bcs_basename(bcs): while bcs[-1] == '/': bcs = bcs[0:-1] # remove extra '/' return os.path.basename(bcs) -def we_default(tag): +def we_default(bc_version): default_ = '26' - if tag =='NLv3' : default_ = '13' - if tag in NewStructureBCTag : default_ = '13' + if bc_version =='NLv3' : default_ = '13' + if bc_version in NewStructureBCList : default_ = '13' return default_ def zoom_default(x): @@ -336,10 +336,10 @@ def get_command_line_from_answers(answers): if newid.strip(): out_newid = " -newid " + newid - in_tagin = '' - if answers.get("input:shared:tag"): - in_tagin = " -tagin " + answers["input:shared:tag"] - tagout = " -tagout " + answers["output:shared:tag"] + bcvin = '' + if answers.get("input:shared:bc_version"): + bcvin = " -bcvin " + answers["input:shared:bc_version"] + bcvout = " -bcvout " + answers["output:shared:bc_version"] ocnmdlin = '' if answers.get("input:shared:model"): @@ -402,8 +402,8 @@ def get_command_line_from_answers(answers): ocnmdlout + \ oceanin + \ oceanout + \ - in_tagin + \ - tagout + \ + bcvin + \ + bcvout + \ rst_dir + \ out_dir + \ in_bcsdir + \ @@ -443,19 +443,19 @@ def get_config_from_answers(answers): return config -def get_bcTag(tag, ogrid): - bctag = BCSTag[tag] - if bctag in NewStructureBCTag : return bctag +def get_bcVersion(bc_version, ogrid): + bcVersion = BcsVersionMap[bc_version] + if bcVersion in NewStructureBCList : return bcVersion if ogrid[0].upper() == "C": - bctag=bctag.replace('_Reynolds','_Ostia') + bcVersion=bcVersion.replace('_Reynolds','_Ostia') else: xy = ogrid.upper().split('X') x = int(xy[0]) - if x == 1440: bctag=bctag.replace('_Reynolds','_MERRA-2') + if x == 1440: bcVersion=bcVersion.replace('_Reynolds','_MERRA-2') if x == 2880: - bctag=bctag.replace('_Reynolds','_Ostia') - bctag=bctag.replace('_M','_D') - return bctag + bcVersion=bcVersion.replace('_Reynolds','_Ostia') + bcVersion=bcVersion.replace('_M','_D') + return bcVersion def get_grid_subdir(bcdir, agrid, ogrid, model): def get_name_with_grid( grid, names, a_o): @@ -516,13 +516,13 @@ def get_name_with_grid( grid, names, a_o): return g def get_bcsdir(x, opt): - init_tags() - tag = x.get('input:shared:tag') + init_bc_versions() + bc_version = x.get('input:shared:bc_version') agrid = x.get('input:shared:agrid') ogrid = x.get('input:shared:ogrid') model = x.get('input:shared:model') if opt.upper() == "OUT": - tag = x.get('output:shared:tag') + bc_version = x.get('output:shared:bc_version') agrid = x.get('output:shared:agrid') ogrid = x.get('output:shared:ogrid') model = x.get('output:shared:model') @@ -531,35 +531,35 @@ def get_bcsdir(x, opt): base = 'discover_legacy' if model != "data": base = 'discover_couple' - if tag in NewStructureBCTag: + if bc_version in NewStructureBCList: base = 'discover_ns' if x.get('input:shared:MERRA-2') and opt.upper() == "IN": base = 'discover_ops' bc_base = BCBase[base] - bctag = get_bcTag(tag,ogrid) - tagrank = TagsRank[bctag] + bcVersion = get_bcVersion(bc_version,ogrid) + bc_versionrank = BcsRank[bcVersion] - if (tagrank >= TagsRank['NL3']) : - bcdir = bc_base+'/'+ bctag+'/geometry/' - elif (tagrank >= TagsRank['Icarus-NLv3_Reynolds']) : - bcdir = bc_base+'/Icarus-NLv3/'+bctag+'/' + if (bc_versionrank >= BcsRank['NL3']) : + bcdir = bc_base+'/'+ bcVersion+'/geometry/' + elif (bc_versionrank >= BcsRank['Icarus-NLv3_Reynolds']) : + bcdir = bc_base+'/Icarus-NLv3/'+bcVersion+'/' if model == 'MOM6' or model == 'MOM5': bcdir = bc_base+'/Icarus-NLv3/'+model+'/' - elif (tagrank >= TagsRank['Icarus_Reynolds']): + elif (bc_versionrank >= BcsRank['Icarus_Reynolds']): if bc_base == BCBase['discover_ops']: - bcdir = bc_base+'/Icarus_Updated/'+bctag+'/' + bcdir = bc_base+'/Icarus_Updated/'+bcVersion+'/' else: - bcdir = bc_base+'/Icarus/'+bctag+'/' + bcdir = bc_base+'/Icarus/'+bcVersion+'/' if model == 'MOM6' or model == 'MOM5': bcdir = bc_base+'/Icarus/'+model+'/' - elif(tagrank >= TagsRank["Ganymed-4_0_Reynolds"]): - bcdir = bc_base + '/Ganymed-4_0/'+bctag+'/' + elif(bc_versionrank >= BcsRank["Ganymed-4_0_Reynolds"]): + bcdir = bc_base + '/Ganymed-4_0/'+bcVersion+'/' if model == 'MOM6' or model == 'MOM5': bcdir = bc_base+'/Ganymed/'+model+'/' else: - bcdir = bc_base + '/' + bctag + '/' + bcdir = bc_base + '/' + bcVersion + '/' if model == 'MOM6' or model == 'MOM5': bcdir = bc_base+'/Ganymed/'+model+'/' From 2e8beacbcf76c61938443fde847484857c6f65af Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 29 Jun 2023 11:18:16 -0400 Subject: [PATCH 21/88] use triple ''' to make string cleaner --- pre/remap_restart/remap_questions.py | 72 ++++++++++++++-------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 031d477..d0ecc67 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -80,11 +80,11 @@ def ask_questions(): { "type": "text", "name": "input:shared:agrid", - "message": "Enter input atmospheric grid: \n \ - C12 C180 C1000 C270\n \ - C24 C360 C1440 C540\n \ - C48 C500 C2880 C1080\n \ - C90 C720 C5760 C2160 C1536\n ", + "message": f'''Enter input atmospheric grid: + C12 C180 C1000 C270 + C24 C360 C1440 C540 + C48 C500 C2880 C1080 + C90 C720 C5760 C2160 C1536\n''', "default": 'C360', # if it is merra-2 or has_fvcore, agrid is deduced "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_name(x), @@ -128,11 +128,11 @@ def ask_questions(): { "type": "text", "name": "output:shared:agrid", - "message": "Enter new atmospheric grid: \n \ - C12 C180 C1000 C270\n \ - C24 C360 C1440 C540\n \ - C48 C500 C2880 C1080\n \ - C90 C720 C5760 C2160 C1536\n ", + "message": f'''Enter new atmospheric grid: + C12 C180 C1000 C270 + C24 C360 C1440 C540 + C48 C500 C2880 C1080 + C90 C720 C5760 C2160 C1536\n''', "default": 'C360', }, @@ -177,19 +177,19 @@ def ask_questions(): { "type": "text", "name": "input:shared:bc_version", - "message": "Enter BC version that matches input restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) \n \ -\n \ - BC version \n \ - --------------- \n \ - ICA : Icarus \n \ - NLv3 : New land version 3 \n \ -\n \ - New directory structures: \n \ -\n \ - NL3 : Newland version 3 \n \ - NL4 : Newland version 4 \n \ - NL5 : Newland version 5 \n \ - v06, v07, v08, v09: Not generated yet \n", + "message": f'''Enter BC version that matches input restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) + + BC version + --------------- + ICA : Icarus + NLv3 : New land version 3 + + New directory structures: + + NL3 : Newland version 3 + NL4 : Newland version 4 + NL5 : Newland version 5 + v06, v07, v08, v09: Not generated yet \n''', "validate": lambda text : text in ["ICA", "NLv3", "NL3", "NL4", "NL5", "v06", "v07"], "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -206,19 +206,19 @@ def ask_questions(): { "type": "text", "name": "output:shared:bc_version", - "message": " Enter BC version for new restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) \n \ -\n \ - BC version \n \ - --------------- \n \ - ICA : Icarus \n \ - NLv3 : New land version 3 \n \ -\n \ - New directory structures: \n \ -\n \ - NL3 : Newland version 3 \n \ - NL4 : Newland version 4 \n \ - NL5 : Newland version 5 \n \ - v06, v07, v08, v09: Not generated yet \n", + "message": f'''Enter BC version that matches input restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) + + BC version + --------------- + ICA : Icarus + NLv3 : New land version 3 + + New directory structures: + + NL3 : Newland version 3 + NL4 : Newland version 4 + NL5 : Newland version 5 + v06, v07, v08, v09: Not generated yet \n''', "validate": lambda text : text in ["ICA", "NLv3", "NL3", "NL4", "NL5", "v06", "v07"], "default": "NLv3", "when": lambda x: x["input:shared:MERRA-2"], From ef135bdc32778359d6e643d7db6177419b2c286c Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang <52509753+weiyuan-jiang@users.noreply.github.com> Date: Thu, 29 Jun 2023 11:27:07 -0400 Subject: [PATCH 22/88] Update remap_questions.py --- pre/remap_restart/remap_questions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index d0ecc67..1f59bca 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -206,7 +206,7 @@ def ask_questions(): { "type": "text", "name": "output:shared:bc_version", - "message": f'''Enter BC version that matches input restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) + "message": f'''Enter BC version for new restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) BC version --------------- From 88232db6d3e2612bae3d2da7f062f544a2b8a2b2 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 6 Jul 2023 10:33:49 -0400 Subject: [PATCH 23/88] more cleanup. BC version, catch_model ... --- pre/remap_restart/remap_command.py | 4 +- pre/remap_restart/remap_questions.py | 90 +++++++++++++--------------- pre/remap_restart/remap_utils.py | 29 ++++++++- 3 files changed, 71 insertions(+), 52 deletions(-) diff --git a/pre/remap_restart/remap_command.py b/pre/remap_restart/remap_command.py index a8d7e50..4cd1268 100755 --- a/pre/remap_restart/remap_command.py +++ b/pre/remap_restart/remap_command.py @@ -139,12 +139,12 @@ def get_answers_from_command_line(cml): if cml.in_wemin : answers["input:surface:wemin"] = cml.in_wemin else: - answers["input:surface:wemin"] = we_default(answers['input:shared:bc_version']) + answers["input:surface:wemin"] = wemin_default(answers['input:shared:bc_version']) if cml.out_wemin : answers["output:surface:wemin"] = cml.out_wemin else: - answers["output:surface:wemin"] = we_default(answers['output:shared:bc_version']) + answers["output:surface:wemin"] = wemin_default(answers['output:shared:bc_version']) answers["slurm:account"] = cml.account answers["slurm:qos"] = cml.qos diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 1f59bca..a170d1c 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -85,7 +85,7 @@ def ask_questions(): C24 C360 C1440 C540 C48 C500 C2880 C1080 C90 C720 C5760 C2160 C1536\n''', - "default": 'C360', + "validate": lambda text : text in ['C12','C180','C1000','C270','C24','C360','C1440','C540','C48','C500','C2880','C1080','C90','C720','C5760','C2160','C1536'], # if it is merra-2 or has_fvcore, agrid is deduced "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_name(x), }, @@ -134,6 +134,7 @@ def ask_questions(): C48 C500 C2880 C1080 C90 C720 C5760 C2160 C1536\n''', "default": 'C360', + "validate": lambda text : text in ['C12','C180','C1000','C270','C24','C360','C1440','C540','C48','C500','C2880','C1080','C90','C720','C5760','C2160','C1536'], }, { @@ -175,52 +176,51 @@ def ask_questions(): }, { - "type": "text", + "type": "select", "name": "input:shared:bc_version", - "message": f'''Enter BC version that matches input restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) - - BC version - --------------- - ICA : Icarus - NLv3 : New land version 3 - - New directory structures: + "message": f'''Enter BC version that matches input restarts: - NL3 : Newland version 3 - NL4 : Newland version 4 - NL5 : Newland version 5 - v06, v07, v08, v09: Not generated yet \n''', - "validate": lambda text : text in ["ICA", "NLv3", "NL3", "NL4", "NL5", "v06", "v07"], + BC version Sample GCM and DAS tags + ---------- ----------------------- + Ganymed-4_0: Ganymed-4_0 ...... Heracles-5_4_p3 + Ganymed-4_0: GEOSadas-5_12_2 .. GEOSadas-5_16_5 + ICA: Icarus ........... Jason + ICA: 10.3 ............. 10.18 + ICA: GEOSadas-5_17_0 .. GEOSadas-5_24_0_p1 + NL3: Icarus-NL ........ Jason-NL + NL3: 10.19 ............ 10.23 + NL3: 11.00 ............ 11.1 + NL3: GEOSadas-5_25_1 .. GEOSadas-5_29_4\n''', + "choices": ['NL3', 'ICA','Ganymed-4_0'], "when": lambda x: not x["input:shared:MERRA-2"], }, { - "type": "text", + "type": "select", "name": "output:shared:bc_version", "message": "Enter BC version for new restarts:", - "validate": lambda text : text in ["ICA", "NLv3", "NL3", "NL4", "NL5", "v06", "v07"], - "default": "NLv3", + "choices": ['NL3', 'ICA','Ganymed-4_0'], + "default": "NL3", "when": lambda x: not x["input:shared:MERRA-2"], }, # show the message if it is merra2 { - "type": "text", + "type": "select", "name": "output:shared:bc_version", - "message": f'''Enter BC version for new restarts: (ICA, NLv3, NL3, NL4, NL5, v06, v07,...) - - BC version - --------------- - ICA : Icarus - NLv3 : New land version 3 - - New directory structures: - - NL3 : Newland version 3 - NL4 : Newland version 4 - NL5 : Newland version 5 - v06, v07, v08, v09: Not generated yet \n''', - "validate": lambda text : text in ["ICA", "NLv3", "NL3", "NL4", "NL5", "v06", "v07"], - "default": "NLv3", + "message": f'''Enter BC version for new restarts: + BC version Sample GCM and DAS tags + ---------- ----------------------- + Ganymed-4_0: Ganymed-4_0 ...... Heracles-5_4_p3 + Ganymed-4_0: GEOSadas-5_12_2 .. GEOSadas-5_16_5 + ICA: Icarus ........... Jason + ICA: 10.3 ............. 10.18 + ICA: GEOSadas-5_17_0 .. GEOSadas-5_24_0_p1 + NL3: Icarus-NL ........ Jason-NL + NL3: 10.19 ............ 10.23 + NL3: 11.00 ............ 11.1 + NL3: GEOSadas-5_25_1 .. GEOSadas-5_29_4\n''', + "choices": ['NL3', 'ICA','Ganymed-4_0'], + "default": "NL3", "when": lambda x: x["input:shared:MERRA-2"], }, @@ -251,15 +251,6 @@ def ask_questions(): "default": True, }, - { - "type": "select", - "name": "input:surface:catch_model", - "message": "What is the catchment model? ", - "choices": ['catch','catchcnclm40','catchcnclm45'], - "default": 'catch', - "when": lambda x: x["output:surface:remap"] and not x["input:shared:MERRA-2"] - }, - { "type": "confirm", "name": "output:analysis:bkg", @@ -276,18 +267,18 @@ def ask_questions(): "type": "text", "name": "input:surface:wemin", "message": "What is value of wemin (minimum snow water equivalent parameter) for surface inputs?", - "default": lambda x: we_default(x.get('input:shared:bc_version')) + "default": lambda x: wemin_default(x.get('input:shared:bc_version')) }, { "type": "text", "name": "output:surface:wemin", "message": "What is value of wemin (minimum snow water equivalent parameter) for new surface restarts?", - "default": lambda x: we_default(x.get('output:shared:bc_version')) + "default": lambda x: wemin_default(x.get('output:shared:bc_version')) }, { "type": "text", "name": "input:surface:zoom", - "message": "What is value of zoom (paremeter of radius search, smaller value means larger radius) for surface inputs [1-8]?", + "message": "What is value of zoom (parameter of radius search, smaller value means larger radius) for surface inputs [1-8]?", "default": lambda x: zoom_default(x) }, { @@ -299,7 +290,7 @@ def ask_questions(): { "type": "confirm", "name": "output:shared:label", - "message": "Would you like to add labels (bc_versions,resoultions) to restarts' names?", + "message": "Would you like to add labels (bc_versions,resolutions) to restarts' names?", "default": False, }, @@ -328,7 +319,10 @@ def ask_questions(): answers['input:shared:yyyymmddhh'] = answers['input:shared:yyyymmddhh']+ '21' answers['input:shared:rst_dir'] = os.path.abspath(answers['input:shared:rst_dir']) answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) - + + if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"]: + answers["input:surface:catch_model"] = catch_model(answers) + return answers if __name__ == "__main__": diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 308b124..8dfca55 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -144,7 +144,7 @@ def init_bc_versions(): # for bc_version in GITOL: BcsVersionMap[bc_version]= "Icarus_Reynolds" # for bc_version in INL: BcsVersionMap[bc_version]= "Icarus-NLv3_Reynolds" # for bc_version in GITNL: BcsVersionMap[bc_version]= "Icarus-NLv3_Reynolds" -# for bc_version in NewStructureBCList: BcsVersionMap[bc_version]= bc_version + for bc_version in NewStructureBCList: BcsVersionMap[bc_version]= bc_version # # for bc_version in D214: BcsVersionMap[bc_version]= "Fortuna-1_4" @@ -239,6 +239,31 @@ def fvcore_name(x): x['input:shared:expid'] = expid return fname +def catch_model(x): + ymdh = x['input:shared:yyyymmddhh'] + time = ymdh[0:8] + '_'+ymdh[8:10] + rst_dir = x.get('input:shared:rst_dir') + if not rst_dir : return False + + x['input:shared:rst_dir'] = rst_dir.strip() # remove extra space + + files = glob.glob(rst_dir+'/*catch*') + + if len (files) == 0 : return False + fname= '' + if len(files) == 1: + fname = os.path.basename(files[0]) + + if len(files) > 1 : + files = glob.glob(rst_dir+'/*fvcore_*'+time+'*') + fname = os.path.basename(files[0]) + model = 'catch' + if 'cnclm40' in fname.lower(): + model = 'catchcnclm40' + if 'cnclm45' in fname.lower(): + model = 'catchcnclm45' + return model + def data_ocean_default(resolution): # the default string should match the choice in remapl_question.py default_ = 'CS (same as atmosphere OSTIA cubed-sphere grid)' @@ -253,7 +278,7 @@ def get_bcs_basename(bcs): while bcs[-1] == '/': bcs = bcs[0:-1] # remove extra '/' return os.path.basename(bcs) -def we_default(bc_version): +def wemin_default(bc_version): default_ = '26' if bc_version =='NLv3' : default_ = '13' if bc_version in NewStructureBCList : default_ = '13' From 1a316d323eaf60fbb946dd124eea0e86970ce963 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 12 Jul 2023 07:48:13 -0400 Subject: [PATCH 24/88] remove extra line --- pre/remap_restart/remap_catchANDcn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index abb2bff..5c2bfca 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -57,7 +57,6 @@ def remap(self): surflay = config['output']['surface']['surflay'] in_tilefile = config['input']['surface']['catch_tilefile'] - in_bcsdir = get_landdir(in_bcsdir) if not in_tilefile : if not in_bcsdir: exit("Must provide either input tile file or input bcs directory") From d744f27fe8f4b1963795954bbdada9a77cd55f2a Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 13 Jul 2023 10:21:45 -0400 Subject: [PATCH 25/88] fix output label --- pre/remap_restart/remap_analysis.py | 14 ++++---------- pre/remap_restart/remap_catchANDcn.py | 15 ++++----------- pre/remap_restart/remap_lake_landice_saltwater.py | 10 +++------- pre/remap_restart/remap_restarts.py | 2 +- pre/remap_restart/remap_upper.py | 9 +++------ pre/remap_restart/remap_utils.py | 14 +++++++++++--- 6 files changed, 26 insertions(+), 38 deletions(-) diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index 32834c4..9588228 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -15,7 +15,7 @@ import fileinput import ruamel.yaml from remap_base import remap_base -from remap_utils import get_bcs_basename +from remap_utils import get_label class analysis(remap_base): def __init__(self, **configs): @@ -62,12 +62,7 @@ def remap(self): else: expid_out = '' - in_bcsdir = config['input']['shared']['bcs_dir'] - out_bcsdir = config['output']['shared']['bcs_dir'] - label = '' - if config['output']['shared']['label']: - label = '.' + config['input']['shared']['bc_version'] + '.' + get_bcs_basename(in_bcsdir) + \ - '.' + config['output']['shared']['bc_version']+ '.' + get_bcs_basename(out_bcsdir) + label = get_label(config) aqua = config['output']['analysis']['aqua'] local_fs=[] @@ -98,8 +93,7 @@ def remap(self): for f in local_fs: fname = os.path.basename(f) - k = fname.rfind('.') - fname = fname[0:k] + label + fname[k:] + fname = fname + label shutil.move(f, out_dir+'/'+fname) # write lcv lcv = config['output']['analysis']['lcv'] @@ -107,7 +101,7 @@ def remap(self): ymd_ = yyyymmddhh_[0:8] hh_ = yyyymmddhh_[8:10] hms_ = hh_+'0000' - rstlcvOut = out_dir+'/'+expid_out+'rst.lcv.'+ymd_+'_'+hh_+'z'+label +'.bin' + rstlcvOut = out_dir+'/'+expid_out+'rst.lcv.'+ymd_+'_'+hh_+'z.bin' + label cmd = bindir+'/mkdrstdate.x ' + ymd_ + ' ' + hms_ +' ' + rstlcvOut print(cmd) subprocess.call(shlex.split(cmd)) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 5c2bfca..1d88ee7 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -10,15 +10,11 @@ import mimetypes import netCDF4 as nc from remap_base import remap_base -from remap_utils import get_bcs_basename +from remap_utils import get_label def get_landdir(bcsdir): if bcsdir : - k = bcsdir.find('/geometry/') - if k != -1 : - while bcsdir[-1] == '/': bcsdir = bcsdir[0:-1] # remove extra '/' - sub_grids = os.path.basename(bcsdir) - bcsdir = bcsdir[0:k]+'/land/'+ sub_grids + bcsdir = bcsdir.replace('/geometry/','/land/') return bcsdir class catchANDcn(remap_base): @@ -94,12 +90,9 @@ def remap(self): account = config['slurm']['account'] # even the input is binary, the output would be nc4 - label = '' - if config['output']['shared']['label']: - label = '.' + config['input']['shared']['bc_version'] + '.' + get_bcs_basename(in_bcsdir) + \ - '.' + config['output']['shared']['bc_version']+ '.' + get_bcs_basename(out_bcsdir) + label = get_label(config) - suffix = time+'z' + label + '.nc4' + suffix = time+'z.nc4' + label if (expid) : expid = expid + '.' diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index 5faf113..1383c45 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -7,7 +7,7 @@ import ruamel.yaml import shlex from remap_base import remap_base -from remap_utils import get_bcs_basename +from remap_utils import get_label from remap_bin2nc import bin2nc class lake_landice_saltwater(remap_base): @@ -52,12 +52,8 @@ def remap(self): types = '.nc4' yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) - label = '' - if config['output']['shared']['label']: - label = '.' + config['input']['shared']['bc_version'] + '.' + get_bcs_basename(in_bcsdir) + \ - '.' + config['output']['shared']['bc_version']+ '.' + get_bcs_basename(out_bcsdir) - - suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + label + types + label = get_label(config) + suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + types + label saltwater = '' seaice = '' diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index fcf2cfc..ba89d6e 100644 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -101,7 +101,7 @@ def main(): # write config to yaml file config_to_yaml(config, config_yaml,noprompt = noprompt) - + print(config_yaml) # upper air upper = upperair(params_file=config_yaml) upper.remap() diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 4daa954..1088e08 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -7,7 +7,7 @@ import shutil import glob from remap_base import remap_base -from remap_utils import get_bcs_basename +from remap_utils import get_label from remap_bin2nc import bin2nc def get_topodir(bcsdir): @@ -79,12 +79,9 @@ def remap(self): if type_str.find('Hierarchical') >=0: types = '.nc4' yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) - label = '' - if config['output']['shared']['label']: - label = '.' + config['input']['shared']['bc_version'] + '.' + get_bcs_basename(in_bcsdir) + \ - '.' + config['output']['shared']['bc_version']+ '.' + get_bcs_basename(out_bcsdir) - suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + label + types + label = get_label(config) + suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + types + label for rst in restarts_in : f = os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_restart_in' cmd = '/bin/ln -s ' + rst + ' ' + f diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 8dfca55..207fbc6 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -272,12 +272,20 @@ def data_ocean_default(resolution): def get_bcs_basename(bcs): if not bcs: return "" - k = bcs.find('/geometry') - if k != -1 : - bcs = bcs[0:k] while bcs[-1] == '/': bcs = bcs[0:-1] # remove extra '/' return os.path.basename(bcs) +def get_label(config): + label = '' + if config['output']['shared']['label']: + in_resolution = get_bcs_basename(config['input']['shared']['bcs_dir']) + out_resolution = get_bcs_basename(config['output']['shared']['bcs_dir']) + in_bcv = config['input']['shared']['bc_version'] + out_bcv = config['output']['shared']['bc_version'] + label = '.' + in_bcv + '.' + in_resolution + \ + '.' + out_bcv + '.' + out_resolution + return label + def wemin_default(bc_version): default_ = '26' if bc_version =='NLv3' : default_ = '13' From df4fe288bfbc80e4063786fd9aaffac2de5497ca Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 9 Aug 2023 12:45:26 -0400 Subject: [PATCH 26/88] support all the new structure of BCS --- pre/remap_restart/remap_params.py | 29 +--- pre/remap_restart/remap_questions.py | 38 ++--- pre/remap_restart/remap_utils.py | 220 +-------------------------- 3 files changed, 24 insertions(+), 263 deletions(-) diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index 2253130..5c04368 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -63,15 +63,8 @@ def __init__(self, config_from_question): def params_for_air(self, config_tpl): if self.common_in['MERRA-2']: return config_tpl - - ogrid = config_tpl['input']['shared']['ogrid'] - bcvout = self.common_out['bc_version'] - bcVersion = get_bcVersion(bcvout, ogrid) - bc_versionrank = BcsRank[bcVersion] if ( not config_tpl['input']['air']['drymass']) : - config_tpl['input']['air']['drymass'] = 0 - if bc_versionrank >=12 : - config_tpl['input']['air']['drymass'] = 1 + config_tpl['input']['air']['drymass'] = 1 return config_tpl @@ -82,15 +75,12 @@ def options_for_slurm(self, config_tpl): return config_tpl def params_for_surface(self, config_tpl): - config_tpl['output']['surface']['surflay'] = 20. - bcvout = self.common_out['bc_version'] + config_tpl['output']['surface']['surflay'] = 50. + bc_version = self.common_out['bc_version'] ogrid = self.common_out['ogrid'] - bcVersion = get_bcVersion(bcvout, ogrid) - bc_versionrank = BcsRank[bcVersion] - if bc_versionrank >=12 : - config_tpl['output']['surface']['surflay'] = 50. - if bc_versionrank >= BcsRank["Icarus_Reynolds"]: - config_tpl['output']['surface']['split_saltwater'] = True + config_tpl['output']['surface']['split_saltwater'] = True + if 'Ganymed' in bc_version : + config_tpl['output']['surface']['split_saltwater'] = False config_tpl['input']['surface']['zoom']= self.surf_in['zoom'] config_tpl['input']['surface']['wemin']= self.surf_in['wemin'] config_tpl['output']['surface']['wemin']= self.surf_out['wemin'] @@ -100,13 +90,8 @@ def params_for_surface(self, config_tpl): def params_for_analysis(self, config_tpl): config_tpl['output']['analysis']['lcv'] = self.ana_out.get('lcv') config_tpl['output']['analysis']['bkg'] = self.ana_out.get('bkg') + config_tpl['output']['analysis']['aqua'] = True - ogrid = self.common_out['ogrid'] - bcvout = self.common_out['bc_version'] - bcVersion = get_bcVersion(bcvout, ogrid) - bc_versionrank = BcsRank[bcVersion] - if bc_versionrank >= BcsRank["Ganymed-4_0_Reynolds"] : - config_tpl['output']['analysis']['aqua'] = True return config_tpl if __name__ == "__main__": diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index a170d1c..261db4d 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -180,18 +180,12 @@ def ask_questions(): "name": "input:shared:bc_version", "message": f'''Enter BC version that matches input restarts: - BC version Sample GCM and DAS tags - ---------- ----------------------- - Ganymed-4_0: Ganymed-4_0 ...... Heracles-5_4_p3 - Ganymed-4_0: GEOSadas-5_12_2 .. GEOSadas-5_16_5 - ICA: Icarus ........... Jason - ICA: 10.3 ............. 10.18 - ICA: GEOSadas-5_17_0 .. GEOSadas-5_24_0_p1 - NL3: Icarus-NL ........ Jason-NL - NL3: 10.19 ............ 10.23 - NL3: 11.00 ............ 11.1 - NL3: GEOSadas-5_25_1 .. GEOSadas-5_29_4\n''', - "choices": ['NL3', 'ICA','Ganymed-4_0'], + BC version ADAS tags GCM tags + ---------- --------------- ------------------ + GM4: 5_12_4 Ganymed-4_0 - + ICA: 5_13 ... 5_22 Icarus.....v10.18 + NL3: 5_25 - Icarus_NL, v10.19- \n ''', + "choices": ['NL3', 'ICA','GM4'], "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -199,7 +193,7 @@ def ask_questions(): "type": "select", "name": "output:shared:bc_version", "message": "Enter BC version for new restarts:", - "choices": ['NL3', 'ICA','Ganymed-4_0'], + "choices": ['NL3', 'ICA','GM4'], "default": "NL3", "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -208,18 +202,12 @@ def ask_questions(): "type": "select", "name": "output:shared:bc_version", "message": f'''Enter BC version for new restarts: - BC version Sample GCM and DAS tags - ---------- ----------------------- - Ganymed-4_0: Ganymed-4_0 ...... Heracles-5_4_p3 - Ganymed-4_0: GEOSadas-5_12_2 .. GEOSadas-5_16_5 - ICA: Icarus ........... Jason - ICA: 10.3 ............. 10.18 - ICA: GEOSadas-5_17_0 .. GEOSadas-5_24_0_p1 - NL3: Icarus-NL ........ Jason-NL - NL3: 10.19 ............ 10.23 - NL3: 11.00 ............ 11.1 - NL3: GEOSadas-5_25_1 .. GEOSadas-5_29_4\n''', - "choices": ['NL3', 'ICA','Ganymed-4_0'], + BC version ADAS tags GCM tags + ---------- --------------- ------------------ + GM4: 5_12_4 Ganymed-4_0 - + ICA: 5_13 ... 5_22 Icarus.....v10.18 + NL3: 5_25 - Icarus_NL, v10.19- \n ''', + "choices": ['NL3', 'ICA','GM4'], "default": "NL3", "when": lambda x: x["input:shared:MERRA-2"], }, diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 207fbc6..a597d59 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -9,167 +9,6 @@ import glob import shlex -# define some constants for bcs - -NewStructureBCList = ['NL3', 'NL4', 'NL5', 'v06', 'v07', 'v08', 'v09'] -BCBase={} -BCBase['discover_ops'] = "/discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs" -BCBase['discover_legacy'] = "/discover/nobackup/projects/gmao/bcs_shared/legacy_bcs" -BCBase['discover_couple'] = "/discover/nobackup/projects/gmao/ssd/aogcm/atmosphere_bcs" -BCBase['discover_ns'] = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" - -BcsVersionMap = {} -BcsRank = {} -bc_version_initialized = False - -def init_bc_versions(): - # copy and paste from rigrid.pl - # minor change. Add "D" to the number for each group - # BCS Tag: Fortuna-1_4 - global bc_version_initialized - if bc_version_initialized : return -# F14 = ( 'F14', 'Fortuna-1_4', 'Fortuna-1_4_p1' ) -# D214 = ( 'D214', 'GEOSdas-2_1_4', 'GEOSdas-2_1_4-m1', -# 'GEOSdas-2_1_4-m2', 'GEOSdas-2_1_4-m3', 'GEOSdas-2_1_4-m4' ) -# D540 = ( 'D540', 'GEOSadas-5_4_0', 'GEOSadas-5_4_0_p1', -# 'GEOSadas-5_4_0_p2', 'GEOSadas-5_4_0_p3', 'GEOSadas-5_4_0_p4', -# 'GEOSadas-5_4_1', 'GEOSadas-5_4_1_p1', 'GEOSadas-5_4_2', -# 'GEOSadas-5_4_3', 'GEOSadas-5_4_4', 'GEOSadas-5_5_0', -# 'GEOSadas-5_5_1', 'GEOSadas-5_5_2', 'GEOSadas-5_5_3' ) - - # BCS Tag: Fortuna-2_0 - #--------------------- -# F20 = ( 'F20', 'Fortuna-2_0') - - # BCS Tag: Fortuna-2_1 - #--------------------- -# F21 = ( 'F21', 'Fortuna-2_1', 'Fortuna-2_1_p1', -# 'Fortuna-2_1_p2', 'Fortuna-2_1_p3', 'Fortuna-2_2', -# 'Fortuna-2_2_p1', 'Fortuna-2_2_p2', 'Fortuna-2_3', -# 'Fortuna-2_3_p1', 'Fortuna-2_4', 'Fortuna-2_4_p1', -# 'Fortuna-2_4_p2', 'Fortuna-2_5', 'Fortuna-2_5_BETA0', -# 'Fortuna-2_5_p1', 'Fortuna-2_5_p2', 'Fortuna-2_5_p3', -# 'Fortuna-2_5_p4', 'Fortuna-2_5_p5', 'Fortuna-2_5_p6', -# 'Fortuna-2_5_pp2' ) -# D561 = ( 'D561', 'GEOSadas-5_6_1', 'GEOSadas-5_6_1_p1', -# 'GEOSadas-5_6_1_p2', 'GEOSadas-5_6_1_p3', 'GEOSadas-5_6_1_p4', -# 'GEOSadas-5_6_2', 'GEOSadas-5_6_2_p1', 'GEOSadas-5_6_2_p2', -# 'GEOSadas-5_6_2_p3', 'GEOSadas-5_6_2_p4', 'GEOSadas-5_6_2_p5', -# 'GEOSadas-5_6_2_p6', 'GEOSadas-5_7_1', 'GEOSadas-5_7_1_p1', -# 'GEOSadas-5_7_1_p2', 'GEOSadas-5_7_2', 'GEOSadas-5_7_2_p1', -# 'GEOSadas-5_7_2_p2', 'GEOSadas-5_7_2_p2_m1','GEOSadas-5_7_2_p3', -# 'GEOSadas-5_7_2_p3_m1', 'GEOSadas-5_7_2_p3_m2','GEOSadas-5_7_2_p4', -# 'GEOSadas-5_7_2_p5', 'GEOSadas-5_7_2_p5_m1','GEOSadas-5_7_3', -# 'GEOSadas-5_7_3_p1', 'GEOSadas-5_7_3_p2', 'GEOSadas-5_7_3_p2' ) - - # BCS Tag: Ganymed-1_0 - #--------------------- -# G10 = ( 'G10', 'Ganymed-1_0', 'Ganymed-1_0_BETA', -# 'Ganymed-1_0_BETA1', 'Ganymed-1_0_BETA2', 'Ganymed-1_0_BETA3', -# 'Ganymed-1_0_BETA4' ) -# -# D580 = ( 'D580', 'GEOSadas-5_8_0', 'GEOSadas-5_9_0', -# 'GEOSadas-5_9_1' ) -# -# # BCS Tags: Ganymed-1_0_M and Ganymed-1_0_D -# #------------------------------------------ -# G10p = ( 'G10p', 'Ganymed-1_0_p1', 'Ganymed-1_0_p2', -# 'Ganymed-1_0_p3', 'Ganymed-1_0_p4', 'Ganymed-1_0_p5', -# 'Ganymed-1_0_p6' ) -# -# D591p= ( 'D591p', 'GEOSadas-5_9_1_p1', 'GEOSadas-5_9_1_p2', -# 'GEOSadas-5_9_1_p3', 'GEOSadas-5_9_1_p4', 'GEOSadas-5_9_1_p5', -# 'GEOSadas-5_9_1_p6', 'GEOSadas-5_9_1_p7', 'GEOSadas-5_9_1_p8', -# 'GEOSadas-5_9_1_p9' ) -# -# # BCS Tags: Ganymed-1_0_M and Ganymed-1_0_D w/ new landice rst -# #------------------------------------------------------------------------ -# G20 = ( 'G20', 'Ganymed-2_0', 'Ganymed-2_1', -# 'Ganymed-2_1_p1', 'Ganymed-2_1_p2', 'Ganymed-2_1_p3', -# 'Ganymed-2_1_p4', 'Ganymed-2_1_p5', 'Ganymed-2_1_p6' ) -# D5A0 = ( 'D5A0', 'GEOSadas-5_10_0', 'GEOSadas-5_10_0_p1' ) -# -# -# BCS Tags: Ganymed-1_0_Reynolds and Ganymed-1_0_Ostia -#----------------------------------------------------- -# G30 = ( 'G30', 'Ganymed-3_0', 'Ganymed-3_0_p1' ) -# D5B0 = ( '5B0', 'GEOSadas-5_10_0_p2', 'GEOSadas-5_11_0' ) -# -# # BCS Tags: Ganymed-4_0_Reynolds, Ganymed-4_0_MERRA-2, and Ganymed-4_0_Ostia -# #--------------------------------------------------------------------------- - G40 = ( 'G40', 'Ganymed-4_0', 'Ganymed-4_0_p1', - 'Ganymed-4_1', 'Heracles-1_0', 'Heracles-1_1', - 'Heracles-2_0', 'Heracles-2_1', 'Heracles-3_0', - 'Heracles-4_0', 'Heracles-5_4_p3' ) -# D512 = ( '512', 'GEOSadas-5_12_2', 'GEOSadas-5_12_4', -# 'GEOSadas-5_12_4_p1', 'GEOSadas-5_12_4_p2', 'GEOSadas-5_12_4_p3', -# 'GEOSadas-5_12_5', 'GEOSadas-5_13_0_p1', 'GEOSadas-5_13_0_p2', -# 'GEOSadas-5_13_1', 'GEOSadas-5_16_5' ) -# -# # BCS Tags: Icarus (New Land Parameters, New Topography) -# #--------------------------------------------------------------------------- - ICA = ( 'ICA', 'Icarus', 'Jason' ) - NLv3 = ( 'NLv3',) -# D517 = ( '517', 'GEOSadas-5_17_0', 'GEOSadas-5_17_1', 'GEOSadas-5_18_0', -# 'GEOSadas-5_18_1', 'GEOSadas-5_18_2', 'GEOSadas-5_18_3', -# 'GEOSadas-5_18_3_p1', 'GEOSadas-5_19_0', 'GEOSadas-5_20_0', -# 'GEOSadas-5_20_0_p1', 'GEOSadas-5_20_0_p2', 'GEOSadas-5_21_0', -# 'GEOSadas-5_21_2', 'GEOSadas-5_21_3_p1', 'GEOSadas-5_22_0', -# 'GEOSadas-5_22_0_p1', 'GEOSadas-5_22_0_p2', 'GEOSadas-5_23_0', -# 'GEOSadas-5_23_0_p1', 'GEOSadas-5_24_0', 'GEOSadas-5_24_0_p1' ) -# GITOL = ( 'GITOL', '10.3', '10.4', '10.5', -# '10.6', '10.7', '10.8', -# '10.9', '10.10', '10.11', -# '10.12', '10.13', '10.14', -# '10.15', '10.16', '10.17', -# '10.18' ) - - # BCS Tags: Icarus-NLv3 (New Land Parameters) - #--------------------------------------------------------------------------- -# INL = ( 'INL', 'Icarus-NL', 'Icarus-NLv3', 'Jason-NL' ) -# GITNL = ( 'GITNL', '10.19', '10.20', '10.21', '10.22', '10.23' ) -# D525 = ( '525', 'GEOSadas-5_25_1', 'GEOSadas-5_25_1_p5', 'GEOSadas-5_25_p7', -# 'GEOSadas-5_27_1', 'GEOSadas-5_29_3', 'GEOSadas-5_29_4' ) - -# for bc_version in F14: BcsVersionMap[bc_version]= "Fortuna-1_4" -# for bc_version in F20: BcsVersionMap[bc_version]= "Fortuna-2_0" -# for bc_version in F21: BcsVersionMap[bc_version]= "Fortuna-2_1" -# for bc_version in G10: BcsVersionMap[bc_version]= "Ganymed-1_0" -# for bc_version in G10p: BcsVersionMap[bc_version]= "Ganymed-1_0_M" -# for bc_version in G20: BcsVersionMap[bc_version]= "Ganymed-1_0_M" -# for bc_version in G30: BcsVersionMap[bc_version]= "Ganymed-1_0_Reynolds" - for bc_version in G40: BcsVersionMap[bc_version]= "Ganymed-4_0_Reynolds" - for bc_version in ICA: BcsVersionMap[bc_version]= "Icarus_Reynolds" - for bc_version in NLv3: BcsVersionMap[bc_version]= "Icarus-NLv3_Reynolds" -# for bc_version in GITOL: BcsVersionMap[bc_version]= "Icarus_Reynolds" -# for bc_version in INL: BcsVersionMap[bc_version]= "Icarus-NLv3_Reynolds" -# for bc_version in GITNL: BcsVersionMap[bc_version]= "Icarus-NLv3_Reynolds" - for bc_version in NewStructureBCList: BcsVersionMap[bc_version]= bc_version -# - -# for bc_version in D214: BcsVersionMap[bc_version]= "Fortuna-1_4" -# for bc_version in D540: BcsVersionMap[bc_version]= "Fortuna-1_4" -# for bc_version in D561: BcsVersionMap[bc_version]= "Fortuna-2_1" -# for bc_version in D580: BcsVersionMap[bc_version]= "Ganymed-1_0" -# for bc_version in D591p: BcsVersionMap[bc_version]= "Ganymed-1_0_M" -# for bc_version in D5A0: BcsVersionMap[bc_version]= "Ganymed-1_0_M" -# for bc_version in D5B0: BcsVersionMap[bc_version]= "Ganymed-1_0_Reynolds" -# for bc_version in D512: BcsVersionMap[bc_version]= "Ganymed-4_0_Reynolds" -# for bc_version in D517: BcsVersionMap[bc_version]= "Icarus_Reynolds" -# for bc_version in D525: BcsVersionMap[bc_version]= "Icarus-NLv3_Reynolds" -# - BcsRank['Ganymed-4_0_Reynolds'] = 12 - BcsRank['Ganymed-4_0_Ostia'] = 13 - BcsRank['Ganymed-4_0_MERRA-2'] = 14 - BcsRank['Icarus_Reynolds'] = 15 - BcsRank['Icarus_MERRA-2'] = 16 - BcsRank['Icarus_Ostia'] = 17 - BcsRank['Icarus-NLv3_Reynolds'] = 18 - BcsRank['Icarus-NLv3_MERRA-2'] = 19 - BcsRank['Icarus-NLv3_Ostia'] = 20 - for bc_version in NewStructureBCList: BcsRank[bc_version] = 21 - bc_version_initialized = True - def init_merra2(x): if not x.get('input:shared:MERRA-2') : return False @@ -193,7 +32,7 @@ def init_merra2(x): x['input:shared:model'] = 'data' x['input:shared:agrid'] = 'C180' x['input:shared:ogrid'] = '1440x720' - x['input:shared:bc_version'] = 'Ganymed-4_0' + x['input:shared:bc_version'] = 'GM4' x['input:surface:catch_model'] = 'catch' x['input:shared:rst_dir'] = x['output:shared:out_dir'] + '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' @@ -288,8 +127,7 @@ def get_label(config): def wemin_default(bc_version): default_ = '26' - if bc_version =='NLv3' : default_ = '13' - if bc_version in NewStructureBCList : default_ = '13' + if bc_version =='NL3' : default_ = '13' return default_ def zoom_default(x): @@ -476,20 +314,6 @@ def get_config_from_answers(answers): return config -def get_bcVersion(bc_version, ogrid): - bcVersion = BcsVersionMap[bc_version] - if bcVersion in NewStructureBCList : return bcVersion - if ogrid[0].upper() == "C": - bcVersion=bcVersion.replace('_Reynolds','_Ostia') - else: - xy = ogrid.upper().split('X') - x = int(xy[0]) - if x == 1440: bcVersion=bcVersion.replace('_Reynolds','_MERRA-2') - if x == 2880: - bcVersion=bcVersion.replace('_Reynolds','_Ostia') - bcVersion=bcVersion.replace('_M','_D') - return bcVersion - def get_grid_subdir(bcdir, agrid, ogrid, model): def get_name_with_grid( grid, names, a_o): if not grid : @@ -549,7 +373,6 @@ def get_name_with_grid( grid, names, a_o): return g def get_bcsdir(x, opt): - init_bc_versions() bc_version = x.get('input:shared:bc_version') agrid = x.get('input:shared:agrid') ogrid = x.get('input:shared:ogrid') @@ -560,46 +383,11 @@ def get_bcsdir(x, opt): ogrid = x.get('output:shared:ogrid') model = x.get('output:shared:model') - bcdir = '' - base = 'discover_legacy' - if model != "data": - base = 'discover_couple' - if bc_version in NewStructureBCList: - base = 'discover_ns' - if x.get('input:shared:MERRA-2') and opt.upper() == "IN": - base = 'discover_ops' - - bc_base = BCBase[base] - - bcVersion = get_bcVersion(bc_version,ogrid) - bc_versionrank = BcsRank[bcVersion] - - if (bc_versionrank >= BcsRank['NL3']) : - bcdir = bc_base+'/'+ bcVersion+'/geometry/' - elif (bc_versionrank >= BcsRank['Icarus-NLv3_Reynolds']) : - bcdir = bc_base+'/Icarus-NLv3/'+bcVersion+'/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Icarus-NLv3/'+model+'/' - elif (bc_versionrank >= BcsRank['Icarus_Reynolds']): - if bc_base == BCBase['discover_ops']: - bcdir = bc_base+'/Icarus_Updated/'+bcVersion+'/' - else: - bcdir = bc_base+'/Icarus/'+bcVersion+'/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Icarus/'+model+'/' - elif(bc_versionrank >= BcsRank["Ganymed-4_0_Reynolds"]): - bcdir = bc_base + '/Ganymed-4_0/'+bcVersion+'/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Ganymed/'+model+'/' - else: - bcdir = bc_base + '/' + bcVersion + '/' - if model == 'MOM6' or model == 'MOM5': - bcdir = bc_base+'/Ganymed/'+model+'/' - + bc_base = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" + bcdir = bc_base+'/'+ bc_version+'/geometry/' if not os.path.exists(bcdir): exit("Cannot find bc dir " + bcdir) - gridStr = get_grid_subdir(bcdir,agrid, ogrid, model) bcdir = bcdir + gridStr From c56bf5de24a95ec2edccc9d7843832c746ab615d Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Mon, 14 Aug 2023 11:58:47 -0400 Subject: [PATCH 27/88] edit table --- pre/remap_restart/remap_questions.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 261db4d..18ba694 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -180,11 +180,11 @@ def ask_questions(): "name": "input:shared:bc_version", "message": f'''Enter BC version that matches input restarts: - BC version ADAS tags GCM tags - ---------- --------------- ------------------ - GM4: 5_12_4 Ganymed-4_0 - - ICA: 5_13 ... 5_22 Icarus.....v10.18 - NL3: 5_25 - Icarus_NL, v10.19- \n ''', + BC version ADAS tags GCM tags + ---------- --------------- ------------------ + GM4: Ganymed-4_0 5_12_2 ... 5_16_5 Ganymed-4_0 ... Heracles-5_4_p3 + ICA: Icarus 5_17_0 ... 5_24_0_p1 Icarus, Jason ... 10.18 + NL3: Icarus-NLv3 5_25_1 ... 5_29_4 (now) Icarus_NL, 10.19 ... 11.1 (now) \n ''', "choices": ['NL3', 'ICA','GM4'], "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -202,11 +202,11 @@ def ask_questions(): "type": "select", "name": "output:shared:bc_version", "message": f'''Enter BC version for new restarts: - BC version ADAS tags GCM tags - ---------- --------------- ------------------ - GM4: 5_12_4 Ganymed-4_0 - - ICA: 5_13 ... 5_22 Icarus.....v10.18 - NL3: 5_25 - Icarus_NL, v10.19- \n ''', + BC version ADAS tags GCM tags + ---------- --------------- ------------------ + GM4: Ganymed-4_0 5_12_2 ... 5_16_5 Ganymed-4_0 ... Heracles-5_4_p3 + ICA: Icarus 5_17_0 ... 5_24_0_p1 Icarus, Jason ... 10.18 + NL3: Icarus-NLv3 5_25_1 ... 5_29_4 (now) Icarus_NL, 10.19 ... 11.1 (now) \n ''', "choices": ['NL3', 'ICA','GM4'], "default": "NL3", "when": lambda x: x["input:shared:MERRA-2"], From aec6d18a0cc3f4ae9be846c2462b92b75ff9b2a7 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Mon, 14 Aug 2023 13:16:48 -0400 Subject: [PATCH 28/88] GM4 without 'smoothed' directory --- pre/remap_restart/remap_upper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 1088e08..ee6c2f5 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -15,7 +15,10 @@ def get_topodir(bcsdir): if k != -1 : while bcsdir[-1] == '/': bcsdir = bcsdir[0:-1] # remove extra '/' agrid_name = os.path.basename(bcsdir).split('_')[0] - bcsdir = bcsdir[0:k]+'/TOPO/TOPO_'+agrid_name + '/smoothed' + if '/GM4/' in bcsdir : + bcsdir = bcsdir[0:k]+'/TOPO/TOPO_'+agrid_name + else: + bcsdir = bcsdir[0:k]+'/TOPO/TOPO_'+agrid_name + '/smoothed' return bcsdir class upperair(remap_base): From ecc69473d527c32ee894a736dd2f1f4215a0b50f Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 24 Aug 2023 10:40:11 -0400 Subject: [PATCH 29/88] add develop choice; add cap_restart --- pre/remap_restart/remap_params.tpl | 2 +- pre/remap_restart/remap_questions.py | 70 +++++++++++++++++----------- pre/remap_restart/remap_upper.py | 15 ++++-- pre/remap_restart/remap_utils.py | 4 +- 4 files changed, 59 insertions(+), 32 deletions(-) diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index cbabcda..ce0e320 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -40,7 +40,7 @@ output: nlevel: surface: split_saltwater: false - surflay: 20. + surflay: 50. wemin: # remap lake, saltwater, landicet remap_water: true diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 18ba694..3f9f949 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -35,6 +35,16 @@ def remove_ogrid_comment(x, opt): return False +def echo_bcs(x,opt): + if opt == "IN": + x['input:shared:bcs_dir'] = get_bcsdir(x, 'IN') + print("\nBc path for the input restart: " + x['input:shared:bcs_dir']) + if opt == "OUT": + x['output:shared:bcs_dir'] = get_bcsdir(x, 'OUT') + print("\nBc path for the output restart: " + x['output:shared:bcs_dir']) + print("\nUsers can change the paths in the generated remap_params.yaml file later on") + return False + def ask_questions(): questions = [ @@ -47,7 +57,7 @@ def ask_questions(): { "type": "path", "name": "input:shared:rst_dir", - "message": "Enter the directory containing restart files to be remapped:", + "message": "Enter the input directory containing restart files to be remapped:", "when": lambda x: not x['input:shared:MERRA-2'], }, { @@ -67,7 +77,7 @@ def ask_questions(): { "type": "path", "name": "output:shared:out_dir", - "message": "Enter the directory for new restarts:\n" + "message": "Enter the output directory for new restarts:\n" }, { "type": "path", @@ -175,54 +185,62 @@ def ask_questions(): "default": "72", }, + # to show the message, we ask output first { "type": "select", - "name": "input:shared:bc_version", - "message": f'''Enter BC version that matches input restarts: - + "name": "output:shared:bc_version", + "message": f'''Enter BC version for new restarts: BC version ADAS tags GCM tags ---------- --------------- ------------------ GM4: Ganymed-4_0 5_12_2 ... 5_16_5 Ganymed-4_0 ... Heracles-5_4_p3 ICA: Icarus 5_17_0 ... 5_24_0_p1 Icarus, Jason ... 10.18 - NL3: Icarus-NLv3 5_25_1 ... 5_29_4 (now) Icarus_NL, 10.19 ... 11.1 (now) \n ''', - "choices": ['NL3', 'ICA','GM4'], - "when": lambda x: not x["input:shared:MERRA-2"], + NL3: Icarus-NLv3 5_25_1 ... present Icarus_NL, 10.19 ... present + + develop: notes + ------------------- + v06: + v07: + v08: + \n ''', + "choices": ['NL3', 'ICA','GM4','develop'], + "default": "NL3", }, { "type": "select", "name": "output:shared:bc_version", - "message": "Enter BC version for new restarts:", - "choices": ['NL3', 'ICA','GM4'], - "default": "NL3", + "message": "Enter develop BC version for new restarts:", + "choices": ['v06', 'v07','v08'], + "when": lambda x: x["output:shared:bc_version"] == 'develop', + }, + + { + "type": "select", + "name": "input:shared:bc_version", + "message": "Enter BC version that matches input restarts:", + "choices": ['NL3', 'ICA','GM4','develop'], "when": lambda x: not x["input:shared:MERRA-2"], }, - # show the message if it is merra2 + { "type": "select", - "name": "output:shared:bc_version", - "message": f'''Enter BC version for new restarts: - BC version ADAS tags GCM tags - ---------- --------------- ------------------ - GM4: Ganymed-4_0 5_12_2 ... 5_16_5 Ganymed-4_0 ... Heracles-5_4_p3 - ICA: Icarus 5_17_0 ... 5_24_0_p1 Icarus, Jason ... 10.18 - NL3: Icarus-NLv3 5_25_1 ... 5_29_4 (now) Icarus_NL, 10.19 ... 11.1 (now) \n ''', - "choices": ['NL3', 'ICA','GM4'], - "default": "NL3", - "when": lambda x: x["input:shared:MERRA-2"], + "name": "input:shared:bc_version", + "message": "Enter develop BC version that matches input restarts:", + "choices": ['v06', 'v07','v08'], + "when": lambda x: not x["input:shared:MERRA-2"] and x["input:shared:bc_version"] == 'develop', }, { "type": "path", "name": "input:shared:bcs_dir", "message": "Is this BCS matching input restarts? If no, enter your own absolute path: \n", - "default": lambda x: get_bcsdir(x, "IN"), + "when": lambda x: echo_bcs(x, "IN"), }, { "type": "path", "name": "output:shared:bcs_dir", "message": "Is this BCS for new restarts? If no, enter your own absolute path: \n", - "default": lambda x: get_bcsdir(x, "OUT"), + "when": lambda x: echo_bcs(x, "OUT"), }, { @@ -242,7 +260,7 @@ def ask_questions(): { "type": "confirm", "name": "output:analysis:bkg", - "message": "Regrid bkg files?", + "message": "Remap bkg files?", "default": False, }, { @@ -285,7 +303,7 @@ def ask_questions(): { "type": "text", "name": "slurm:qos", - "message": "qos?", + "message": "qos? If the resolution is c1440 or higher, enter allnccs ", "default": "debug", }, diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index ee6c2f5..1e8859e 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -136,7 +136,11 @@ def remap(self): NPE = 5400; nwrit= 6 QOS = "#SBATCH --qos="+config['slurm']['qos'] - if NPE > 532: QOS = "###" + QOS + TIME ="#SBATCH --time=1:00:00" + if NPE > 532: + assert config['slurm']['qos'] != 'debug', "qos should be allnccs" + TIME = "#SBATCH --time=12:00:00" + CONSTR = "#SBATCH --constraint=" + config['slurm']['constraint'] log_name = out_dir+'/remap_upper_log' @@ -181,10 +185,10 @@ def remap(self): remap_template="""#!/bin/csh -xf #SBATCH --account={account} -#SBATCH --time=1:00:00 #SBATCH --ntasks={NPE} #SBATCH --job-name=remap_upper #SBATCH --output={log_name} +{TIME} {QOS} {CONSTR} @@ -250,7 +254,7 @@ def remap(self): remap_upper_script = remap_template.format(Bin=bindir, account = account, \ out_dir = out_dir, log_name = log_name, drymassFLG = drymassFLG, \ imout = imout, nwrit = nwrit, NPE = NPE, \ - QOS = QOS,CONSTR = CONSTR, nlevel = nlevel, hydrostatic = hydrostatic, + QOS = QOS, TIME = TIME, CONSTR = CONSTR, nlevel = nlevel, hydrostatic = hydrostatic, stretch_str = stretch_str) script_name = './remap_upper.j' @@ -296,6 +300,11 @@ def remap(self): print('\n Move remap_upper.j to ' + out_dir) shutil.move('remap_upper.j', out_dir+"/remap_upper.j") + with open(out_dir+'/cap_restart', 'w') as f: + yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) + time = yyyymmddhh_[0:8]+' '+yyyymmddhh_[8:10]+'0000' + print('Create cap_restart') + f.write(time) print('cd ' + cwdir) os.chdir(cwdir) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index a597d59..ac60c2d 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -126,8 +126,8 @@ def get_label(config): return label def wemin_default(bc_version): - default_ = '26' - if bc_version =='NL3' : default_ = '13' + default_ = '13' + if bc_version =='GM4' or bc_version == 'ICA' : default_ = '26' return default_ def zoom_default(x): From a3ea2454db422b305d17f992087b6e409d9847a3 Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Tue, 29 Aug 2023 12:59:42 -0400 Subject: [PATCH 30/88] edit v06 notes and options --- pre/remap_restart/remap_questions.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 3f9f949..19dbfd9 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -196,38 +196,36 @@ def ask_questions(): ICA: Icarus 5_17_0 ... 5_24_0_p1 Icarus, Jason ... 10.18 NL3: Icarus-NLv3 5_25_1 ... present Icarus_NL, 10.19 ... present - develop: notes + Development: Notes ------------------- - v06: - v07: - v08: + v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb \n ''', - "choices": ['NL3', 'ICA','GM4','develop'], + "choices": ['NL3', 'ICA','GM4','Development'], "default": "NL3", }, { "type": "select", "name": "output:shared:bc_version", - "message": "Enter develop BC version for new restarts:", - "choices": ['v06', 'v07','v08'], - "when": lambda x: x["output:shared:bc_version"] == 'develop', + "message": "Enter BC version for new restarts; this option is in testing stage:", + "choices": ['v06'], + "when": lambda x: x["output:shared:bc_version"] == 'Development', }, { "type": "select", "name": "input:shared:bc_version", "message": "Enter BC version that matches input restarts:", - "choices": ['NL3', 'ICA','GM4','develop'], + "choices": ['NL3', 'ICA','GM4','Development'], "when": lambda x: not x["input:shared:MERRA-2"], }, { "type": "select", "name": "input:shared:bc_version", - "message": "Enter develop BC version that matches input restarts:", - "choices": ['v06', 'v07','v08'], - "when": lambda x: not x["input:shared:MERRA-2"] and x["input:shared:bc_version"] == 'develop', + "message": "Enter BC version for new restarts; this option is in testing stage:", + "choices": ['v06'], + "when": lambda x: not x["input:shared:MERRA-2"] and x["input:shared:bc_version"] == 'Development', }, { From 65e38ff913adcc1f6855584a78bace9dd0a1dd50 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 7 Sep 2023 12:24:16 -0400 Subject: [PATCH 31/88] Addressing more issues.... --- pre/remap_restart/remap_catchANDcn.py | 14 +++- pre/remap_restart/remap_command.py | 4 +- pre/remap_restart/remap_params.py | 2 +- pre/remap_restart/remap_params.tpl | 2 +- pre/remap_restart/remap_questions.py | 108 +++++++++++++++++--------- pre/remap_restart/remap_restarts.py | 15 +++- pre/remap_restart/remap_upper.py | 6 +- pre/remap_restart/remap_utils.py | 54 ++++++++++++- 8 files changed, 152 insertions(+), 53 deletions(-) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 1d88ee7..cfee453 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -87,6 +87,13 @@ def remap(self): NPE = 120 else: NPE = 160 + + QOS = "#SBATCH --qos="+config['slurm']['qos'] + TIME ="#SBATCH --time=1:00:00" + if NPE >= 160: + assert config['slurm']['qos'] != 'debug', "qos should be allnccs" + TIME = "#SBATCH --time=12:00:00" + PARTITION = "#SBATCH --partition=" + config['slurm']['partition'] account = config['slurm']['account'] # even the input is binary, the output would be nc4 @@ -120,10 +127,11 @@ def remap(self): mk_catch_j_template = """#!/bin/csh -f #SBATCH --account={account} #SBATCH --ntasks={NPE} -#SBATCH --time=1:00:00 #SBATCH --job-name=mk_catchANDcn -#SBATCH --qos=debug #SBATCH --output={log_name} +{TIME} +{QOS} +{PARTITION} # source {Bin}/g5_modules @@ -143,7 +151,7 @@ def remap(self): catch1script = mk_catch_j_template.format(Bin = bindir, account = account, out_bcs = out_bcsdir, \ model = model, out_dir = out_dir, surflay = surflay, log_name = log_name, NPE = NPE, \ in_wemin = in_wemin, out_wemin = out_wemin, out_tilefile = out_tilefile, in_tilefile = in_tilefile, \ - in_rstfile = in_rstfile, out_rstfile = out_rstfile, time = yyyymmddhh_ ) + in_rstfile = in_rstfile, out_rstfile = out_rstfile, time = yyyymmddhh_, TIME = TIME, PARTITION = PARTITION, QOS=QOS ) script_name = './mk_catchANDcn.j' diff --git a/pre/remap_restart/remap_command.py b/pre/remap_restart/remap_command.py index 4cd1268..fd88b10 100755 --- a/pre/remap_restart/remap_command.py +++ b/pre/remap_restart/remap_command.py @@ -72,7 +72,7 @@ def parse_args(program_description): p_command.add_argument('-qos', default = "debug", help= "queue of slurm job", choices=['debug', 'allnccs']) account = get_account() p_command.add_argument('-account', default = account, help= "account of slurm job") - p_command.add_argument('-constraint', default= 'sky', help= "machine of slurm job") + p_command.add_argument('-partition', default= 'compute', help= "partition of slurm job") p_command.add_argument('-rs', default= '3', help='flag indicating which restarts to regrid: 1 (upper air); 2 (surface) 3 (both)', choices=['1','2','3']) # Parse using parse_known_args so we can pass the rest to the remap scripts @@ -148,7 +148,7 @@ def get_answers_from_command_line(cml): answers["slurm:account"] = cml.account answers["slurm:qos"] = cml.qos - answers["slurm:constraint"] = cml.constraint + answers["slurm:partition"] = cml.partition return answers diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index 5c04368..01309d4 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -71,7 +71,7 @@ def params_for_air(self, config_tpl): def options_for_slurm(self, config_tpl): config_tpl['slurm']['account'] = self.slurm_options['account'] config_tpl['slurm']['qos'] = self.slurm_options['qos'] - config_tpl['slurm']['constraint'] = self.slurm_options['constraint'] + config_tpl['slurm']['partition'] = self.slurm_options['partition'] return config_tpl def params_for_surface(self, config_tpl): diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index ce0e320..1dcce1b 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -55,4 +55,4 @@ output: slurm: account: qos: - constraint: + partition: diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 19dbfd9..23a8803 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -45,32 +45,38 @@ def echo_bcs(x,opt): print("\nUsers can change the paths in the generated remap_params.yaml file later on") return False +def default_partition(x): + if x['slurm:qos'] == 'debug': + x['slurm:partition'] = 'compute' + return False + return True + def ask_questions(): questions = [ { "type": "confirm", "name": "input:shared:MERRA-2", - "message": "Would you like to remap archived MERRA-2 restarts?", + "message": "Would you like to remap archived MERRA-2 restarts?\n", "default": False, }, { "type": "path", "name": "input:shared:rst_dir", - "message": "Enter the input directory containing restart files to be remapped:", + "message": "Enter the input directory containing restart files to be remapped:\n", "when": lambda x: not x['input:shared:MERRA-2'], }, { "type": "text", "name": "input:shared:yyyymmddhh", - "message": "From what restart date/time would you like to remap? (must be 10 digits: yyyymmddhh)", + "message": "From what restart date/time would you like to remap? (must be 10 digits: yyyymmddhh)\n", "validate": lambda text: len(text)==10 , - "when": lambda x: not x['input:shared:MERRA-2'], + "when": lambda x: not x['input:shared:MERRA-2'] and fvcore_time(x), }, { "type": "text", "name": "input:shared:yyyymmddhh", - "message": "From what restart date would you like to remap? (must be 8 digits: yyyymmdd, hour=21z)", + "message": "From what restart date would you like to remap? (must be 8 digits: yyyymmdd, hour=21z)\n", "validate": lambda text: len(text)==8 , "when": lambda x: x['input:shared:MERRA-2'], }, @@ -103,7 +109,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:model", - "message": "Select ocean model that matches input restarts:", + "message": "Select ocean model that matches input restarts:\n", "choices": ["data", "MOM5", "MOM6"], "default": "data", "when": lambda x: not x['input:shared:MERRA-2'] @@ -112,7 +118,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:ogrid", - "message": "Select data ocean grid that matches input restarts:", + "message": "Select data ocean grid that matches input restarts:\n", "choices": ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'], "default": lambda x: data_ocean_default(x.get('input:shared:agrid')), "when": lambda x: x.get('input:shared:model') == 'data' and not x['input:shared:MERRA-2'], @@ -130,7 +136,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:ogrid", - "message": "Select coupled (MOM5, MOM6) ocean grid that matches input restarts:", + "message": "Select coupled (MOM5, MOM6) ocean grid that matches input restarts:\n", "choices": ['72x36','360x200','720x410','1440x1080'], "when": lambda x: x.get('input:shared:model') == 'MOM5' or x.get('input:shared:model')== 'MOM6' }, @@ -150,14 +156,14 @@ def ask_questions(): { "type": "select", "name": "output:shared:model", - "message": "Select ocean model for new restarts:", + "message": "Select ocean model for new restarts:\n", "choices": ["data", "MOM5", "MOM6"], "default": "data", }, { "type": "select", "name": "output:shared:ogrid", - "message": "Select data ocean grid for new restarts:", + "message": "Select data ocean grid for new restarts:\n", "choices": ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'], "default": lambda x: data_ocean_default(x.get('output:shared:agrid')), "when": lambda x: x['output:shared:model'] == 'data', @@ -173,7 +179,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:ogrid", - "message": "Select couple ocean grid for new restarts:", + "message": "Select couple ocean grid for new restarts:\n", "choices": ['72x36','360x200','720x410','1440x1080'], "when": lambda x: x['output:shared:model'] != 'data', }, @@ -181,51 +187,77 @@ def ask_questions(): { "type": "text", "name": "output:air:nlevel", - "message": "Enter new atmospheric levels: (71 72 91 127 132 137 144 181)", + "message": "Enter new atmospheric levels: (71 72 91 127 132 137 144 181)\n", "default": "72", }, # to show the message, we ask output first { "type": "select", - "name": "output:shared:bc_version", - "message": f'''Enter BC version for new restarts: + "name": "input:shared:bc_version", + "message": f'''\nEnter BC version that matches input restarts: BC version ADAS tags GCM tags ---------- --------------- ------------------ GM4: Ganymed-4_0 5_12_2 ... 5_16_5 Ganymed-4_0 ... Heracles-5_4_p3 ICA: Icarus 5_17_0 ... 5_24_0_p1 Icarus, Jason ... 10.18 NL3: Icarus-NLv3 5_25_1 ... present Icarus_NL, 10.19 ... present - - Development: Notes - ------------------- - v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb - \n ''', + Development: more choices + \n\n ''', "choices": ['NL3', 'ICA','GM4','Development'], - "default": "NL3", + "when": lambda x: not x["input:shared:MERRA-2"], }, { "type": "select", - "name": "output:shared:bc_version", - "message": "Enter BC version for new restarts; this option is in testing stage:", + "name": "input:shared:bc_version", + "message": f'''\nMore choices of BC versions that macthes input restarts (this option is in testing stage): + + v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''', "choices": ['v06'], - "when": lambda x: x["output:shared:bc_version"] == 'Development', + "when": lambda x: x["input:shared:bc_version"] == 'Development', }, { "type": "select", - "name": "input:shared:bc_version", - "message": "Enter BC version that matches input restarts:", + "name": "output:shared:bc_version", + "message": f'''\nEnter BC version for new restarts: + BC version ADAS tags GCM tags + ---------- --------------- ------------------ + GM4: Ganymed-4_0 5_12_2 ... 5_16_5 Ganymed-4_0 ... Heracles-5_4_p3 + ICA: Icarus 5_17_0 ... 5_24_0_p1 Icarus, Jason ... 10.18 + NL3: Icarus-NLv3 5_25_1 ... present Icarus_NL, 10.19 ... present + Development: more choices + \n\n''', + "choices": ['NL3', 'ICA','GM4','Development'], + "default": "NL3", + "when": lambda x: x["input:shared:MERRA-2"], + }, + + { + "type": "select", + "name": "output:shared:bc_version", + "message": "Enter BC version for new restarts:\n", "choices": ['NL3', 'ICA','GM4','Development'], + "default": "NL3", "when": lambda x: not x["input:shared:MERRA-2"], }, { "type": "select", - "name": "input:shared:bc_version", - "message": "Enter BC version for new restarts; this option is in testing stage:", + "name": "output:shared:bc_version", + "message": f'''\nMore choices of BC versions for new restarts (this option is in testing stage): + + v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''', "choices": ['v06'], - "when": lambda x: not x["input:shared:MERRA-2"] and x["input:shared:bc_version"] == 'Development', + "when": lambda x: x["output:shared:bc_version"] == 'Development' and x["input:shared:bc_version"] not in ['v06'], + }, + + { + "type": "select", + "name": "output:shared:bc_version", + "message": "\nMore choices of BC version for new restarts (this option is in testing stage):\n", + "choices": ['v06'], + "when": lambda x: x["output:shared:bc_version"] == 'Development' and x["input:shared:bc_version"] in ['v06'], }, { @@ -270,25 +302,26 @@ def ask_questions(): { "type": "text", "name": "input:surface:wemin", - "message": "What is value of wemin (minimum snow water equivalent parameter) for surface inputs?", - "default": lambda x: wemin_default(x.get('input:shared:bc_version')) + "message": "What is value of wemin (minimum snow water equivalent parameter) for surface inputs?\n", + "default": lambda x: wemin_default(x.get('input:shared:bc_version')), + "when": lambda x: show_wemin_default(x), }, { "type": "text", "name": "output:surface:wemin", - "message": "What is value of wemin (minimum snow water equivalent parameter) for new surface restarts?", + "message": "What is value of wemin (minimum snow water equivalent parameter) for new surface restarts?\n", "default": lambda x: wemin_default(x.get('output:shared:bc_version')) }, { "type": "text", "name": "input:surface:zoom", - "message": "What is value of zoom (parameter of radius search, smaller value means larger radius) for surface inputs [1-8]?", + "message": "What is value of zoom (parameter of radius search, smaller value means larger radius) for surface inputs [1-8]?\n", "default": lambda x: zoom_default(x) }, { "type": "text", "name": "output:shared:expid", - "message": "Enter new restarts expid:", + "message": "Enter new restarts expid (prefix of the output restart names): \n", "default": "", }, { @@ -312,10 +345,11 @@ def ask_questions(): "default": get_account(), }, { - "type": "select", - "name": "slurm:constraint", - "message": "constraint?", - "choices": ['sky', 'cas'], + "type": "text", + "name": "slurm:partition", + "message": "partition?", + "default": "compute", + "when": lambda x : default_partition(x), }, ] answers = questionary.prompt(questions) diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index ba89d6e..1f07193 100644 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -75,11 +75,12 @@ def main(): raw_config = get_config_from_answers(answers) cmd = get_command_line_from_answers(answers) write_cmd(answers['output:shared:out_dir'], cmd) - with open("raw_answers.yaml", "w") as f: - yaml.dump(raw_config, f) + # just for debugging + # with open("raw_answers.yaml", "w") as f: + # yaml.dump(raw_config, f) params = remap_params(raw_config) config = params.config - config_yaml = 'remap_params.yaml' + config_yaml = answers['output:shared:out_dir']+'/remap_params.yaml' print('\n') if config: @@ -101,6 +102,14 @@ def main(): # write config to yaml file config_to_yaml(config, config_yaml,noprompt = noprompt) + + if not noprompt : + submit = questionary.confirm("Submit the jobs now ?" , default=True).ask() + if not submit : + print("\nYou can submit the jobs by the command later on: \n") + print("./remap_restarts.py config_file -c " + config_yaml + "\n") + sys.exit(0) + print(config_yaml) # upper air upper = upperair(params_file=config_yaml) diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 1e8859e..81a5afd 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -141,7 +141,7 @@ def remap(self): assert config['slurm']['qos'] != 'debug', "qos should be allnccs" TIME = "#SBATCH --time=12:00:00" - CONSTR = "#SBATCH --constraint=" + config['slurm']['constraint'] + PARTITION = "#SBATCH --partition=" + config['slurm']['partition'] log_name = out_dir+'/remap_upper_log' @@ -190,7 +190,7 @@ def remap(self): #SBATCH --output={log_name} {TIME} {QOS} -{CONSTR} +{PARTITION} unlimit @@ -254,7 +254,7 @@ def remap(self): remap_upper_script = remap_template.format(Bin=bindir, account = account, \ out_dir = out_dir, log_name = log_name, drymassFLG = drymassFLG, \ imout = imout, nwrit = nwrit, NPE = NPE, \ - QOS = QOS, TIME = TIME, CONSTR = CONSTR, nlevel = nlevel, hydrostatic = hydrostatic, + QOS = QOS, TIME = TIME, PARTITION = PARTITION, nlevel = nlevel, hydrostatic = hydrostatic, stretch_str = stretch_str) script_name = './remap_upper.j' diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index ac60c2d..c0c4563 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -39,9 +39,12 @@ def init_merra2(x): return False def fvcore_name(x): + if 'input:shared:agrid' in x.keys(): + return True ymdh = x['input:shared:yyyymmddhh'] time = ymdh[0:8] + '_'+ymdh[8:10] rst_dir = x.get('input:shared:rst_dir') + if not rst_dir : return False x['input:shared:rst_dir'] = rst_dir.strip() # remove extra space @@ -78,6 +81,43 @@ def fvcore_name(x): x['input:shared:expid'] = expid return fname +def fvcore_time(x): + rst_dir = x.get('input:shared:rst_dir') + if not rst_dir : + sys.exit('Input restart directory does not exist.\n') + + x['input:shared:rst_dir'] = rst_dir.strip() # remove extra space + + files = glob.glob(rst_dir+'/*fvcore_*') + + if len (files) == 0 : return True + if len (files) > 1 : return True + fname ='' + if len(files) == 1: + fname = files[0] + + fvrst = os.path.dirname(os.path.realpath(__file__)) + '/fvrst.x -h ' + cmd = fvrst + fname + #print(cmd +'\n') + p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + (output, err) = p.communicate() + p_status = p.wait() + ss = output.decode().split() + lat = int(ss[0]) + lon = int(ss[1]) + if (lon != lat*6) : + sys.exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') + ymdh_ = str(ss[3]) + str(ss[4])[0:2] + x['input:shared:yyyymmddhh'] = ymdh_ + print("\nThe fvore has restart time " + ymdh_) + x['input:shared:agrid'] = "C"+ss[0] # save for air parameter + print("The input fvcore restart has air grid: " + "C" + ss[0] + '\n') + expid = os.path.basename(fname).split('fvcore')[0] + expid = expid[0:-1] + x['input:shared:expid'] = expid + if (expid) : print("The input fvcore's experiment ID " + expid + '\n') + return False + def catch_model(x): ymdh = x['input:shared:yyyymmddhh'] time = ymdh[0:8] + '_'+ymdh[8:10] @@ -130,6 +170,13 @@ def wemin_default(bc_version): if bc_version =='GM4' or bc_version == 'ICA' : default_ = '26' return default_ +def show_wemin_default(x): + if not x['input:shared:MERRA-2']: + return True + else: + x['input:surface:wemin'] = '26' + return False + def zoom_default(x): zoom_ = '8' cxx = x.get('input:shared:agrid') @@ -182,7 +229,8 @@ def write_cmd( out_dir, cmdl) : cmdl = bin_path+'/'+ cmdl with open(out_dir + '/remap_restarts.CMD', 'w') as f: f.write(cmdl) - subprocess.call(['chmod', '+x',out_dir + '/remap_restarts.CMD']) + # Don't run remap_ressatrs.CMD directly. It may overwrite itself + #subprocess.call(['chmod', '+x',out_dir + '/remap_restarts.CMD']) def print_config( config, indent = 0 ): for k, v in config.items(): @@ -261,7 +309,7 @@ def get_command_line_from_answers(answers): account = " -account " + answers["slurm:account"] qos = " -qos " + answers["slurm:qos"] - constraint = " -constraint " + answers["slurm:constraint"] + partition = " -partition " + answers["slurm:partition"] cmdl = "remap_restarts.py command_line " + merra2 + \ @@ -289,7 +337,7 @@ def get_command_line_from_answers(answers): out_rs + \ account + \ qos + \ - constraint + partition return cmdl From 73ded40c9a04592971217ccd9904ffc641bf3229 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Fri, 8 Sep 2023 12:03:28 -0400 Subject: [PATCH 32/88] change word to select --- pre/remap_restart/remap_questions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 23a8803..c2e8be5 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -195,7 +195,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:bc_version", - "message": f'''\nEnter BC version that matches input restarts: + "message": f'''\nSelect BC version that matches input restarts: BC version ADAS tags GCM tags ---------- --------------- ------------------ GM4: Ganymed-4_0 5_12_2 ... 5_16_5 Ganymed-4_0 ... Heracles-5_4_p3 @@ -220,7 +220,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": f'''\nEnter BC version for new restarts: + "message": f'''\nSelect BC version for new restarts: BC version ADAS tags GCM tags ---------- --------------- ------------------ GM4: Ganymed-4_0 5_12_2 ... 5_16_5 Ganymed-4_0 ... Heracles-5_4_p3 @@ -236,7 +236,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": "Enter BC version for new restarts:\n", + "message": "Select BC version for new restarts:\n", "choices": ['NL3', 'ICA','GM4','Development'], "default": "NL3", "when": lambda x: not x["input:shared:MERRA-2"], From 01b6fc4ce8e8e8101d07650121cc12eb4ae18e79 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 13 Sep 2023 14:37:08 -0400 Subject: [PATCH 33/88] rename yaml files and add more comments --- .../{remap_catch.yaml => bin2nc_catch.yaml} | 0 .../{remap_fv.yaml => bin2nc_fv.yaml} | 0 .../{remap_gocart.yaml => bin2nc_gocart.yaml} | 0 .../{remap_lake.yaml => bin2nc_lake.yaml} | 0 .../{remap_landice.yaml => bin2nc_landice.yaml} | 0 .../{remap_moist.yaml => bin2nc_moist.yaml} | 0 .../{remap_pchem.yaml => bin2nc_pchem.yaml} | 0 .../{remap_salt.yaml => bin2nc_salt.yaml} | 0 pre/remap_restart/remap_bin2nc.py | 16 ++++++++-------- .../{remap_command.py => remap_command_line.py} | 2 ++ pre/remap_restart/remap_questions.py | 14 +++++++++----- pre/remap_restart/remap_restarts.py | 2 +- 12 files changed, 20 insertions(+), 14 deletions(-) rename pre/remap_restart/{remap_catch.yaml => bin2nc_catch.yaml} (100%) rename pre/remap_restart/{remap_fv.yaml => bin2nc_fv.yaml} (100%) rename pre/remap_restart/{remap_gocart.yaml => bin2nc_gocart.yaml} (100%) rename pre/remap_restart/{remap_lake.yaml => bin2nc_lake.yaml} (100%) rename pre/remap_restart/{remap_landice.yaml => bin2nc_landice.yaml} (100%) rename pre/remap_restart/{remap_moist.yaml => bin2nc_moist.yaml} (100%) rename pre/remap_restart/{remap_pchem.yaml => bin2nc_pchem.yaml} (100%) rename pre/remap_restart/{remap_salt.yaml => bin2nc_salt.yaml} (100%) rename pre/remap_restart/{remap_command.py => remap_command_line.py} (98%) diff --git a/pre/remap_restart/remap_catch.yaml b/pre/remap_restart/bin2nc_catch.yaml similarity index 100% rename from pre/remap_restart/remap_catch.yaml rename to pre/remap_restart/bin2nc_catch.yaml diff --git a/pre/remap_restart/remap_fv.yaml b/pre/remap_restart/bin2nc_fv.yaml similarity index 100% rename from pre/remap_restart/remap_fv.yaml rename to pre/remap_restart/bin2nc_fv.yaml diff --git a/pre/remap_restart/remap_gocart.yaml b/pre/remap_restart/bin2nc_gocart.yaml similarity index 100% rename from pre/remap_restart/remap_gocart.yaml rename to pre/remap_restart/bin2nc_gocart.yaml diff --git a/pre/remap_restart/remap_lake.yaml b/pre/remap_restart/bin2nc_lake.yaml similarity index 100% rename from pre/remap_restart/remap_lake.yaml rename to pre/remap_restart/bin2nc_lake.yaml diff --git a/pre/remap_restart/remap_landice.yaml b/pre/remap_restart/bin2nc_landice.yaml similarity index 100% rename from pre/remap_restart/remap_landice.yaml rename to pre/remap_restart/bin2nc_landice.yaml diff --git a/pre/remap_restart/remap_moist.yaml b/pre/remap_restart/bin2nc_moist.yaml similarity index 100% rename from pre/remap_restart/remap_moist.yaml rename to pre/remap_restart/bin2nc_moist.yaml diff --git a/pre/remap_restart/remap_pchem.yaml b/pre/remap_restart/bin2nc_pchem.yaml similarity index 100% rename from pre/remap_restart/remap_pchem.yaml rename to pre/remap_restart/bin2nc_pchem.yaml diff --git a/pre/remap_restart/remap_salt.yaml b/pre/remap_restart/bin2nc_salt.yaml similarity index 100% rename from pre/remap_restart/remap_salt.yaml rename to pre/remap_restart/bin2nc_salt.yaml diff --git a/pre/remap_restart/remap_bin2nc.py b/pre/remap_restart/remap_bin2nc.py index a5d87a9..d56278c 100755 --- a/pre/remap_restart/remap_bin2nc.py +++ b/pre/remap_restart/remap_bin2nc.py @@ -159,11 +159,11 @@ def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=Fal if __name__ == "__main__": - bin2nc('fvcore_internal_rst', 'fvcore_internal_rst.nc4', 'remap_fv.yaml', isDouble=True, hasHeader=True) - bin2nc('moist_internal_rst', 'moist_internal_rst.nc4', 'remap_moist.yaml') - bin2nc('pchem_internal_rst', 'pchem_internal_rst.nc4', 'remap_pchem.yaml') - bin2nc('gocart_internal_rst', 'gocart_internal_rst.nc4', 'remap_gocart.yaml') - bin2nc('lake_internal_rst', 'lake_internal_rst.nc4', 'remap_lake.yaml') - bin2nc('landice_internal_rst', 'landice_internal_rst.nc4', 'remap_landice.yaml') - bin2nc('saltwater_internal_rst','saltwater_internal_rst.nc4', 'remap_saltwater.yaml') - bin2nc('catch_internal_rst', 'catch_internal_rst.nc4', 'remap_catch.yaml') + bin2nc('fvcore_internal_rst', 'fvcore_internal_rst.nc4', 'bin2nc_fv.yaml', isDouble=True, hasHeader=True) + bin2nc('moist_internal_rst', 'moist_internal_rst.nc4', 'bin2nc_moist.yaml') + bin2nc('pchem_internal_rst', 'pchem_internal_rst.nc4', 'bin2nc_pchem.yaml') + bin2nc('gocart_internal_rst', 'gocart_internal_rst.nc4', 'bin2nc_gocart.yaml') + bin2nc('lake_internal_rst', 'lake_internal_rst.nc4', 'bin2nc_lake.yaml') + bin2nc('landice_internal_rst', 'landice_internal_rst.nc4', 'bin2nc_landice.yaml') + bin2nc('saltwater_internal_rst','saltwater_internal_rst.nc4', 'bin2nc_saltwater.yaml') + bin2nc('catch_internal_rst', 'catch_internal_rst.nc4', 'bin2nc_catch.yaml') diff --git a/pre/remap_restart/remap_command.py b/pre/remap_restart/remap_command_line.py similarity index 98% rename from pre/remap_restart/remap_command.py rename to pre/remap_restart/remap_command_line.py index fd88b10..191ad1f 100755 --- a/pre/remap_restart/remap_command.py +++ b/pre/remap_restart/remap_command_line.py @@ -5,6 +5,8 @@ # Newer GEOS code should load a module with GEOSpyD Python3 if not run: # module load python/GEOSpyD/Min4.10.3_py3.9 # +# This script parses and converts the command-line arguments to answers equivalent to that from remap_questions.py + import os import subprocess diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index c2e8be5..50dca9f 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -53,6 +53,9 @@ def default_partition(x): def ask_questions(): + bc_versions = ['NL3', 'ICA','GM4','Development'] + develop_bcs = ['v06'] + questions = [ { "type": "confirm", @@ -168,9 +171,10 @@ def ask_questions(): "default": lambda x: data_ocean_default(x.get('output:shared:agrid')), "when": lambda x: x['output:shared:model'] == 'data', }, + + # This is not a real question. "when" always return false. It is used to remove the comment in the ogrid choices. { "type": "text", - # remove the comments "name": "output:shared:ogrid", "message": "remove the comment of ogrid", # always return false, so it never shows @@ -228,7 +232,7 @@ def ask_questions(): NL3: Icarus-NLv3 5_25_1 ... present Icarus_NL, 10.19 ... present Development: more choices \n\n''', - "choices": ['NL3', 'ICA','GM4','Development'], + "choices": bc_versions , "default": "NL3", "when": lambda x: x["input:shared:MERRA-2"], }, @@ -237,7 +241,7 @@ def ask_questions(): "type": "select", "name": "output:shared:bc_version", "message": "Select BC version for new restarts:\n", - "choices": ['NL3', 'ICA','GM4','Development'], + "choices": bc_versions , "default": "NL3", "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -248,7 +252,7 @@ def ask_questions(): "message": f'''\nMore choices of BC versions for new restarts (this option is in testing stage): v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''', - "choices": ['v06'], + "choices": develop_bcs , "when": lambda x: x["output:shared:bc_version"] == 'Development' and x["input:shared:bc_version"] not in ['v06'], }, @@ -256,7 +260,7 @@ def ask_questions(): "type": "select", "name": "output:shared:bc_version", "message": "\nMore choices of BC version for new restarts (this option is in testing stage):\n", - "choices": ['v06'], + "choices": develop_bcs , "when": lambda x: x["output:shared:bc_version"] == 'Development' and x["input:shared:bc_version"] in ['v06'], }, diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index 1f07193..c341b3f 100644 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -13,7 +13,7 @@ import questionary from remap_utils import * from remap_questions import * -from remap_command import * +from remap_command_line import * from remap_params import * from remap_upper import * from remap_lake_landice_saltwater import * From d2ceb9a2f71fb094d1093f8bdfd20f76b12849b9 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Wed, 13 Sep 2023 17:08:52 -0400 Subject: [PATCH 34/88] document "lcv" restart file from ADAS (remap_analysis.py, remap_questions.py) --- pre/remap_restart/remap_analysis.py | 9 ++++++++- pre/remap_restart/remap_questions.py | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index 9588228..990e242 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -95,7 +95,14 @@ def remap(self): fname = os.path.basename(f) fname = fname + label shutil.move(f, out_dir+'/'+fname) - # write lcv + # write lcv file + # (Info about "lcv" file provided by Ricardo Todling via email, 1 Sep 2023, + # paraphrased by Rolf Reichle: + # The lcv files is binary and inherits the nomenclature of the old GCM primary restarts [GEOS-4]. + # The file simply carries the date of the restarts, e.g., 20231201 210000. + # The file is required by the ADAS. + # LCV stands for Lagrangian Control Volume – which is what the grid coordinates of the model are + # [on the cubed in the GEOS-5 case].) lcv = config['output']['analysis']['lcv'] if lcv : ymd_ = yyyymmddhh_[0:8] diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 50dca9f..e8b1de6 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -294,13 +294,13 @@ def ask_questions(): { "type": "confirm", "name": "output:analysis:bkg", - "message": "Remap bkg files?", + "message": "Remap bkg files? (required by ADAS but not mapped onto ADAS grid; run one ADAS cycle to spin up) ", "default": False, }, { "type": "confirm", "name": "output:analysis:lcv", - "message": "Write lcv?", + "message": "Write lcv file? (required by ADAS) ", "default": False, }, { From 2285bae94503fa3bc8e564109d9959a167bb38f0 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Wed, 13 Sep 2023 17:13:27 -0400 Subject: [PATCH 35/88] renamed bin2nc_*.yaml files to clarify that they are specific to MERRA2 (bin2nc_*.yaml) --- pre/remap_restart/{bin2nc_catch.yaml => bin2nc_merra2_catch.yaml} | 0 pre/remap_restart/{bin2nc_fv.yaml => bin2nc_merra2_fv.yaml} | 0 .../{bin2nc_gocart.yaml => bin2nc_merra2_gocart.yaml} | 0 pre/remap_restart/{bin2nc_lake.yaml => bin2nc_merra2_lake.yaml} | 0 .../{bin2nc_landice.yaml => bin2nc_merra2_landice.yaml} | 0 pre/remap_restart/{bin2nc_moist.yaml => bin2nc_merra2_moist.yaml} | 0 pre/remap_restart/{bin2nc_pchem.yaml => bin2nc_merra2_pchem.yaml} | 0 pre/remap_restart/{bin2nc_salt.yaml => bin2nc_merra2_salt.yaml} | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename pre/remap_restart/{bin2nc_catch.yaml => bin2nc_merra2_catch.yaml} (100%) rename pre/remap_restart/{bin2nc_fv.yaml => bin2nc_merra2_fv.yaml} (100%) rename pre/remap_restart/{bin2nc_gocart.yaml => bin2nc_merra2_gocart.yaml} (100%) rename pre/remap_restart/{bin2nc_lake.yaml => bin2nc_merra2_lake.yaml} (100%) rename pre/remap_restart/{bin2nc_landice.yaml => bin2nc_merra2_landice.yaml} (100%) rename pre/remap_restart/{bin2nc_moist.yaml => bin2nc_merra2_moist.yaml} (100%) rename pre/remap_restart/{bin2nc_pchem.yaml => bin2nc_merra2_pchem.yaml} (100%) rename pre/remap_restart/{bin2nc_salt.yaml => bin2nc_merra2_salt.yaml} (100%) diff --git a/pre/remap_restart/bin2nc_catch.yaml b/pre/remap_restart/bin2nc_merra2_catch.yaml similarity index 100% rename from pre/remap_restart/bin2nc_catch.yaml rename to pre/remap_restart/bin2nc_merra2_catch.yaml diff --git a/pre/remap_restart/bin2nc_fv.yaml b/pre/remap_restart/bin2nc_merra2_fv.yaml similarity index 100% rename from pre/remap_restart/bin2nc_fv.yaml rename to pre/remap_restart/bin2nc_merra2_fv.yaml diff --git a/pre/remap_restart/bin2nc_gocart.yaml b/pre/remap_restart/bin2nc_merra2_gocart.yaml similarity index 100% rename from pre/remap_restart/bin2nc_gocart.yaml rename to pre/remap_restart/bin2nc_merra2_gocart.yaml diff --git a/pre/remap_restart/bin2nc_lake.yaml b/pre/remap_restart/bin2nc_merra2_lake.yaml similarity index 100% rename from pre/remap_restart/bin2nc_lake.yaml rename to pre/remap_restart/bin2nc_merra2_lake.yaml diff --git a/pre/remap_restart/bin2nc_landice.yaml b/pre/remap_restart/bin2nc_merra2_landice.yaml similarity index 100% rename from pre/remap_restart/bin2nc_landice.yaml rename to pre/remap_restart/bin2nc_merra2_landice.yaml diff --git a/pre/remap_restart/bin2nc_moist.yaml b/pre/remap_restart/bin2nc_merra2_moist.yaml similarity index 100% rename from pre/remap_restart/bin2nc_moist.yaml rename to pre/remap_restart/bin2nc_merra2_moist.yaml diff --git a/pre/remap_restart/bin2nc_pchem.yaml b/pre/remap_restart/bin2nc_merra2_pchem.yaml similarity index 100% rename from pre/remap_restart/bin2nc_pchem.yaml rename to pre/remap_restart/bin2nc_merra2_pchem.yaml diff --git a/pre/remap_restart/bin2nc_salt.yaml b/pre/remap_restart/bin2nc_merra2_salt.yaml similarity index 100% rename from pre/remap_restart/bin2nc_salt.yaml rename to pre/remap_restart/bin2nc_merra2_salt.yaml From fc7e2e2657eac522237cb49ab5bd2b65cd37296f Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Wed, 13 Sep 2023 17:36:34 -0400 Subject: [PATCH 36/88] clarified comments/variable names re. WEMIN parameter, binary restarts, dummy questions, and ocean model (remap_catchANDcn.py, remap_questions.py, remap_utils.py) --- pre/remap_restart/remap_catchANDcn.py | 2 +- pre/remap_restart/remap_questions.py | 24 ++++++++++++++---------- pre/remap_restart/remap_utils.py | 5 +++++ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index cfee453..63304d1 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -96,7 +96,7 @@ def remap(self): PARTITION = "#SBATCH --partition=" + config['slurm']['partition'] account = config['slurm']['account'] - # even the input is binary, the output would be nc4 + # even if the (MERRA-2) input restarts are binary, the output restarts will always be nc4 label = get_label(config) suffix = time+'z.nc4' + label diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index e8b1de6..1c7aff9 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -111,7 +111,7 @@ def ask_questions(): { "type": "select", - "name": "input:shared:model", + "name": "input:shared:omodel", "message": "Select ocean model that matches input restarts:\n", "choices": ["data", "MOM5", "MOM6"], "default": "data", @@ -124,12 +124,14 @@ def ask_questions(): "message": "Select data ocean grid that matches input restarts:\n", "choices": ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'], "default": lambda x: data_ocean_default(x.get('input:shared:agrid')), - "when": lambda x: x.get('input:shared:model') == 'data' and not x['input:shared:MERRA-2'], + "when": lambda x: x.get('input:shared:omodel') == 'data' and not x['input:shared:MERRA-2'], }, + # The following is not a real question; "when" always return false. Removes the parenthetical comment + # from the ogrid "choice" that was selected in answer to the previous question. + # (hack that uses the questionary framework to manipulate data inbetween questions) { "type": "text", - # remove the comments "name": "input:shared:ogrid", "message": "remove the comment of ogrid", # always return false, so it never shows @@ -141,7 +143,7 @@ def ask_questions(): "name": "input:shared:ogrid", "message": "Select coupled (MOM5, MOM6) ocean grid that matches input restarts:\n", "choices": ['72x36','360x200','720x410','1440x1080'], - "when": lambda x: x.get('input:shared:model') == 'MOM5' or x.get('input:shared:model')== 'MOM6' + "when": lambda x: x.get('input:shared:omodel') == 'MOM5' or x.get('input:shared:omodel')== 'MOM6' }, { @@ -158,7 +160,7 @@ def ask_questions(): { "type": "select", - "name": "output:shared:model", + "name": "output:shared:omodel", "message": "Select ocean model for new restarts:\n", "choices": ["data", "MOM5", "MOM6"], "default": "data", @@ -169,10 +171,12 @@ def ask_questions(): "message": "Select data ocean grid for new restarts:\n", "choices": ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'], "default": lambda x: data_ocean_default(x.get('output:shared:agrid')), - "when": lambda x: x['output:shared:model'] == 'data', + "when": lambda x: x['output:shared:omodel'] == 'data', }, - # This is not a real question. "when" always return false. It is used to remove the comment in the ogrid choices. + # The following is not a real question; "when" always return false. Removes the parenthetical comment + # from the ogrid "choice" that was selected in answer to the previous question. + # (hack that uses the questionary framework to manipulate data inbetween questions) { "type": "text", "name": "output:shared:ogrid", @@ -185,7 +189,7 @@ def ask_questions(): "name": "output:shared:ogrid", "message": "Select couple ocean grid for new restarts:\n", "choices": ['72x36','360x200','720x410','1440x1080'], - "when": lambda x: x['output:shared:model'] != 'data', + "when": lambda x: x['output:shared:omodel'] != 'data', }, { @@ -306,14 +310,14 @@ def ask_questions(): { "type": "text", "name": "input:surface:wemin", - "message": "What is value of wemin (minimum snow water equivalent parameter) for surface inputs?\n", + "message": "Enter value of wemin (min. snow water equivalent parameter) used in simulation that produced input restarts.\n", "default": lambda x: wemin_default(x.get('input:shared:bc_version')), "when": lambda x: show_wemin_default(x), }, { "type": "text", "name": "output:surface:wemin", - "message": "What is value of wemin (minimum snow water equivalent parameter) for new surface restarts?\n", + "message": "Enter value of wemin (min. snow water equivalent parameter) to be used in simulation with new restarts.\n", "default": lambda x: wemin_default(x.get('output:shared:bc_version')) }, { diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index c0c4563..2eef135 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -165,6 +165,11 @@ def get_label(config): '.' + out_bcv + '.' + out_resolution return label +# NOTE: "wemin" is a configurable parameter that can be set to anything, independent +# of the bcs version. The default set here is simply the "wemin" value that is +# typically used with the bcs version. The user needs to confirm or overwrite +# the default value, depending on the "wemin" value used in the simulation +# that is associated with the set of restarts. def wemin_default(bc_version): default_ = '13' if bc_version =='GM4' or bc_version == 'ICA' : default_ = '26' From 0e8a9ce2deb3f97246b1c60df62863c81d416e4f Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang <52509753+weiyuan-jiang@users.noreply.github.com> Date: Thu, 14 Sep 2023 09:13:10 -0400 Subject: [PATCH 37/88] Update remap_bin2nc.py rename bin2nc yaml files --- pre/remap_restart/remap_bin2nc.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pre/remap_restart/remap_bin2nc.py b/pre/remap_restart/remap_bin2nc.py index d56278c..ba1b025 100755 --- a/pre/remap_restart/remap_bin2nc.py +++ b/pre/remap_restart/remap_bin2nc.py @@ -159,11 +159,11 @@ def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=Fal if __name__ == "__main__": - bin2nc('fvcore_internal_rst', 'fvcore_internal_rst.nc4', 'bin2nc_fv.yaml', isDouble=True, hasHeader=True) - bin2nc('moist_internal_rst', 'moist_internal_rst.nc4', 'bin2nc_moist.yaml') - bin2nc('pchem_internal_rst', 'pchem_internal_rst.nc4', 'bin2nc_pchem.yaml') - bin2nc('gocart_internal_rst', 'gocart_internal_rst.nc4', 'bin2nc_gocart.yaml') - bin2nc('lake_internal_rst', 'lake_internal_rst.nc4', 'bin2nc_lake.yaml') - bin2nc('landice_internal_rst', 'landice_internal_rst.nc4', 'bin2nc_landice.yaml') - bin2nc('saltwater_internal_rst','saltwater_internal_rst.nc4', 'bin2nc_saltwater.yaml') - bin2nc('catch_internal_rst', 'catch_internal_rst.nc4', 'bin2nc_catch.yaml') + bin2nc('fvcore_internal_rst', 'fvcore_internal_rst.nc4', 'bin2nc_merra2_fv.yaml', isDouble=True, hasHeader=True) + bin2nc('moist_internal_rst', 'moist_internal_rst.nc4', 'bin2nc_merra2_moist.yaml') + bin2nc('pchem_internal_rst', 'pchem_internal_rst.nc4', 'bin2nc_merra2_pchem.yaml') + bin2nc('gocart_internal_rst', 'gocart_internal_rst.nc4', 'bin2nc_merra2_gocart.yaml') + bin2nc('lake_internal_rst', 'lake_internal_rst.nc4', 'bin2nc_merra2_lake.yaml') + bin2nc('landice_internal_rst', 'landice_internal_rst.nc4', 'bin2nc_merra2_landice.yaml') + bin2nc('saltwater_internal_rst','saltwater_internal_rst.nc4', 'bin2nc_merra2_saltwater.yaml') + bin2nc('catch_internal_rst', 'catch_internal_rst.nc4', 'bin2nc_merra2_catch.yaml') From 7463c1eae5ed20bacf2c0d46248c4f0d4836d5a0 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Thu, 14 Sep 2023 18:16:34 -0400 Subject: [PATCH 38/88] cleanup of comments, "messages" for questions, hardcoded lists (remap_restarts.py, remap_questions.py, remap_command_line.py) --- pre/remap_restart/remap_command_line.py | 67 +++++------ pre/remap_restart/remap_questions.py | 150 ++++++++++++------------ pre/remap_restart/remap_restarts.py | 40 +++---- 3 files changed, 132 insertions(+), 125 deletions(-) diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 191ad1f..edb15ec 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -36,46 +36,47 @@ def parse_args(program_description): help = "Use command line as input", ) p_command.add_argument('-merra2', action='store_true', default= False, help='use merra2 restarts') - p_command.add_argument('-ymdh', help='yyyymmddhh year month date hour of input and output restarts') - p_command.add_argument('-grout', help='Grid ID of the output restart, format Cxx') - p_command.add_argument('-levsout', help='levels of output restarts') + p_command.add_argument('-ymdh', help='yyyymmddhh year month date hour of input and new restarts') + p_command.add_argument('-grout', help='Grid ID of new restarts, format Cxx') + p_command.add_argument('-levsout', help='Levels for new restarts') - p_command.add_argument('-out_dir', help='directory for output restarts') - p_command.add_argument('-rst_dir', help='directory for input restarts') + p_command.add_argument('-out_dir', help='Directory for new restarts') + p_command.add_argument('-rst_dir', help='Directory of input restarts') - p_command.add_argument('-expid', help='restart id for input restarts') - p_command.add_argument('-newid', default="",help='restart id for output restarts') + p_command.add_argument('-expid', help='Experiment ID of input restarts') + p_command.add_argument('-newid', default="", help='Experiment ID for new restarts') - p_command.add_argument('-bcvin', help='bc_version associated with inputs') - p_command.add_argument('-bcvout', help='bc_version associated with outputs') + p_command.add_argument('-bcvin', help='Boundary conditions version of input restarts') + p_command.add_argument('-bcvout', help='Boundary conditions version for new restarts') - p_command.add_argument('-in_wemin', help='minimum water snow water equivalent for input catch/cn') - p_command.add_argument('-out_wemin', help='minimum water snow water equivalent for output catch/cn') + p_command.add_argument('-in_wemin', help='Min. water snow water equivalent param. used with input restarts') + p_command.add_argument('-out_wemin', help='Min. water snow water equivalent param. to be used with new restarts') ocean_grids=['360x180','1440x720','2880x1440','CS', '72x36', '360x200','720x410','1440x1080'] - p_command.add_argument('-oceanin', help='ocean horizontal grid of inputs. \n \ - data model choices: 360x180,1440x720,2880x1440,CS. \n \ - coupled model choices: 72x36, 360x200,720x410,1440x1080', choices=ocean_grids) - p_command.add_argument('-oceanout', help='ocean horizontal grid of outputs. \n \ - choices are the same as option "oceanin"', choices=ocean_grids) - - p_command.add_argument('-ocnmdlin', default='data', help='ocean input model', choices=['data', 'MOM5', 'MOM6']) - p_command.add_argument('-ocnmdlout',default='data', help='ocean output model', choices=['data', 'MOM5', 'MOM6']) - p_command.add_argument('-catch_model',default='catch', help='catchment model', choices=['catch', 'catchcnclm40', 'catchcnclm45']) - - p_command.add_argument('-nobkg', action='store_true', help="Don't remap bkg files") - p_command.add_argument('-nolcv', action='store_true', help="Don't remap lcv files") - p_command.add_argument('-np', action='store_true', help="No prompt. Overwrite config files without prompting questions") - p_command.add_argument('-lbl', action='store_true', help="Label output restarts with bc_versions and resolutions") - p_command.add_argument('-in_bcsdir', default="", help= "users' alternative boundary condition files for input. If not specified (default), it will be deduced from bc_version and resolution information") - p_command.add_argument('-out_bcsdir', default="", help= "users' alternative boundary condition files for output. If not specified (default), it will be deduced from bc_version and resolution information") - p_command.add_argument('-zoom', help= "zoom for the surface input") - - p_command.add_argument('-qos', default = "debug", help= "queue of slurm job", choices=['debug', 'allnccs']) + + p_command.add_argument('-oceanin', help='Ocean horizontal grid of input restarts. \n \ + data model choices: 360x180,1440x720,2880x1440,CS. \n \ + coupled model choices: 72x36, 360x200,720x410,1440x1080', choices=ocean_grids) + p_command.add_argument('-oceanout', help='Ocean horizontal grid of new restarts. \n \ + choices are the same as option "oceanin"', choices=ocean_grids) + + p_command.add_argument('-ocnmdlin', default='data', help='Ocean model of input restarts', choices=['data', 'MOM5', 'MOM6']) + p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=['data', 'MOM5', 'MOM6']) + p_command.add_argument('-catch_model',default='catch', help='Catchment[CN] model', choices=['catch', 'catchcnclm40', 'catchcnclm45']) + + p_command.add_argument('-nobkg', action='store_true', help="Do not remap bkg files") + p_command.add_argument('-nolcv', action='store_true', help="Do not write lcv file") + p_command.add_argument('-np', action='store_true', help="No prompt. Overwrite config files without prompting questions") + p_command.add_argument('-lbl', action='store_true', help="Label output restarts with bc_versions and resolutions") + p_command.add_argument('-in_bcsdir', default="", help="User-supplied directory with boundary conditions of input restarts. If not specified (default), directory deduced from bc_version and resolution info") + p_command.add_argument('-out_bcsdir', default="", help="User-supplied directory with boundary conditions for new restarts. If not specified (default), directory deduced from bc_version and resolution info") + p_command.add_argument('-zoom', help= "Zoom parameter for the input surface restarts") + + p_command.add_argument('-qos', default = "debug", help="Quality-of-service of slurm job", choices=['debug', 'allnccs']) account = get_account() - p_command.add_argument('-account', default = account, help= "account of slurm job") - p_command.add_argument('-partition', default= 'compute', help= "partition of slurm job") - p_command.add_argument('-rs', default= '3', help='flag indicating which restarts to regrid: 1 (upper air); 2 (surface) 3 (both)', choices=['1','2','3']) + p_command.add_argument('-account', default = account, help="Account of slurm job") + p_command.add_argument('-partition', default= 'compute', help="Partition of slurm job") + p_command.add_argument('-rs', default= '3', help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface) 3 (both)", choices=['1','2','3']) # Parse using parse_known_args so we can pass the rest to the remap scripts args, extra_args = parser.parse_known_args() diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 1c7aff9..9d70eff 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -53,53 +53,61 @@ def default_partition(x): def ask_questions(): - bc_versions = ['NL3', 'ICA','GM4','Development'] - develop_bcs = ['v06'] + choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] + choices_bc_other = ['v06'] + + choices_omodel = ['data', 'MOM5', 'MOM6'] + + choices_ogrid_data = ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'] + + choices_ogrid_cpld = ['72x36', '360x200', '720x410', '1440x1080'] questions = [ { "type": "confirm", "name": "input:shared:MERRA-2", - "message": "Would you like to remap archived MERRA-2 restarts?\n", + "message": "Remap archived MERRA-2 restarts?\n", "default": False, }, { "type": "path", "name": "input:shared:rst_dir", - "message": "Enter the input directory containing restart files to be remapped:\n", + "message": "Enter input directory with restart files to be remapped:\n", "when": lambda x: not x['input:shared:MERRA-2'], }, { "type": "text", "name": "input:shared:yyyymmddhh", - "message": "From what restart date/time would you like to remap? (must be 10 digits: yyyymmddhh)\n", + "message": "Enter restart date/time: (Must be 10 digits: yyyymmddhh.)\n", "validate": lambda text: len(text)==10 , "when": lambda x: not x['input:shared:MERRA-2'] and fvcore_time(x), }, { "type": "text", "name": "input:shared:yyyymmddhh", - "message": "From what restart date would you like to remap? (must be 8 digits: yyyymmdd, hour=21z)\n", + "message": "Enter restart date: (Must be 8 digits: yyyymmdd; hour=21z.)\n", "validate": lambda text: len(text)==8 , "when": lambda x: x['input:shared:MERRA-2'], }, { "type": "path", "name": "output:shared:out_dir", - "message": "Enter the output directory for new restarts:\n" + "message": "Enter output directory for new restarts:\n" }, + + # dummy (invisible) question to run function that initializes MERRA-2 config { "type": "path", "name": "output:shared:out_dir", - "message": "init merra2 \n", - # always return false, no show but change x + "message": "init merra2\n", + # always return false, so question never shows but changes x "when": lambda x: init_merra2(x), }, { "type": "text", "name": "input:shared:agrid", - "message": f'''Enter input atmospheric grid: + "message": f'''Enter atmospheric grid of input restarts: C12 C180 C1000 C270 C24 C360 C1440 C540 C48 C500 C2880 C1080 @@ -112,8 +120,8 @@ def ask_questions(): { "type": "select", "name": "input:shared:omodel", - "message": "Select ocean model that matches input restarts:\n", - "choices": ["data", "MOM5", "MOM6"], + "message": "Select ocean model of input restarts:\n", + "choices": choices_omodel, "default": "data", "when": lambda x: not x['input:shared:MERRA-2'] }, @@ -121,35 +129,33 @@ def ask_questions(): { "type": "select", "name": "input:shared:ogrid", - "message": "Select data ocean grid that matches input restarts:\n", - "choices": ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'], + "message": "Select data ocean grid/resolution of input restarts:\n", + "choices": choices_ogrid_data, "default": lambda x: data_ocean_default(x.get('input:shared:agrid')), "when": lambda x: x.get('input:shared:omodel') == 'data' and not x['input:shared:MERRA-2'], }, - # The following is not a real question; "when" always return false. Removes the parenthetical comment - # from the ogrid "choice" that was selected in answer to the previous question. - # (hack that uses the questionary framework to manipulate data inbetween questions) + # dummy (invisible) question to remove parenthetical comments from selected input:shared:ogrid { "type": "text", "name": "input:shared:ogrid", "message": "remove the comment of ogrid", - # always return false, so it never shows + # always return false, so question never shows but changes ogrid "when": lambda x: remove_ogrid_comment(x, 'IN') }, { "type": "select", "name": "input:shared:ogrid", - "message": "Select coupled (MOM5, MOM6) ocean grid that matches input restarts:\n", - "choices": ['72x36','360x200','720x410','1440x1080'], + "message": "Select coupled ocean resolution of input restarts:\n", + "choices": choices_ogrid_cpld "when": lambda x: x.get('input:shared:omodel') == 'MOM5' or x.get('input:shared:omodel')== 'MOM6' }, { "type": "text", "name": "output:shared:agrid", - "message": f'''Enter new atmospheric grid: + "message": f'''Enter atmospheric grid for new restarts: C12 C180 C1000 C270 C24 C360 C1440 C540 C48 C500 C2880 C1080 @@ -162,40 +168,38 @@ def ask_questions(): "type": "select", "name": "output:shared:omodel", "message": "Select ocean model for new restarts:\n", - "choices": ["data", "MOM5", "MOM6"], + "choices": choices_omodel, "default": "data", }, { "type": "select", "name": "output:shared:ogrid", - "message": "Select data ocean grid for new restarts:\n", - "choices": ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'], + "message": "Select data ocean grid/resolution for new restarts:\n", + "choices": choices_ogrid_data, "default": lambda x: data_ocean_default(x.get('output:shared:agrid')), "when": lambda x: x['output:shared:omodel'] == 'data', }, - # The following is not a real question; "when" always return false. Removes the parenthetical comment - # from the ogrid "choice" that was selected in answer to the previous question. - # (hack that uses the questionary framework to manipulate data inbetween questions) + # dummy (invisible) question to remove parenthetical comments from selected output:shared:ogrid { "type": "text", "name": "output:shared:ogrid", "message": "remove the comment of ogrid", - # always return false, so it never shows + # always return false, so questions never shows but changes ogrid "when": lambda x: remove_ogrid_comment(x, 'OUT') }, { "type": "select", "name": "output:shared:ogrid", - "message": "Select couple ocean grid for new restarts:\n", - "choices": ['72x36','360x200','720x410','1440x1080'], + "message": "Select coupled ocean resolution for new restarts:\n", + "choices": choices_ogrid_cpld, "when": lambda x: x['output:shared:omodel'] != 'data', }, { "type": "text", "name": "output:air:nlevel", - "message": "Enter new atmospheric levels: (71 72 91 127 132 137 144 181)\n", + "message": "Enter number of atmospheric levels for new restarts: (71 72 91 127 132 137 144 181)\n", "default": "72", }, @@ -203,40 +207,42 @@ def ask_questions(): { "type": "select", "name": "input:shared:bc_version", - "message": f'''\nSelect BC version that matches input restarts: - BC version ADAS tags GCM tags - ---------- --------------- ------------------ - GM4: Ganymed-4_0 5_12_2 ... 5_16_5 Ganymed-4_0 ... Heracles-5_4_p3 - ICA: Icarus 5_17_0 ... 5_24_0_p1 Icarus, Jason ... 10.18 - NL3: Icarus-NLv3 5_25_1 ... present Icarus_NL, 10.19 ... present - Development: more choices + "message": f'''\nSelect boundary conditions (BCs) version of input restarts: + BCs version | ADAS tags | GCM tags typically used with BCs version + -----------------|----------------------|----------------------------------------- + GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5 | Ganymed-4_0 ... Heracles-5_4_p3 + ICA: Icarus | 5_17_0 ... 5_24_0_p1 | Icarus, Jason ... 10.18 + NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... present + ---------------------------------------------------------------------------------- + Other: Additional choices used in model or DAS development. \n\n ''', - "choices": ['NL3', 'ICA','GM4','Development'], + "choices": choices_bc_ops, "when": lambda x: not x["input:shared:MERRA-2"], }, { "type": "select", "name": "input:shared:bc_version", - "message": f'''\nMore choices of BC versions that macthes input restarts (this option is in testing stage): + "message": f'''\nSelect BCs version of input restarts: v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''', - "choices": ['v06'], - "when": lambda x: x["input:shared:bc_version"] == 'Development', + "choices": choices_bc_other, + "when": lambda x: x["input:shared:bc_version"] == 'Other', }, { "type": "select", "name": "output:shared:bc_version", - "message": f'''\nSelect BC version for new restarts: - BC version ADAS tags GCM tags - ---------- --------------- ------------------ - GM4: Ganymed-4_0 5_12_2 ... 5_16_5 Ganymed-4_0 ... Heracles-5_4_p3 - ICA: Icarus 5_17_0 ... 5_24_0_p1 Icarus, Jason ... 10.18 - NL3: Icarus-NLv3 5_25_1 ... present Icarus_NL, 10.19 ... present - Development: more choices + "message": f'''\nSelect BCs version for new restarts: + BCs version | ADAS tags | GCM tags typically used with BCs version + -----------------|----------------------|----------------------------------------- + GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5 | Ganymed-4_0 ... Heracles-5_4_p3 + ICA: Icarus | 5_17_0 ... 5_24_0_p1 | Icarus, Jason ... 10.18 + NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... present + ---------------------------------------------------------------------------------- + Other: Additional choices used in model or DAS development. \n\n''', - "choices": bc_versions , + "choices": choices_bc_ops, "default": "NL3", "when": lambda x: x["input:shared:MERRA-2"], }, @@ -244,8 +250,8 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": "Select BC version for new restarts:\n", - "choices": bc_versions , + "message": "Select BCs version for new restarts:\n", + "choices": choices_bc_ops, "default": "NL3", "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -253,109 +259,109 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": f'''\nMore choices of BC versions for new restarts (this option is in testing stage): + "message": f'''\nSelect BCs version of input restarts: v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''', - "choices": develop_bcs , - "when": lambda x: x["output:shared:bc_version"] == 'Development' and x["input:shared:bc_version"] not in ['v06'], + "choices": choices_bc_other, + "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06'], }, { "type": "select", "name": "output:shared:bc_version", - "message": "\nMore choices of BC version for new restarts (this option is in testing stage):\n", - "choices": develop_bcs , - "when": lambda x: x["output:shared:bc_version"] == 'Development' and x["input:shared:bc_version"] in ['v06'], + "message": "\nSelect BCs version of input restarts:\n", + "choices": choices_bc_other, + "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06'], }, { "type": "path", "name": "input:shared:bcs_dir", - "message": "Is this BCS matching input restarts? If no, enter your own absolute path: \n", + "message": "Is this BCs version correct for input restarts? If no, enter your own absolute path: \n", "when": lambda x: echo_bcs(x, "IN"), }, { "type": "path", "name": "output:shared:bcs_dir", - "message": "Is this BCS for new restarts? If no, enter your own absolute path: \n", + "message": "Is this BCs version correct for new restarts? If no, enter your own absolute path: \n", "when": lambda x: echo_bcs(x, "OUT"), }, { "type": "confirm", "name": "output:air:remap", - "message": "Would you like to remap upper air?", + "message": "Remap upper air restarts?", "default": True, }, { "type": "confirm", "name": "output:surface:remap", - "message": "Would you like to remap surface?", + "message": "Remap surface restarts?", "default": True, }, { "type": "confirm", "name": "output:analysis:bkg", - "message": "Remap bkg files? (required by ADAS but not mapped onto ADAS grid; run one ADAS cycle to spin up) ", + "message": "Remap bkg files? (Required by ADAS but not mapped onto ADAS grid; run one ADAS cycle to spin up.) ", "default": False, }, { "type": "confirm", "name": "output:analysis:lcv", - "message": "Write lcv file? (required by ADAS) ", + "message": "Write lcv file? (Required by ADAS.) ", "default": False, }, { "type": "text", "name": "input:surface:wemin", - "message": "Enter value of wemin (min. snow water equivalent parameter) used in simulation that produced input restarts.\n", + "message": "Enter value of WEMIN (min. snow water equivalent parameter) used in simulation that produced input restarts.\n", "default": lambda x: wemin_default(x.get('input:shared:bc_version')), "when": lambda x: show_wemin_default(x), }, { "type": "text", "name": "output:surface:wemin", - "message": "Enter value of wemin (min. snow water equivalent parameter) to be used in simulation with new restarts.\n", + "message": "Enter value of WEMIN (min. snow water equivalent parameter) to be used in simulation with new restarts.\n", "default": lambda x: wemin_default(x.get('output:shared:bc_version')) }, { "type": "text", "name": "input:surface:zoom", - "message": "What is value of zoom (parameter of radius search, smaller value means larger radius) for surface inputs [1-8]?\n", + "message": "Enter value of zoom parameter for surface restarts [1-8]? (Search radius, smaller value means larger radius.)\n", "default": lambda x: zoom_default(x) }, { "type": "text", "name": "output:shared:expid", - "message": "Enter new restarts expid (prefix of the output restart names): \n", + "message": "Enter experiment ID for new restarts: (Added as prefix to new restart file names; can leave blank.)\n", "default": "", }, { "type": "confirm", "name": "output:shared:label", - "message": "Would you like to add labels (bc_versions,resolutions) to restarts' names?", + "message": "Add labels for BCs version and atm/ocean resolutions to restart file names?", "default": False, }, { "type": "text", "name": "slurm:qos", - "message": "qos? If the resolution is c1440 or higher, enter allnccs ", + "message": "SLURM quality-of-service (qos)? (If on NCCS and atm resolution is c1440 or higher, enter allnccs.) ", "default": "debug", }, { "type": "text", "name": "slurm:account", - "message": "account?", + "message": "SLURM account?", "default": get_account(), }, { "type": "text", "name": "slurm:partition", - "message": "partition?", + "message": "SLURM partition?", "default": "compute", "when": lambda x : default_partition(x), }, diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index c341b3f..2e3156c 100644 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -24,33 +24,33 @@ program_description = textwrap.dedent(f''' USAGE: - There are three ways to use this script to remap restarts. + This script provides three options to remap restart files: - 1. Use questionary to convert template remap_params.tpl to - remap_params.yaml and then remap: + 1. Use questionary to create a yaml config file (remap_params.yaml): + ./remap_questions.py + and then remap: ./remap_restarts.py - 2. Use an existing config file to remap: + 2. Use an existing yaml config file to remap: ./remap_restarts.py config_file -c my_config.yaml - 3. Use command line's options as inputs: - ./remap_restarts.py command_line -ymdh 2004041421 .... - To see more command options, please use - ./remap_restarts.py command_line -h + 3. Use command line options: + ./remap_restarts.py command_line -ymdh 2004041421 .... + To see more command line options, please use + ./remap_restarts.py command_line -h + Note that the questionary also generates command line inputs. There are three help commands: - ./remap_restarts.py -h - ./remap_restarts.py config_file -h - ./remap_restarts.py command_line -h - - NOTE: Each individual script can be executed independently - 1. remap_questions.py generates raw_answer.yaml - 2. If command_line option is used, the command_line option is converted raw_answers.yaml - 3. remap_params.py uses raw_answer.yaml and remap_params.tpl as inputs and generates remap_params.yaml - 4. remap_upper.py uses remap_params.yaml as input for remapping - 5. remap_lake_landice_saltwater.py uses remap_params.yaml as input for remapping - 6. remap_catchANDcn.py uses remap_params.yaml as input for remapping - 7. remap_analysis.py uses remap_params.yaml as input for remapping + ./remap_restarts.py -h + ./remap_restarts.py config_file -h + ./remap_restarts.py command_line -h + + NOTE: Individual scripts can be executed independently: + 1. remap_questions.py generates raw_answers.yaml. + 2. If the command_line option is used with remap_restarts.py, the command line options are converted raw_answers.yaml. + 3. remap_params.py uses raw_answer.yaml and remap_params.tpl as inputs and generates remap_params.yaml. + 4. Sets of restarts can be remapped separately with remap_upper.py, remap_lake_landice_saltwater.py, + remap_catchANDcn.py, and remap_analysis.py, all of which use remap_params.yaml as input for remapping. ''') def main(): From 1f189c66e5b4af4023e2ad5418dc2689020ea063 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 14 Sep 2023 20:13:47 -0400 Subject: [PATCH 39/88] simplify choices for command_line --- pre/remap_restart/remap_command_line.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index edb15ec..3691332 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -46,8 +46,9 @@ def parse_args(program_description): p_command.add_argument('-expid', help='Experiment ID of input restarts') p_command.add_argument('-newid', default="", help='Experiment ID for new restarts') - p_command.add_argument('-bcvin', help='Boundary conditions version of input restarts') - p_command.add_argument('-bcvout', help='Boundary conditions version for new restarts') + choices_bc = ['NL3', 'ICA', 'GM4', 'v06'] + p_command.add_argument('-bcvin', help='Boundary conditions version of input restarts', choices= choices_bc) + p_command.add_argument('-bcvout', help='Boundary conditions version for new restarts', choices= choices_bc) p_command.add_argument('-in_wemin', help='Min. water snow water equivalent param. used with input restarts') p_command.add_argument('-out_wemin', help='Min. water snow water equivalent param. to be used with new restarts') @@ -60,9 +61,13 @@ def parse_args(program_description): p_command.add_argument('-oceanout', help='Ocean horizontal grid of new restarts. \n \ choices are the same as option "oceanin"', choices=ocean_grids) - p_command.add_argument('-ocnmdlin', default='data', help='Ocean model of input restarts', choices=['data', 'MOM5', 'MOM6']) - p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=['data', 'MOM5', 'MOM6']) - p_command.add_argument('-catch_model',default='catch', help='Catchment[CN] model', choices=['catch', 'catchcnclm40', 'catchcnclm45']) + + choices_omodel = ['data', 'MOM5', 'MOM6'] + p_command.add_argument('-ocnmdlin', default='data', help='Ocean model of input restarts', choices=choices_omodel) + p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=choices_omodel) + + choices_catchmodel = ['catch', 'catchcnclm40', 'catchcnclm45'] + p_command.add_argument('-catch_model',default='catch', help='Catchment[CN] model', choices=choices_catchmodel) p_command.add_argument('-nobkg', action='store_true', help="Do not remap bkg files") p_command.add_argument('-nolcv', action='store_true', help="Do not write lcv file") From 19c24aa1d6137683d4e8f63ca50c2c06f3290dbb Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 14 Sep 2023 20:24:01 -0400 Subject: [PATCH 40/88] more refactoring... --- pre/remap_restart/remap_questions.py | 66 +++++++++++++--------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 9d70eff..5787ab4 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -62,6 +62,28 @@ def ask_questions(): choices_ogrid_cpld = ['72x36', '360x200', '720x410', '1440x1080'] + bc_message = f'''\nSelect boundary conditions (BCs) version of input restarts: + BCs version | ADAS tags | GCM tags typically used with BCs version + -----------------|----------------------|----------------------------------------- + GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5 | Ganymed-4_0 ... Heracles-5_4_p3 + ICA: Icarus | 5_17_0 ... 5_24_0_p1 | Icarus, Jason ... 10.18 + NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... present + ---------------------------------------------------------------------------------- + Other: Additional choices used in model or DAS development. + \n\n ''' + + other_bc_message = f'''\nSelect BCs version of input restarts: + + v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''' + + agrid_message = f'''Enter atmospheric grid of input restarts: + C12 C180 C1000 C270 + C24 C360 C1440 C540 + C48 C500 C2880 C1080 + C90 C720 C5760 C2160 C1536\n''' + + valid_agrid = ['C12','C180','C1000','C270','C24','C360','C1440','C540','C48','C500','C2880','C1080','C90','C720','C5760','C2160','C1536'] + questions = [ { "type": "confirm", @@ -107,12 +129,8 @@ def ask_questions(): { "type": "text", "name": "input:shared:agrid", - "message": f'''Enter atmospheric grid of input restarts: - C12 C180 C1000 C270 - C24 C360 C1440 C540 - C48 C500 C2880 C1080 - C90 C720 C5760 C2160 C1536\n''', - "validate": lambda text : text in ['C12','C180','C1000','C270','C24','C360','C1440','C540','C48','C500','C2880','C1080','C90','C720','C5760','C2160','C1536'], + "message": agrid_message, + "validate": lambda text : text in valid_agrid, # if it is merra-2 or has_fvcore, agrid is deduced "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_name(x), }, @@ -155,13 +173,9 @@ def ask_questions(): { "type": "text", "name": "output:shared:agrid", - "message": f'''Enter atmospheric grid for new restarts: - C12 C180 C1000 C270 - C24 C360 C1440 C540 - C48 C500 C2880 C1080 - C90 C720 C5760 C2160 C1536\n''', + "message": agrid_message, "default": 'C360', - "validate": lambda text : text in ['C12','C180','C1000','C270','C24','C360','C1440','C540','C48','C500','C2880','C1080','C90','C720','C5760','C2160','C1536'], + "validate": lambda text : text in valid_agrid, }, { @@ -207,15 +221,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:bc_version", - "message": f'''\nSelect boundary conditions (BCs) version of input restarts: - BCs version | ADAS tags | GCM tags typically used with BCs version - -----------------|----------------------|----------------------------------------- - GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5 | Ganymed-4_0 ... Heracles-5_4_p3 - ICA: Icarus | 5_17_0 ... 5_24_0_p1 | Icarus, Jason ... 10.18 - NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... present - ---------------------------------------------------------------------------------- - Other: Additional choices used in model or DAS development. - \n\n ''', + "message": bc_message, "choices": choices_bc_ops, "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -223,9 +229,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:bc_version", - "message": f'''\nSelect BCs version of input restarts: - - v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''', + "message": other_bc_message, "choices": choices_bc_other, "when": lambda x: x["input:shared:bc_version"] == 'Other', }, @@ -233,15 +237,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": f'''\nSelect BCs version for new restarts: - BCs version | ADAS tags | GCM tags typically used with BCs version - -----------------|----------------------|----------------------------------------- - GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5 | Ganymed-4_0 ... Heracles-5_4_p3 - ICA: Icarus | 5_17_0 ... 5_24_0_p1 | Icarus, Jason ... 10.18 - NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... present - ---------------------------------------------------------------------------------- - Other: Additional choices used in model or DAS development. - \n\n''', + "message": bc_message, "choices": choices_bc_ops, "default": "NL3", "when": lambda x: x["input:shared:MERRA-2"], @@ -259,9 +255,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": f'''\nSelect BCs version of input restarts: - - v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''', + "message": other_bc_message, "choices": choices_bc_other, "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06'], }, From 61e5a4769cb1433b33dca7d0c1d0298eeb8fd8d2 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Fri, 15 Sep 2023 08:14:10 -0400 Subject: [PATCH 41/88] typo --- pre/remap_restart/remap_questions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 5787ab4..a2fb7b9 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -166,7 +166,7 @@ def ask_questions(): "type": "select", "name": "input:shared:ogrid", "message": "Select coupled ocean resolution of input restarts:\n", - "choices": choices_ogrid_cpld + "choices": choices_ogrid_cpld, "when": lambda x: x.get('input:shared:omodel') == 'MOM5' or x.get('input:shared:omodel')== 'MOM6' }, From 1e4ac3c66c750b6ce1c2fec38ce1941f421439fc Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Fri, 15 Sep 2023 08:23:13 -0400 Subject: [PATCH 42/88] complete name change shared:model to shared:omodel --- pre/remap_restart/remap_command_line.py | 4 ++-- pre/remap_restart/remap_utils.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 3691332..7372415 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -93,7 +93,7 @@ def get_answers_from_command_line(cml): answers = {} answers["input:shared:MERRA-2"] = cml.merra2 answers["input:shared:yyyymmddhh"] = cml.ymdh - answers["input:shared:model"] = cml.ocnmdlin + answers["input:shared:omodel"] = cml.ocnmdlin answers["output:shared:out_dir"] = os.path.abspath(cml.out_dir + '/') if cml.merra2: init_merra2(answers) @@ -111,7 +111,7 @@ def get_answers_from_command_line(cml): answers["output:air:nlevel"] = cml.levsout answers["output:shared:expid"] = cml.newid answers["output:shared:bc_version"] = cml.bcvout - answers["output:shared:model"] = cml.ocnmdlout + answers["output:shared:omodel"] = cml.ocnmdlout answers["output:shared:label"] = cml.lbl ogrid = cml.oceanout if ogrid == "CS": diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 2eef135..fac01b0 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -29,7 +29,7 @@ def init_merra2(x): else: expid = "d5124_m2_jan10" x['input:shared:expid'] = expid - x['input:shared:model'] = 'data' + x['input:shared:omodel'] = 'data' x['input:shared:agrid'] = 'C180' x['input:shared:ogrid'] = '1440x720' x['input:shared:bc_version'] = 'GM4' @@ -266,9 +266,9 @@ def get_command_line_from_answers(answers): bcvout = " -bcvout " + answers["output:shared:bc_version"] ocnmdlin = '' - if answers.get("input:shared:model"): - ocnmdlin = ' -ocnmdlin ' + answers.get("input:shared:model") - ocnmdlout = ' -ocnmdlout ' + answers["output:shared:model"] + if answers.get("input:shared:omodel"): + ocnmdlin = ' -ocnmdlin ' + answers.get("input:shared:omodel") + ocnmdlout = ' -ocnmdlout ' + answers["output:shared:omodel"] oceanin='' ogrid = answers.get("input:shared:ogrid") @@ -429,12 +429,12 @@ def get_bcsdir(x, opt): bc_version = x.get('input:shared:bc_version') agrid = x.get('input:shared:agrid') ogrid = x.get('input:shared:ogrid') - model = x.get('input:shared:model') + model = x.get('input:shared:omodel') if opt.upper() == "OUT": bc_version = x.get('output:shared:bc_version') agrid = x.get('output:shared:agrid') ogrid = x.get('output:shared:ogrid') - model = x.get('output:shared:model') + model = x.get('output:shared:omodel') bc_base = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" bcdir = bc_base+'/'+ bc_version+'/geometry/' From f5594edfd99b0f879c02e77653f6b94a877b539e Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Fri, 15 Sep 2023 08:29:29 -0400 Subject: [PATCH 43/88] default on omodel --- pre/remap_restart/remap_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index fac01b0..d912e9c 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -265,10 +265,13 @@ def get_command_line_from_answers(answers): bcvin = " -bcvin " + answers["input:shared:bc_version"] bcvout = " -bcvout " + answers["output:shared:bc_version"] - ocnmdlin = '' + ocnmdlin = '-ocnmdlin data' if answers.get("input:shared:omodel"): ocnmdlin = ' -ocnmdlin ' + answers.get("input:shared:omodel") - ocnmdlout = ' -ocnmdlout ' + answers["output:shared:omodel"] + + ocnmdlout = ' -ocnmdlout data' + if answers.get("output:shared:omodel"): + ocnmdlout = ' -ocnmdlout ' + answers["output:shared:omodel"] oceanin='' ogrid = answers.get("input:shared:ogrid") From 553554e2418b97d84583f5acf571fd2729cf4aa5 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Fri, 15 Sep 2023 09:23:59 -0400 Subject: [PATCH 44/88] added comment, minor variable name changes for consistency (remap_questions.py, remap_command_line.py) --- pre/remap_restart/remap_command_line.py | 2 ++ pre/remap_restart/remap_questions.py | 26 +++++++++++++------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 7372415..b5b7c73 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -66,6 +66,8 @@ def parse_args(program_description): p_command.add_argument('-ocnmdlin', default='data', help='Ocean model of input restarts', choices=choices_omodel) p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=choices_omodel) + + # Unlike remap_questions.py, command-line feature does not deduce Catch vs. CatchCN[40,45] for simplicity, thus requires input argument choices_catchmodel = ['catch', 'catchcnclm40', 'catchcnclm45'] p_command.add_argument('-catch_model',default='catch', help='Catchment[CN] model', choices=choices_catchmodel) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index a2fb7b9..5b1bc38 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -53,6 +53,8 @@ def default_partition(x): def ask_questions(): + # define choices, messages, and validation lists that are common to multiple questions + choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] choices_bc_other = ['v06'] @@ -62,7 +64,7 @@ def ask_questions(): choices_ogrid_cpld = ['72x36', '360x200', '720x410', '1440x1080'] - bc_message = f'''\nSelect boundary conditions (BCs) version of input restarts: + message_bc_ops = f'''\nSelect boundary conditions (BCs) version of input restarts: BCs version | ADAS tags | GCM tags typically used with BCs version -----------------|----------------------|----------------------------------------- GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5 | Ganymed-4_0 ... Heracles-5_4_p3 @@ -72,17 +74,17 @@ def ask_questions(): Other: Additional choices used in model or DAS development. \n\n ''' - other_bc_message = f'''\nSelect BCs version of input restarts: + message_bc_other = f'''\nSelect BCs version of input restarts: v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''' - agrid_message = f'''Enter atmospheric grid of input restarts: + message_agrid = f'''Enter atmospheric grid of input restarts: C12 C180 C1000 C270 C24 C360 C1440 C540 C48 C500 C2880 C1080 C90 C720 C5760 C2160 C1536\n''' - valid_agrid = ['C12','C180','C1000','C270','C24','C360','C1440','C540','C48','C500','C2880','C1080','C90','C720','C5760','C2160','C1536'] + validate_agrid = ['C12','C180','C1000','C270','C24','C360','C1440','C540','C48','C500','C2880','C1080','C90','C720','C5760','C2160','C1536'] questions = [ { @@ -129,8 +131,8 @@ def ask_questions(): { "type": "text", "name": "input:shared:agrid", - "message": agrid_message, - "validate": lambda text : text in valid_agrid, + "message": message_agrid, + "validate": lambda text : text in validate_agrid, # if it is merra-2 or has_fvcore, agrid is deduced "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_name(x), }, @@ -173,9 +175,9 @@ def ask_questions(): { "type": "text", "name": "output:shared:agrid", - "message": agrid_message, + "message": message_agrid, "default": 'C360', - "validate": lambda text : text in valid_agrid, + "validate": lambda text : text in validate_agrid, }, { @@ -221,7 +223,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:bc_version", - "message": bc_message, + "message": message_bc_ops, "choices": choices_bc_ops, "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -229,7 +231,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:bc_version", - "message": other_bc_message, + "message": message_bc_other, "choices": choices_bc_other, "when": lambda x: x["input:shared:bc_version"] == 'Other', }, @@ -237,7 +239,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": bc_message, + "message": message_bc_ops, "choices": choices_bc_ops, "default": "NL3", "when": lambda x: x["input:shared:MERRA-2"], @@ -255,7 +257,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": other_bc_message, + "message": message_bc_other, "choices": choices_bc_other, "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06'], }, From d8d49e1c7ae9f61e3edd14b939115ee8ff90b481 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Fri, 15 Sep 2023 13:09:19 -0400 Subject: [PATCH 45/88] consolidate some globale variables --- pre/remap_restart/remap_command_line.py | 13 +++------ pre/remap_restart/remap_questions.py | 33 ----------------------- pre/remap_restart/remap_utils.py | 36 +++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 42 deletions(-) diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index b5b7c73..cf1e79c 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -46,29 +46,24 @@ def parse_args(program_description): p_command.add_argument('-expid', help='Experiment ID of input restarts') p_command.add_argument('-newid', default="", help='Experiment ID for new restarts') - choices_bc = ['NL3', 'ICA', 'GM4', 'v06'] - p_command.add_argument('-bcvin', help='Boundary conditions version of input restarts', choices= choices_bc) - p_command.add_argument('-bcvout', help='Boundary conditions version for new restarts', choices= choices_bc) + p_command.add_argument('-bcvin', help='Boundary conditions version of input restarts', choices= choices_bc_cmd) + p_command.add_argument('-bcvout', help='Boundary conditions version for new restarts', choices= choices_bc_cmd) p_command.add_argument('-in_wemin', help='Min. water snow water equivalent param. used with input restarts') p_command.add_argument('-out_wemin', help='Min. water snow water equivalent param. to be used with new restarts') - ocean_grids=['360x180','1440x720','2880x1440','CS', '72x36', '360x200','720x410','1440x1080'] - p_command.add_argument('-oceanin', help='Ocean horizontal grid of input restarts. \n \ data model choices: 360x180,1440x720,2880x1440,CS. \n \ - coupled model choices: 72x36, 360x200,720x410,1440x1080', choices=ocean_grids) + coupled model choices: 72x36, 360x200,720x410,1440x1080', choices=ocean_grids_cmd) p_command.add_argument('-oceanout', help='Ocean horizontal grid of new restarts. \n \ - choices are the same as option "oceanin"', choices=ocean_grids) + choices are the same as option "oceanin"', choices=ocean_grids_cmd) - choices_omodel = ['data', 'MOM5', 'MOM6'] p_command.add_argument('-ocnmdlin', default='data', help='Ocean model of input restarts', choices=choices_omodel) p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=choices_omodel) # Unlike remap_questions.py, command-line feature does not deduce Catch vs. CatchCN[40,45] for simplicity, thus requires input argument - choices_catchmodel = ['catch', 'catchcnclm40', 'catchcnclm45'] p_command.add_argument('-catch_model',default='catch', help='Catchment[CN] model', choices=choices_catchmodel) p_command.add_argument('-nobkg', action='store_true', help="Do not remap bkg files") diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 5b1bc38..dde1d5c 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -53,39 +53,6 @@ def default_partition(x): def ask_questions(): - # define choices, messages, and validation lists that are common to multiple questions - - choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] - choices_bc_other = ['v06'] - - choices_omodel = ['data', 'MOM5', 'MOM6'] - - choices_ogrid_data = ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'] - - choices_ogrid_cpld = ['72x36', '360x200', '720x410', '1440x1080'] - - message_bc_ops = f'''\nSelect boundary conditions (BCs) version of input restarts: - BCs version | ADAS tags | GCM tags typically used with BCs version - -----------------|----------------------|----------------------------------------- - GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5 | Ganymed-4_0 ... Heracles-5_4_p3 - ICA: Icarus | 5_17_0 ... 5_24_0_p1 | Icarus, Jason ... 10.18 - NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... present - ---------------------------------------------------------------------------------- - Other: Additional choices used in model or DAS development. - \n\n ''' - - message_bc_other = f'''\nSelect BCs version of input restarts: - - v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''' - - message_agrid = f'''Enter atmospheric grid of input restarts: - C12 C180 C1000 C270 - C24 C360 C1440 C540 - C48 C500 C2880 C1080 - C90 C720 C5760 C2160 C1536\n''' - - validate_agrid = ['C12','C180','C1000','C270','C24','C360','C1440','C540','C48','C500','C2880','C1080','C90','C720','C5760','C2160','C1536'] - questions = [ { "type": "confirm", diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index d912e9c..d0c572e 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -9,6 +9,42 @@ import glob import shlex +#shared global veriables + +choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] +choices_bc_other = ['v06'] +choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v06'] +choices_omodel = ['data', 'MOM5', 'MOM6'] +choices_catchmodel = ['catch', 'catchcnclm40', 'catchcnclm45'] +choices_ogrid_data = ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'] + +choices_ogrid_cpld = ['72x36', '360x200', '720x410', '1440x1080'] +ocean_grids_cmd = ['360x180','1440x720','2880x1440','CS', '72x36', '360x200','720x410','1440x1080'] + + +message_bc_ops = f'''\nSelect boundary conditions (BCs) version of input restarts: + BCs version | ADAS tags | GCM tags typically used with BCs version + -----------------|----------------------|----------------------------------------- + GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5 | Ganymed-4_0 ... Heracles-5_4_p3 + ICA: Icarus | 5_17_0 ... 5_24_0_p1 | Icarus, Jason ... 10.18 + NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... present + ---------------------------------------------------------------------------------- + Other: Additional choices used in model or DAS development. + \n\n ''' + +message_bc_other = f'''\nSelect BCs version of input restarts: + + v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''' + +message_agrid = f'''Enter atmospheric grid of input restarts: + C12 C180 C1000 C270 + C24 C360 C1440 C540 + C48 C500 C2880 C1080 + C90 C720 C5760 C2160 C1536\n''' + +validate_agrid = ['C12','C180','C1000','C270','C24','C360','C1440','C540','C48','C500','C2880','C1080','C90','C720','C5760','C2160','C1536'] + + def init_merra2(x): if not x.get('input:shared:MERRA-2') : return False From 4bc33f4f35a578cf91a89391b0629525c920ed19 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Fri, 15 Sep 2023 17:37:39 -0400 Subject: [PATCH 46/88] fixing bugs in centrally defined "messages"; revised documentation; additional cleanup (remap_restarts.py, remap_questions.py, remap_command_line.py, remap_utils.py) --- pre/remap_restart/remap_command_line.py | 43 ++++++++++++------------- pre/remap_restart/remap_questions.py | 15 +++++---- pre/remap_restart/remap_restarts.py | 11 ++++--- pre/remap_restart/remap_utils.py | 37 +++++++++++++++------ 4 files changed, 63 insertions(+), 43 deletions(-) diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index cf1e79c..a74a80e 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -21,6 +21,9 @@ def parse_args(program_description): + # See remap_utils.py for definitions of "choices", "message" strings, and "validate" lists + # that are used multiple times. + parser = argparse.ArgumentParser(description='Remap restarts',epilog=program_description,formatter_class=argparse.RawDescriptionHelpFormatter) p_sub = parser.add_subparsers(help = 'Sub command help') @@ -52,12 +55,8 @@ def parse_args(program_description): p_command.add_argument('-in_wemin', help='Min. water snow water equivalent param. used with input restarts') p_command.add_argument('-out_wemin', help='Min. water snow water equivalent param. to be used with new restarts') - p_command.add_argument('-oceanin', help='Ocean horizontal grid of input restarts. \n \ - data model choices: 360x180,1440x720,2880x1440,CS. \n \ - coupled model choices: 72x36, 360x200,720x410,1440x1080', choices=ocean_grids_cmd) - p_command.add_argument('-oceanout', help='Ocean horizontal grid of new restarts. \n \ - choices are the same as option "oceanin"', choices=ocean_grids_cmd) - + p_command.add_argument('-oceanin', help='Ocean resolution of input restarts. See remap_questions.py for choices.\n', choices=choices_ogrid_cmd) + p_command.add_argument('-oceanout', help='Ocean resolution of new restarts. See remap_questions.py for choices.\n', choices=choices_ogrid_cmd) p_command.add_argument('-ocnmdlin', default='data', help='Ocean model of input restarts', choices=choices_omodel) p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=choices_omodel) @@ -90,14 +89,14 @@ def get_answers_from_command_line(cml): answers = {} answers["input:shared:MERRA-2"] = cml.merra2 answers["input:shared:yyyymmddhh"] = cml.ymdh - answers["input:shared:omodel"] = cml.ocnmdlin + answers["input:shared:omodel"] = cml.ocnmdlin answers["output:shared:out_dir"] = os.path.abspath(cml.out_dir + '/') if cml.merra2: init_merra2(answers) else: - answers["input:shared:bc_version"] = cml.bcvin + answers["input:shared:bc_version"] = cml.bcvin answers["input:surface:catch_model"] = cml.catch_model - answers["input:shared:rst_dir"] = os.path.abspath(cml.rst_dir + '/') + answers["input:shared:rst_dir"] = os.path.abspath(cml.rst_dir + '/') fvcore_name(answers) ogrid = cml.oceanin if ogrid == "CS": @@ -107,8 +106,8 @@ def get_answers_from_command_line(cml): answers["output:shared:agrid"] = cml.grout answers["output:air:nlevel"] = cml.levsout answers["output:shared:expid"] = cml.newid - answers["output:shared:bc_version"] = cml.bcvout - answers["output:shared:omodel"] = cml.ocnmdlout + answers["output:shared:bc_version"] = cml.bcvout + answers["output:shared:omodel"] = cml.ocnmdlout answers["output:shared:label"] = cml.lbl ogrid = cml.oceanout if ogrid == "CS": @@ -123,33 +122,33 @@ def get_answers_from_command_line(cml): if cml.out_bcsdir.strip(): answers["output:shared:bcs_dir"] = cml.out_bcsdir else: - answers["output:shared:bcs_dir"] = get_bcsdir(answers, "OUT") + answers["output:shared:bcs_dir"] = get_bcsdir(answers, "OUT") answers["output:analysis:bkg"] = not cml.nobkg answers["output:analysis:lcv"] = not cml.nolcv if cml.rs == '1': - answers["output:air:remap"] = True + answers["output:air:remap"] = True if cml.rs == '2': - answers["output:surface:remap"] = True + answers["output:surface:remap"] = True if cml.rs == '3': - answers["output:surface:remap"] = True - answers["output:air:remap"] = True + answers["output:surface:remap"] = True + answers["output:air:remap"] = True if cml.zoom: - answers["input:surface:zoom"] = cml.zoom + answers["input:surface:zoom"] = cml.zoom else: # zoom_default fills 'input:shared:agrid' - answers["input:surface:zoom"] = zoom_default(answers) + answers["input:surface:zoom"] = zoom_default(answers) if cml.in_wemin : - answers["input:surface:wemin"] = cml.in_wemin + answers["input:surface:wemin"] = cml.in_wemin else: - answers["input:surface:wemin"] = wemin_default(answers['input:shared:bc_version']) + answers["input:surface:wemin"] = wemin_default(answers['input:shared:bc_version']) if cml.out_wemin : - answers["output:surface:wemin"] = cml.out_wemin + answers["output:surface:wemin"] = cml.out_wemin else: - answers["output:surface:wemin"] = wemin_default(answers['output:shared:bc_version']) + answers["output:surface:wemin"] = wemin_default(answers['output:shared:bc_version']) answers["slurm:account"] = cml.account answers["slurm:qos"] = cml.qos diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index dde1d5c..4efd4e0 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -53,6 +53,9 @@ def default_partition(x): def ask_questions(): + # See remap_utils.py for definitions of "choices", "message" strings, and "validate" lists + # that are used multiple times. + questions = [ { "type": "confirm", @@ -98,7 +101,7 @@ def ask_questions(): { "type": "text", "name": "input:shared:agrid", - "message": message_agrid, + "message": message_agrid_in, "validate": lambda text : text in validate_agrid, # if it is merra-2 or has_fvcore, agrid is deduced "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_name(x), @@ -142,7 +145,7 @@ def ask_questions(): { "type": "text", "name": "output:shared:agrid", - "message": message_agrid, + "message": message_agrid_new, "default": 'C360', "validate": lambda text : text in validate_agrid, }, @@ -190,7 +193,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:bc_version", - "message": message_bc_ops, + "message": message_bc_ops_in, "choices": choices_bc_ops, "when": lambda x: not x["input:shared:MERRA-2"], }, @@ -198,7 +201,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:bc_version", - "message": message_bc_other, + "message": message_bc_other_in, "choices": choices_bc_other, "when": lambda x: x["input:shared:bc_version"] == 'Other', }, @@ -206,7 +209,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": message_bc_ops, + "message": message_bc_ops_new, "choices": choices_bc_ops, "default": "NL3", "when": lambda x: x["input:shared:MERRA-2"], @@ -224,7 +227,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": message_bc_other, + "message": message_bc_other_new, "choices": choices_bc_other, "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06'], }, diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index 2e3156c..e0c3c75 100644 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 # +# main script for remapping GEOS restart files to a new resolution and/or boundary conditions version +# # source install/bin/g5_modules # # Newer GEOS code should load a module with GEOSpyD Python3 if not run: @@ -46,11 +48,10 @@ ./remap_restarts.py command_line -h NOTE: Individual scripts can be executed independently: - 1. remap_questions.py generates raw_answers.yaml. - 2. If the command_line option is used with remap_restarts.py, the command line options are converted raw_answers.yaml. - 3. remap_params.py uses raw_answer.yaml and remap_params.tpl as inputs and generates remap_params.yaml. - 4. Sets of restarts can be remapped separately with remap_upper.py, remap_lake_landice_saltwater.py, - remap_catchANDcn.py, and remap_analysis.py, all of which use remap_params.yaml as input for remapping. + 1. remap_questions.py generates remap_params.yaml. + 2. If the command_line option is used with remap_restarts.py, the command line options are converted remap_params.yaml. + 3. Subsets of restarts can be remapped separately with remap_upper.py, remap_lake_landice_saltwater.py, + remap_catchANDcn.py, and remap_analysis.py, all of which use remap_params.yaml as configuration input. ''') def main(): diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index d0c572e..8410a92 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -9,20 +9,27 @@ import glob import shlex -#shared global veriables +# shared global variables +# +# define "choices", "message" strings, and "validate" lists that are used multiple times. choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] + choices_bc_other = ['v06'] + choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v06'] + choices_omodel = ['data', 'MOM5', 'MOM6'] + choices_catchmodel = ['catch', 'catchcnclm40', 'catchcnclm45'] + choices_ogrid_data = ['360x180 (Reynolds)','1440x720 (MERRA-2)','2880x1440 (OSTIA)','CS (same as atmosphere OSTIA cubed-sphere grid)'] choices_ogrid_cpld = ['72x36', '360x200', '720x410', '1440x1080'] -ocean_grids_cmd = ['360x180','1440x720','2880x1440','CS', '72x36', '360x200','720x410','1440x1080'] +choices_ogrid_cmd = ['360x180', '1440x720', '2880x1440', 'CS'] + choices_ogrid_cpld -message_bc_ops = f'''\nSelect boundary conditions (BCs) version of input restarts: +message_bc_ops = f'''\n BCs version | ADAS tags | GCM tags typically used with BCs version -----------------|----------------------|----------------------------------------- GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5 | Ganymed-4_0 ... Heracles-5_4_p3 @@ -32,17 +39,27 @@ Other: Additional choices used in model or DAS development. \n\n ''' -message_bc_other = f'''\nSelect BCs version of input restarts: +message_bc_ops_in = ("Select boundary conditions (BCs) version of input restarts:\n" + message_bc_ops) +message_bc_ops_new = ("Select boundary conditions (BCs) version for new restarts:\n" + message_bc_ops) + +message_bc_other = f'''\n v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''' -message_agrid = f'''Enter atmospheric grid of input restarts: - C12 C180 C1000 C270 - C24 C360 C1440 C540 - C48 C500 C2880 C1080 - C90 C720 C5760 C2160 C1536\n''' +message_bc_other_in = ("Select BCs version of input restarts:\n" + message_bc_other) +message_bc_other_new = ("Select BCs version for new restarts:\n" + message_bc_other) + +message_agrid_list = f''' + C12 C180 C540 C1440 + C24 C270 C720 C1536 + C48 C360 C1000 C2160 + C90 C500 C1080 C2880 C5760\n''' + +message_agrid_in = ("Enter atmospheric grid of input restarts:\n" + message_agrid_list) + +message_agrid_new = ("Enter atmospheric grid for new restarts:\n" + message_agrid_list) -validate_agrid = ['C12','C180','C1000','C270','C24','C360','C1440','C540','C48','C500','C2880','C1080','C90','C720','C5760','C2160','C1536'] +validate_agrid = ['C12','C24','C48','C90','C180','C270','C360','C500','C540','C720','C1000','C1080','C1440','C1536','C2160','C2880','C5760'] def init_merra2(x): From 43543aef01cf6a8104f151cd86eb78dbd1ea928d Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Mon, 18 Sep 2023 14:18:54 -0400 Subject: [PATCH 47/88] minor cleanup of comment strings and white space (various remap_*.[py,tpl]) --- pre/remap_restart/remap_analysis.py | 6 +-- pre/remap_restart/remap_bin2nc.py | 6 +++ pre/remap_restart/remap_catchANDcn.py | 2 +- pre/remap_restart/remap_command_line.py | 27 +++++----- pre/remap_restart/remap_params.py | 65 +++++++++++++------------ pre/remap_restart/remap_params.tpl | 4 +- pre/remap_restart/remap_questions.py | 6 +-- pre/remap_restart/remap_utils.py | 6 +-- 8 files changed, 65 insertions(+), 57 deletions(-) diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index 990e242..cb797c1 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -98,11 +98,11 @@ def remap(self): # write lcv file # (Info about "lcv" file provided by Ricardo Todling via email, 1 Sep 2023, # paraphrased by Rolf Reichle: - # The lcv files is binary and inherits the nomenclature of the old GCM primary restarts [GEOS-4]. + # The lcv file is binary and inherits the nomenclature of the old GCM primary restarts [GEOS-4]. # The file simply carries the date of the restarts, e.g., 20231201 210000. # The file is required by the ADAS. - # LCV stands for Lagrangian Control Volume – which is what the grid coordinates of the model are - # [on the cubed in the GEOS-5 case].) + # LCV stands for Lagrangian Control Volume, which is what the grid coordinates of the model are + # [on the cubed-sphere for GEOS-5].) lcv = config['output']['analysis']['lcv'] if lcv : ymd_ = yyyymmddhh_[0:8] diff --git a/pre/remap_restart/remap_bin2nc.py b/pre/remap_restart/remap_bin2nc.py index ba1b025..2a48040 100755 --- a/pre/remap_restart/remap_bin2nc.py +++ b/pre/remap_restart/remap_bin2nc.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +# +# script to convert binary MERRA-2 restarts to nc4 +# +# As of Sep 2023, MERRA-2 restarts are the only binary restarts that may need to be +# remapped. The set of yaml config files "bin2nc_merra2_*.yaml" include hard-wired +# variables names, long names, and tile/grid dimensions specific to MERRA-2 restarts. from yaml import safe_load, load, dump from netCDF4 import Dataset diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 63304d1..2e53670 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -96,7 +96,7 @@ def remap(self): PARTITION = "#SBATCH --partition=" + config['slurm']['partition'] account = config['slurm']['account'] - # even if the (MERRA-2) input restarts are binary, the output restarts will always be nc4 + # even if the (MERRA-2) input restarts are binary, the output restarts will always be nc4 (remap_bin2nc.py) label = get_label(config) suffix = time+'z.nc4' + label diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index a74a80e..6e6c488 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -5,7 +5,7 @@ # Newer GEOS code should load a module with GEOSpyD Python3 if not run: # module load python/GEOSpyD/Min4.10.3_py3.9 # -# This script parses and converts the command-line arguments to answers equivalent to that from remap_questions.py +# This script parses and converts the command-line arguments to answers equivalent to those from remap_questions.py import os @@ -39,12 +39,13 @@ def parse_args(program_description): help = "Use command line as input", ) p_command.add_argument('-merra2', action='store_true', default= False, help='use merra2 restarts') + p_command.add_argument('-ymdh', help='yyyymmddhh year month date hour of input and new restarts') - p_command.add_argument('-grout', help='Grid ID of new restarts, format Cxx') - p_command.add_argument('-levsout', help='Levels for new restarts') + p_command.add_argument('-grout', help='Grid ID/resolution of new restarts, format C[xxx] (cubed-sphere only for now)') + p_command.add_argument('-levsout', help='No. of levels for new restarts') - p_command.add_argument('-out_dir', help='Directory for new restarts') p_command.add_argument('-rst_dir', help='Directory of input restarts') + p_command.add_argument('-out_dir', help='Directory for new restarts') p_command.add_argument('-expid', help='Experiment ID of input restarts') p_command.add_argument('-newid', default="", help='Experiment ID for new restarts') @@ -52,8 +53,8 @@ def parse_args(program_description): p_command.add_argument('-bcvin', help='Boundary conditions version of input restarts', choices= choices_bc_cmd) p_command.add_argument('-bcvout', help='Boundary conditions version for new restarts', choices= choices_bc_cmd) - p_command.add_argument('-in_wemin', help='Min. water snow water equivalent param. used with input restarts') - p_command.add_argument('-out_wemin', help='Min. water snow water equivalent param. to be used with new restarts') + p_command.add_argument('-in_wemin', help='Min. snow water equivalent param. used with input restarts') + p_command.add_argument('-out_wemin', help='Min. snow water equivalent param. to be used with new restarts') p_command.add_argument('-oceanin', help='Ocean resolution of input restarts. See remap_questions.py for choices.\n', choices=choices_ogrid_cmd) p_command.add_argument('-oceanout', help='Ocean resolution of new restarts. See remap_questions.py for choices.\n', choices=choices_ogrid_cmd) @@ -69,15 +70,15 @@ def parse_args(program_description): p_command.add_argument('-nolcv', action='store_true', help="Do not write lcv file") p_command.add_argument('-np', action='store_true', help="No prompt. Overwrite config files without prompting questions") p_command.add_argument('-lbl', action='store_true', help="Label output restarts with bc_versions and resolutions") - p_command.add_argument('-in_bcsdir', default="", help="User-supplied directory with boundary conditions of input restarts. If not specified (default), directory deduced from bc_version and resolution info") - p_command.add_argument('-out_bcsdir', default="", help="User-supplied directory with boundary conditions for new restarts. If not specified (default), directory deduced from bc_version and resolution info") - p_command.add_argument('-zoom', help= "Zoom parameter for the input surface restarts") + p_command.add_argument('-in_bcsdir', default="", help="User-supplied directory with boundary conditions of input restarts. If not specified (default), dir. is deduced from bc_version and resolution info") + p_command.add_argument('-out_bcsdir', default="", help="User-supplied directory with boundary conditions for new restarts. If not specified (default), dir. is deduced from bc_version and resolution info") + p_command.add_argument('-zoom', help= "Zoom parameter (search radius) for input surface restarts") - p_command.add_argument('-qos', default = "debug", help="Quality-of-service of slurm job", choices=['debug', 'allnccs']) + p_command.add_argument('-qos', default = "debug", help="SLURM quality-of-service", choices=['debug', 'allnccs']) account = get_account() - p_command.add_argument('-account', default = account, help="Account of slurm job") - p_command.add_argument('-partition', default= 'compute', help="Partition of slurm job") - p_command.add_argument('-rs', default= '3', help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface) 3 (both)", choices=['1','2','3']) + p_command.add_argument('-account', default = account, help="SLURM account") + p_command.add_argument('-partition', default= 'compute', help="SLURM partition") + p_command.add_argument('-rs', default= '3', help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3']) # Parse using parse_known_args so we can pass the rest to the remap scripts args, extra_args = parser.parse_known_args() diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index 01309d4..e8f732c 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -17,9 +17,9 @@ def __init__(self, config_from_question): self.common_out = config_from_question['output']['shared'] self.upper_out = config_from_question['output']['air'] self.slurm_options = config_from_question['slurm'] - self.surf_in = config_from_question['input']['surface'] - self.surf_out = config_from_question['output']['surface'] - self.ana_out = config_from_question['output']['analysis'] + self.surf_in = config_from_question['input']['surface'] + self.surf_out = config_from_question['output']['surface'] + self.ana_out = config_from_question['output']['analysis'] # load input yaml yaml = ruamel.yaml.YAML() @@ -30,28 +30,29 @@ def __init__(self, config_from_question): config_tpl = yaml.load(stream) # params for shared - config_tpl['input']['shared']['MERRA-2'] = self.common_in.get('MERRA-2') - config_tpl['input']['shared']['agrid'] = self.common_in.get('agrid') - config_tpl['input']['shared']['ogrid'] = self.common_in.get('ogrid') - config_tpl['input']['shared']['rst_dir'] = self.common_in['rst_dir']+'/' - config_tpl['input']['shared']['expid'] = self.common_in.get('expid') - config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] - config_tpl['input']['shared']['bc_version'] = self.common_in.get('bc_version') - config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') + config_tpl['input']['shared']['MERRA-2'] = self.common_in.get('MERRA-2') + config_tpl['input']['shared']['agrid'] = self.common_in.get('agrid') + config_tpl['input']['shared']['ogrid'] = self.common_in.get('ogrid') + config_tpl['input']['shared']['rst_dir'] = self.common_in['rst_dir']+'/' + config_tpl['input']['shared']['expid'] = self.common_in.get('expid') + config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] + config_tpl['input']['shared']['bc_version'] = self.common_in.get('bc_version') + config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') - config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') - config_tpl['output']['air']['remap'] = self.upper_out.get('remap') - config_tpl['output']['surface']['remap_water'] = self.surf_out.get('remap') - config_tpl['output']['surface']['remap_catch'] = self.surf_out.get('remap') - config_tpl['output']['shared']['agrid'] = self.common_out['agrid'] - config_tpl['output']['shared']['ogrid'] = self.common_out['ogrid'] - config_tpl['output']['shared']['out_dir'] = self.common_out['out_dir'] + '/' - config_tpl['output']['shared']['expid'] = self.common_out['expid'] - config_tpl['output']['shared']['bc_version'] = self.common_out.get('bc_version') - config_tpl['output']['shared']['label'] = self.common_out.get('label') + config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') + config_tpl['output']['air']['remap'] = self.upper_out.get('remap') + config_tpl['output']['surface']['remap_water'] = self.surf_out.get('remap') + config_tpl['output']['surface']['remap_catch'] = self.surf_out.get('remap') + config_tpl['output']['shared']['agrid'] = self.common_out['agrid'] + config_tpl['output']['shared']['ogrid'] = self.common_out['ogrid'] + config_tpl['output']['shared']['out_dir'] = self.common_out['out_dir'] + '/' + config_tpl['output']['shared']['expid'] = self.common_out['expid'] + config_tpl['output']['shared']['bc_version'] = self.common_out.get('bc_version') + config_tpl['output']['shared']['label'] = self.common_out.get('label') + + config_tpl['input']['shared']['bcs_dir'] = self.common_in['bcs_dir'] + config_tpl['output']['shared']['bcs_dir'] = self.common_out['bcs_dir'] - config_tpl['input']['shared']['bcs_dir'] = self.common_in['bcs_dir'] - config_tpl['output']['shared']['bcs_dir'] = self.common_out['bcs_dir'] # params for upper air config_tpl = self.params_for_air(config_tpl) config_tpl = self.params_for_surface(config_tpl) @@ -69,27 +70,27 @@ def params_for_air(self, config_tpl): return config_tpl def options_for_slurm(self, config_tpl): - config_tpl['slurm']['account'] = self.slurm_options['account'] - config_tpl['slurm']['qos'] = self.slurm_options['qos'] + config_tpl['slurm']['account'] = self.slurm_options['account'] + config_tpl['slurm']['qos'] = self.slurm_options['qos'] config_tpl['slurm']['partition'] = self.slurm_options['partition'] return config_tpl def params_for_surface(self, config_tpl): config_tpl['output']['surface']['surflay'] = 50. bc_version = self.common_out['bc_version'] - ogrid = self.common_out['ogrid'] + ogrid = self.common_out['ogrid'] config_tpl['output']['surface']['split_saltwater'] = True if 'Ganymed' in bc_version : config_tpl['output']['surface']['split_saltwater'] = False - config_tpl['input']['surface']['zoom']= self.surf_in['zoom'] - config_tpl['input']['surface']['wemin']= self.surf_in['wemin'] - config_tpl['output']['surface']['wemin']= self.surf_out['wemin'] - config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') + config_tpl['input']['surface']['zoom'] = self.surf_in['zoom'] + config_tpl['input']['surface']['wemin'] = self.surf_in['wemin'] + config_tpl['output']['surface']['wemin'] = self.surf_out['wemin'] + config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') return config_tpl def params_for_analysis(self, config_tpl): - config_tpl['output']['analysis']['lcv'] = self.ana_out.get('lcv') - config_tpl['output']['analysis']['bkg'] = self.ana_out.get('bkg') + config_tpl['output']['analysis']['lcv'] = self.ana_out.get('lcv') + config_tpl['output']['analysis']['bkg'] = self.ana_out.get('bkg') config_tpl['output']['analysis']['aqua'] = True return config_tpl diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index 1dcce1b..6ae9dfe 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -8,7 +8,7 @@ input: drymass: 1 hydrostatic: 0 shared: - #the bc_version info here is not critical. It is used in command_line options + # bc_version info here is not critical. It is used in command_line options bc_version: none MERRA-2: false agrid: @@ -26,7 +26,7 @@ input: catch_tilefile: null output: shared: - #the bc_version info here is not critical. It is used in command_line options + # bc_version info here is not critical. It is used in command_line options bc_version: none label: false agrid: diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 4efd4e0..f75154a 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -38,11 +38,11 @@ def remove_ogrid_comment(x, opt): def echo_bcs(x,opt): if opt == "IN": x['input:shared:bcs_dir'] = get_bcsdir(x, 'IN') - print("\nBc path for the input restart: " + x['input:shared:bcs_dir']) + print("\nBc path for input restart: " + x['input:shared:bcs_dir']) if opt == "OUT": x['output:shared:bcs_dir'] = get_bcsdir(x, 'OUT') - print("\nBc path for the output restart: " + x['output:shared:bcs_dir']) - print("\nUsers can change the paths in the generated remap_params.yaml file later on") + print("\nBc path for output restart: " + x['output:shared:bcs_dir']) + print("\nLater on, user can edit paths in generated remap_params.yaml file") return False def default_partition(x): diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 8410a92..1e2961b 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -220,9 +220,9 @@ def get_label(config): # NOTE: "wemin" is a configurable parameter that can be set to anything, independent # of the bcs version. The default set here is simply the "wemin" value that is -# typically used with the bcs version. The user needs to confirm or overwrite -# the default value, depending on the "wemin" value used in the simulation -# that is associated with the set of restarts. +# typically used with the bcs version. The user needs to confirm the default +# value or overwrite it with the "wemin" value used in the simulation that is +# associated with the given set of restarts. def wemin_default(bc_version): default_ = '13' if bc_version =='GM4' or bc_version == 'ICA' : default_ = '26' From dc9fb2ae4e1f6f11d928dcee581ae45c071be177 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 20 Sep 2023 09:38:40 -0400 Subject: [PATCH 48/88] complete bin2nc file names change --- pre/CMakeLists.txt | 2 +- pre/remap_restart/remap_lake_landice_saltwater.py | 2 +- pre/remap_restart/remap_upper.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pre/CMakeLists.txt b/pre/CMakeLists.txt index 4f61326..4653c7c 100644 --- a/pre/CMakeLists.txt +++ b/pre/CMakeLists.txt @@ -1,7 +1,7 @@ add_subdirectory(NSIDC-OSTIA_SST-ICE_blend) add_subdirectory(prepare_ocnExtData) -file(GLOB pythonscripts CONFIGURE_DEPENDS ./remap_restart/remap* ./remap_restart/tests/*.*) +file(GLOB pythonscripts CONFIGURE_DEPENDS ./remap_restart/remap* ./remap_restart/bin2nc_merra2* ./remap_restart/tests/*.*) install( PROGRAMS ${pythonscripts} DESTINATION bin) diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index 1383c45..28d7fd9 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -191,7 +191,7 @@ def copy_merra2(self): surfin = [ merra_2_rst_dir + expid+'.lake_internal_rst.' + suffix, merra_2_rst_dir + expid+'.landice_internal_rst.' + suffix, merra_2_rst_dir + expid+'.saltwater_internal_rst.'+ suffix] - bin2nc_yaml = ['remap_lake.yaml', 'remap_landice.yaml','remap_salt.yaml'] + bin2nc_yaml = ['bin2nc_merra2_lake.yaml', 'bin2nc_merra2_landice.yaml','bin2nc_merra2_salt.yaml'] bin_path = os.path.dirname(os.path.realpath(__file__)) for (f,yf) in zip(surfin, bin2nc_yaml): fname = os.path.basename(f) diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 81a5afd..08eaa35 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -349,7 +349,7 @@ def copy_merra2(self): merra_2_rst_dir + expid+'.moist_internal_rst.' + suffix, merra_2_rst_dir + expid+'.gocart_internal_rst.' + suffix, merra_2_rst_dir + expid+'.pchem_internal_rst.' + suffix ] - bin2nc_yaml = ['remap_fv.yaml', 'remap_moist.yaml', 'remap_gocart.yaml', 'remap_pchem.yaml'] + bin2nc_yaml = ['bin2nc_merra2_fv.yaml', 'bin2nc_merra2_moist.yaml', 'bin2nc_merra2_gocart.yaml', 'bin2nc_merra2_pchem.yaml'] bin_path = os.path.dirname(os.path.realpath(__file__)) for (f, yf) in zip(upperin,bin2nc_yaml) : fname = os.path.basename(f) From f576c9f1d9bf83fda716282c04ea2169bad3fdee Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 20 Sep 2023 11:34:35 -0400 Subject: [PATCH 49/88] remove C500 --- pre/remap_restart/remap_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 1e2961b..07a8855 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -53,13 +53,13 @@ C12 C180 C540 C1440 C24 C270 C720 C1536 C48 C360 C1000 C2160 - C90 C500 C1080 C2880 C5760\n''' + C90 C1080 C2880 C5760\n''' message_agrid_in = ("Enter atmospheric grid of input restarts:\n" + message_agrid_list) message_agrid_new = ("Enter atmospheric grid for new restarts:\n" + message_agrid_list) -validate_agrid = ['C12','C24','C48','C90','C180','C270','C360','C500','C540','C720','C1000','C1080','C1440','C1536','C2160','C2880','C5760'] +validate_agrid = ['C12','C24','C48','C90','C180','C270','C360','C540','C720','C1000','C1080','C1440','C1536','C2160','C2880','C5760'] def init_merra2(x): From 2a94a6714c6e6d2fcb1cdc0b2d519e780f350ba5 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 21 Sep 2023 22:50:02 -0400 Subject: [PATCH 50/88] added support to stretched grid --- pre/remap_restart/remap_analysis.py | 15 ++- pre/remap_restart/remap_command_line.py | 6 +- pre/remap_restart/remap_params.py | 2 + pre/remap_restart/remap_params.tpl | 3 + pre/remap_restart/remap_questions.py | 39 ++++++- pre/remap_restart/remap_upper.py | 30 +++--- pre/remap_restart/remap_utils.py | 130 +++++++++++------------- 7 files changed, 133 insertions(+), 92 deletions(-) diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index cb797c1..9f0c4fb 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -16,6 +16,7 @@ import ruamel.yaml from remap_base import remap_base from remap_utils import get_label +import fnmatch class analysis(remap_base): def __init__(self, **configs): @@ -67,11 +68,16 @@ def remap(self): aqua = config['output']['analysis']['aqua'] local_fs=[] for f in analysis_in: - print(f) fname = os.path.basename(f) - out_name = fname.replace(expid_in + '.', expid_out) + out_name = '' + if (expid_in) : + out_name = fname.replace(expid_in + '.', expid_out) + else: + out_name = expid_out+fname + f_tmp = tmpdir+'/'+out_name local_fs.append(f_tmp) + print("Copy " + f + ' to ' + f_tmp) shutil.copy(f,f_tmp) if out_name.find('satbias') != -1 : if (aqua): @@ -83,8 +89,9 @@ def remap(self): nlevel = config['output']['air']['nlevel'] agrid_out = config['output']['shared']['agrid'] flags = "-g5 -res " + self.get_grid_kind(agrid_out.upper()) + " -nlevs " + str(nlevel) - bkg_files = glob.glob(tmpdir+'/*.bkg??_eta_rst*') - for f in bkg_files: + + dyn2dyn = fnmatch.filter(local_fs, '*bkg??_eta_rst*') + for f in dyn2dyn: f_orig = f + ".orig" shutil.move(f,f_orig) cmd = bindir + '/dyn2dyn.x ' + flags + ' -o ' + f + ' ' + f_orig diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 6e6c488..3cf57f7 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -61,6 +61,8 @@ def parse_args(program_description): p_command.add_argument('-ocnmdlin', default='data', help='Ocean model of input restarts', choices=choices_omodel) p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=choices_omodel) + p_command.add_argument('-out_stretch', default= False, help='Output stretch params', choices=choices_stretch) + p_command.add_argument('-in_stretch', default= False, help='Output stretch params', choices=choices_stretch) # Unlike remap_questions.py, command-line feature does not deduce Catch vs. CatchCN[40,45] for simplicity, thus requires input argument @@ -98,7 +100,7 @@ def get_answers_from_command_line(cml): answers["input:shared:bc_version"] = cml.bcvin answers["input:surface:catch_model"] = cml.catch_model answers["input:shared:rst_dir"] = os.path.abspath(cml.rst_dir + '/') - fvcore_name(answers) + fvcore_info(answers) ogrid = cml.oceanin if ogrid == "CS": ogrid = answers["input:shared:agrid"] @@ -125,6 +127,8 @@ def get_answers_from_command_line(cml): else: answers["output:shared:bcs_dir"] = get_bcsdir(answers, "OUT") + answers["output:shared:stretch"] = cml.out_stretch + answers["input:shared:stretch"] = cml.in_stretch answers["output:analysis:bkg"] = not cml.nobkg answers["output:analysis:lcv"] = not cml.nolcv if cml.rs == '1': diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index e8f732c..459296b 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -37,6 +37,7 @@ def __init__(self, config_from_question): config_tpl['input']['shared']['expid'] = self.common_in.get('expid') config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] config_tpl['input']['shared']['bc_version'] = self.common_in.get('bc_version') + config_tpl['input']['shared']['stretch'] = self.common_in.get('stretch') config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') @@ -49,6 +50,7 @@ def __init__(self, config_from_question): config_tpl['output']['shared']['expid'] = self.common_out['expid'] config_tpl['output']['shared']['bc_version'] = self.common_out.get('bc_version') config_tpl['output']['shared']['label'] = self.common_out.get('label') + config_tpl['output']['shared']['stretch'] = self.common_out.get('stretch') config_tpl['input']['shared']['bcs_dir'] = self.common_in['bcs_dir'] config_tpl['output']['shared']['bcs_dir'] = self.common_out['bcs_dir'] diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index 6ae9dfe..e670bd4 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -11,6 +11,7 @@ input: # bc_version info here is not critical. It is used in command_line options bc_version: none MERRA-2: false + stretch: false agrid: bcs_dir: expid: @@ -29,6 +30,8 @@ output: # bc_version info here is not critical. It is used in command_line options bc_version: none label: false + # SG001,SG002 + stretch: false agrid: bcs_dir: expid: diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index f75154a..259632d 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -74,7 +74,7 @@ def ask_questions(): "name": "input:shared:yyyymmddhh", "message": "Enter restart date/time: (Must be 10 digits: yyyymmddhh.)\n", "validate": lambda text: len(text)==10 , - "when": lambda x: not x['input:shared:MERRA-2'] and fvcore_time(x), + "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_info(x), }, { "type": "text", @@ -104,7 +104,7 @@ def ask_questions(): "message": message_agrid_in, "validate": lambda text : text in validate_agrid, # if it is merra-2 or has_fvcore, agrid is deduced - "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_name(x), + "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_info(x), }, { @@ -142,12 +142,47 @@ def ask_questions(): "when": lambda x: x.get('input:shared:omodel') == 'MOM5' or x.get('input:shared:omodel')== 'MOM6' }, + { + "type": "confirm", + "name": "output:shared:stretch", + "message": "Remap to a stretch grid?", + "default": False, + }, { "type": "text", "name": "output:shared:agrid", "message": message_agrid_new, "default": 'C360', "validate": lambda text : text in validate_agrid, + "when": lambda x : not x['output:shared:stretch'], + }, + + { + "type": "select", + "name": "output:shared:stretch", + "message": f'''Select stretch parameters: + Stretch factor Lat lon + -------------- ----- ------ + SG001 2.5 39.5 -98.35 + SG002 3.0 39.5 -98.35 \n''', + "choices": ['SG001','SG002'], + "when": lambda x : x['output:shared:stretch'], + }, + + { + "type": "select", + "name": "output:shared:agrid", + "message": "Select a stretched grid for the new restart: \n", + "choices": ['C270', 'C540', 'C1080', 'C2160'], + "when": lambda x : x.get('output:shared:stretch') == 'SG001', + }, + + { + "type": "select", + "name": "output:shared:agrid", + "message": "Select a stretched grid for the new restart: \n", + "choices": ['C1536'], + "when": lambda x : x.get('output:shared:stretch') == 'SG002', }, { diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 08eaa35..36599a3 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -148,27 +148,25 @@ def remap(self): # We need to create an input.nml file which is different if we are running stretched grid # First, let's define a boolean for whether we are running stretched grid # If we are running with imout of 270, 540, 1080, 1536 or 2160, then we are running stretched grid - if imout in [270, 540, 1080, 2160]: - stretched_grid = True - target_lat = 39.5 - target_lon = -98.35 - stretch_fac = 2.5 - elif imout in [1536]: - stretched_grid = True - target_lat = 39.5 - target_lon = -98.35 - stretch_fac = 3.0 - else: - stretched_grid = False - # If we are running stretched grid, we need to pass in the target lat, lon, and stretch factor # to interp_restarts.x. Per the code we use: # -stretched_grid target_lon target_lat stretch_fac # If we are not running stretched grid, we should pass in a blank string - if stretched_grid: + stretch = config['output']['shared']['stretch'] + stretch_str = "" + if stretch: + target_lat = 0.0 + target_lon = 0.0 + stretch_fac = 1.0 + if stretch == 'SG001': + target_lat = 39.5 + target_lon = -98.35 + stretch_fac = 2.5 + if stretch == 'SG002': + target_lat = 39.5 + target_lon = -98.35 + stretch_fac = 3.0 stretch_str = "-stretched_grid " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac) - else: - stretch_str = "" # Now, let's create the input.nml file # We need to create a namelist for the upper air remapping diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 07a8855..7f46d41 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -8,7 +8,7 @@ import questionary import glob import shlex - +import netCDF4 as nc # shared global variables # # define "choices", "message" strings, and "validate" lists that are used multiple times. @@ -29,6 +29,8 @@ choices_ogrid_cmd = ['360x180', '1440x720', '2880x1440', 'CS'] + choices_ogrid_cpld +choices_stretch = [False, 'SG001', 'SG002'] + message_bc_ops = f'''\n BCs version | ADAS tags | GCM tags typically used with BCs version -----------------|----------------------|----------------------------------------- @@ -50,16 +52,16 @@ message_bc_other_new = ("Select BCs version for new restarts:\n" + message_bc_other) message_agrid_list = f''' - C12 C180 C540 C1440 - C24 C270 C720 C1536 - C48 C360 C1000 C2160 - C90 C1080 C2880 C5760\n''' + C12 C180 C1440 + C24 C360 C2880 + C48 C720 C5760 + C90 C1000 \n''' message_agrid_in = ("Enter atmospheric grid of input restarts:\n" + message_agrid_list) message_agrid_new = ("Enter atmospheric grid for new restarts:\n" + message_agrid_list) -validate_agrid = ['C12','C24','C48','C90','C180','C270','C360','C540','C720','C1000','C1080','C1440','C1536','C2160','C2880','C5760'] +validate_agrid = ['C12','C24','C48','C90','C180','C360','C720','C1000','C1440','C2880','C5760'] def init_merra2(x): @@ -87,89 +89,66 @@ def init_merra2(x): x['input:shared:ogrid'] = '1440x720' x['input:shared:bc_version'] = 'GM4' x['input:surface:catch_model'] = 'catch' + x['input:shared:stretch'] = False x['input:shared:rst_dir'] = x['output:shared:out_dir'] + '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' return False -def fvcore_name(x): +def fvcore_info(x): if 'input:shared:agrid' in x.keys(): return True - ymdh = x['input:shared:yyyymmddhh'] - time = ymdh[0:8] + '_'+ymdh[8:10] rst_dir = x.get('input:shared:rst_dir') - if not rst_dir : return False - x['input:shared:rst_dir'] = rst_dir.strip() # remove extra space files = glob.glob(rst_dir+'/*fvcore_*') - if len (files) == 0 : return False - if len(files) == 1: - fname = files[0] + fname ='' + ymdh ='' if len(files) > 1 : + ymdh = x.get('input:shared:yyyymmddhh') + if (not ymdh): return False + time = ymdh[0:8] + '_'+ymdh[8:10] files = glob.glob(rst_dir+'/*fvcore_*'+time+'*') fname = files[0] - fvrst = os.path.dirname(os.path.realpath(__file__)) + '/fvrst.x -h ' - cmd = fvrst + fname - #print(cmd +'\n') - p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - (output, err) = p.communicate() - p_status = p.wait() - ss = output.decode().split() - lat = int(ss[0]) - lon = int(ss[1]) - if (lon != lat*6) : - sys.exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') - ymdh = x.get('input:shared:yyyymmddhh') - ymdh_ = str(ss[3]) + str(ss[4])[0:2] - if (ymdh_ != ymdh) : - print("Warning: The date in fvcore is " + ymdh_ ) - x['input:shared:agrid'] = "C"+ss[0] # save for air parameter - print("The input fvcore restart has air grid: " + "C" + ss[0] + '\n') - expid = os.path.basename(fname).split('fvcore')[0] - expid = expid[0:-1] - x['input:shared:expid'] = expid - return fname -def fvcore_time(x): - rst_dir = x.get('input:shared:rst_dir') - if not rst_dir : - sys.exit('Input restart directory does not exist.\n') - - x['input:shared:rst_dir'] = rst_dir.strip() # remove extra space - - files = glob.glob(rst_dir+'/*fvcore_*') - - if len (files) == 0 : return True - if len (files) > 1 : return True - fname ='' if len(files) == 1: fname = files[0] - - fvrst = os.path.dirname(os.path.realpath(__file__)) + '/fvrst.x -h ' - cmd = fvrst + fname - #print(cmd +'\n') - p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - (output, err) = p.communicate() - p_status = p.wait() - ss = output.decode().split() - lat = int(ss[0]) - lon = int(ss[1]) - if (lon != lat*6) : - sys.exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') - ymdh_ = str(ss[3]) + str(ss[4])[0:2] - x['input:shared:yyyymmddhh'] = ymdh_ - print("\nThe fvore has restart time " + ymdh_) - x['input:shared:agrid'] = "C"+ss[0] # save for air parameter - print("The input fvcore restart has air grid: " + "C" + ss[0] + '\n') + + fvrst = nc.Dataset(fname) + lon = fvrst.dimensions['lon'].size + lat = fvrst.dimensions['lat'].size + ymdh = fvrst.variables['time'].units.split('since ')[1].split(":")[0].replace('-','').replace(' ', "") + if (lat != lon*6) : + print("jiangjiang", lon, lat) + exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') + x['input:shared:yyyymmddhh'] = ymdh + x['input:shared:agrid'] = "C"+str(lon) + print("The input fvcore restart has air grid: " + "C" + str(lon) + '\n') + print("The input fvcore restart has time: " + ymdh + '\n') expid = os.path.basename(fname).split('fvcore')[0] expid = expid[0:-1] - x['input:shared:expid'] = expid - if (expid) : print("The input fvcore's experiment ID " + expid + '\n') - return False + x['input:shared:expid'] = expid + if (expid) : print("The input fvcore restart has experiment ID " + expid + '\n') + + x['input:shared:stretch'] = False + stretch_factor = fvrst.__dict__.get('STRETCH_FACTOR') + if (stretch_factor) : + sg001 = ['2.50', '39.50', '-98.35'] + sg002 = ['3.00', '39.50', '-98.35'] + target_lat = fvrst.__dict__.get('TARGET_LAT') + target_lon = fvrst.__dict__.get('TARGET_LON') + sg = [stretch_factor, target_lat, target_lon] + f_ = [f"{number:.{2}f}" for number in sg] + print("[stretch_factor, target_lat, target_lon]: ", f_) + if f_ == sg001 : + x['input:shared:stretch'] = 'SG001' + if f_ == sg002 : + x['input:shared:stretch'] = 'SG002' + + return True def catch_model(x): ymdh = x['input:shared:yyyymmddhh'] @@ -353,6 +332,13 @@ def get_command_line_from_answers(answers): if not os.path.samefile(default_bc, answers.get("output:shared:bcs_dir").strip()) : out_bcsdir = ' -out_bcsdir ' + answers.get("output:shared:bcs_dir") + out_stretch = '' + if answers["output:shared:stretch"]: + out_stretch = ' -out_stretch ' + answers["output:shared:stretch"] + in_stretch = '' + if answers["input:shared:stretch"]: + in_stretch = ' -in_stretch ' + answers["input:shared:stretch"] + zoom = " -zoom " + answers["input:surface:zoom"] wemin = " -in_wemin " + answers["input:surface:wemin"] wemout = " -out_wemin " + answers["output:surface:wemin"] @@ -388,6 +374,8 @@ def get_command_line_from_answers(answers): out_dir + \ in_bcsdir + \ out_bcsdir + \ + out_stretch + \ + in_stretch + \ catch_model + \ zoom + \ wemin + \ @@ -423,7 +411,7 @@ def get_config_from_answers(answers): return config -def get_grid_subdir(bcdir, agrid, ogrid, model): +def get_grid_subdir(bcdir, agrid, ogrid, model, stretch): def get_name_with_grid( grid, names, a_o): if not grid : return names @@ -459,6 +447,8 @@ def get_name_with_grid( grid, names, a_o): #dirnames = [ f.name for f in os.scandir(bcdir) if f.is_dir()] #v2.7 dirnames = [f for f in os.listdir(bcdir) if os.path.isdir(os.path.join(bcdir,f))] + if stretch: + dirnames = [dname for dname in dirnames if stretch in dname] anames = get_name_with_grid(agrid, dirnames, 'a') gridID = get_name_with_grid(ogrid, anames, 'o') if len(gridID) == 0 : @@ -486,18 +476,20 @@ def get_bcsdir(x, opt): agrid = x.get('input:shared:agrid') ogrid = x.get('input:shared:ogrid') model = x.get('input:shared:omodel') + stretch = x.get('input:shared:stretch') if opt.upper() == "OUT": bc_version = x.get('output:shared:bc_version') agrid = x.get('output:shared:agrid') ogrid = x.get('output:shared:ogrid') model = x.get('output:shared:omodel') + stretch = x.get('output:shared:stretch') bc_base = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" bcdir = bc_base+'/'+ bc_version+'/geometry/' if not os.path.exists(bcdir): exit("Cannot find bc dir " + bcdir) - gridStr = get_grid_subdir(bcdir,agrid, ogrid, model) + gridStr = get_grid_subdir(bcdir,agrid, ogrid, model,stretch) bcdir = bcdir + gridStr return bcdir From c5c99d6ee4fedf1ce110c781532e6b925448f993 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Fri, 22 Sep 2023 12:27:20 -0400 Subject: [PATCH 51/88] fixed some comments, white-space changes (various remap_*.py) --- pre/remap_restart/remap_command_line.py | 17 ++++++++--------- pre/remap_restart/remap_params.py | 2 +- pre/remap_restart/remap_questions.py | 16 ++++++++-------- pre/remap_restart/remap_upper.py | 8 ++++---- pre/remap_restart/remap_utils.py | 6 +++--- 5 files changed, 24 insertions(+), 25 deletions(-) diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 3cf57f7..6d71797 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -59,12 +59,11 @@ def parse_args(program_description): p_command.add_argument('-oceanin', help='Ocean resolution of input restarts. See remap_questions.py for choices.\n', choices=choices_ogrid_cmd) p_command.add_argument('-oceanout', help='Ocean resolution of new restarts. See remap_questions.py for choices.\n', choices=choices_ogrid_cmd) - p_command.add_argument('-ocnmdlin', default='data', help='Ocean model of input restarts', choices=choices_omodel) - p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=choices_omodel) - p_command.add_argument('-out_stretch', default= False, help='Output stretch params', choices=choices_stretch) - p_command.add_argument('-in_stretch', default= False, help='Output stretch params', choices=choices_stretch) + p_command.add_argument('-ocnmdlin', default='data', help='Ocean model of input restarts', choices=choices_omodel) + p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=choices_omodel) + p_command.add_argument('-in_stretch', default=False, help='Stretched CS params of input restarts', choices=choices_stretch) + p_command.add_argument('-out_stretch', default=False, help='Stretched CS params for new restarts', choices=choices_stretch) - # Unlike remap_questions.py, command-line feature does not deduce Catch vs. CatchCN[40,45] for simplicity, thus requires input argument p_command.add_argument('-catch_model',default='catch', help='Catchment[CN] model', choices=choices_catchmodel) @@ -76,11 +75,11 @@ def parse_args(program_description): p_command.add_argument('-out_bcsdir', default="", help="User-supplied directory with boundary conditions for new restarts. If not specified (default), dir. is deduced from bc_version and resolution info") p_command.add_argument('-zoom', help= "Zoom parameter (search radius) for input surface restarts") - p_command.add_argument('-qos', default = "debug", help="SLURM quality-of-service", choices=['debug', 'allnccs']) + p_command.add_argument('-qos', default="debug", help="SLURM quality-of-service", choices=['debug', 'allnccs']) account = get_account() - p_command.add_argument('-account', default = account, help="SLURM account") - p_command.add_argument('-partition', default= 'compute', help="SLURM partition") - p_command.add_argument('-rs', default= '3', help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3']) + p_command.add_argument('-account', default=account, help="SLURM account") + p_command.add_argument('-partition', default='compute', help="SLURM partition") + p_command.add_argument('-rs', default='3', help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3']) # Parse using parse_known_args so we can pass the rest to the remap scripts args, extra_args = parser.parse_known_args() diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index 459296b..229783c 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -37,7 +37,7 @@ def __init__(self, config_from_question): config_tpl['input']['shared']['expid'] = self.common_in.get('expid') config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] config_tpl['input']['shared']['bc_version'] = self.common_in.get('bc_version') - config_tpl['input']['shared']['stretch'] = self.common_in.get('stretch') + config_tpl['input']['shared']['stretch'] = self.common_in.get('stretch') config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 259632d..0a613b4 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -145,7 +145,7 @@ def ask_questions(): { "type": "confirm", "name": "output:shared:stretch", - "message": "Remap to a stretch grid?", + "message": "Remap to a stretched cubed-sphere grid?", "default": False, }, { @@ -160,11 +160,11 @@ def ask_questions(): { "type": "select", "name": "output:shared:stretch", - "message": f'''Select stretch parameters: - Stretch factor Lat lon - -------------- ----- ------ - SG001 2.5 39.5 -98.35 - SG002 3.0 39.5 -98.35 \n''', + "message": f'''Select parameters of stretched cubed-sphere grid for new restarts: + Name Stretch_Factor Focus_Lat Focus_Lon + ----- -------------- --------- --------- + SG001 2.5 39.5 -98.35 + SG002 3.0 39.5 -98.35 \n''', "choices": ['SG001','SG002'], "when": lambda x : x['output:shared:stretch'], }, @@ -172,7 +172,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:agrid", - "message": "Select a stretched grid for the new restart: \n", + "message": "Select resolution of SG001 grid for new restarts: \n", "choices": ['C270', 'C540', 'C1080', 'C2160'], "when": lambda x : x.get('output:shared:stretch') == 'SG001', }, @@ -180,7 +180,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:agrid", - "message": "Select a stretched grid for the new restart: \n", + "message": "Select resolution of SG002 grid for new restarts: \n", "choices": ['C1536'], "when": lambda x : x.get('output:shared:stretch') == 'SG002', }, diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 36599a3..3a1d775 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -159,13 +159,13 @@ def remap(self): target_lon = 0.0 stretch_fac = 1.0 if stretch == 'SG001': - target_lat = 39.5 + target_lat = 39.5 target_lon = -98.35 - stretch_fac = 2.5 + stretch_fac = 2.5 if stretch == 'SG002': - target_lat = 39.5 + target_lat = 39.5 target_lon = -98.35 - stretch_fac = 3.0 + stretch_fac = 3.0 stretch_str = "-stretched_grid " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac) # Now, let's create the input.nml file diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 7f46d41..40efb91 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -29,7 +29,7 @@ choices_ogrid_cmd = ['360x180', '1440x720', '2880x1440', 'CS'] + choices_ogrid_cpld -choices_stretch = [False, 'SG001', 'SG002'] +choices_stretch = [False, 'SG001', 'SG002'] message_bc_ops = f'''\n BCs version | ADAS tags | GCM tags typically used with BCs version @@ -122,11 +122,11 @@ def fvcore_info(x): lat = fvrst.dimensions['lat'].size ymdh = fvrst.variables['time'].units.split('since ')[1].split(":")[0].replace('-','').replace(' ', "") if (lat != lon*6) : - print("jiangjiang", lon, lat) + print("(N_lon,N_lat) = ", lon, lat) exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') x['input:shared:yyyymmddhh'] = ymdh x['input:shared:agrid'] = "C"+str(lon) - print("The input fvcore restart has air grid: " + "C" + str(lon) + '\n') + print("The input fvcore restart has atm grid: " + "C" + str(lon) + '\n') print("The input fvcore restart has time: " + ymdh + '\n') expid = os.path.basename(fname).split('fvcore')[0] expid = expid[0:-1] From 2ed46bc41dd57094f1f541faa64a1ef45d175214 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Sat, 23 Sep 2023 16:58:20 -0400 Subject: [PATCH 52/88] center SGxxx information --- pre/remap_restart/remap_upper.py | 5 ++++- pre/remap_restart/remap_utils.py | 12 +++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 3a1d775..0014c3f 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -162,10 +162,13 @@ def remap(self): target_lat = 39.5 target_lon = -98.35 stretch_fac = 2.5 - if stretch == 'SG002': + elif stretch == 'SG002': target_lat = 39.5 target_lon = -98.35 stretch_fac = 3.0 + else: + exit("This stretched grid option is not supported " + str(stretch)) + stretch_str = "-stretched_grid " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac) # Now, let's create the input.nml file diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 40efb91..098b42d 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -63,6 +63,9 @@ validate_agrid = ['C12','C24','C48','C90','C180','C360','C720','C1000','C1440','C2880','C5760'] +STRETCH_GRID = {} +STRETCH_GRID['SG001'] = ['2.50', '39.50', '-98.35'] +STRETCH_GRID['SG002'] = ['3.00', '39.50', '-98.35'] def init_merra2(x): if not x.get('input:shared:MERRA-2') : return False @@ -136,18 +139,17 @@ def fvcore_info(x): x['input:shared:stretch'] = False stretch_factor = fvrst.__dict__.get('STRETCH_FACTOR') if (stretch_factor) : - sg001 = ['2.50', '39.50', '-98.35'] - sg002 = ['3.00', '39.50', '-98.35'] target_lat = fvrst.__dict__.get('TARGET_LAT') target_lon = fvrst.__dict__.get('TARGET_LON') sg = [stretch_factor, target_lat, target_lon] f_ = [f"{number:.{2}f}" for number in sg] print("[stretch_factor, target_lat, target_lon]: ", f_) - if f_ == sg001 : + if f_ == STRETCH_GRID['SG001']: x['input:shared:stretch'] = 'SG001' - if f_ == sg002 : + elif f_ == STRETCH_GRID['SG002'] : x['input:shared:stretch'] = 'SG002' - + else: + exit("This stretched cubed-sphere grid is not supported ") return True def catch_model(x): From 40e43d66028852f696b8fa26189e6dd5ff36af51 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Mon, 25 Sep 2023 17:26:20 -0400 Subject: [PATCH 53/88] more cleanup of stretched cubed-sphere parameter handling (remap_questions.py, remap_upper.py, remap_utils.py) --- pre/remap_restart/remap_questions.py | 12 +++----- pre/remap_restart/remap_upper.py | 24 ++++++++-------- pre/remap_restart/remap_utils.py | 41 ++++++++++++++++++++-------- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 0a613b4..b70f293 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -160,12 +160,8 @@ def ask_questions(): { "type": "select", "name": "output:shared:stretch", - "message": f'''Select parameters of stretched cubed-sphere grid for new restarts: - Name Stretch_Factor Focus_Lat Focus_Lon - ----- -------------- --------- --------- - SG001 2.5 39.5 -98.35 - SG002 3.0 39.5 -98.35 \n''', - "choices": ['SG001','SG002'], + "message": message_stretch, + "choices": choices_stretch[1:2], "when": lambda x : x['output:shared:stretch'], }, @@ -173,7 +169,7 @@ def ask_questions(): "type": "select", "name": "output:shared:agrid", "message": "Select resolution of SG001 grid for new restarts: \n", - "choices": ['C270', 'C540', 'C1080', 'C2160'], + "choices": choices_res_SG001, "when": lambda x : x.get('output:shared:stretch') == 'SG001', }, @@ -181,7 +177,7 @@ def ask_questions(): "type": "select", "name": "output:shared:agrid", "message": "Select resolution of SG002 grid for new restarts: \n", - "choices": ['C1536'], + "choices": choices_res_SG002, "when": lambda x : x.get('output:shared:stretch') == 'SG002', }, diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 0014c3f..2fb2bde 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -7,7 +7,8 @@ import shutil import glob from remap_base import remap_base -from remap_utils import get_label +from remap_utils import get_label +from remap_utils import STRETCH_GRID from remap_bin2nc import bin2nc def get_topodir(bcsdir): @@ -146,29 +147,26 @@ def remap(self): log_name = out_dir+'/remap_upper_log' # We need to create an input.nml file which is different if we are running stretched grid - # First, let's define a boolean for whether we are running stretched grid - # If we are running with imout of 270, 540, 1080, 1536 or 2160, then we are running stretched grid - # If we are running stretched grid, we need to pass in the target lat, lon, and stretch factor + # If we are running stretched grid, we need to pass in the target lon+lat and stretch factor # to interp_restarts.x. Per the code we use: # -stretched_grid target_lon target_lat stretch_fac # If we are not running stretched grid, we should pass in a blank string stretch = config['output']['shared']['stretch'] stretch_str = "" if stretch: - target_lat = 0.0 - target_lon = 0.0 - stretch_fac = 1.0 if stretch == 'SG001': - target_lat = 39.5 - target_lon = -98.35 - stretch_fac = 2.5 + stretch_fac = STRETCH_GRID['SG001'][0] + target_lat = STRETCH_GRID['SG001'][1] + target_lon = STRETCH_GRID['SG001'][2] elif stretch == 'SG002': - target_lat = 39.5 - target_lon = -98.35 - stretch_fac = 3.0 + stretch_fac = STRETCH_GRID['SG002'][0] + target_lat = STRETCH_GRID['SG002'][1] + target_lon = STRETCH_GRID['SG002'][2] else: exit("This stretched grid option is not supported " + str(stretch)) + # note "reversed" order of args (relative to order in definition of STRETCH_GRID) + stretch_str = "-stretched_grid " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac) # Now, let's create the input.nml file diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 098b42d..cb709e4 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -9,9 +9,11 @@ import glob import shlex import netCDF4 as nc + # shared global variables # -# define "choices", "message" strings, and "validate" lists that are used multiple times. +# define "choices", "message" strings, and "validate" lists that are used multiple times +# (and related definitions, even if they are used just once). choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] @@ -29,8 +31,25 @@ choices_ogrid_cmd = ['360x180', '1440x720', '2880x1440', 'CS'] + choices_ogrid_cpld +# the following needs more cleanup; e.g., first define list of SGxxx names and parameters (i.e., STRETCH_GRID), +# then assemble message_stretch and choices_stretch using this definition + +message_stretch = f'''Select parameters of stretched cubed-sphere grid for new restarts: + Name Stretch_Factor Focus_Lat Focus_Lon + ----- -------------- --------- --------- + SG001 2.5 39.5 -98.35 + SG002 3.0 39.5 -98.35 \n''', + +STRETCH_GRID = {} +STRETCH_GRID['SG001'] = ['2.50', '39.50', '-98.35'] +STRETCH_GRID['SG002'] = ['3.00', '39.50', '-98.35'] + choices_stretch = [False, 'SG001', 'SG002'] +choices_res_SG001 = ['C270', 'C540', 'C1080', 'C2160'] + +choices_res_SG002 = ['C1536'] + message_bc_ops = f'''\n BCs version | ADAS tags | GCM tags typically used with BCs version -----------------|----------------------|----------------------------------------- @@ -63,9 +82,7 @@ validate_agrid = ['C12','C24','C48','C90','C180','C360','C720','C1000','C1440','C2880','C5760'] -STRETCH_GRID = {} -STRETCH_GRID['SG001'] = ['2.50', '39.50', '-98.35'] -STRETCH_GRID['SG002'] = ['3.00', '39.50', '-98.35'] +# -------------------------------------------------------------------------------- def init_merra2(x): if not x.get('input:shared:MERRA-2') : return False @@ -86,14 +103,14 @@ def init_merra2(x): expid = "d5124_m2_jun21" else: expid = "d5124_m2_jan10" - x['input:shared:expid'] = expid - x['input:shared:omodel'] = 'data' - x['input:shared:agrid'] = 'C180' - x['input:shared:ogrid'] = '1440x720' + x['input:shared:expid'] = expid + x['input:shared:omodel'] = 'data' + x['input:shared:agrid'] = 'C180' + x['input:shared:ogrid'] = '1440x720' x['input:shared:bc_version'] = 'GM4' - x['input:surface:catch_model'] = 'catch' - x['input:shared:stretch'] = False - x['input:shared:rst_dir'] = x['output:shared:out_dir'] + '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' + x['input:surface:catch_model'] = 'catch' + x['input:shared:stretch'] = False + x['input:shared:rst_dir'] = x['output:shared:out_dir'] + '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' return False @@ -136,12 +153,14 @@ def fvcore_info(x): x['input:shared:expid'] = expid if (expid) : print("The input fvcore restart has experiment ID " + expid + '\n') + # get stretch parameters from input restart file x['input:shared:stretch'] = False stretch_factor = fvrst.__dict__.get('STRETCH_FACTOR') if (stretch_factor) : target_lat = fvrst.__dict__.get('TARGET_LAT') target_lon = fvrst.__dict__.get('TARGET_LON') sg = [stretch_factor, target_lat, target_lon] + # verify that stretched cubed-sphere grid is supported by remap_restarts.py f_ = [f"{number:.{2}f}" for number in sg] print("[stretch_factor, target_lat, target_lon]: ", f_) if f_ == STRETCH_GRID['SG001']: From 492465318043bba13a15329484a8f98f9ecf1526 Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Wed, 27 Sep 2023 09:41:59 -0400 Subject: [PATCH 54/88] edit stretched question --- pre/remap_restart/remap_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index cb709e4..7b32f9f 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -34,11 +34,14 @@ # the following needs more cleanup; e.g., first define list of SGxxx names and parameters (i.e., STRETCH_GRID), # then assemble message_stretch and choices_stretch using this definition -message_stretch = f'''Select parameters of stretched cubed-sphere grid for new restarts: +message_stretch = f'''\n + Select parameters of stretched cubed-sphere grid for new restarts: + Name Stretch_Factor Focus_Lat Focus_Lon ----- -------------- --------- --------- SG001 2.5 39.5 -98.35 - SG002 3.0 39.5 -98.35 \n''', + SG002 3.0 39.5 -98.35 + \n\n''' STRETCH_GRID = {} STRETCH_GRID['SG001'] = ['2.50', '39.50', '-98.35'] From 896da9d3db9311e7046fe5a1e2bd11c05436d1c5 Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Wed, 27 Sep 2023 10:29:28 -0400 Subject: [PATCH 55/88] enable 3 options --- pre/remap_restart/remap_questions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index b70f293..4ff9c7b 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -161,7 +161,7 @@ def ask_questions(): "type": "select", "name": "output:shared:stretch", "message": message_stretch, - "choices": choices_stretch[1:2], + "choices": choices_stretch[1:3], "when": lambda x : x['output:shared:stretch'], }, From e76f269fe17a72d7b169ef945b0b93d23bd86bde Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Mon, 9 Oct 2023 13:04:51 -0400 Subject: [PATCH 56/88] copy cbkg and satbias and satbang files --- pre/remap_restart/remap_analysis.py | 36 ++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index 9f0c4fb..8c53a1b 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -77,12 +77,12 @@ def remap(self): f_tmp = tmpdir+'/'+out_name local_fs.append(f_tmp) - print("Copy " + f + ' to ' + f_tmp) shutil.copy(f,f_tmp) if out_name.find('satbias') != -1 : if (aqua): f_ = open(f_tmp, 'w') for line in fileinput.input(f): + # This is unlikely to be the case with new files f_.write(line.replace('airs281SUBSET_aqua', 'airs281_aqua ')) f_.close() @@ -90,13 +90,14 @@ def remap(self): agrid_out = config['output']['shared']['agrid'] flags = "-g5 -res " + self.get_grid_kind(agrid_out.upper()) + " -nlevs " + str(nlevel) - dyn2dyn = fnmatch.filter(local_fs, '*bkg??_eta_rst*') + dyn2dyn = fnmatch.filter(local_fs, '*bkg??_eta_rst*') for f in dyn2dyn: - f_orig = f + ".orig" - shutil.move(f,f_orig) - cmd = bindir + '/dyn2dyn.x ' + flags + ' -o ' + f + ' ' + f_orig - print(cmd) - subprocess.call(shlex.split(cmd)) + if "cbkg" not in f: + f_orig = f + ".orig" + shutil.move(f,f_orig) + cmd = bindir + '/dyn2dyn.x ' + flags + ' -o ' + f + ' ' + f_orig + print(cmd) + subprocess.call(shlex.split(cmd)) for f in local_fs: fname = os.path.basename(f) @@ -174,6 +175,7 @@ def copy_merra2(self): rst_time = datetime(year=int(yyyy_), month=int(mm_), day=int(dd_), hour = int(hh_)) anafiles=[] + canafiles=[] for h in [3,4,5,6,7,8,9]: delt = timedelta(hours = h-3) new_time = rst_time + delt @@ -200,6 +202,24 @@ def copy_merra2(self): anafiles.append(f) else: print('Warning: Cannot find '+f) + + # cbkg files + fname = expid + '.cbkg'+hh+'_eta_rst.'+ymd+'_'+newhh+'z.nc4' + f = m2_rst_dir+'/'+fname + if(os.path.isfile(f)): + canafiles.append(f) + else: + print('Warning: Cannot find '+f) + + # satb files + for ftype in ['satbias', 'satbang']: + fname = expid + '.ana_'+ftype+'_rst.'+ymd+'_'+newhh+'z.txt' + f = m2_rst_dir+'/'+fname + if(os.path.isfile(f)): + canafiles.append(f) + else: + print('Warning: Cannot find '+f) + # trak.GDA.rst file delt = timedelta(hours = 3) new_time = rst_time - delt @@ -211,7 +231,7 @@ def copy_merra2(self): f = m2_rst_dir+'/'+fname if (os.path.isfile(f)): anafiles.append(f) - for f in anafiles: + for f in anafiles + canafiles: fname = os.path.basename(f) f_tmp = rst_dir+'/'+fname print("Copy file "+f +" to " + rst_dir) From b376108602980d639ad6b4484bd5437b8395f9f0 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 11 Oct 2023 11:26:49 -0400 Subject: [PATCH 57/88] remove trailing spaces in yaml files --- pre/remap_restart/bin2nc_merra2_catch.yaml | 320 +++++++++---------- pre/remap_restart/bin2nc_merra2_fv.yaml | 1 - pre/remap_restart/bin2nc_merra2_gocart.yaml | 1 - pre/remap_restart/bin2nc_merra2_lake.yaml | 1 - pre/remap_restart/bin2nc_merra2_landice.yaml | 1 - pre/remap_restart/bin2nc_merra2_moist.yaml | 5 +- pre/remap_restart/bin2nc_merra2_pchem.yaml | 1 - pre/remap_restart/bin2nc_merra2_salt.yaml | 38 +-- 8 files changed, 181 insertions(+), 187 deletions(-) diff --git a/pre/remap_restart/bin2nc_merra2_catch.yaml b/pre/remap_restart/bin2nc_merra2_catch.yaml index 26d7fbf..077ca94 100644 --- a/pre/remap_restart/bin2nc_merra2_catch.yaml +++ b/pre/remap_restart/bin2nc_merra2_catch.yaml @@ -1,287 +1,287 @@ variables: - - short_name: 'BF1' - long_name: 'topo_baseflow_param_1' - units: 'kg m-4' + - short_name: 'BF1' + long_name: 'topo_baseflow_param_1' + units: 'kg m-4' dimension: - tile - - short_name: 'BF2' - long_name: 'topo_baseflow_param_2' - units: 'm' + - short_name: 'BF2' + long_name: 'topo_baseflow_param_2' + units: 'm' dimension: - tile - - short_name: 'BF3' - long_name: 'topo_baseflow_param_3' - units: 'log(m)' + - short_name: 'BF3' + long_name: 'topo_baseflow_param_3' + units: 'log(m)' dimension: - tile - - short_name: 'VGWMAX' + - short_name: 'VGWMAX' long_name: 'max_rootzone_water_content' - units: 'kg m-2' + units: 'kg m-2' dimension: - tile - - short_name: 'CDCR1' - long_name: 'moisture_threshold' - units: 'kg m-2' + - short_name: 'CDCR1' + long_name: 'moisture_threshold' + units: 'kg m-2' dimension: - tile - - short_name: 'CDCR2' - long_name: 'max_water_content' - units: 'kg m-2' + - short_name: 'CDCR2' + long_name: 'max_water_content' + units: 'kg m-2' dimension: - tile - - short_name: 'PSIS' + - short_name: 'PSIS' long_name: 'saturated_matric_potential' - units: 'm' + units: 'm' dimension: - tile - - short_name: 'BEE' - long_name: 'clapp_hornberger_b' - units: '1' + - short_name: 'BEE' + long_name: 'clapp_hornberger_b' + units: '1' dimension: - tile - - short_name: 'POROS' - long_name: 'soil_porosity' - units: '1' + - short_name: 'POROS' + long_name: 'soil_porosity' + units: '1' dimension: - tile - - short_name: 'WPWET' - long_name: 'wetness_at_wilting_point' - units: '1' + - short_name: 'WPWET' + long_name: 'wetness_at_wilting_point' + units: '1' dimension: - tile - - short_name: 'COND' - long_name: 'sfc_sat_hydraulic_conduct' - units: 'm s-1' + - short_name: 'COND' + long_name: 'sfc_sat_hydraulic_conduct' + units: 'm s-1' dimension: - tile - - short_name: 'GNU' - long_name: 'vertical_transmissivity' - units: 'm-1' + - short_name: 'GNU' + long_name: 'vertical_transmissivity' + units: 'm-1' dimension: - tile - - short_name: 'ARS1' - long_name: 'wetness_param_1' - units: 'm+2 kg-1' + - short_name: 'ARS1' + long_name: 'wetness_param_1' + units: 'm+2 kg-1' dimension: - tile - - short_name: 'ARS2' - long_name: 'wetness_param_2' - units: 'm+2 kg-1' + - short_name: 'ARS2' + long_name: 'wetness_param_2' + units: 'm+2 kg-1' dimension: - tile - - short_name: 'ARS3' - long_name: 'wetness_param_3' - units: 'm+4 kg-2' + - short_name: 'ARS3' + long_name: 'wetness_param_3' + units: 'm+4 kg-2' dimension: - tile - - short_name: 'ARA1' - long_name: 'shape_param_1' - units: 'm+2 kg-1' + - short_name: 'ARA1' + long_name: 'shape_param_1' + units: 'm+2 kg-1' dimension: - tile - - short_name: 'ARA2' - long_name: 'shape_param_2' - units: '1' + - short_name: 'ARA2' + long_name: 'shape_param_2' + units: '1' dimension: - tile - - short_name: 'ARA3' - long_name: 'shape_param_3' - units: 'm+2 kg-1' + - short_name: 'ARA3' + long_name: 'shape_param_3' + units: 'm+2 kg-1' dimension: - tile - - short_name: 'ARA4' - long_name: 'shape_param_4' - units: '1' + - short_name: 'ARA4' + long_name: 'shape_param_4' + units: '1' dimension: - tile - - short_name: 'ARW1' - long_name: 'min_theta_param_1' - units: 'm+2 kg-1' + - short_name: 'ARW1' + long_name: 'min_theta_param_1' + units: 'm+2 kg-1' dimension: - tile - - short_name: 'ARW2' - long_name: 'min_theta_param_2' - units: 'm+2 kg-1' + - short_name: 'ARW2' + long_name: 'min_theta_param_2' + units: 'm+2 kg-1' dimension: - tile - - short_name: 'ARW3' - long_name: 'min_theta_param_3' - units: 'm+4 kg-2' + - short_name: 'ARW3' + long_name: 'min_theta_param_3' + units: 'm+4 kg-2' dimension: - tile - - short_name: 'ARW4' - long_name: 'min_theta_param_4' - units: '1' + - short_name: 'ARW4' + long_name: 'min_theta_param_4' + units: '1' dimension: - tile - - short_name: 'TSA1' - long_name: 'water_transfer_param_1' - units: '1' + - short_name: 'TSA1' + long_name: 'water_transfer_param_1' + units: '1' dimension: - tile - - short_name: 'TSA2' - long_name: 'water_transfer_param_2' - units: '1' + - short_name: 'TSA2' + long_name: 'water_transfer_param_2' + units: '1' dimension: - tile - - short_name: 'TSB1' - long_name: 'water_transfer_param_3' - units: '1' + - short_name: 'TSB1' + long_name: 'water_transfer_param_3' + units: '1' dimension: - tile - - short_name: 'TSB2' - long_name: 'water_transfer_param_4' - units: '1' + - short_name: 'TSB2' + long_name: 'water_transfer_param_4' + units: '1' dimension: - tile - - short_name: 'ATAU' - long_name: 'water_transfer_param_5' - units: '1' + - short_name: 'ATAU' + long_name: 'water_transfer_param_5' + units: '1' dimension: - tile - - short_name: 'BTAU' - long_name: 'water_transfer_param_6' - units: '1' + - short_name: 'BTAU' + long_name: 'water_transfer_param_6' + units: '1' dimension: - tile - - short_name: 'OLD_ITY' + - short_name: 'OLD_ITY' long_name: 'Placeholder. Used to be vegetation_type.' - units: '1' + units: '1' dimension: - tile - - short_name: 'TC' - long_name: 'canopy_temperature' - units: 'K' + - short_name: 'TC' + long_name: 'canopy_temperature' + units: 'K' dimension: - subtile - tile - - short_name: 'QC' - long_name: 'canopy_specific_humidity' - units: 'kg kg-1' + - short_name: 'QC' + long_name: 'canopy_specific_humidity' + units: 'kg kg-1' dimension: - subtile - tile - - short_name: 'CAPAC' + - short_name: 'CAPAC' long_name: 'interception_reservoir_capac' - units: 'kg m-2' + units: 'kg m-2' dimension: - tile - - short_name: 'CATDEF' - long_name: 'catchment_deficit' - units: 'kg m-2' + - short_name: 'CATDEF' + long_name: 'catchment_deficit' + units: 'kg m-2' dimension: - tile - - short_name: 'RZEXC' - long_name: 'root_zone_excess' - units: 'kg m-2' + - short_name: 'RZEXC' + long_name: 'root_zone_excess' + units: 'kg m-2' dimension: - tile - - short_name: 'SRFEXC' - long_name: 'surface_excess' - units: 'kg m-2' + - short_name: 'SRFEXC' + long_name: 'surface_excess' + units: 'kg m-2' dimension: - tile - - short_name: 'GHTCNT1' - long_name: 'soil_heat_content_layer_1' - units: 'J m-2' + - short_name: 'GHTCNT1' + long_name: 'soil_heat_content_layer_1' + units: 'J m-2' dimension: - tile - - short_name: 'GHTCNT2' - long_name: 'soil_heat_content_layer_2' - units: 'J_m-2' + - short_name: 'GHTCNT2' + long_name: 'soil_heat_content_layer_2' + units: 'J_m-2' dimension: - tile - - short_name: 'GHTCNT3' - long_name: 'soil_heat_content_layer_3' - units: 'J m-2' + - short_name: 'GHTCNT3' + long_name: 'soil_heat_content_layer_3' + units: 'J m-2' dimension: - tile - - short_name: 'GHTCNT4' - long_name: 'soil_heat_content_layer_4' - units: 'J m-2' + - short_name: 'GHTCNT4' + long_name: 'soil_heat_content_layer_4' + units: 'J m-2' dimension: - tile - - short_name: 'GHTCNT5' - long_name: 'soil_heat_content_layer_5' - units: 'J m-2' + - short_name: 'GHTCNT5' + long_name: 'soil_heat_content_layer_5' + units: 'J m-2' dimension: - tile - - short_name: 'GHTCNT6' - long_name: 'soil_heat_content_layer_6' - units: 'J m-2' + - short_name: 'GHTCNT6' + long_name: 'soil_heat_content_layer_6' + units: 'J m-2' dimension: - tile - - short_name: 'TSURF' + - short_name: 'TSURF' long_name: 'mean_catchment_temp_incl_snw' - units: 'K' + units: 'K' dimension: - tile - - short_name: 'WESNN1' - long_name: 'snow_mass_layer_1' - units: 'kg m-2' + - short_name: 'WESNN1' + long_name: 'snow_mass_layer_1' + units: 'kg m-2' dimension: - tile - - short_name: 'WESNN2' - long_name: 'snow_mass_layer_2' - units: 'kg m-2' + - short_name: 'WESNN2' + long_name: 'snow_mass_layer_2' + units: 'kg m-2' dimension: - tile - - short_name: 'WESNN3' - long_name: 'snow_mass_layer_3' - units: 'kg m-2' + - short_name: 'WESNN3' + long_name: 'snow_mass_layer_3' + units: 'kg m-2' dimension: - tile - - short_name: 'HTSNNN1' - long_name: 'heat_content_snow_layer_1' - units: 'J m-2' + - short_name: 'HTSNNN1' + long_name: 'heat_content_snow_layer_1' + units: 'J m-2' dimension: - tile - - short_name: 'HTSNNN2' - long_name: 'heat_content_snow_layer_2' - units: 'J m-2' + - short_name: 'HTSNNN2' + long_name: 'heat_content_snow_layer_2' + units: 'J m-2' dimension: - tile - - short_name: 'HTSNNN3' - long_name: 'heat_content_snow_layer_3' - units: 'J m-2' + - short_name: 'HTSNNN3' + long_name: 'heat_content_snow_layer_3' + units: 'J m-2' dimension: - tile - - short_name: 'SNDZN1' - long_name: 'snow_depth_layer_1' - units: 'm' + - short_name: 'SNDZN1' + long_name: 'snow_depth_layer_1' + units: 'm' dimension: - tile - - short_name: 'SNDZN2' - long_name: 'snow_depth_layer_2' - units: 'm' + - short_name: 'SNDZN2' + long_name: 'snow_depth_layer_2' + units: 'm' dimension: - tile - - short_name: 'SNDZN3' - long_name: 'snow_depth_layer_3' - units: 'm' + - short_name: 'SNDZN3' + long_name: 'snow_depth_layer_3' + units: 'm' dimension: - tile - - short_name: 'CH' + - short_name: 'CH' long_name: 'surface_heat_exchange_coefficient' - units: 'kg m-2 s-1' + units: 'kg m-2 s-1' dimension: - subtile - tile - - short_name: 'CM' + - short_name: 'CM' long_name: 'surface_momentum_exchange_coefficient' - units: 'kg m-2 s-1' + units: 'kg m-2 s-1' dimension: - subtile - tile - - short_name: 'CQ' + - short_name: 'CQ' long_name: 'surface_moisture_exchange_coffiecient' - units: 'kg m-2 s-1' + units: 'kg m-2 s-1' dimension: - subtile - tile - - short_name: 'FR' - long_name: 'subtile_fractions' - units: '1' + - short_name: 'FR' + long_name: 'subtile_fractions' + units: '1' dimension: - subtile - tile diff --git a/pre/remap_restart/bin2nc_merra2_fv.yaml b/pre/remap_restart/bin2nc_merra2_fv.yaml index 8a2c699..4756f29 100644 --- a/pre/remap_restart/bin2nc_merra2_fv.yaml +++ b/pre/remap_restart/bin2nc_merra2_fv.yaml @@ -64,4 +64,3 @@ dimensions: lev: 72 edges: 73 time: 1 - diff --git a/pre/remap_restart/bin2nc_merra2_gocart.yaml b/pre/remap_restart/bin2nc_merra2_gocart.yaml index 4faf5de..6140738 100644 --- a/pre/remap_restart/bin2nc_merra2_gocart.yaml +++ b/pre/remap_restart/bin2nc_merra2_gocart.yaml @@ -46,4 +46,3 @@ dimensions: lat: 1080 lev: 72 time: 1 - diff --git a/pre/remap_restart/bin2nc_merra2_lake.yaml b/pre/remap_restart/bin2nc_merra2_lake.yaml index 65dcef1..0719e94 100644 --- a/pre/remap_restart/bin2nc_merra2_lake.yaml +++ b/pre/remap_restart/bin2nc_merra2_lake.yaml @@ -39,4 +39,3 @@ dimensions: tile: 14910 subtile: 2 time: 1 - diff --git a/pre/remap_restart/bin2nc_merra2_landice.yaml b/pre/remap_restart/bin2nc_merra2_landice.yaml index 159f4b3..6d66370 100644 --- a/pre/remap_restart/bin2nc_merra2_landice.yaml +++ b/pre/remap_restart/bin2nc_merra2_landice.yaml @@ -65,4 +65,3 @@ dimensions: unknown_dim1: 2 unknown_dim2: 15 time: 1 - diff --git a/pre/remap_restart/bin2nc_merra2_moist.yaml b/pre/remap_restart/bin2nc_merra2_moist.yaml index 7085f38..5fae226 100644 --- a/pre/remap_restart/bin2nc_merra2_moist.yaml +++ b/pre/remap_restart/bin2nc_merra2_moist.yaml @@ -6,7 +6,7 @@ variables: - lev - lat - lon - - short_name: QLLS + - short_name: QLLS long_name: mass_fraction_of_large_scale_cloud_liquid_water units: 'kg kg-1' dimension: @@ -41,7 +41,7 @@ variables: - lev - lat - lon - - short_name: QICN + - short_name: QICN long_name: mass_fraction_of_convective_cloud_ice_water units: '1' dimension: @@ -53,4 +53,3 @@ dimensions: lat: 1080 lev: 72 time: 1 - diff --git a/pre/remap_restart/bin2nc_merra2_pchem.yaml b/pre/remap_restart/bin2nc_merra2_pchem.yaml index 4faf5de..6140738 100644 --- a/pre/remap_restart/bin2nc_merra2_pchem.yaml +++ b/pre/remap_restart/bin2nc_merra2_pchem.yaml @@ -46,4 +46,3 @@ dimensions: lat: 1080 lev: 72 time: 1 - diff --git a/pre/remap_restart/bin2nc_merra2_salt.yaml b/pre/remap_restart/bin2nc_merra2_salt.yaml index af3f8af..c65e09a 100644 --- a/pre/remap_restart/bin2nc_merra2_salt.yaml +++ b/pre/remap_restart/bin2nc_merra2_salt.yaml @@ -1,32 +1,32 @@ variables: - - short_name: 'HSKINW' - long_name: 'water_skin_layer_mass' - units: 'kg m-2' + - short_name: 'HSKINW' + long_name: 'water_skin_layer_mass' + units: 'kg m-2' dimension: - tile - - short_name: 'TSKINW' - long_name: 'water_skin_temperature' - units: 'K' + - short_name: 'TSKINW' + long_name: 'water_skin_temperature' + units: 'K' dimension: - tile - - short_name: 'SSKINW' - long_name: 'water_skin_salinity' - units: 'psu' + - short_name: 'SSKINW' + long_name: 'water_skin_salinity' + units: 'psu' dimension: - tile - - short_name: 'HSKINI' + - short_name: 'HSKINI' long_name: 'ice_skin_layer_mass' - units: 'kg m-2' + units: 'kg m-2' dimension: - tile - - short_name: 'TSKINI' + - short_name: 'TSKINI' long_name: 'ice_skin_temperature' - units: 'K' + units: 'K' dimension: - tile - - short_name: 'SSKINI' + - short_name: 'SSKINI' long_name: 'ice_skin_salinity' - units: 'psu' + units: 'psu' dimension: - tile - short_name: 'QS' @@ -65,14 +65,14 @@ variables: dimension: - subtile - tile - - short_name: 'TWMTS' + - short_name: 'TWMTS' long_name: 'departure_of_skin_temperature_from_mean_interface_temperature' - units: 'K' + units: 'K' dimension: - tile - - short_name: 'DTWARM' + - short_name: 'DTWARM' long_name: 'departure_of_departure_of_TDEL_from_TS_FOUND' - units: 'K' + units: 'K' dimension: - tile dimensions: From 42fdf279aacced64ec4b07a849667119ae4c6d35 Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Mon, 16 Oct 2023 09:12:26 -0400 Subject: [PATCH 58/88] adding agcm_import_rst as option to remap --- pre/remap_restart/bin2nc_merra2_agcm.yaml | 48 +++++++++++++++++++++++ pre/remap_restart/remap_analysis.py | 4 +- pre/remap_restart/remap_bin2nc.py | 1 + pre/remap_restart/remap_command_line.py | 3 ++ pre/remap_restart/remap_params.py | 1 + pre/remap_restart/remap_questions.py | 10 +++++ pre/remap_restart/remap_upper.py | 11 +++--- pre/remap_restart/remap_utils.py | 4 ++ 8 files changed, 75 insertions(+), 7 deletions(-) create mode 100755 pre/remap_restart/bin2nc_merra2_agcm.yaml diff --git a/pre/remap_restart/bin2nc_merra2_agcm.yaml b/pre/remap_restart/bin2nc_merra2_agcm.yaml new file mode 100755 index 0000000..9c48ea8 --- /dev/null +++ b/pre/remap_restart/bin2nc_merra2_agcm.yaml @@ -0,0 +1,48 @@ +variables: + - short_name: DMDTANA + long_name: vertically_integrated_mass_tendency_due_to_analysis + units: 'kg m-2 s-1' + dimension: + - lev + - lat + - lon + - short_name: DQDTANA + long_name: total_specific_humidity_vapor_analysis_tendency + units: 'kg kg-1 s-1' + dimension: + - lev + - lat + - lon + - short_name: DTDTANA + long_name: total_temperature_analysis_tendency + units: 'K s-1' + dimension: + - lev + - lat + - lon + - short_name: DUDTANA + long_name: total_eastward_wind_analysis_tendency + units: 'm s-2' + dimension: + - lev + - lat + - lon + - short_name: DVDTANA + long_name: total_northward_wind_analysis_tendency + units: 'm s-2' + dimension: + - lev + - lat + - lon + - short_name: QVBKG + long_name: specific_humidity_background + units: 'kg kg-1' + dimension: + - lev + - lat + - lon +dimensions: + lon: 180 + lat: 1080 + lev: 72 + time: 1 diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index 8c53a1b..f830ae3 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -170,7 +170,7 @@ def copy_merra2(self): merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' rst_dir = self.config['input']['shared']['rst_dir'] + '/' os.makedirs(rst_dir, exist_ok = True) - print(' Copy MERRA-2 analysis files \n from \n ' + merra_2_rst_dir + '\n to\n '+ rst_dir +'\n') + print(' Stage MERRA-2 analysis files \n from \n ' + merra_2_rst_dir + '\n to\n '+ rst_dir +'\n') rst_time = datetime(year=int(yyyy_), month=int(mm_), day=int(dd_), hour = int(hh_)) @@ -234,7 +234,7 @@ def copy_merra2(self): for f in anafiles + canafiles: fname = os.path.basename(f) f_tmp = rst_dir+'/'+fname - print("Copy file "+f +" to " + rst_dir) + print("Stage file "+f +" to " + rst_dir) shutil.copy(f,f_tmp) if __name__ == '__main__' : diff --git a/pre/remap_restart/remap_bin2nc.py b/pre/remap_restart/remap_bin2nc.py index 2a48040..fc873d6 100755 --- a/pre/remap_restart/remap_bin2nc.py +++ b/pre/remap_restart/remap_bin2nc.py @@ -168,6 +168,7 @@ def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=Fal bin2nc('fvcore_internal_rst', 'fvcore_internal_rst.nc4', 'bin2nc_merra2_fv.yaml', isDouble=True, hasHeader=True) bin2nc('moist_internal_rst', 'moist_internal_rst.nc4', 'bin2nc_merra2_moist.yaml') bin2nc('pchem_internal_rst', 'pchem_internal_rst.nc4', 'bin2nc_merra2_pchem.yaml') + bin2nc('agcm_import_rst', 'agcm_import_rst.nc4', 'bin2nc_merra2_agcm.yaml') bin2nc('gocart_internal_rst', 'gocart_internal_rst.nc4', 'bin2nc_merra2_gocart.yaml') bin2nc('lake_internal_rst', 'lake_internal_rst.nc4', 'bin2nc_merra2_lake.yaml') bin2nc('landice_internal_rst', 'landice_internal_rst.nc4', 'bin2nc_merra2_landice.yaml') diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 6d71797..2e4b455 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -71,6 +71,7 @@ def parse_args(program_description): p_command.add_argument('-nolcv', action='store_true', help="Do not write lcv file") p_command.add_argument('-np', action='store_true', help="No prompt. Overwrite config files without prompting questions") p_command.add_argument('-lbl', action='store_true', help="Label output restarts with bc_versions and resolutions") + p_command.add_argument('-noagcm_import_rst', action='store_true', help="Do not remap agcm_import_rst file") p_command.add_argument('-in_bcsdir', default="", help="User-supplied directory with boundary conditions of input restarts. If not specified (default), dir. is deduced from bc_version and resolution info") p_command.add_argument('-out_bcsdir', default="", help="User-supplied directory with boundary conditions for new restarts. If not specified (default), dir. is deduced from bc_version and resolution info") p_command.add_argument('-zoom', help= "Zoom parameter (search radius) for input surface restarts") @@ -138,6 +139,8 @@ def get_answers_from_command_line(cml): answers["output:surface:remap"] = True answers["output:air:remap"] = True + answers["output:air:agcm_import_rst"] = not cml.noagcm_import_rst + if cml.zoom: answers["input:surface:zoom"] = cml.zoom else: diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index 229783c..a19be5a 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -41,6 +41,7 @@ def __init__(self, config_from_question): config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') + config_tpl['output']['air']['agcm_import_rst'] = self.upper_out.get('agcm_import_rst') config_tpl['output']['air']['remap'] = self.upper_out.get('remap') config_tpl['output']['surface']['remap_water'] = self.surf_out.get('remap') config_tpl['output']['surface']['remap_catch'] = self.surf_out.get('remap') diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 4ff9c7b..4032f05 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -291,6 +291,15 @@ def ask_questions(): "default": True, }, + { + "type": "confirm", + "name": "output:air:agcm_import_rst", + "message": "Remap agcm_import_rst increment file needed for REPLAY runs? \n \ + NOTE: Prefered method is to regenerate this file. \n \ + Cannot regrid agcm_import_rst to different number of atmosphere levels.", + "default": False, + }, + { "type": "confirm", "name": "output:surface:remap", @@ -304,6 +313,7 @@ def ask_questions(): "message": "Remap bkg files? (Required by ADAS but not mapped onto ADAS grid; run one ADAS cycle to spin up.) ", "default": False, }, + { "type": "confirm", "name": "output:analysis:lcv", diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 2fb2bde..c8d5437 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -216,7 +216,7 @@ def remap(self): set outfils = () foreach infile ( *_restart_in ) if ( $infile == fvcore_internal_restart_in ) continue - if ( $infile == moist_internal_restart_in ) continue + if ( $infile == moist_internal_restart_in ) continue set infiles = ( $infiles $infile ) set outfil = `echo $infile | sed "s/restart_in/rst_out/"` @@ -342,18 +342,19 @@ def copy_merra2(self): merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' rst_dir = self.config['input']['shared']['rst_dir'] + '/' os.makedirs(rst_dir, exist_ok = True) - print(' Copy MERRA-2 upper air restarts \n from \n ' + merra_2_rst_dir + '\n to\n '+ rst_dir +'\n') + print(' Stage MERRA-2 upper air restarts \n from \n ' + merra_2_rst_dir + '\n to\n '+ rst_dir +'\n') upperin =[merra_2_rst_dir + expid+'.fvcore_internal_rst.' + suffix, merra_2_rst_dir + expid+'.moist_internal_rst.' + suffix, merra_2_rst_dir + expid+'.gocart_internal_rst.' + suffix, - merra_2_rst_dir + expid+'.pchem_internal_rst.' + suffix ] - bin2nc_yaml = ['bin2nc_merra2_fv.yaml', 'bin2nc_merra2_moist.yaml', 'bin2nc_merra2_gocart.yaml', 'bin2nc_merra2_pchem.yaml'] + merra_2_rst_dir + expid+'.pchem_internal_rst.' + suffix, + merra_2_rst_dir + expid+'.agcm_import_rst.' + suffix ] + bin2nc_yaml = ['bin2nc_merra2_fv.yaml', 'bin2nc_merra2_moist.yaml', 'bin2nc_merra2_gocart.yaml', 'bin2nc_merra2_pchem.yaml','bin2nc_merra2_agcm.yaml'] bin_path = os.path.dirname(os.path.realpath(__file__)) for (f, yf) in zip(upperin,bin2nc_yaml) : fname = os.path.basename(f) dest = rst_dir + '/'+fname - print("Copy file "+f +" to " + rst_dir) + print("Stage file "+f +" to " + rst_dir) shutil.copy(f, dest) ncdest = dest.replace('z.bin', 'z.nc4') yaml_file = bin_path + '/'+yf diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 7b32f9f..d8d3668 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -309,6 +309,7 @@ def get_command_line_from_answers(answers): grout = ' -grout ' + answers["output:shared:agrid"] levsout = ' -levsout ' + answers["output:air:nlevel"] + out_dir = ' -out_dir ' + answers["output:shared:out_dir"] newid = answers["output:shared:expid"] @@ -378,6 +379,8 @@ def get_command_line_from_answers(answers): rs = 2 out_rs = out_rs + str(rs) + noagcm_import_rst = '' if answers["output:air:agcm_import_rst"] else " -noagcm_import_rst " + account = " -account " + answers["slurm:account"] qos = " -qos " + answers["slurm:qos"] partition = " -partition " + answers["slurm:partition"] @@ -406,6 +409,7 @@ def get_command_line_from_answers(answers): wemout + \ label + \ nobkg + \ + noagcm_import_rst + \ nolcv + \ out_rs + \ account + \ From 4a369fb3c27e3b2b28b3bfff510cd36cb7c1b673 Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Mon, 16 Oct 2023 16:26:11 -0400 Subject: [PATCH 59/88] edit var names --- pre/remap_restart/bin2nc_merra2_agcm.yaml | 36 +++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/pre/remap_restart/bin2nc_merra2_agcm.yaml b/pre/remap_restart/bin2nc_merra2_agcm.yaml index 9c48ea8..075b914 100755 --- a/pre/remap_restart/bin2nc_merra2_agcm.yaml +++ b/pre/remap_restart/bin2nc_merra2_agcm.yaml @@ -1,42 +1,42 @@ variables: - - short_name: DMDTANA - long_name: vertically_integrated_mass_tendency_due_to_analysis - units: 'kg m-2 s-1' + - short_name: DQVDT + long_name: specific_humidity_analysis_increment + units: 'kg kg-1' dimension: - lev - lat - lon - - short_name: DQDTANA - long_name: total_specific_humidity_vapor_analysis_tendency - units: 'kg kg-1 s-1' + - short_name: DTDT + long_name: temperature_analysis_increment + units: 'K' dimension: - lev - lat - lon - - short_name: DTDTANA - long_name: total_temperature_analysis_tendency - units: 'K s-1' + - short_name: DPEDT + long_name: edge_pressure_analysis_increment + units: 'Pa' dimension: - lev - lat - lon - - short_name: DUDTANA - long_name: total_eastward_wind_analysis_tendency - units: 'm s-2' + - short_name: DTSDT + long_name: skin_temparature_increment + units: 'K' dimension: - lev - lat - lon - - short_name: DVDTANA - long_name: total_northward_wind_analysis_tendency - units: 'm s-2' + - short_name: DUDT + long_name: eastward_wind_analysis_increment + units: 'm s-1' dimension: - lev - lat - lon - - short_name: QVBKG - long_name: specific_humidity_background - units: 'kg kg-1' + - short_name: DVDT + long_name: northward_wind_analysis_increment + units: 'm s-1' dimension: - lev - lat From e8dc1683d9a624b6c8b317e1a81ef88ddc65ea6c Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Mon, 16 Oct 2023 19:56:18 -0400 Subject: [PATCH 60/88] right order --- pre/remap_restart/bin2nc_merra2_agcm.yaml | 32 +++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pre/remap_restart/bin2nc_merra2_agcm.yaml b/pre/remap_restart/bin2nc_merra2_agcm.yaml index 075b914..d2c12e7 100755 --- a/pre/remap_restart/bin2nc_merra2_agcm.yaml +++ b/pre/remap_restart/bin2nc_merra2_agcm.yaml @@ -1,7 +1,14 @@ variables: - - short_name: DQVDT - long_name: specific_humidity_analysis_increment - units: 'kg kg-1' + - short_name: DUDT + long_name: eastward_wind_analysis_increment + units: 'm s-1' + dimension: + - lev + - lat + - lon + - short_name: DVDT + long_name: northward_wind_analysis_increment + units: 'm s-1' dimension: - lev - lat @@ -20,23 +27,16 @@ variables: - lev - lat - lon - - short_name: DTSDT - long_name: skin_temparature_increment - units: 'K' - dimension: - - lev - - lat - - lon - - short_name: DUDT - long_name: eastward_wind_analysis_increment - units: 'm s-1' + - short_name: DQVDT + long_name: specific_humidity_analysis_increment + units: 'kg kg-1' dimension: - lev - lat - lon - - short_name: DVDT - long_name: northward_wind_analysis_increment - units: 'm s-1' + - short_name: DO3DT + long_name: ozone_analysis_increment + units: 'ppmv' dimension: - lev - lat From a58454a1ccfbb2e243e307479a2664e15ca1aa5d Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Tue, 17 Oct 2023 08:06:43 -0400 Subject: [PATCH 61/88] change comment --- pre/remap_restart/remap_questions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 4032f05..868571d 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -295,8 +295,8 @@ def ask_questions(): "type": "confirm", "name": "output:air:agcm_import_rst", "message": "Remap agcm_import_rst increment file needed for REPLAY runs? \n \ - NOTE: Prefered method is to regenerate this file. \n \ - Cannot regrid agcm_import_rst to different number of atmosphere levels.", + NOTE: Prefered method is to regenerate IAU file. \n \ + Cannot regrid IAU file to different number of atmosphere levels.", "default": False, }, From f392a6a1481d00b879a72a333570d8085ad95325 Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Tue, 17 Oct 2023 08:10:04 -0400 Subject: [PATCH 62/88] one line for comment --- pre/remap_restart/remap_questions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 868571d..35cadda 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -295,8 +295,7 @@ def ask_questions(): "type": "confirm", "name": "output:air:agcm_import_rst", "message": "Remap agcm_import_rst increment file needed for REPLAY runs? \n \ - NOTE: Prefered method is to regenerate IAU file. \n \ - Cannot regrid IAU file to different number of atmosphere levels.", + NOTE: Prefered method is to regenerate IAU file. Cannot regrid IAU file to different number of atmosphere levels.", "default": False, }, From f93b3776a582efb024b87abf73f230664946af7d Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Thu, 19 Oct 2023 09:25:13 -0400 Subject: [PATCH 63/88] edit question --- pre/remap_restart/remap_questions.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 35cadda..48e25b9 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -35,6 +35,13 @@ def remove_ogrid_comment(x, opt): return False +def echo_level(x): + if x["output:air:nlevel"] != "72": + print("NOTE: The number of atmosphere levels at input and output does not match! Cannot regrid IAU file.") + return False + if x["output:air:nlevel"] == "72": + return True + def echo_bcs(x,opt): if opt == "IN": x['input:shared:bcs_dir'] = get_bcsdir(x, 'IN') @@ -295,10 +302,12 @@ def ask_questions(): "type": "confirm", "name": "output:air:agcm_import_rst", "message": "Remap agcm_import_rst increment file needed for REPLAY runs? \n \ - NOTE: Prefered method is to regenerate IAU file. Cannot regrid IAU file to different number of atmosphere levels.", + (NOTE: Prefered method is to regenerate IAU file, but IF requested remapping will be performed)", "default": False, + "when": lambda x: echo_level(x), }, + { "type": "confirm", "name": "output:surface:remap", From 92467b9d66c29c416bfecc6bc0b55729198aaab2 Mon Sep 17 00:00:00 2001 From: Rolf Reichle <54944691+gmao-rreichle@users.noreply.github.com> Date: Thu, 19 Oct 2023 09:37:24 -0400 Subject: [PATCH 64/88] Edited agcm_import_rst/IAU echo & question (remap_questions.py) --- pre/remap_restart/remap_questions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 48e25b9..2de2e6a 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -37,7 +37,7 @@ def remove_ogrid_comment(x, opt): def echo_level(x): if x["output:air:nlevel"] != "72": - print("NOTE: The number of atmosphere levels at input and output does not match! Cannot regrid IAU file.") + print("NOTE: Different # atm levels in input and new restarts. Cannot remap agcm_import_rst (a.k.a. IAU) file.") return False if x["output:air:nlevel"] == "72": return True @@ -301,8 +301,8 @@ def ask_questions(): { "type": "confirm", "name": "output:air:agcm_import_rst", - "message": "Remap agcm_import_rst increment file needed for REPLAY runs? \n \ - (NOTE: Prefered method is to regenerate IAU file, but IF requested remapping will be performed)", + "message": "Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs? \n \ + (NOTE: Preferred method is to regenerate IAU file, but IF requested remapping will be performed.)", "default": False, "when": lambda x: echo_level(x), }, From 61d10cd408e5f7a43d8210366c5e93242b5e575f Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 19 Oct 2023 10:25:16 -0400 Subject: [PATCH 65/88] conditions to show agcm_import_rst question --- pre/remap_restart/remap_questions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 2de2e6a..e225c00 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -36,11 +36,11 @@ def remove_ogrid_comment(x, opt): return False def echo_level(x): - if x["output:air:nlevel"] != "72": + if x["output:air:nlevel"] != x["input:air:nlevel"] print("NOTE: Different # atm levels in input and new restarts. Cannot remap agcm_import_rst (a.k.a. IAU) file.") + x['output:air:agcm_import_rst'] = False return False - if x["output:air:nlevel"] == "72": - return True + return True def echo_bcs(x,opt): if opt == "IN": From d7c89e4593ed13ae16a9e9ac6c208547fa9120fb Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 19 Oct 2023 10:54:06 -0400 Subject: [PATCH 66/88] typo --- pre/remap_restart/remap_questions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index e225c00..f467a81 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -36,7 +36,7 @@ def remove_ogrid_comment(x, opt): return False def echo_level(x): - if x["output:air:nlevel"] != x["input:air:nlevel"] + if x["output:air:nlevel"] != x["input:air:nlevel"] : print("NOTE: Different # atm levels in input and new restarts. Cannot remap agcm_import_rst (a.k.a. IAU) file.") x['output:air:agcm_import_rst'] = False return False From dde40f779d01c757f4783791119b8c799aa3d59e Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 19 Oct 2023 12:11:06 -0400 Subject: [PATCH 67/88] add input:air:nlevel --- pre/remap_restart/remap_questions.py | 3 ++- pre/remap_restart/remap_utils.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index f467a81..618934d 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -36,7 +36,7 @@ def remove_ogrid_comment(x, opt): return False def echo_level(x): - if x["output:air:nlevel"] != x["input:air:nlevel"] : + if x["output:air:nlevel"] != str(x.get("input:air:nlevel")) : print("NOTE: Different # atm levels in input and new restarts. Cannot remap agcm_import_rst (a.k.a. IAU) file.") x['output:air:agcm_import_rst'] = False return False @@ -387,6 +387,7 @@ def ask_questions(): answers['input:shared:rst_dir'] = os.path.abspath(answers['input:shared:rst_dir']) answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) + if answers.get('input:air:nlevel') : del answers['input:air:nlevel'] if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"]: answers["input:surface:catch_model"] = catch_model(answers) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index d8d3668..1ebe254 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -114,6 +114,7 @@ def init_merra2(x): x['input:surface:catch_model'] = 'catch' x['input:shared:stretch'] = False x['input:shared:rst_dir'] = x['output:shared:out_dir'] + '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' + x['input:air:nlevel'] = 72 return False @@ -143,6 +144,8 @@ def fvcore_info(x): fvrst = nc.Dataset(fname) lon = fvrst.dimensions['lon'].size lat = fvrst.dimensions['lat'].size + lev = fvrst.dimensions['lev'].size + x['input:air:nlevel'] = lev ymdh = fvrst.variables['time'].units.split('since ')[1].split(":")[0].replace('-','').replace(' ', "") if (lat != lon*6) : print("(N_lon,N_lat) = ", lon, lat) From 4bcabc4682170043637768e70c0fe4ca53c91b22 Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Thu, 19 Oct 2023 13:50:39 -0400 Subject: [PATCH 68/88] add one more in development land version v11 --- pre/remap_restart/remap_questions.py | 4 ++-- pre/remap_restart/remap_utils.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 618934d..7f92280 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -267,7 +267,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": message_bc_other_new, "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06'], + "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11'], }, { @@ -275,7 +275,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": "\nSelect BCs version of input restarts:\n", "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06'], + "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11'], }, { diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 1ebe254..bc33197 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -17,9 +17,9 @@ choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] -choices_bc_other = ['v06'] +choices_bc_other = ['v06','v11'] -choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v06'] +choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v06', 'v11'] choices_omodel = ['data', 'MOM5', 'MOM6'] @@ -68,7 +68,8 @@ message_bc_other = f'''\n - v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n\n''' + v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n + v11: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2\n\n'''\ message_bc_other_in = ("Select BCs version of input restarts:\n" + message_bc_other) message_bc_other_new = ("Select BCs version for new restarts:\n" + message_bc_other) From eed60a84ef1987ea3dd9594081f02794925baed7 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 26 Oct 2023 12:04:57 -0400 Subject: [PATCH 69/88] update testing cases --- pre/remap_restart/tests/amip_c180Toc90.yaml | 8 +-- pre/remap_restart/tests/c180Toc360.yaml | 8 +-- pre/remap_restart/tests/c24Toc12.yaml | 8 +-- pre/remap_restart/tests/c360Toc24.yaml | 8 +-- pre/remap_restart/tests/f522Toc360.yaml | 8 +-- pre/remap_restart/tests/s2sv3Toc12MOM6.yaml | 54 ------------------- pre/remap_restart/tests/test_remap_cases.yaml | 3 -- 7 files changed, 25 insertions(+), 72 deletions(-) delete mode 100644 pre/remap_restart/tests/s2sv3Toc12MOM6.yaml diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index 6a2c374..28013f2 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -9,8 +9,9 @@ input: hydrostatic: 0 shared: MERRA-2: false + stretch: false agrid: C180 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/legacy_bcs/Icarus-NLv3/Icarus-NLv3_MERRA-2//CF0180x6C_DE1440xPE0720/ + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0180x6C_DE1440xPE0720/ expid: JM_v10.22.2_L072_C180_AMIP ogrid: 1440X720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/amip_c180Toc90/inputs/ @@ -24,7 +25,8 @@ input: output: shared: agrid: C90 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/legacy_bcs/Icarus-NLv3/Icarus-NLv3_Ostia//CF0090x6C_CF0090x6C/ + stretch: false + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0090x6C_CF0090x6C/ expid: C90CS_JM_v10.22.2_L072_C180_AMIP ogrid: C90 out_dir: $NOBACKUP/REMAP_TESTS/amip_c180Toc90/ @@ -50,4 +52,4 @@ output: slurm: account: g0620 qos: debug - constraint: sky + partition: compute diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index 93368c6..835507d 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -9,8 +9,9 @@ input: hydrostatic: 0 shared: MERRA-2: false + stretch: false agrid: C180 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/legacy_bcs/Icarus-NLv3/Icarus-NLv3_MERRA-2//CF0180x6C_DE1440xPE0720/ + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0180x6C_DE1440xPE0720/ expid: Jason-3_4_NL_REAMIP_MERRA2_C180 ogrid: 1440X720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c180Toc360/inputs/ @@ -24,7 +25,8 @@ input: output: shared: agrid: C360 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/legacy_bcs/Icarus-NLv3/Icarus-NLv3_Ostia//CF0360x6C_CF0360x6C/ + stretch: false + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0360x6C_CF0360x6C/ expid: C360CS_Jason-3_4_NL_REAMIP_MERRA2_C180 ogrid: C360 out_dir: $NOBACKUP/REMAP_TESTS/c360Toc24/ @@ -50,4 +52,4 @@ output: slurm: account: g0620 qos: debug - constraint: sky + partition: compute diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index 868ac12..bad4182 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -9,8 +9,9 @@ input: hydrostatic: 0 shared: MERRA-2: false + stretch: false agrid: C24 - bcs_dir: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs/Icarus-NLv3/Icarus-NLv3_Reynolds//CF0024x6C_DE0360xPE0180/ + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0024x6C_DE0360xPE0180/ expid: '' ogrid: 360X180 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c24Toc12/inputs @@ -24,7 +25,8 @@ input: output: shared: agrid: C12 - bcs_dir: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs/Icarus-NLv3/Icarus-NLv3_Reynolds//CF0012x6C_DE0360xPE0180/ + stretch: false + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0012x6C_DE0360xPE0180/ expid: hello ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/c24Toc12/ @@ -50,4 +52,4 @@ output: slurm: account: g0610 qos: debug - constraint: hasw + partition: compute diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index ba84b30..c5736ae 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -10,7 +10,8 @@ input: shared: MERRA-2: false agrid: C360 - bcs_dir: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs/Icarus-NLv3/Icarus-NLv3_Ostia//CF0360x6C_CF0360x6C/ + stretch: false + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0360x6C_CF0360x6C/ expid: x0046a ogrid: C360 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c360Toc24/inputs @@ -24,7 +25,8 @@ input: output: shared: agrid: C24 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/legacy_bcs/Icarus-NLv3/Icarus-NLv3_Reynolds//CF0024x6C_DE0360xPE0180/ + stretch: false + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0024x6C_DE0360xPE0180/ expid: C24c_x0046a ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/c360Toc24/ @@ -50,4 +52,4 @@ output: slurm: account: g0620 qos: debug - constraint: sky + partition: compute diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index abf4908..b0a70c6 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -9,8 +9,9 @@ input: hydrostatic: 0 shared: MERRA-2: false + stretch: false agrid: C720 - bcs_dir: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs/Icarus_Updated/Icarus_Ostia//CF0720x6C_CF0720x6C/ + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/ICA/geometry/CF0720x6C_CF0720x6C/ expid: f522_fp ogrid: C720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/f522Toc360/inputs/ @@ -23,7 +24,8 @@ input: output: shared: agrid: C360 - bcs_dir: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs/Icarus-NLv3/Icarus-NLv3_Reynolds//CF0360x6C_DE0360xPE0180/ + stretch: false + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0360x6C_DE0360xPE0180/ expid: remap ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/f522Toc360/ @@ -46,4 +48,4 @@ output: slurm: account: g0610 qos: debug - constraint: hasw + partition: compute diff --git a/pre/remap_restart/tests/s2sv3Toc12MOM6.yaml b/pre/remap_restart/tests/s2sv3Toc12MOM6.yaml deleted file mode 100644 index d280414..0000000 --- a/pre/remap_restart/tests/s2sv3Toc12MOM6.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# -# This template file can be filled with questionary or manually -# -# - -input: - air: - drymass: 1 - hydrostatic: 0 - shared: - MERRA-2: false - agrid: C180 - bcs_dir: /discover/nobackup/projects/gmao/ssd/aogcm/atmosphere_bcs/Icarus/MOM5//CF0180x6C_TM1440xTM1080/ - expid: M2OCEAN_S2SV3 - ogrid: 1440X1080 - rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/s2sv3Toc12MOM6/inputs - yyyymmddhh: '1982010100' - surface: - zoom: '2' - wemin: '26' - # it supports three models: catch, catchcnclm40, catchcnclm45 - catch_model: catch - # if catch_tilefile is null, it searches bcs_dir - catch_tilefile: -output: - shared: - agrid: C12 - bcs_dir: /discover/nobackup/projects/gmao/ssd/aogcm/atmosphere_bcs/Icarus-NLv3/MOM6//CF0012x6C_TM0072xTM0036/ - expid: C12-MOM6-from-S2Sv3 - ogrid: 72X36 - out_dir: $NOBACKUP/REMAP_TESTS/s2sv3Toc12MOM6/ - air: - # remap upper air or not - remap: true - nlevel: '72' - surface: - split_saltwater: true - surflay: 50.0 - wemin: '13' - # remap lake, saltwater, landicet - remap_water: true - # remap catch(cn) - remap_catch: true - # if catch_tilefile is null, it searches bcs_dir - catch_tilefile: - analysis: - bkg: false - aqua: true - lcv: false - -slurm: - account: g0620 - qos: debug - constraint: cas diff --git a/pre/remap_restart/tests/test_remap_cases.yaml b/pre/remap_restart/tests/test_remap_cases.yaml index 5563d26..b4df816 100644 --- a/pre/remap_restart/tests/test_remap_cases.yaml +++ b/pre/remap_restart/tests/test_remap_cases.yaml @@ -13,6 +13,3 @@ c180Toc360: c360Toc24: base_line: '/discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c360Toc24/baseline/' config: 'c360Toc24.yaml' -#s2sv3Toc12MOM6: -# base_line: '/discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/s2sv3Toc12MOM6/baseline/' -# config: 's2sv3Toc12MOM6.yaml' From 0ba5b79ba047e995d208278b1940625d320da767 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Fri, 27 Oct 2023 10:52:10 -0400 Subject: [PATCH 70/88] 1) change the bcs path 2)change the way to get the grid directory. 3) correct tests --- pre/remap_restart/remap_catchANDcn.py | 32 +++-- .../remap_lake_landice_saltwater.py | 16 ++- pre/remap_restart/remap_params.py | 2 + pre/remap_restart/remap_params.tpl | 4 + pre/remap_restart/remap_questions.py | 26 +++- pre/remap_restart/remap_upper.py | 33 +++-- pre/remap_restart/remap_utils.py | 133 ++++++++---------- pre/remap_restart/tests/amip_c180Toc90.yaml | 8 +- pre/remap_restart/tests/c180Toc360.yaml | 8 +- pre/remap_restart/tests/c24Toc12.yaml | 8 +- pre/remap_restart/tests/c360Toc24.yaml | 8 +- pre/remap_restart/tests/f522Toc360.yaml | 8 +- 12 files changed, 168 insertions(+), 118 deletions(-) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 2e53670..a8d04a5 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -10,13 +10,10 @@ import mimetypes import netCDF4 as nc from remap_base import remap_base +from remap_utils import get_landdir +from remap_utils import get_geomdir from remap_utils import get_label -def get_landdir(bcsdir): - if bcsdir : - bcsdir = bcsdir.replace('/geometry/','/land/') - return bcsdir - class catchANDcn(remap_base): def __init__(self, **configs): super().__init__(**configs) @@ -56,13 +53,26 @@ def remap(self): if not in_tilefile : if not in_bcsdir: exit("Must provide either input tile file or input bcs directory") - in_tilefile = glob.glob(in_bcsdir+ '/*.til')[0] - + agrid = config['input']['shared']['agrid'] + ogrid = config['input']['shared']['ogrid'] + omodel = config['input']['shared']['omodel'] + stretch = config['input']['shared']['stretch'] + bcs_geomdir = get_geomdir(in_bcsdir, agrid, ogrid, omodel, stretch) + in_tilefile = glob.glob(bcs_geomdir + '/*.til')[0] + + agrid = config['output']['shared']['agrid'] + ogrid = config['output']['shared']['ogrid'] + omodel = config['output']['shared']['omodel'] + stretch = config['output']['shared']['stretch'] out_tilefile = config['output']['surface']['catch_tilefile'] + if not out_tilefile : - out_tilefile = glob.glob(out_bcsdir+ '/*.til')[0] + if not out_bcsdir: + exit("Must provide either input tile file or input bcs directory") + bcs_geomdir = get_geomdir(out_bcsdir, agrid, ogrid, omodel, stretch) + out_tilefile = glob.glob(bcs_geomdir+ '/*.til')[0] - out_bcsdir = get_landdir(out_bcsdir) + out_bcs_landdir = get_landdir(out_bcsdir, agrid, ogrid, omodel, stretch) # determine NPE based on *approximate* number of input and output tile @@ -75,7 +85,7 @@ def remap(self): in_Ntile = ds.dimensions['tile'].size out_Ntile = 0 - with open( out_bcsdir+'/clsm/catchment.def') as f: + with open( out_bcs_landdir+'/clsm/catchment.def') as f: out_Ntile = int(next(f)) max_Ntile = max(in_Ntile, out_Ntile) NPE = 0 @@ -148,7 +158,7 @@ def remap(self): $esma_mpirun_X $mk_catchANDcnRestarts_X $params """ - catch1script = mk_catch_j_template.format(Bin = bindir, account = account, out_bcs = out_bcsdir, \ + catch1script = mk_catch_j_template.format(Bin = bindir, account = account, out_bcs = out_bcs_landdir, \ model = model, out_dir = out_dir, surflay = surflay, log_name = log_name, NPE = NPE, \ in_wemin = in_wemin, out_wemin = out_wemin, out_tilefile = out_tilefile, in_tilefile = in_tilefile, \ in_rstfile = in_rstfile, out_rstfile = out_rstfile, time = yyyymmddhh_, TIME = TIME, PARTITION = PARTITION, QOS=QOS ) diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index 28d7fd9..3305536 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -8,6 +8,7 @@ import shlex from remap_base import remap_base from remap_utils import get_label +from remap_utils import get_geomdir from remap_bin2nc import bin2nc class lake_landice_saltwater(remap_base): @@ -74,8 +75,19 @@ def remap(self): if 'roue' in f : route = f if 'openwater' in f : openwater = f - in_tile_file = glob.glob(in_bcsdir+ '/*-Pfafstetter.til')[0] - out_tile_file = glob.glob(out_bcsdir+ '/*-Pfafstetter.til')[0] + agrid = config['input']['shared']['agrid'] + ogrid = config['input']['shared']['ogrid'] + omodel = config['input']['shared']['omodel'] + stretch = config['input']['shared']['stretch'] + in_geomdir= get_geomdir(in_bcsdir, agrid, ogrid, omodel, stretch) + in_tile_file = glob.glob(in_geomdir+ '/*-Pfafstetter.til')[0] + + agrid = config['output']['shared']['agrid'] + ogrid = config['output']['shared']['ogrid'] + omodel = config['output']['shared']['omodel'] + stretch = config['output']['shared']['stretch'] + out_geomdir= get_geomdir(out_bcsdir, agrid, ogrid, omodel, stretch) + out_tile_file = glob.glob(out_geomdir+ '/*-Pfafstetter.til')[0] in_til = InData_dir+'/' + os.path.basename(in_tile_file) out_til = OutData_dir+'/'+ os.path.basename(out_tile_file) diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index a19be5a..0f42c53 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -33,6 +33,7 @@ def __init__(self, config_from_question): config_tpl['input']['shared']['MERRA-2'] = self.common_in.get('MERRA-2') config_tpl['input']['shared']['agrid'] = self.common_in.get('agrid') config_tpl['input']['shared']['ogrid'] = self.common_in.get('ogrid') + config_tpl['input']['shared']['omodel'] = self.common_in.get('omodel') config_tpl['input']['shared']['rst_dir'] = self.common_in['rst_dir']+'/' config_tpl['input']['shared']['expid'] = self.common_in.get('expid') config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] @@ -47,6 +48,7 @@ def __init__(self, config_from_question): config_tpl['output']['surface']['remap_catch'] = self.surf_out.get('remap') config_tpl['output']['shared']['agrid'] = self.common_out['agrid'] config_tpl['output']['shared']['ogrid'] = self.common_out['ogrid'] + config_tpl['output']['shared']['omodel'] = self.common_out['omodel'] config_tpl['output']['shared']['out_dir'] = self.common_out['out_dir'] + '/' config_tpl['output']['shared']['expid'] = self.common_out['expid'] config_tpl['output']['shared']['bc_version'] = self.common_out.get('bc_version') diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index e670bd4..d88716d 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -16,6 +16,8 @@ input: bcs_dir: expid: ogrid: + # ocean couple model: data, MOM5, MO6 + omodel: data rst_dir: yyyymmddhh: surface: @@ -36,6 +38,8 @@ output: bcs_dir: expid: ogrid: + # ocean couple model: data, MOM5, MO6 + omodel: data out_dir: air: # remap upper air or not diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 7f92280..ea10783 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -44,12 +44,23 @@ def echo_level(x): def echo_bcs(x,opt): if opt == "IN": - x['input:shared:bcs_dir'] = get_bcsdir(x, 'IN') - print("\nBc path for input restart: " + x['input:shared:bcs_dir']) + in_bcsdir = get_bcsdir(x, 'IN') + agrid = x.get('input:shared:agrid') + ogrid = x.get('input:shared:ogrid') + model = x.get('input:shared:omodel') + stretch = x.get('input:shared:stretch') + land_dir = get_landdir(in_bcsdir, agrid, ogrid, model, stretch) + x['input:shared:bcs_dir'] = in_bcsdir + print("\n BC of land for input restart: " + land_dir ) if opt == "OUT": - x['output:shared:bcs_dir'] = get_bcsdir(x, 'OUT') - print("\nBc path for output restart: " + x['output:shared:bcs_dir']) - print("\nLater on, user can edit paths in generated remap_params.yaml file") + out_bcsdir = get_bcsdir(x, 'OUT') + agrid = x.get('output:shared:agrid') + ogrid = x.get('output:shared:ogrid') + model = x.get('output:shared:omodel') + stretch = x.get('output:shared:stretch') + land_dir = get_landdir(out_bcsdir, agrid, ogrid, model, stretch) + x['output:shared:bcs_dir'] = out_bcsdir + print("\n BC of land for output restart: " + land_dir) return False def default_partition(x): @@ -301,8 +312,9 @@ def ask_questions(): { "type": "confirm", "name": "output:air:agcm_import_rst", - "message": "Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs? \n \ - (NOTE: Preferred method is to regenerate IAU file, but IF requested remapping will be performed.)", + "message": f'''Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs? + (NOTE: Preferred method is to regenerate IAU file, + but IF requested, remapping will be performed.)''', "default": False, "when": lambda x: echo_level(x), }, diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index c8d5437..e983bcb 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -9,19 +9,9 @@ from remap_base import remap_base from remap_utils import get_label from remap_utils import STRETCH_GRID +from remap_utils import get_topodir from remap_bin2nc import bin2nc -def get_topodir(bcsdir): - k = bcsdir.find('/geometry/') - if k != -1 : - while bcsdir[-1] == '/': bcsdir = bcsdir[0:-1] # remove extra '/' - agrid_name = os.path.basename(bcsdir).split('_')[0] - if '/GM4/' in bcsdir : - bcsdir = bcsdir[0:k]+'/TOPO/TOPO_'+agrid_name - else: - bcsdir = bcsdir[0:k]+'/TOPO/TOPO_'+agrid_name + '/smoothed' - return bcsdir - class upperair(remap_base): def __init__(self, **configs): super().__init__(**configs) @@ -92,19 +82,28 @@ def remap(self): print('\n'+cmd) subprocess.call(shlex.split(cmd)) - - in_bcsdir=get_topodir(in_bcsdir) - - topoin = glob.glob(in_bcsdir+'/topo_DYN_ave*.data')[0] + agrid = config['input']['shared']['agrid'] + ogrid = config['input']['shared']['ogrid'] + omodel = config['input']['shared']['omodel'] + stretch = config['input']['shared']['stretch'] + print("get topo:") + topo_bcsdir= get_topodir(in_bcsdir, agrid, ogrid, omodel, stretch) + print("get topo:") + + topoin = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] # link topo file cmd = '/bin/ln -s ' + topoin + ' .' print('\n'+cmd) subprocess.call(shlex.split(cmd)) + agrid = config['output']['shared']['agrid'] + ogrid = config['output']['shared']['ogrid'] + omodel = config['output']['shared']['omodel'] + stretch = config['output']['shared']['stretch'] + topo_bcsdir= get_topodir(out_bcsdir, agrid, ogrid, omodel, stretch) - out_bcsdir=get_topodir(out_bcsdir) - topoout = glob.glob(out_bcsdir+'/topo_DYN_ave*.data')[0] + topoout = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] cmd = '/bin/ln -s ' + topoout + ' topo_dynave.data' print('\n'+cmd) subprocess.call(shlex.split(cmd)) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index bc33197..04aee35 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -12,9 +12,11 @@ # shared global variables # -# define "choices", "message" strings, and "validate" lists that are used multiple times +# define "bc_base", "choices", "message" strings, and "validate" lists that are used multiple times # (and related definitions, even if they are used just once). +bc_base = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" + choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] choices_bc_other = ['v06','v11'] @@ -111,6 +113,7 @@ def init_merra2(x): x['input:shared:omodel'] = 'data' x['input:shared:agrid'] = 'C180' x['input:shared:ogrid'] = '1440x720' + x['input:shared:omodel'] = 'data' x['input:shared:bc_version'] = 'GM4' x['input:surface:catch_model'] = 'catch' x['input:shared:stretch'] = False @@ -444,88 +447,76 @@ def get_config_from_answers(answers): return config def get_grid_subdir(bcdir, agrid, ogrid, model, stretch): - def get_name_with_grid( grid, names, a_o): - if not grid : - return names - namex = [] - if (grid[0].upper() == 'C'): - n = int(grid[1:]) - s1 ='{:04d}x6C'.format(n) - j=n*6 - s2 =str(n) - s3 =str(j) - # first try - for aoname in names: - name = '' - if(a_o == 'a'): - name = aoname.split('_')[0] - else: - name = aoname.split('_')[-1] - if (name.find(s1) != -1 or (name.find(s2) != -1 and name.find(s3) != -1 )): - namex.append(aoname) - else: - xy = grid.upper().split('X') - s2 = xy[0] - s3 = xy[1] - for aoname in names: - name = '' - if(a_o == 'a'): - name = aoname.split('_')[0] - else: - name = aoname.split('_')[-1] - if (name.find(s2) != -1 and name.find(s3) != -1): namex.append(aoname) - return namex - #v3.5 - #dirnames = [ f.name for f in os.scandir(bcdir) if f.is_dir()] - #v2.7 - dirnames = [f for f in os.listdir(bcdir) if os.path.isdir(os.path.join(bcdir,f))] + aname = '' + oname = '' + if (agrid[0].upper() == 'C'): + n = int(agrid[1:]) + aname ='CF{:04d}x6C'.format(n) + + if (ogrid[0].upper() == 'C'): + oname = aname + else: + xy = ogrid.upper().split('X') + s0 = int(xy[0]) + s1 = int(xy[1]) + if model == 'data': + oname = 'DE{:04d}xPE{:04d}'.format(s0,s1) + if model == 'MOM5': + oname = 'M5TP{:04d}x{:04d}'.format(s0,s1) + if model == 'MOM6': + oname = 'M6TP{:04d}x{:04d}'.format(s0,s1) if stretch: - dirnames = [dname for dname in dirnames if stretch in dname] - anames = get_name_with_grid(agrid, dirnames, 'a') - gridID = get_name_with_grid(ogrid, anames, 'o') - if len(gridID) == 0 : - exit("cannot find the grid subdirctory of agrid: " +agrid+ " and ogrid " + ogrid + " under "+ bcdir) - g = '' - gridID.sort(key=len) - g = gridID[0] - - # For new structure BC - for g_ in gridID : - if model == 'MOM5': - if 'M5' in g_ : g = g_ - if model == 'MOM6': - if 'M6' in g_ : g = g_ - - if len(gridID) >= 2 : - print("\n Warning! Find many GridIDs in " + bcdir) - print(" GridIDs found: ", gridID) - #WY note, found many string in the directory - print(" This GridID is chosen: " + g) - return g + aname = aname + '-' + stretch + + grid_directory = aname +'_' + oname + + if not os.path.isdir(os.path.join(bcdir,grid_directory)): + exit("cannot find the grid subdirctory of agrid: " + agrid+ " and ogrid " + ogrid + " under "+ bcdir) + + return grid_directory def get_bcsdir(x, opt): bc_version = x.get('input:shared:bc_version') - agrid = x.get('input:shared:agrid') - ogrid = x.get('input:shared:ogrid') - model = x.get('input:shared:omodel') - stretch = x.get('input:shared:stretch') if opt.upper() == "OUT": bc_version = x.get('output:shared:bc_version') - agrid = x.get('output:shared:agrid') - ogrid = x.get('output:shared:ogrid') - model = x.get('output:shared:omodel') - stretch = x.get('output:shared:stretch') - bc_base = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" - bcdir = bc_base+'/'+ bc_version+'/geometry/' + bcdir = bc_base+'/'+ bc_version+'/' if not os.path.exists(bcdir): exit("Cannot find bc dir " + bcdir) - gridStr = get_grid_subdir(bcdir,agrid, ogrid, model,stretch) - bcdir = bcdir + gridStr - return bcdir +def get_topodir(bcsdir, agrid, ogrid, model, stretch): + k = bcsdir.find('/land/') + if k != -1 : + bcsdir = bcsdir[0:k] + bcs_geom = bcsdir+'/geometry/' + gridStr = get_grid_subdir(bcs_geom, agrid, ogrid, model,stretch) + agrid_name = gridStr.split('_')[0] + bcs_topo = '' + if '/GM4/' in bcsdir : + bcs_topo = bcsdir+'/TOPO/TOPO_'+agrid_name + else: + bcs_topo = bcsdir+'/TOPO/TOPO_'+agrid_name + '/smoothed' + + return bcs_topo + +def get_landdir(bcsdir, agrid, ogrid, model, stretch): + if '/land/' in bcsdir: + return bcsdir + bcs_land = bcsdir+'/land/' + gridStr = get_grid_subdir(bcs_land, agrid, ogrid, model,stretch) + bcs_land = bcs_land + gridStr + return bcs_land + +def get_geomdir(bcsdir, agrid, ogrid, model, stretch): + if '/land/' in bcsdir: + bcs_geom = bcsdir.replace('/land/', '/geometry/') + return bcs_geom + bcs_geom = bcsdir+'/geometry/' + gridStr = get_grid_subdir(bcs_geom, agrid, ogrid, model,stretch) + bcs_geom = bcs_geom + gridStr + return bcs_geom if __name__ == '__main__' : config = yaml_to_config('c24Toc12.yaml') diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index 28013f2..2d9ae73 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -11,7 +11,9 @@ input: MERRA-2: false stretch: false agrid: C180 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0180x6C_DE1440xPE0720/ + # ocean couple model: data, MOM5, MO6 + omodel: data + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: JM_v10.22.2_L072_C180_AMIP ogrid: 1440X720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/amip_c180Toc90/inputs/ @@ -26,7 +28,9 @@ output: shared: agrid: C90 stretch: false - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0090x6C_CF0090x6C/ + # ocean couple model: data, MOM5, MO6 + omodel: data + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: C90CS_JM_v10.22.2_L072_C180_AMIP ogrid: C90 out_dir: $NOBACKUP/REMAP_TESTS/amip_c180Toc90/ diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index 835507d..dc44483 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -10,8 +10,10 @@ input: shared: MERRA-2: false stretch: false + # ocean couple model: data, MOM5, MO6 + omodel: data agrid: C180 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0180x6C_DE1440xPE0720/ + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: Jason-3_4_NL_REAMIP_MERRA2_C180 ogrid: 1440X720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c180Toc360/inputs/ @@ -26,7 +28,9 @@ output: shared: agrid: C360 stretch: false - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0360x6C_CF0360x6C/ + # ocean couple model: data, MOM5, MO6 + omodel: data + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: C360CS_Jason-3_4_NL_REAMIP_MERRA2_C180 ogrid: C360 out_dir: $NOBACKUP/REMAP_TESTS/c360Toc24/ diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index bad4182..2ecebe3 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -10,8 +10,10 @@ input: shared: MERRA-2: false stretch: false + # ocean couple model: data, MOM5, MO6 + omodel: data agrid: C24 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0024x6C_DE0360xPE0180/ + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: '' ogrid: 360X180 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c24Toc12/inputs @@ -26,7 +28,9 @@ output: shared: agrid: C12 stretch: false - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0012x6C_DE0360xPE0180/ + # ocean couple model: data, MOM5, MO6 + omodel: data + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: hello ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/c24Toc12/ diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index c5736ae..1b6f821 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -11,7 +11,9 @@ input: MERRA-2: false agrid: C360 stretch: false - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0360x6C_CF0360x6C/ + # ocean couple model: data, MOM5, MO6 + omodel: data + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: x0046a ogrid: C360 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c360Toc24/inputs @@ -26,7 +28,9 @@ output: shared: agrid: C24 stretch: false - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0024x6C_DE0360xPE0180/ + # ocean couple model: data, MOM5, MO6 + omodel: data + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: C24c_x0046a ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/c360Toc24/ diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index b0a70c6..cdb0139 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -10,8 +10,10 @@ input: shared: MERRA-2: false stretch: false + # ocean couple model: data, MOM5, MO6 + omodel: data agrid: C720 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/ICA/geometry/CF0720x6C_CF0720x6C/ + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/ICA/ expid: f522_fp ogrid: C720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/f522Toc360/inputs/ @@ -25,7 +27,9 @@ output: shared: agrid: C360 stretch: false - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/geometry/CF0360x6C_DE0360xPE0180/ + # ocean couple model: data, MOM5, MO6 + omodel: data + bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: remap ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/f522Toc360/ From 5d1af2df7cfc83f3b80dbcc7b5995c8441e9073c Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 1 Nov 2023 10:08:48 -0400 Subject: [PATCH 71/88] fix test for different accounts --- pre/remap_restart/tests/test_remap_restarts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pre/remap_restart/tests/test_remap_restarts.py b/pre/remap_restart/tests/test_remap_restarts.py index 197e302..f31e3d1 100755 --- a/pre/remap_restart/tests/test_remap_restarts.py +++ b/pre/remap_restart/tests/test_remap_restarts.py @@ -12,6 +12,7 @@ import glob import subprocess as sp import remap_restarts +import remap_utils from remap_params import * from remap_upper import * from remap_lake_landice_saltwater import * @@ -38,7 +39,7 @@ def compare(base, result): if "identical." in out : print('identical') else: - print ( f + ' is different from ' + r) + print ( b + ' is different from ' + r) return False return True @@ -81,6 +82,7 @@ def test_remap(config): out_dir = '/discover/nobackup/'+user+'/REMAP_TESTS/'+case+'/' config['output']['shared']['out_dir'] = out_dir + config['slurm']['account'] = get_account() test_remap(config) From cc415b5ade293085c11210ce2be28d83da75e7f6 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 1 Nov 2023 10:29:14 -0400 Subject: [PATCH 72/88] more merra2 hours (03, 09, 15, 21) --- pre/remap_restart/remap_questions.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index ea10783..2060dc3 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -69,6 +69,16 @@ def default_partition(x): return False return True +def validate_merra2_time(text): + if len(text) == 10 : + hh = text[8:] + if hh in ['03','09','15','21']: + return True + else: + return False + else: + return False + def ask_questions(): # See remap_utils.py for definitions of "choices", "message" strings, and "validate" lists @@ -97,8 +107,8 @@ def ask_questions(): { "type": "text", "name": "input:shared:yyyymmddhh", - "message": "Enter restart date: (Must be 8 digits: yyyymmdd; hour=21z.)\n", - "validate": lambda text: len(text)==8 , + "message": "Enter restart date: (Must be 10 digits: yyyymmddhh; hour=03,09,15,21z.)\n", + "validate": lambda text: validate_merra2_time(text) , "when": lambda x: x['input:shared:MERRA-2'], }, { @@ -394,8 +404,6 @@ def ask_questions(): }, ] answers = questionary.prompt(questions) - if answers['input:shared:MERRA-2'] : - answers['input:shared:yyyymmddhh'] = answers['input:shared:yyyymmddhh']+ '21' answers['input:shared:rst_dir'] = os.path.abspath(answers['input:shared:rst_dir']) answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) From ae672eeb4dd5ef7b8da2f4e8d7d96e16773e3d19 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 2 Nov 2023 15:44:45 -0400 Subject: [PATCH 73/88] use baselib's nccmp --- pre/remap_restart/tests/test_remap_restarts.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pre/remap_restart/tests/test_remap_restarts.py b/pre/remap_restart/tests/test_remap_restarts.py index f31e3d1..05d0a30 100755 --- a/pre/remap_restart/tests/test_remap_restarts.py +++ b/pre/remap_restart/tests/test_remap_restarts.py @@ -5,7 +5,7 @@ # Newer GEOS code should load a module with GEOSpyD Python3 if not run: # module load python/GEOSpyD/Min4.10.3_py3.9 # - +import os import sys, getopt import ruamel.yaml import questionary @@ -29,8 +29,10 @@ def compare(base, result): return False bases.sort() results.sort() + basedir = os.environ['BASEDIR'] + NCCMP = basedir+ '/Linux/bin/nccmp' for b, r in zip(bases, results): - cmd = 'nccmp -dmgfs '+ b + ' ' + r + cmd = NCCMP + ' -dmgfs '+ b + ' ' + r print(cmd) p = sp.Popen(shlex.split(cmd), stdout=subprocess.PIPE) (out, err) = p.communicate() From 43817fcfbab618566e9b3fecf331156c4cf4d3cc Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Tue, 7 Nov 2023 14:48:58 -0500 Subject: [PATCH 74/88] minimal edits of comments for clarity; white-space changes to improve readability --- pre/remap_restart/remap_catchANDcn.py | 40 +++++++++---------- .../remap_lake_landice_saltwater.py | 22 +++++----- pre/remap_restart/remap_params.tpl | 4 +- pre/remap_restart/remap_questions.py | 28 ++++++------- pre/remap_restart/remap_upper.py | 36 ++++++++--------- pre/remap_restart/remap_utils.py | 6 +-- pre/remap_restart/tests/amip_c180Toc90.yaml | 4 +- pre/remap_restart/tests/c180Toc360.yaml | 4 +- pre/remap_restart/tests/c24Toc12.yaml | 4 +- pre/remap_restart/tests/c360Toc24.yaml | 4 +- pre/remap_restart/tests/f522Toc360.yaml | 4 +- .../tests/test_remap_restarts.py | 2 +- 12 files changed, 79 insertions(+), 79 deletions(-) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index a8d04a5..694fd42 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -39,37 +39,37 @@ def remap(self): print("\nRemapping " + model + ".....\n") - cwdir = os.getcwd() - bindir = os.path.dirname(os.path.realpath(__file__)) - in_bcsdir = config['input']['shared']['bcs_dir'] - out_bcsdir = config['output']['shared']['bcs_dir'] - out_dir = config['output']['shared']['out_dir'] - expid = config['output']['shared']['expid'] - in_wemin = config['input']['surface']['wemin'] - out_wemin = config['output']['surface']['wemin'] - surflay = config['output']['surface']['surflay'] - in_tilefile = config['input']['surface']['catch_tilefile'] + cwdir = os.getcwd() + bindir = os.path.dirname(os.path.realpath(__file__)) + in_bcsdir = config['input']['shared']['bcs_dir'] + out_bcsdir = config['output']['shared']['bcs_dir'] + out_dir = config['output']['shared']['out_dir'] + expid = config['output']['shared']['expid'] + in_wemin = config['input']['surface']['wemin'] + out_wemin = config['output']['surface']['wemin'] + surflay = config['output']['surface']['surflay'] + in_tilefile = config['input']['surface']['catch_tilefile'] if not in_tilefile : if not in_bcsdir: exit("Must provide either input tile file or input bcs directory") - agrid = config['input']['shared']['agrid'] - ogrid = config['input']['shared']['ogrid'] - omodel = config['input']['shared']['omodel'] - stretch = config['input']['shared']['stretch'] - bcs_geomdir = get_geomdir(in_bcsdir, agrid, ogrid, omodel, stretch) + agrid = config['input']['shared']['agrid'] + ogrid = config['input']['shared']['ogrid'] + omodel = config['input']['shared']['omodel'] + stretch = config['input']['shared']['stretch'] + bcs_geomdir = get_geomdir(in_bcsdir, agrid, ogrid, omodel, stretch) in_tilefile = glob.glob(bcs_geomdir + '/*.til')[0] - agrid = config['output']['shared']['agrid'] - ogrid = config['output']['shared']['ogrid'] - omodel = config['output']['shared']['omodel'] - stretch = config['output']['shared']['stretch'] + agrid = config['output']['shared']['agrid'] + ogrid = config['output']['shared']['ogrid'] + omodel = config['output']['shared']['omodel'] + stretch = config['output']['shared']['stretch'] out_tilefile = config['output']['surface']['catch_tilefile'] if not out_tilefile : if not out_bcsdir: exit("Must provide either input tile file or input bcs directory") - bcs_geomdir = get_geomdir(out_bcsdir, agrid, ogrid, omodel, stretch) + bcs_geomdir = get_geomdir(out_bcsdir, agrid, ogrid, omodel, stretch) out_tilefile = glob.glob(bcs_geomdir+ '/*.til')[0] out_bcs_landdir = get_landdir(out_bcsdir, agrid, ogrid, omodel, stretch) diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index 3305536..141af32 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -75,21 +75,21 @@ def remap(self): if 'roue' in f : route = f if 'openwater' in f : openwater = f - agrid = config['input']['shared']['agrid'] - ogrid = config['input']['shared']['ogrid'] - omodel = config['input']['shared']['omodel'] - stretch = config['input']['shared']['stretch'] - in_geomdir= get_geomdir(in_bcsdir, agrid, ogrid, omodel, stretch) + agrid = config['input']['shared']['agrid'] + ogrid = config['input']['shared']['ogrid'] + omodel = config['input']['shared']['omodel'] + stretch = config['input']['shared']['stretch'] + in_geomdir = get_geomdir(in_bcsdir, agrid, ogrid, omodel, stretch) in_tile_file = glob.glob(in_geomdir+ '/*-Pfafstetter.til')[0] - agrid = config['output']['shared']['agrid'] - ogrid = config['output']['shared']['ogrid'] - omodel = config['output']['shared']['omodel'] - stretch = config['output']['shared']['stretch'] - out_geomdir= get_geomdir(out_bcsdir, agrid, ogrid, omodel, stretch) + agrid = config['output']['shared']['agrid'] + ogrid = config['output']['shared']['ogrid'] + omodel = config['output']['shared']['omodel'] + stretch = config['output']['shared']['stretch'] + out_geomdir = get_geomdir(out_bcsdir, agrid, ogrid, omodel, stretch) out_tile_file = glob.glob(out_geomdir+ '/*-Pfafstetter.til')[0] - in_til = InData_dir+'/' + os.path.basename(in_tile_file) + in_til = InData_dir+'/' + os.path.basename(in_tile_file) out_til = OutData_dir+'/'+ os.path.basename(out_tile_file) if os.path.exists(in_til) : shutil.remove(in_til) diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index d88716d..16cfc4b 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -16,7 +16,7 @@ input: bcs_dir: expid: ogrid: - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data rst_dir: yyyymmddhh: @@ -38,7 +38,7 @@ output: bcs_dir: expid: ogrid: - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data out_dir: air: diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 2060dc3..490dd15 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -44,23 +44,23 @@ def echo_level(x): def echo_bcs(x,opt): if opt == "IN": - in_bcsdir = get_bcsdir(x, 'IN') - agrid = x.get('input:shared:agrid') - ogrid = x.get('input:shared:ogrid') - model = x.get('input:shared:omodel') - stretch = x.get('input:shared:stretch') - land_dir = get_landdir(in_bcsdir, agrid, ogrid, model, stretch) + in_bcsdir = get_bcsdir(x, 'IN') + agrid = x.get('input:shared:agrid') + ogrid = x.get('input:shared:ogrid') + model = x.get('input:shared:omodel') + stretch = x.get('input:shared:stretch') + land_dir = get_landdir(in_bcsdir, agrid, ogrid, model, stretch) x['input:shared:bcs_dir'] = in_bcsdir - print("\n BC of land for input restart: " + land_dir ) + print("\n Land BCs for input restarts: " + land_dir ) if opt == "OUT": out_bcsdir = get_bcsdir(x, 'OUT') - agrid = x.get('output:shared:agrid') - ogrid = x.get('output:shared:ogrid') - model = x.get('output:shared:omodel') - stretch = x.get('output:shared:stretch') - land_dir = get_landdir(out_bcsdir, agrid, ogrid, model, stretch) + agrid = x.get('output:shared:agrid') + ogrid = x.get('output:shared:ogrid') + model = x.get('output:shared:omodel') + stretch = x.get('output:shared:stretch') + land_dir = get_landdir(out_bcsdir, agrid, ogrid, model, stretch) x['output:shared:bcs_dir'] = out_bcsdir - print("\n BC of land for output restart: " + land_dir) + print("\n Land BCs for new restarts: " + land_dir) return False def default_partition(x): @@ -107,7 +107,7 @@ def ask_questions(): { "type": "text", "name": "input:shared:yyyymmddhh", - "message": "Enter restart date: (Must be 10 digits: yyyymmddhh; hour=03,09,15,21z.)\n", + "message": "Enter restart date: (Must be 10 digits: yyyymmddhh; hour = 03, 09, 15, or 21 [z].)\n", "validate": lambda text: validate_merra2_time(text) , "when": lambda x: x['input:shared:MERRA-2'], }, diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index e983bcb..c97e077 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -82,12 +82,12 @@ def remap(self): print('\n'+cmd) subprocess.call(shlex.split(cmd)) - agrid = config['input']['shared']['agrid'] - ogrid = config['input']['shared']['ogrid'] - omodel = config['input']['shared']['omodel'] - stretch = config['input']['shared']['stretch'] + agrid = config['input']['shared']['agrid'] + ogrid = config['input']['shared']['ogrid'] + omodel = config['input']['shared']['omodel'] + stretch = config['input']['shared']['stretch'] print("get topo:") - topo_bcsdir= get_topodir(in_bcsdir, agrid, ogrid, omodel, stretch) + topo_bcsdir = get_topodir(in_bcsdir, agrid, ogrid, omodel, stretch) print("get topo:") topoin = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] @@ -97,11 +97,11 @@ def remap(self): print('\n'+cmd) subprocess.call(shlex.split(cmd)) - agrid = config['output']['shared']['agrid'] - ogrid = config['output']['shared']['ogrid'] - omodel = config['output']['shared']['omodel'] - stretch = config['output']['shared']['stretch'] - topo_bcsdir= get_topodir(out_bcsdir, agrid, ogrid, omodel, stretch) + agrid = config['output']['shared']['agrid'] + ogrid = config['output']['shared']['ogrid'] + omodel = config['output']['shared']['omodel'] + stretch = config['output']['shared']['stretch'] + topo_bcsdir = get_topodir(out_bcsdir, agrid, ogrid, omodel, stretch) topoout = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] cmd = '/bin/ln -s ' + topoout + ' topo_dynave.data' @@ -119,21 +119,21 @@ def remap(self): exit("Only support cs grid so far") if (imout <90): - NPE = 12; nwrit = 1 + NPE = 12; nwrit = 1 elif (imout<=180): - NPE = 24; nwrit = 1 + NPE = 24; nwrit = 1 elif (imout<=540): - NPE = 96; nwrit = 1 + NPE = 96; nwrit = 1 elif (imout<=720): - NPE = 192; nwrit = 2 + NPE = 192; nwrit = 2 elif (imout<=1080): - NPE = 384; nwrit = 2 + NPE = 384; nwrit = 2 elif (imout<=1440): - NPE = 576; nwrit = 2 + NPE = 576; nwrit = 2 elif (imout< 2880): - NPE = 768; nwrit = 2 + NPE = 768; nwrit = 2 elif (imout>=2880): - NPE = 5400; nwrit= 6 + NPE = 5400; nwrit = 6 QOS = "#SBATCH --qos="+config['slurm']['qos'] TIME ="#SBATCH --time=1:00:00" diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 04aee35..4d53970 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -118,7 +118,7 @@ def init_merra2(x): x['input:surface:catch_model'] = 'catch' x['input:shared:stretch'] = False x['input:shared:rst_dir'] = x['output:shared:out_dir'] + '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' - x['input:air:nlevel'] = 72 + x['input:air:nlevel'] = 72 return False @@ -152,7 +152,7 @@ def fvcore_info(x): x['input:air:nlevel'] = lev ymdh = fvrst.variables['time'].units.split('since ')[1].split(":")[0].replace('-','').replace(' ', "") if (lat != lon*6) : - print("(N_lon,N_lat) = ", lon, lat) + print("(N_lon, N_lat) = ", lon, lat) exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') x['input:shared:yyyymmddhh'] = ymdh x['input:shared:agrid'] = "C"+str(lon) @@ -471,7 +471,7 @@ def get_grid_subdir(bcdir, agrid, ogrid, model, stretch): grid_directory = aname +'_' + oname if not os.path.isdir(os.path.join(bcdir,grid_directory)): - exit("cannot find the grid subdirctory of agrid: " + agrid+ " and ogrid " + ogrid + " under "+ bcdir) + exit("cannot find grid subdirectory for agrid=" + agrid + " and ogrid=" + ogrid + " under " + bcdir) return grid_directory diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index 2d9ae73..2eef779 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -11,7 +11,7 @@ input: MERRA-2: false stretch: false agrid: C180 - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: JM_v10.22.2_L072_C180_AMIP @@ -28,7 +28,7 @@ output: shared: agrid: C90 stretch: false - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: C90CS_JM_v10.22.2_L072_C180_AMIP diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index dc44483..bb93c27 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -10,7 +10,7 @@ input: shared: MERRA-2: false stretch: false - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data agrid: C180 bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ @@ -28,7 +28,7 @@ output: shared: agrid: C360 stretch: false - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: C360CS_Jason-3_4_NL_REAMIP_MERRA2_C180 diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index 2ecebe3..91c25a0 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -10,7 +10,7 @@ input: shared: MERRA-2: false stretch: false - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data agrid: C24 bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ @@ -28,7 +28,7 @@ output: shared: agrid: C12 stretch: false - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: hello diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index 1b6f821..91edf86 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -11,7 +11,7 @@ input: MERRA-2: false agrid: C360 stretch: false - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: x0046a @@ -28,7 +28,7 @@ output: shared: agrid: C24 stretch: false - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: C24c_x0046a diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index cdb0139..6492225 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -10,7 +10,7 @@ input: shared: MERRA-2: false stretch: false - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data agrid: C720 bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/ICA/ @@ -27,7 +27,7 @@ output: shared: agrid: C360 stretch: false - # ocean couple model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MO6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: remap diff --git a/pre/remap_restart/tests/test_remap_restarts.py b/pre/remap_restart/tests/test_remap_restarts.py index 05d0a30..0fcb075 100755 --- a/pre/remap_restart/tests/test_remap_restarts.py +++ b/pre/remap_restart/tests/test_remap_restarts.py @@ -30,7 +30,7 @@ def compare(base, result): bases.sort() results.sort() basedir = os.environ['BASEDIR'] - NCCMP = basedir+ '/Linux/bin/nccmp' + NCCMP = basedir+'/Linux/bin/nccmp' for b, r in zip(bases, results): cmd = NCCMP + ' -dmgfs '+ b + ' ' + r print(cmd) From 0bd0a2367431b11d1350cc8a9e8e62c4f5f55766 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Tue, 7 Nov 2023 15:08:22 -0500 Subject: [PATCH 75/88] more comments and white-space changes --- pre/remap_restart/remap_questions.py | 2 +- pre/remap_restart/remap_utils.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 490dd15..f102bd2 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -53,7 +53,7 @@ def echo_bcs(x,opt): x['input:shared:bcs_dir'] = in_bcsdir print("\n Land BCs for input restarts: " + land_dir ) if opt == "OUT": - out_bcsdir = get_bcsdir(x, 'OUT') + out_bcsdir = get_bcsdir(x, 'OUT') agrid = x.get('output:shared:agrid') ogrid = x.get('output:shared:ogrid') model = x.get('output:shared:omodel') diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 4d53970..fcb318d 100644 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -357,9 +357,9 @@ def get_command_line_from_answers(answers): in_bcsdir = '' default_bc = get_bcsdir(answers, "IN") if not os.path.samefile(default_bc, answers.get("input:shared:bcs_dir").strip()) : - in_bcsdir = ' -in_bcsdir ' + answers.get("input:shared:bcs_dir") + in_bcsdir = ' -in_bcsdir ' + answers.get("input:shared:bcs_dir") - out_bcsdir = '' + out_bcsdir = '' default_bc = get_bcsdir(answers, "OUT") if not os.path.samefile(default_bc, answers.get("output:shared:bcs_dir").strip()) : out_bcsdir = ' -out_bcsdir ' + answers.get("output:shared:bcs_dir") @@ -367,11 +367,11 @@ def get_command_line_from_answers(answers): out_stretch = '' if answers["output:shared:stretch"]: out_stretch = ' -out_stretch ' + answers["output:shared:stretch"] - in_stretch = '' + in_stretch = '' if answers["input:shared:stretch"]: - in_stretch = ' -in_stretch ' + answers["input:shared:stretch"] + in_stretch = ' -in_stretch ' + answers["input:shared:stretch"] - zoom = " -zoom " + answers["input:surface:zoom"] + zoom = " -zoom " + answers["input:surface:zoom"] wemin = " -in_wemin " + answers["input:surface:wemin"] wemout = " -out_wemin " + answers["output:surface:wemin"] catch_model ='' @@ -482,7 +482,7 @@ def get_bcsdir(x, opt): bcdir = bc_base+'/'+ bc_version+'/' if not os.path.exists(bcdir): - exit("Cannot find bc dir " + bcdir) + exit("Cannot find BCs dir " + bcdir) return bcdir From af4ecf3173ff4d5f73d268d51e3a7195fdff2c6a Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Tue, 7 Nov 2023 15:21:30 -0500 Subject: [PATCH 76/88] fixed typo in comment --- pre/remap_restart/remap_params.tpl | 4 ++-- pre/remap_restart/tests/amip_c180Toc90.yaml | 4 ++-- pre/remap_restart/tests/c180Toc360.yaml | 4 ++-- pre/remap_restart/tests/c24Toc12.yaml | 4 ++-- pre/remap_restart/tests/c360Toc24.yaml | 4 ++-- pre/remap_restart/tests/f522Toc360.yaml | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index 16cfc4b..1aa18cd 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -16,7 +16,7 @@ input: bcs_dir: expid: ogrid: - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data rst_dir: yyyymmddhh: @@ -38,7 +38,7 @@ output: bcs_dir: expid: ogrid: - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data out_dir: air: diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index 2eef779..63d9670 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -11,7 +11,7 @@ input: MERRA-2: false stretch: false agrid: C180 - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: JM_v10.22.2_L072_C180_AMIP @@ -28,7 +28,7 @@ output: shared: agrid: C90 stretch: false - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: C90CS_JM_v10.22.2_L072_C180_AMIP diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index bb93c27..73f017f 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -10,7 +10,7 @@ input: shared: MERRA-2: false stretch: false - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: C180 bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ @@ -28,7 +28,7 @@ output: shared: agrid: C360 stretch: false - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: C360CS_Jason-3_4_NL_REAMIP_MERRA2_C180 diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index 91c25a0..dc6b8db 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -10,7 +10,7 @@ input: shared: MERRA-2: false stretch: false - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: C24 bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ @@ -28,7 +28,7 @@ output: shared: agrid: C12 stretch: false - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: hello diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index 91edf86..40f089f 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -11,7 +11,7 @@ input: MERRA-2: false agrid: C360 stretch: false - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: x0046a @@ -28,7 +28,7 @@ output: shared: agrid: C24 stretch: false - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: C24c_x0046a diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index 6492225..e8bf2d4 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -10,7 +10,7 @@ input: shared: MERRA-2: false stretch: false - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: C720 bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/ICA/ @@ -27,7 +27,7 @@ output: shared: agrid: C360 stretch: false - # (coupled) ocean model: data, MOM5, MO6 + # (coupled) ocean model: data, MOM5, MOM6 omodel: data bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ expid: remap From 9e1a5c8644d4eaa74fe58c36efb8c678f4dcc1d1 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Wed, 8 Nov 2023 18:06:13 -0500 Subject: [PATCH 77/88] added intro comments to remap_*.py scripts; fixed permissions of some files --- pre/remap_restart/bin2nc_merra2_agcm.yaml | 0 pre/remap_restart/remap_analysis.py | 10 +++-- pre/remap_restart/remap_base.py | 3 ++ pre/remap_restart/remap_bin2nc.py | 3 +- pre/remap_restart/remap_catchANDcn.py | 8 ++++ pre/remap_restart/remap_command_line.py | 10 ++--- .../remap_lake_landice_saltwater.py | 9 +++++ pre/remap_restart/remap_params.py | 4 ++ pre/remap_restart/remap_params.tpl | 3 +- pre/remap_restart/remap_questions.py | 7 ++-- pre/remap_restart/remap_restarts.py | 37 +++++++++---------- pre/remap_restart/remap_upper.py | 8 ++++ pre/remap_restart/remap_utils.py | 3 ++ 13 files changed, 68 insertions(+), 37 deletions(-) mode change 100755 => 100644 pre/remap_restart/bin2nc_merra2_agcm.yaml mode change 100644 => 100755 pre/remap_restart/remap_restarts.py mode change 100644 => 100755 pre/remap_restart/remap_utils.py diff --git a/pre/remap_restart/bin2nc_merra2_agcm.yaml b/pre/remap_restart/bin2nc_merra2_agcm.yaml old mode 100755 new mode 100644 diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index f830ae3..5236c66 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 # -# source install/bin/g5_modules +# remap_restarts package: +# remap_analysis.py remaps atmospheric analysis restarts using config inputs from `remap_params.yaml` # -# Newer GEOS code should load a module with GEOSpyD Python3 if not run: -# module load python/GEOSpyD/Min4.10.3_py3.9 +# to run, must first load modules (incl. python3) as follows: +# +# source g5_modules.sh [bash] +# source g5_modules [csh] # - import os from datetime import datetime, timedelta import subprocess diff --git a/pre/remap_restart/remap_base.py b/pre/remap_restart/remap_base.py index 5163c91..990456b 100755 --- a/pre/remap_restart/remap_base.py +++ b/pre/remap_restart/remap_base.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 # +# remap_restarts package: +# remap_base.py does [???] +# import os import ruamel.yaml import shutil diff --git a/pre/remap_restart/remap_bin2nc.py b/pre/remap_restart/remap_bin2nc.py index fc873d6..f7b94ed 100755 --- a/pre/remap_restart/remap_bin2nc.py +++ b/pre/remap_restart/remap_bin2nc.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -# script to convert binary MERRA-2 restarts to nc4 +# remap_restarts package: +# remap_bin2nc.py converts binary MERRA-2 restarts to nc4 # # As of Sep 2023, MERRA-2 restarts are the only binary restarts that may need to be # remapped. The set of yaml config files "bin2nc_merra2_*.yaml" include hard-wired diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 694fd42..f43e188 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -1,5 +1,13 @@ #!/usr/bin/env python3 # +# remap_restarts package: +# remap_catchANDcn.py remaps Catchment[CN] land model restarts using config inputs from `remap_params.yaml` +# +# to run, must first load modules (incl. python3) as follows: +# +# source g5_modules.sh [bash] +# source g5_modules [csh] +# import os import sys import subprocess diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 2e4b455..b4dba0e 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -1,13 +1,9 @@ #!/usr/bin/env python3 # -# source install/bin/g5_modules +# remap_restarts package: +# remap_command_line.py parses and converts the command-line arguments and converts the information +# into matching "Answers" for the questionary (remap_questions.py) # -# Newer GEOS code should load a module with GEOSpyD Python3 if not run: -# module load python/GEOSpyD/Min4.10.3_py3.9 -# -# This script parses and converts the command-line arguments to answers equivalent to those from remap_questions.py - - import os import subprocess import shlex diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index 141af32..edbb9e1 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -1,5 +1,14 @@ #!/usr/bin/env python3 # +# remap_restarts package: +# remap_lake_landice_saltwater.py remaps lake, landice, and (data) ocean restarts +# using config inputs from `remap_params.yaml` +# +# to run, must first load modules (incl. python3) as follows: +# +# source g5_modules.sh [bash] +# source g5_modules [csh] +# import os import subprocess import shutil diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index 0f42c53..4c362e1 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 # +# remap_restarts package: +# remap_params.py uses the dictionary `answers` (from remap_questions.py) as inputs and +# generates a yaml config file named `remap_params.yaml`. +# import os,sys import ruamel.yaml import shutil diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index 1aa18cd..5eb4030 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -1,5 +1,6 @@ # -# This template file can be filled with questionary or manually +# remap_restarts package: +# this template file can be filled via the questionary (remap_questions.py) or manually # # diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index f102bd2..33af37d 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 # -# source install/bin/g5_modules +# remap_restarts package: +# interactive questionary to create a yaml configuration file (remap_params.yaml) and +# a matching command line argument string (remap_restarts.CMD) # -# Newer GEOS code should load a module with GEOSpyD Python3 if not run: -# module load python/GEOSpyD/Min4.10.3_py3.9 # - import os import subprocess import shlex diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py old mode 100644 new mode 100755 index e0c3c75..4f1124c --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 # -# main script for remapping GEOS restart files to a new resolution and/or boundary conditions version +# remap_restarts package: +# remap_restarts.py is the main script for remapping GEOS restart files to a different +# resolution and/or a different version of the associated model boundary conditions # -# source install/bin/g5_modules +# to run, must first load modules (incl. python3) as follows: # -# Newer GEOS code should load a module with GEOSpyD Python3 if not run: -# module load python/GEOSpyD/Min4.11.0_py3.9 +# source g5_modules.sh [bash] +# source g5_modules [csh] # - import sys import argparse import textwrap @@ -26,32 +27,28 @@ program_description = textwrap.dedent(f''' USAGE: - This script provides three options to remap restart files: + This script provides three options for remapping GEOS restart files: - 1. Use questionary to create a yaml config file (remap_params.yaml): - ./remap_questions.py - and then remap: + 1. Use the interactive questionary: ./remap_restarts.py + The questionary concludes with the option to submit the remapping job. + It also creates a yaml configuration file (`remap_params.yaml`) and + a command line options string (`remap_restarts.CMD`), which can be edited + manually and used in the other two ways of running `remap_restarts.py`. - 2. Use an existing yaml config file to remap: + 2. Use an existing yaml config file: ./remap_restarts.py config_file -c my_config.yaml - 3. Use command line options: + 3. Use command line arguments: ./remap_restarts.py command_line -ymdh 2004041421 .... - To see more command line options, please use - ./remap_restarts.py command_line -h - Note that the questionary also generates command line inputs. - There are three help commands: + Help commands: ./remap_restarts.py -h ./remap_restarts.py config_file -h ./remap_restarts.py command_line -h - NOTE: Individual scripts can be executed independently: - 1. remap_questions.py generates remap_params.yaml. - 2. If the command_line option is used with remap_restarts.py, the command line options are converted remap_params.yaml. - 3. Subsets of restarts can be remapped separately with remap_upper.py, remap_lake_landice_saltwater.py, - remap_catchANDcn.py, and remap_analysis.py, all of which use remap_params.yaml as configuration input. + For more information, refer to https://github.com/GEOS-ESM/GEOS_Util/wiki + ''') def main(): diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index c97e077..7e9406d 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -1,5 +1,13 @@ #!/usr/bin/env python3 # +# remap_restarts package: +# remap_upper.py remaps atmospheric model restarts using config inputs from `remap_params.yaml` +# +# to run, must first load modules (incl. python3) as follows: +# +# source g5_modules.sh [bash] +# source g5_modules [csh] +# import os import ruamel.yaml import subprocess diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py old mode 100644 new mode 100755 index fcb318d..6b5a0c1 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 # +# remap_restarts package: +# remap_utils.py contains utility functions and definitions +# import os import subprocess import ruamel.yaml From e84dddee8fbad8b263cfbd1169893d82ffb8b670 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 9 Nov 2023 09:15:26 -0500 Subject: [PATCH 78/88] 1) auto detect discover and nas 2)clean up minor issues --- pre/remap_restart/remap_params.tpl | 8 +++---- pre/remap_restart/remap_utils.py | 24 +++++++++++++++++---- pre/remap_restart/tests/amip_c180Toc90.yaml | 3 ++- pre/remap_restart/tests/c180Toc360.yaml | 3 ++- pre/remap_restart/tests/c24Toc12.yaml | 3 ++- pre/remap_restart/tests/c360Toc24.yaml | 3 ++- pre/remap_restart/tests/f522Toc360.yaml | 3 ++- 7 files changed, 34 insertions(+), 13 deletions(-) diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index 5eb4030..ba65338 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -13,12 +13,12 @@ input: bc_version: none MERRA-2: false stretch: false + # (coupled) ocean model: data, MOM5, MOM6 + omodel: data agrid: bcs_dir: expid: ogrid: - # (coupled) ocean model: data, MOM5, MOM6 - omodel: data rst_dir: yyyymmddhh: surface: @@ -35,12 +35,12 @@ output: label: false # SG001,SG002 stretch: false + # (coupled) ocean model: data, MOM5, MOM6 + omodel: data agrid: bcs_dir: expid: ogrid: - # (coupled) ocean model: data, MOM5, MOM6 - omodel: data out_dir: air: # remap upper air or not diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 6b5a0c1..8c60e07 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -15,11 +15,9 @@ # shared global variables # -# define "bc_base", "choices", "message" strings, and "validate" lists that are used multiple times +# define "choices", "message" strings, and "validate" lists that are used multiple times # (and related definitions, even if they are used just once). -bc_base = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" - choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] choices_bc_other = ['v06','v11'] @@ -478,11 +476,29 @@ def get_grid_subdir(bcdir, agrid, ogrid, model, stretch): return grid_directory +def get_bc_base(): + base = {} + base['NAS'] = '' + base['NCCS'] = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" + cmd = 'uname -n' + p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + (node, err) = p.communicate() + p_status = p.wait() + node = node.decode().split() + node0 = node[0] + site ='NAS' + discover = ['dirac', 'borg','warp', 'discover'] + for node in discover: + if node in node0: + site = 'NCCS' + break + return base[site] + def get_bcsdir(x, opt): bc_version = x.get('input:shared:bc_version') if opt.upper() == "OUT": bc_version = x.get('output:shared:bc_version') - + bc_base = get_bc_base() bcdir = bc_base+'/'+ bc_version+'/' if not os.path.exists(bcdir): exit("Cannot find BCs dir " + bcdir) diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index 63d9670..8f38bdc 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -54,6 +54,7 @@ output: lcv: false slurm: - account: g0620 + # account would be set to users' primary account automatically by test_remap_restarts.py + account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index 73f017f..2fad735 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -54,6 +54,7 @@ output: lcv: false slurm: - account: g0620 + # account would be set to users' primary account automatically by test_remap_restarts.py + account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index dc6b8db..92adea9 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -54,6 +54,7 @@ output: lcv: false slurm: - account: g0610 + # account would be set to users' primary account automatically by test_remap_restarts.py + account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index 40f089f..0ce91a5 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -54,6 +54,7 @@ output: lcv: false slurm: - account: g0620 + # account would be set to users' primary account automatically by test_remap_restarts.py + account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index e8bf2d4..8ea5fa6 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -50,6 +50,7 @@ output: lcv: true slurm: - account: g0610 + # account would be set to users' primary account automatically by test_remap_restarts.py + account: qos: debug partition: compute From 62b738cac83a7292d3b64d28a33c0044ade01e17 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Thu, 9 Nov 2023 09:25:41 -0500 Subject: [PATCH 79/88] minimal edit in comment in yaml files for remap_restart test cases --- pre/remap_restart/tests/amip_c180Toc90.yaml | 2 +- pre/remap_restart/tests/c180Toc360.yaml | 2 +- pre/remap_restart/tests/c24Toc12.yaml | 2 +- pre/remap_restart/tests/c360Toc24.yaml | 2 +- pre/remap_restart/tests/f522Toc360.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index 8f38bdc..e0e6a6e 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -54,7 +54,7 @@ output: lcv: false slurm: - # account would be set to users' primary account automatically by test_remap_restarts.py + # account will be set to user's primary account automatically by test_remap_restarts.py account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index 2fad735..98b0157 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -54,7 +54,7 @@ output: lcv: false slurm: - # account would be set to users' primary account automatically by test_remap_restarts.py + # account will be set to user's primary account automatically by test_remap_restarts.py account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index 92adea9..1c4fbce 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -54,7 +54,7 @@ output: lcv: false slurm: - # account would be set to users' primary account automatically by test_remap_restarts.py + # account will be set to user's primary account automatically by test_remap_restarts.py account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index 0ce91a5..01cc671 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -54,7 +54,7 @@ output: lcv: false slurm: - # account would be set to users' primary account automatically by test_remap_restarts.py + # account will be set to user's primary account automatically by test_remap_restarts.py account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index 8ea5fa6..5c8b50a 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -50,7 +50,7 @@ output: lcv: true slurm: - # account would be set to users' primary account automatically by test_remap_restarts.py + # account will be set to user's primary account automatically by test_remap_restarts.py account: qos: debug partition: compute From 0690a8096a13749cbcbfca1a64b0050831b713ae Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Thu, 9 Nov 2023 09:35:35 -0500 Subject: [PATCH 80/88] moved definition of bc_base dirs to beginning of file (remap_utils.py) --- pre/remap_restart/remap_utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 8c60e07..1b39c02 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -14,7 +14,13 @@ import netCDF4 as nc # shared global variables -# + +# top-level directory for BCs (machine-dependent) + +bc_base_NCCS = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" + +bc_base_NAS = '' + # define "choices", "message" strings, and "validate" lists that are used multiple times # (and related definitions, even if they are used just once). @@ -478,8 +484,8 @@ def get_grid_subdir(bcdir, agrid, ogrid, model, stretch): def get_bc_base(): base = {} - base['NAS'] = '' - base['NCCS'] = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" + base['NAS'] = bc_base_NAS + base['NCCS'] = bc_base_NCCS cmd = 'uname -n' p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) (node, err) = p.communicate() From 03cad9105e1a63cd601c12916774e5e6fe76e63c Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Mon, 13 Nov 2023 18:54:14 -0500 Subject: [PATCH 81/88] add question of bc_base and replace bcs_dir with bc_base --- pre/remap_restart/remap_catchANDcn.py | 24 ++-- pre/remap_restart/remap_command_line.py | 15 +-- .../remap_lake_landice_saltwater.py | 11 +- pre/remap_restart/remap_params.py | 5 +- pre/remap_restart/remap_params.tpl | 14 +- pre/remap_restart/remap_questions.py | 75 +++++++---- pre/remap_restart/remap_upper.py | 12 +- pre/remap_restart/remap_utils.py | 120 +++++++----------- pre/remap_restart/tests/amip_c180Toc90.yaml | 6 +- pre/remap_restart/tests/c180Toc360.yaml | 6 +- pre/remap_restart/tests/c24Toc12.yaml | 6 +- pre/remap_restart/tests/c360Toc24.yaml | 6 +- pre/remap_restart/tests/f522Toc360.yaml | 6 +- 13 files changed, 151 insertions(+), 155 deletions(-) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index f43e188..05ddaa9 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -49,8 +49,10 @@ def remap(self): cwdir = os.getcwd() bindir = os.path.dirname(os.path.realpath(__file__)) - in_bcsdir = config['input']['shared']['bcs_dir'] - out_bcsdir = config['output']['shared']['bcs_dir'] + in_bc_base = config['input']['shared']['bc_base'] + in_bc_version = config['input']['shared']['bc_version'] + out_bc_base = config['output']['shared']['bc_base'] + out_bc_version= config['output']['shared']['bc_version'] out_dir = config['output']['shared']['out_dir'] expid = config['output']['shared']['expid'] in_wemin = config['input']['surface']['wemin'] @@ -59,14 +61,12 @@ def remap(self): in_tilefile = config['input']['surface']['catch_tilefile'] if not in_tilefile : - if not in_bcsdir: - exit("Must provide either input tile file or input bcs directory") agrid = config['input']['shared']['agrid'] ogrid = config['input']['shared']['ogrid'] omodel = config['input']['shared']['omodel'] stretch = config['input']['shared']['stretch'] - bcs_geomdir = get_geomdir(in_bcsdir, agrid, ogrid, omodel, stretch) - in_tilefile = glob.glob(bcs_geomdir + '/*.til')[0] + bc_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid, ogrid, omodel, stretch) + in_tilefile = glob.glob(bc_geomdir + '/*.til')[0] agrid = config['output']['shared']['agrid'] ogrid = config['output']['shared']['ogrid'] @@ -75,12 +75,10 @@ def remap(self): out_tilefile = config['output']['surface']['catch_tilefile'] if not out_tilefile : - if not out_bcsdir: - exit("Must provide either input tile file or input bcs directory") - bcs_geomdir = get_geomdir(out_bcsdir, agrid, ogrid, omodel, stretch) - out_tilefile = glob.glob(bcs_geomdir+ '/*.til')[0] + bc_geomdir = get_geomdir(out_bc_base, out_bc_version, agrid, ogrid, omodel, stretch) + out_tilefile = glob.glob(bc_geomdir+ '/*.til')[0] - out_bcs_landdir = get_landdir(out_bcsdir, agrid, ogrid, omodel, stretch) + out_bc_landdir = get_landdir(out_bc_base, out_bc_version, agrid, ogrid, omodel, stretch) # determine NPE based on *approximate* number of input and output tile @@ -93,7 +91,7 @@ def remap(self): in_Ntile = ds.dimensions['tile'].size out_Ntile = 0 - with open( out_bcs_landdir+'/clsm/catchment.def') as f: + with open( out_bc_landdir+'/clsm/catchment.def') as f: out_Ntile = int(next(f)) max_Ntile = max(in_Ntile, out_Ntile) NPE = 0 @@ -166,7 +164,7 @@ def remap(self): $esma_mpirun_X $mk_catchANDcnRestarts_X $params """ - catch1script = mk_catch_j_template.format(Bin = bindir, account = account, out_bcs = out_bcs_landdir, \ + catch1script = mk_catch_j_template.format(Bin = bindir, account = account, out_bcs = out_bc_landdir, \ model = model, out_dir = out_dir, surflay = surflay, log_name = log_name, NPE = NPE, \ in_wemin = in_wemin, out_wemin = out_wemin, out_tilefile = out_tilefile, in_tilefile = in_tilefile, \ in_rstfile = in_rstfile, out_rstfile = out_rstfile, time = yyyymmddhh_, TIME = TIME, PARTITION = PARTITION, QOS=QOS ) diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index b4dba0e..0c0c883 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -68,8 +68,8 @@ def parse_args(program_description): p_command.add_argument('-np', action='store_true', help="No prompt. Overwrite config files without prompting questions") p_command.add_argument('-lbl', action='store_true', help="Label output restarts with bc_versions and resolutions") p_command.add_argument('-noagcm_import_rst', action='store_true', help="Do not remap agcm_import_rst file") - p_command.add_argument('-in_bcsdir', default="", help="User-supplied directory with boundary conditions of input restarts. If not specified (default), dir. is deduced from bc_version and resolution info") - p_command.add_argument('-out_bcsdir', default="", help="User-supplied directory with boundary conditions for new restarts. If not specified (default), dir. is deduced from bc_version and resolution info") + p_command.add_argument('-in_bc_base', default="", help="Boundary conditions base directory (without bc_version and resolution infomation) for input restart") + p_command.add_argument('-out_bc_base', default="", help="Boundary conditions base directory (without bc_version and resolution infomation) for new restarts") p_command.add_argument('-zoom', help= "Zoom parameter (search radius) for input surface restarts") p_command.add_argument('-qos', default="debug", help="SLURM quality-of-service", choices=['debug', 'allnccs']) @@ -113,15 +113,8 @@ def get_answers_from_command_line(cml): ogrid = answers["output:shared:agrid"] answers["output:shared:ogrid"] = ogrid - if cml.in_bcsdir.strip(): - answers["input:shared:bcs_dir"] = cml.in_bcsdir - else: - answers["input:shared:bcs_dir"] = get_bcsdir(answers, "IN") - - if cml.out_bcsdir.strip(): - answers["output:shared:bcs_dir"] = cml.out_bcsdir - else: - answers["output:shared:bcs_dir"] = get_bcsdir(answers, "OUT") + answers["input:shared:bc_base"] = cml.in_bc_base + answers["output:shared:bc_base"] = cml.out_bc_base answers["output:shared:stretch"] = cml.out_stretch answers["input:shared:stretch"] = cml.in_stretch diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index edbb9e1..486c421 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -37,8 +37,11 @@ def remap(self): config = self.config cwdir = os.getcwd() bindir = os.path.dirname(os.path.realpath(__file__)) - in_bcsdir = config['input']['shared']['bcs_dir'] - out_bcsdir = config['output']['shared']['bcs_dir'] + in_bc_base = config['input']['shared']['bc_base'] + in_bc_version = config['input']['shared']['bc_version'] + out_bc_base = config['output']['shared']['bc_base'] + out_bc_version= config['output']['shared']['bc_version'] + out_dir = config['output']['shared']['out_dir'] if not os.path.exists(out_dir) : os.makedirs(out_dir) @@ -88,14 +91,14 @@ def remap(self): ogrid = config['input']['shared']['ogrid'] omodel = config['input']['shared']['omodel'] stretch = config['input']['shared']['stretch'] - in_geomdir = get_geomdir(in_bcsdir, agrid, ogrid, omodel, stretch) + in_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid, ogrid, omodel, stretch) in_tile_file = glob.glob(in_geomdir+ '/*-Pfafstetter.til')[0] agrid = config['output']['shared']['agrid'] ogrid = config['output']['shared']['ogrid'] omodel = config['output']['shared']['omodel'] stretch = config['output']['shared']['stretch'] - out_geomdir = get_geomdir(out_bcsdir, agrid, ogrid, omodel, stretch) + out_geomdir = get_geomdir(out_bc_base, out_bc_version, agrid, ogrid, omodel, stretch) out_tile_file = glob.glob(out_geomdir+ '/*-Pfafstetter.til')[0] in_til = InData_dir+'/' + os.path.basename(in_tile_file) diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py index 4c362e1..045edc1 100755 --- a/pre/remap_restart/remap_params.py +++ b/pre/remap_restart/remap_params.py @@ -43,6 +43,7 @@ def __init__(self, config_from_question): config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] config_tpl['input']['shared']['bc_version'] = self.common_in.get('bc_version') config_tpl['input']['shared']['stretch'] = self.common_in.get('stretch') + config_tpl['input']['shared']['bc_base'] = self.common_in['bc_base'] config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') @@ -58,9 +59,7 @@ def __init__(self, config_from_question): config_tpl['output']['shared']['bc_version'] = self.common_out.get('bc_version') config_tpl['output']['shared']['label'] = self.common_out.get('label') config_tpl['output']['shared']['stretch'] = self.common_out.get('stretch') - - config_tpl['input']['shared']['bcs_dir'] = self.common_in['bcs_dir'] - config_tpl['output']['shared']['bcs_dir'] = self.common_out['bcs_dir'] + config_tpl['output']['shared']['bc_base'] = self.common_out['bc_base'] # params for upper air config_tpl = self.params_for_air(config_tpl) diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index ba65338..d1d108a 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -9,14 +9,13 @@ input: drymass: 1 hydrostatic: 0 shared: - # bc_version info here is not critical. It is used in command_line options - bc_version: none MERRA-2: false stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: - bcs_dir: + bc_version: none + bc_base: expid: ogrid: rst_dir: @@ -26,19 +25,18 @@ input: wemin: # it supports three models: catch, catchcnclm40, catchcnclm45 catch_model: null - # if catch_tilefile is null, it searches bcs_dir + # if catch_tilefile is null, it searches bc_dir catch_tilefile: null output: shared: - # bc_version info here is not critical. It is used in command_line options - bc_version: none label: false # SG001,SG002 stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: - bcs_dir: + bc_version: none + bc_base: expid: ogrid: out_dir: @@ -54,7 +52,7 @@ output: remap_water: true # remap catch(cn) remap_catch: true - # if catch_tilefile is null, it searches bcs_dir + # if catch_tilefile is null, it searches bc_dir catch_tilefile: null analysis: bkg: true diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 33af37d..afbf592 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -42,24 +42,17 @@ def echo_level(x): return True def echo_bcs(x,opt): - if opt == "IN": - in_bcsdir = get_bcsdir(x, 'IN') - agrid = x.get('input:shared:agrid') - ogrid = x.get('input:shared:ogrid') - model = x.get('input:shared:omodel') - stretch = x.get('input:shared:stretch') - land_dir = get_landdir(in_bcsdir, agrid, ogrid, model, stretch) - x['input:shared:bcs_dir'] = in_bcsdir - print("\n Land BCs for input restarts: " + land_dir ) - if opt == "OUT": - out_bcsdir = get_bcsdir(x, 'OUT') - agrid = x.get('output:shared:agrid') - ogrid = x.get('output:shared:ogrid') - model = x.get('output:shared:omodel') - stretch = x.get('output:shared:stretch') - land_dir = get_landdir(out_bcsdir, agrid, ogrid, model, stretch) - x['output:shared:bcs_dir'] = out_bcsdir - print("\n Land BCs for new restarts: " + land_dir) + base = x.get(opt+':shared:bc_base').split(': ')[-1] + bcv = x.get(opt+':shared:bc_version') + agrid = x.get(opt+':shared:agrid') + ogrid = x.get(opt+':shared:ogrid') + model = x.get(opt+':shared:omodel') + stretch = x.get(opt+':shared:stretch') + x[opt+':shared:bc_base'] = base + land_dir = get_landdir(base, bcv, agrid, ogrid, model, stretch) + if not os.path.isdir(land_dir): + exit("cannot find grid subdirectory for agrid=" + agrid + " and ogrid=" + ogrid + " under " + base+'/'+bcv+'/land/') + print("\n Land BCs for " + opt + " restarts: " + land_dir ) return False def default_partition(x): @@ -298,17 +291,51 @@ def ask_questions(): "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11'], }, + { + "type": "select", + "name": "input:shared:bc_base", + "message": "\nSelect BCs base directory for input restarts: \n", + "choices": choices_bc_base, + "default": get_default_bc_base(), + "when": lambda x: not x.get('input:shared:bc_base'), + }, + { "type": "path", - "name": "input:shared:bcs_dir", - "message": "Is this BCs version correct for input restarts? If no, enter your own absolute path: \n", - "when": lambda x: echo_bcs(x, "IN"), + "name": "input:shared:bc_base", + "message": "\nEnter BCs base directory for input restarts: \n", + "when": lambda x: 'Customize ' in x.get('input:shared:bc_base'), + }, + # dummy (invisible) question to retrieve input:shared:bc_base + { + "type": "text", + "name": "input:shared:bc_base", + "message": "retrieve and echo bcs", + # always return false, so questions never shows but changes ogrid + "when": lambda x: echo_bcs(x, 'input') }, + + { + "type": "select", + "name": "output:shared:bc_base", + "message": "\nSelect BCs base directory for new restarts: \n", + "choices": choices_bc_base, + "default": get_default_bc_base(), + }, + { "type": "path", - "name": "output:shared:bcs_dir", - "message": "Is this BCs version correct for new restarts? If no, enter your own absolute path: \n", - "when": lambda x: echo_bcs(x, "OUT"), + "name": "output:shared:bc_base", + "message": "\nEnter BCs base directory for new restarts: \n", + "when": lambda x: 'Customize ' in x.get('output:shared:bc_base'), + }, + # dummy (invisible) question to retrieve output:shared:bc_base + { + "type": "text", + "name": "output:shared:bc_base", + "message": "retrieve and echo bcs", + # always return false, so questions never shows but changes ogrid + "when": lambda x: echo_bcs(x, 'output') }, { diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 7e9406d..30ceacb 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -58,8 +58,6 @@ def remap(self): config = self.config cwdir = os.getcwd() bindir = os.path.dirname(os.path.realpath(__file__)) - in_bcsdir = config['input']['shared']['bcs_dir'] - out_bcsdir = config['output']['shared']['bcs_dir'] out_dir = config['output']['shared']['out_dir'] if not os.path.exists(out_dir) : os.makedirs(out_dir) @@ -90,13 +88,13 @@ def remap(self): print('\n'+cmd) subprocess.call(shlex.split(cmd)) + in_bc_base = config['input']['shared']['bc_base'] + in_bc_version = config['input']['shared']['bc_version'] agrid = config['input']['shared']['agrid'] ogrid = config['input']['shared']['ogrid'] omodel = config['input']['shared']['omodel'] stretch = config['input']['shared']['stretch'] - print("get topo:") - topo_bcsdir = get_topodir(in_bcsdir, agrid, ogrid, omodel, stretch) - print("get topo:") + topo_bcsdir = get_topodir(in_bc_base, in_bc_version, agrid, ogrid, omodel, stretch) topoin = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] # link topo file @@ -105,11 +103,13 @@ def remap(self): print('\n'+cmd) subprocess.call(shlex.split(cmd)) + out_bc_base = config['output']['shared']['bc_base'] + out_bc_version = config['output']['shared']['bc_version'] agrid = config['output']['shared']['agrid'] ogrid = config['output']['shared']['ogrid'] omodel = config['output']['shared']['omodel'] stretch = config['output']['shared']['stretch'] - topo_bcsdir = get_topodir(out_bcsdir, agrid, ogrid, omodel, stretch) + topo_bcsdir = get_topodir(out_bc_base, out_bc_version, agrid, ogrid, omodel, stretch) topoout = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] cmd = '/bin/ln -s ' + topoout + ' topo_dynave.data' diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 1b39c02..7691c5b 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -17,9 +17,9 @@ # top-level directory for BCs (machine-dependent) -bc_base_NCCS = "/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles" - -bc_base_NAS = '' +choices_bc_base =[ "DISCOVER : /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles", + "NAS : /nobackup/gmao_SIteam/ModelData/bcs_shared/fvInput/ExtData/esm/tiles", + "Customize : " ] # define "choices", "message" strings, and "validate" lists that are used multiple times # (and related definitions, even if they are used just once). @@ -122,6 +122,7 @@ def init_merra2(x): x['input:shared:ogrid'] = '1440x720' x['input:shared:omodel'] = 'data' x['input:shared:bc_version'] = 'GM4' + x['input:shared:bc_base'] = '/discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles' x['input:surface:catch_model'] = 'catch' x['input:shared:stretch'] = False x['input:shared:rst_dir'] = x['output:shared:out_dir'] + '/merra2_tmp_'+x['input:shared:yyyymmddhh']+'/' @@ -219,16 +220,21 @@ def data_ocean_default(resolution): if resolution in ['C12','C24', 'C48'] : default_ = '360x180 (Reynolds)' return default_ -def get_bcs_basename(bcs): - if not bcs: return "" - while bcs[-1] == '/': bcs = bcs[0:-1] # remove extra '/' - return os.path.basename(bcs) - def get_label(config): label = '' if config['output']['shared']['label']: - in_resolution = get_bcs_basename(config['input']['shared']['bcs_dir']) - out_resolution = get_bcs_basename(config['output']['shared']['bcs_dir']) + agrid = config['output']['shared']['agrid'] + ogrid = config['output']['shared']['ogrid'] + model = config['output']['shared']['omodel'] + stretch = config['output']['shared']['stretch'] + out_resolution = get_resolutions(agrid, ogrid, model, stretch) + + agrid = config['input']['shared']['agrid'] + ogrid = config['input']['shared']['ogrid'] + model = config['input']['shared']['omodel'] + stretch = config['input']['shared']['stretch'] + in_resolution = get_resolutions(agrid, ogrid, model, stretch) + in_bcv = config['input']['shared']['bc_version'] out_bcv = config['output']['shared']['bc_version'] label = '.' + in_bcv + '.' + in_resolution + \ @@ -361,15 +367,8 @@ def get_command_line_from_answers(answers): label = ' -lbl ' if answers["output:shared:label"] else "" - in_bcsdir = '' - default_bc = get_bcsdir(answers, "IN") - if not os.path.samefile(default_bc, answers.get("input:shared:bcs_dir").strip()) : - in_bcsdir = ' -in_bcsdir ' + answers.get("input:shared:bcs_dir") - - out_bcsdir = '' - default_bc = get_bcsdir(answers, "OUT") - if not os.path.samefile(default_bc, answers.get("output:shared:bcs_dir").strip()) : - out_bcsdir = ' -out_bcsdir ' + answers.get("output:shared:bcs_dir") + in_bc_base = ' -in_bc_base ' + answers.get("input:shared:bc_base") + out_bc_base = ' -out_bc_base ' + answers.get("output:shared:bc_base") out_stretch = '' if answers["output:shared:stretch"]: @@ -413,8 +412,8 @@ def get_command_line_from_answers(answers): bcvout + \ rst_dir + \ out_dir + \ - in_bcsdir + \ - out_bcsdir + \ + in_bc_base + \ + out_bc_base + \ out_stretch + \ in_stretch + \ catch_model + \ @@ -453,7 +452,7 @@ def get_config_from_answers(answers): return config -def get_grid_subdir(bcdir, agrid, ogrid, model, stretch): +def get_resolutions(agrid, ogrid, model, stretch): aname = '' oname = '' if (agrid[0].upper() == 'C'): @@ -475,73 +474,42 @@ def get_grid_subdir(bcdir, agrid, ogrid, model, stretch): if stretch: aname = aname + '-' + stretch - grid_directory = aname +'_' + oname + resolutions = aname +'_' + oname - if not os.path.isdir(os.path.join(bcdir,grid_directory)): - exit("cannot find grid subdirectory for agrid=" + agrid + " and ogrid=" + ogrid + " under " + bcdir) - - return grid_directory + return resolutions -def get_bc_base(): - base = {} - base['NAS'] = bc_base_NAS - base['NCCS'] = bc_base_NCCS +def get_default_bc_base(): cmd = 'uname -n' p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) (node, err) = p.communicate() p_status = p.wait() node = node.decode().split() node0 = node[0] - site ='NAS' discover = ['dirac', 'borg','warp', 'discover'] for node in discover: if node in node0: - site = 'NCCS' - break - return base[site] - -def get_bcsdir(x, opt): - bc_version = x.get('input:shared:bc_version') - if opt.upper() == "OUT": - bc_version = x.get('output:shared:bc_version') - bc_base = get_bc_base() - bcdir = bc_base+'/'+ bc_version+'/' - if not os.path.exists(bcdir): - exit("Cannot find BCs dir " + bcdir) - - return bcdir - -def get_topodir(bcsdir, agrid, ogrid, model, stretch): - k = bcsdir.find('/land/') - if k != -1 : - bcsdir = bcsdir[0:k] - bcs_geom = bcsdir+'/geometry/' - gridStr = get_grid_subdir(bcs_geom, agrid, ogrid, model,stretch) + return choices_bc_base[0] + return choices_bc_base[1] + +def get_topodir(bc_base, bc_version, agrid, ogrid, model, stretch): + gridStr = get_resolutions(agrid, ogrid, model,stretch) agrid_name = gridStr.split('_')[0] - bcs_topo = '' - if '/GM4/' in bcsdir : - bcs_topo = bcsdir+'/TOPO/TOPO_'+agrid_name + bc_topo = '' + if 'GM4' == bc_version: + bc_topo = bc_base+ '/'+ bc_version + '/TOPO/TOPO_'+agrid_name else: - bcs_topo = bcsdir+'/TOPO/TOPO_'+agrid_name + '/smoothed' - - return bcs_topo - -def get_landdir(bcsdir, agrid, ogrid, model, stretch): - if '/land/' in bcsdir: - return bcsdir - bcs_land = bcsdir+'/land/' - gridStr = get_grid_subdir(bcs_land, agrid, ogrid, model,stretch) - bcs_land = bcs_land + gridStr - return bcs_land - -def get_geomdir(bcsdir, agrid, ogrid, model, stretch): - if '/land/' in bcsdir: - bcs_geom = bcsdir.replace('/land/', '/geometry/') - return bcs_geom - bcs_geom = bcsdir+'/geometry/' - gridStr = get_grid_subdir(bcs_geom, agrid, ogrid, model,stretch) - bcs_geom = bcs_geom + gridStr - return bcs_geom + bc_topo = bc_base+ '/'+ bc_version + '/TOPO/TOPO_'+agrid_name + '/smoothed' + + return bc_topo + +def get_landdir(bc_base, bc_version, agrid, ogrid, model, stretch): + gridStr = get_resolutions(agrid, ogrid, model,stretch) + bc_land = bc_base+'/'+ bc_version+'/land/'+gridStr + return bc_land + +def get_geomdir(bc_base, bc_version, agrid, ogrid, model, stretch): + bc_geom = get_landdir(bc_base, bc_version, agrid, ogrid, model, stretch). replace('/land/', '/geometry/') + return bc_geom if __name__ == '__main__' : config = yaml_to_config('c24Toc12.yaml') diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index e0e6a6e..a4daf98 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -13,7 +13,8 @@ input: agrid: C180 # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ + bc_version: NL3 + bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles expid: JM_v10.22.2_L072_C180_AMIP ogrid: 1440X720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/amip_c180Toc90/inputs/ @@ -30,7 +31,8 @@ output: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ + bc_version: NL3 + bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles expid: C90CS_JM_v10.22.2_L072_C180_AMIP ogrid: C90 out_dir: $NOBACKUP/REMAP_TESTS/amip_c180Toc90/ diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index 98b0157..2819354 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -13,7 +13,8 @@ input: # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: C180 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ + bc_version: NL3 + bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles expid: Jason-3_4_NL_REAMIP_MERRA2_C180 ogrid: 1440X720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c180Toc360/inputs/ @@ -30,7 +31,8 @@ output: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ + bc_version: NL3 + bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles expid: C360CS_Jason-3_4_NL_REAMIP_MERRA2_C180 ogrid: C360 out_dir: $NOBACKUP/REMAP_TESTS/c360Toc24/ diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index 1c4fbce..a55a584 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -13,7 +13,8 @@ input: # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: C24 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ + bc_version: NL3 + bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles expid: '' ogrid: 360X180 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c24Toc12/inputs @@ -30,7 +31,8 @@ output: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ + bc_version: NL3 + bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles expid: hello ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/c24Toc12/ diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index 01cc671..384ace4 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -13,7 +13,8 @@ input: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ + bc_version: NL3 + bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles expid: x0046a ogrid: C360 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c360Toc24/inputs @@ -30,7 +31,8 @@ output: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ + bc_version: NL3 + bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles expid: C24c_x0046a ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/c360Toc24/ diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index 5c8b50a..3816c70 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -13,7 +13,8 @@ input: # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: C720 - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/ICA/ + bc_version: ICA + bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles expid: f522_fp ogrid: C720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/f522Toc360/inputs/ @@ -29,7 +30,8 @@ output: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bcs_dir: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles/NL3/ + bc_version: NL3 + bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles expid: remap ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/f522Toc360/ From 5dd4dfa220d881e9def3b861bbdd317d65784b71 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Tue, 14 Nov 2023 08:56:41 -0500 Subject: [PATCH 82/88] rm trailing space --- pre/remap_restart/tests/amip_c180Toc90.yaml | 2 +- pre/remap_restart/tests/c180Toc360.yaml | 2 +- pre/remap_restart/tests/c24Toc12.yaml | 2 +- pre/remap_restart/tests/c360Toc24.yaml | 2 +- pre/remap_restart/tests/f522Toc360.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index a4daf98..9645e57 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -57,6 +57,6 @@ output: slurm: # account will be set to user's primary account automatically by test_remap_restarts.py - account: + account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index 2819354..ec89e70 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -57,6 +57,6 @@ output: slurm: # account will be set to user's primary account automatically by test_remap_restarts.py - account: + account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index a55a584..f983cec 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -57,6 +57,6 @@ output: slurm: # account will be set to user's primary account automatically by test_remap_restarts.py - account: + account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index 384ace4..250ff0d 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -57,6 +57,6 @@ output: slurm: # account will be set to user's primary account automatically by test_remap_restarts.py - account: + account: qos: debug partition: compute diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index 3816c70..b9664e9 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -53,6 +53,6 @@ output: slurm: # account will be set to user's primary account automatically by test_remap_restarts.py - account: + account: qos: debug partition: compute From 6a9f5169b2e29b74ee4895060dedd2459b776ad1 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Tue, 14 Nov 2023 10:26:50 -0500 Subject: [PATCH 83/88] change order of bc_base and bc_version in yaml template and example/test yaml files (remap_params.tpl, test/*.yaml) --- pre/remap_restart/remap_params.tpl | 4 ++-- pre/remap_restart/tests/amip_c180Toc90.yaml | 4 ++-- pre/remap_restart/tests/c180Toc360.yaml | 4 ++-- pre/remap_restart/tests/c24Toc12.yaml | 4 ++-- pre/remap_restart/tests/c360Toc24.yaml | 4 ++-- pre/remap_restart/tests/f522Toc360.yaml | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index d1d108a..3a78a20 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -14,8 +14,8 @@ input: # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: - bc_version: none bc_base: + bc_version: none expid: ogrid: rst_dir: @@ -35,8 +35,8 @@ output: # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: - bc_version: none bc_base: + bc_version: none expid: ogrid: out_dir: diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index 9645e57..b3b7096 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -13,8 +13,8 @@ input: agrid: C180 # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bc_version: NL3 bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles + bc_version: NL3 expid: JM_v10.22.2_L072_C180_AMIP ogrid: 1440X720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/amip_c180Toc90/inputs/ @@ -31,8 +31,8 @@ output: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bc_version: NL3 bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles + bc_version: NL3 expid: C90CS_JM_v10.22.2_L072_C180_AMIP ogrid: C90 out_dir: $NOBACKUP/REMAP_TESTS/amip_c180Toc90/ diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index ec89e70..10b4c32 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -13,8 +13,8 @@ input: # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: C180 - bc_version: NL3 bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles + bc_version: NL3 expid: Jason-3_4_NL_REAMIP_MERRA2_C180 ogrid: 1440X720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c180Toc360/inputs/ @@ -31,8 +31,8 @@ output: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bc_version: NL3 bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles + bc_version: NL3 expid: C360CS_Jason-3_4_NL_REAMIP_MERRA2_C180 ogrid: C360 out_dir: $NOBACKUP/REMAP_TESTS/c360Toc24/ diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index f983cec..ed9ec24 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -13,8 +13,8 @@ input: # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: C24 - bc_version: NL3 bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles + bc_version: NL3 expid: '' ogrid: 360X180 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c24Toc12/inputs @@ -31,8 +31,8 @@ output: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bc_version: NL3 bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles + bc_version: NL3 expid: hello ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/c24Toc12/ diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index 250ff0d..e566fc9 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -13,8 +13,8 @@ input: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bc_version: NL3 bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles + bc_version: NL3 expid: x0046a ogrid: C360 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c360Toc24/inputs @@ -31,8 +31,8 @@ output: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bc_version: NL3 bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles + bc_version: NL3 expid: C24c_x0046a ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/c360Toc24/ diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index b9664e9..4c58aab 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -13,8 +13,8 @@ input: # (coupled) ocean model: data, MOM5, MOM6 omodel: data agrid: C720 - bc_version: ICA bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles + bc_version: ICA expid: f522_fp ogrid: C720 rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/f522Toc360/inputs/ @@ -30,8 +30,8 @@ output: stretch: false # (coupled) ocean model: data, MOM5, MOM6 omodel: data - bc_version: NL3 bc_base: /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles + bc_version: NL3 expid: remap ogrid: 360X180 out_dir: $NOBACKUP/REMAP_TESTS/f522Toc360/ From 588376877dcef5003d76afec20cfb73e728a0ee1 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Tue, 14 Nov 2023 10:34:20 -0500 Subject: [PATCH 84/88] minor edit to bc_base question (remap_questions.py, remap_utils.py) --- pre/remap_restart/remap_questions.py | 4 ++-- pre/remap_restart/remap_utils.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index afbf592..658cab0 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -304,7 +304,7 @@ def ask_questions(): "type": "path", "name": "input:shared:bc_base", "message": "\nEnter BCs base directory for input restarts: \n", - "when": lambda x: 'Customize ' in x.get('input:shared:bc_base'), + "when": lambda x: 'Custom ' in x.get('input:shared:bc_base'), }, # dummy (invisible) question to retrieve input:shared:bc_base { @@ -327,7 +327,7 @@ def ask_questions(): "type": "path", "name": "output:shared:bc_base", "message": "\nEnter BCs base directory for new restarts: \n", - "when": lambda x: 'Customize ' in x.get('output:shared:bc_base'), + "when": lambda x: 'Custom ' in x.get('output:shared:bc_base'), }, # dummy (invisible) question to retrieve output:shared:bc_base { diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 7691c5b..3fdf31f 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -17,9 +17,9 @@ # top-level directory for BCs (machine-dependent) -choices_bc_base =[ "DISCOVER : /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles", - "NAS : /nobackup/gmao_SIteam/ModelData/bcs_shared/fvInput/ExtData/esm/tiles", - "Customize : " ] +choices_bc_base =[ "NCCS/Discover : /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles", + "NAS : /nobackup/gmao_SIteam/ModelData/bcs_shared/fvInput/ExtData/esm/tiles", + "Custom " ] # define "choices", "message" strings, and "validate" lists that are used multiple times # (and related definitions, even if they are used just once). From 4dc80c0b09df40a2325ecea57269b57b7ad4a235 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Tue, 14 Nov 2023 10:40:23 -0500 Subject: [PATCH 85/88] minor edits in comments and white-space changes (remap_catchANDcn.py, remap_command_line.py, remap_questions.py, remap_utils.py) --- pre/remap_restart/remap_catchANDcn.py | 26 ++++++++++++------------- pre/remap_restart/remap_command_line.py | 4 ++-- pre/remap_restart/remap_questions.py | 4 ++-- pre/remap_restart/remap_utils.py | 4 ++-- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 05ddaa9..c374f49 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -47,25 +47,25 @@ def remap(self): print("\nRemapping " + model + ".....\n") - cwdir = os.getcwd() - bindir = os.path.dirname(os.path.realpath(__file__)) - in_bc_base = config['input']['shared']['bc_base'] - in_bc_version = config['input']['shared']['bc_version'] - out_bc_base = config['output']['shared']['bc_base'] - out_bc_version= config['output']['shared']['bc_version'] - out_dir = config['output']['shared']['out_dir'] - expid = config['output']['shared']['expid'] - in_wemin = config['input']['surface']['wemin'] - out_wemin = config['output']['surface']['wemin'] - surflay = config['output']['surface']['surflay'] - in_tilefile = config['input']['surface']['catch_tilefile'] + cwdir = os.getcwd() + bindir = os.path.dirname(os.path.realpath(__file__)) + in_bc_base = config['input']['shared']['bc_base'] + in_bc_version = config['input']['shared']['bc_version'] + out_bc_base = config['output']['shared']['bc_base'] + out_bc_version = config['output']['shared']['bc_version'] + out_dir = config['output']['shared']['out_dir'] + expid = config['output']['shared']['expid'] + in_wemin = config['input']['surface']['wemin'] + out_wemin = config['output']['surface']['wemin'] + surflay = config['output']['surface']['surflay'] + in_tilefile = config['input']['surface']['catch_tilefile'] if not in_tilefile : agrid = config['input']['shared']['agrid'] ogrid = config['input']['shared']['ogrid'] omodel = config['input']['shared']['omodel'] stretch = config['input']['shared']['stretch'] - bc_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid, ogrid, omodel, stretch) + bc_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid, ogrid, omodel, stretch) in_tilefile = glob.glob(bc_geomdir + '/*.til')[0] agrid = config['output']['shared']['agrid'] diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 0c0c883..2a7693f 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -68,8 +68,8 @@ def parse_args(program_description): p_command.add_argument('-np', action='store_true', help="No prompt. Overwrite config files without prompting questions") p_command.add_argument('-lbl', action='store_true', help="Label output restarts with bc_versions and resolutions") p_command.add_argument('-noagcm_import_rst', action='store_true', help="Do not remap agcm_import_rst file") - p_command.add_argument('-in_bc_base', default="", help="Boundary conditions base directory (without bc_version and resolution infomation) for input restart") - p_command.add_argument('-out_bc_base', default="", help="Boundary conditions base directory (without bc_version and resolution infomation) for new restarts") + p_command.add_argument('-in_bc_base', default="", help="Boundary conditions base dir (w/o bc_version and resolution info) for input restarts") + p_command.add_argument('-out_bc_base',default="", help="Boundary conditions base dir (w/o bc_version and resolution info) for new restarts") p_command.add_argument('-zoom', help= "Zoom parameter (search radius) for input surface restarts") p_command.add_argument('-qos', default="debug", help="SLURM quality-of-service", choices=['debug', 'allnccs']) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 658cab0..083c83d 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -311,7 +311,7 @@ def ask_questions(): "type": "text", "name": "input:shared:bc_base", "message": "retrieve and echo bcs", - # always return false, so questions never shows but changes ogrid + # always return false, so questions never shows "when": lambda x: echo_bcs(x, 'input') }, @@ -334,7 +334,7 @@ def ask_questions(): "type": "text", "name": "output:shared:bc_base", "message": "retrieve and echo bcs", - # always return false, so questions never shows but changes ogrid + # always return false, so questions never shows "when": lambda x: echo_bcs(x, 'output') }, diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 3fdf31f..a7970ea 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -496,9 +496,9 @@ def get_topodir(bc_base, bc_version, agrid, ogrid, model, stretch): agrid_name = gridStr.split('_')[0] bc_topo = '' if 'GM4' == bc_version: - bc_topo = bc_base+ '/'+ bc_version + '/TOPO/TOPO_'+agrid_name + bc_topo = bc_base + '/' + bc_version + '/TOPO/TOPO_' + agrid_name else: - bc_topo = bc_base+ '/'+ bc_version + '/TOPO/TOPO_'+agrid_name + '/smoothed' + bc_topo = bc_base + '/' + bc_version + '/TOPO/TOPO_' + agrid_name + '/smoothed' return bc_topo From bea49ba7780bd725ffaab601a56cecbd816b366b Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Tue, 14 Nov 2023 10:47:33 -0500 Subject: [PATCH 86/88] clarified question about using archived MERRA-2 restarts (remap_questions.py) --- pre/remap_restart/remap_questions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 083c83d..b383900 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -80,7 +80,7 @@ def ask_questions(): { "type": "confirm", "name": "input:shared:MERRA-2", - "message": "Remap archived MERRA-2 restarts?\n", + "message": "Remap archived MERRA-2 restarts? (NCCS/Discover only; elsewhere, select 'N' and complete full config; requires nc4 restarts.)\n", "default": False, }, { From b3c268224de2cfbeac44e9f24e194cc8626c40ce Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Tue, 14 Nov 2023 15:09:19 -0500 Subject: [PATCH 87/88] add key 'air' --- pre/remap_restart/remap_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 7691c5b..5c7b7bb 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -435,6 +435,7 @@ def get_command_line_from_answers(answers): def get_config_from_answers(answers): config = {} config['input'] = {} + config['input']['air'] = {} config['input']['shared'] = {} config['input']['surface'] = {} config['output'] = {} From 39b7dd73d5613715e8c6c21da3276c4ec34bd742 Mon Sep 17 00:00:00 2001 From: Scott Rabenhorst <53346946+sdrabenh@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:24:58 -0500 Subject: [PATCH 88/88] Update CHANGELOG.md for next release --- CHANGELOG.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29116ef..2c9e84b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,21 +9,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +### Changed + +### Fixed + +### Removed + +### Deprecated + +## [2.0.4] - 2023-11-17 + +### Added + - Add CI step to make PR to MAPL3 on push to `main` ### Changed -- Add command_line options. More items like label, altbcs are added to remap_params.tpl -- Updates paths to the legacy bcs data by pointing to the new "bcs_shared" directory in the GMAO project space. +- Add/update command_line options: more items such as label and altbcs were added to `remap_params.tpl` +- Updated paths to the legacy bcs data by pointing to the new "bcs_shared" directory in the GMAO project space. - Support for new boundary conditions package output layout -### Fixed - ### Removed -- Use of haswell nodes on NCCS machines - -### Deprecated +- Use of haswell nodes on NCCS machines for `cube_BCs.pl` ## [2.0.3] - 2023-08-24