Merge pull request #276 from GEOS-ESM/feature/wjiang/improve_merra2

remap_restarts.py: add command line option; move temporary directory to out_dir for merra2; add more test cases

.github/workflows/changelog-enforcer.yml

@@ -4,7 +4,7 @@ on:
       types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
 
 jobs:
-  # Enforces the update of a changelog file on every pull request 
+  # Enforces the update of a changelog file on every pull request
   changelog:
     runs-on: ubuntu-latest
     steps:

.github/workflows/validate_yaml_files.yml

@@ -0,0 +1,31 @@
+---
+
+# Based on code from https://github.com/marketplace/actions/yaml-lint
+
+name: Yaml Lint
+
+on:
+  pull_request:
+      types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+# This validation is equivalent to running on the command line:
+#   yamllint -d relaxed --no-warnings
+# and is controlled by the .yamllint.yml file
+jobs:
+  validate-YAML:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - id: yaml-lint
+        name: yaml-lint
+        uses: ibiqlik/action-yamllint@v3
+        with:
+          no_warnings: true
+          format: colored
+          config_file: .yamllint.yml
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: yamllint-logfile
+          path: ${{ steps.yaml-lint.outputs.logfile }}

.yamllint.yml

@@ -0,0 +1,29 @@
+---
+
+extends: default
+
+rules:
+  braces:
+    level: warning
+    max-spaces-inside: 1
+  brackets:
+    level: warning
+    max-spaces-inside: 1
+  colons:
+    level: warning
+  commas:
+    level: warning
+  comments: disable
+  comments-indentation: disable
+  document-start: disable
+  empty-lines:
+    level: warning
+  hyphens:
+    level: warning
+  indentation:
+    level: warning
+    indent-sequences: consistent
+  line-length:
+    level: warning
+    allow-non-breakable-inline-mappings: true
+  truthy: disable

CHANGELOG.md

@@ -9,8 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add YAML validator
+
 ### Changed
 
+- Generated command line to run the program
+- changed the location of the temporary folder for remap_restarts MERRA-2 case
+- Added new remap tests
+  - amip_c180Toc90
+  - c180Toc360
+  - c360Toc24
+  - s2sv3Toc12MOM6
+
 ### Fixed
 
 ### Removed
@@ -44,7 +54,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- Added statsNx.rc for screen level variable fstats 
+- Added statsNx.rc for screen level variable fstats
 
 ### Changed
 

GEOS_Util/post/remap_restart/remap_analysis.py

@@ -19,6 +19,7 @@
 class analysis(remap_base):
   def __init__(self, **configs):
      super().__init__(**configs)
+     self.copy_merra2()
 
   def remap(self):
      config = self.config
@@ -105,6 +106,8 @@ def remap(self):
      print( "cd " + cwdir)
      os.chdir(cwdir)
 
+     self.remove_merra2()
+
   def get_grid_kind(this, grid):
      hgrd = {}
      hgrd['C12']   = 'a'
@@ -132,6 +135,70 @@ def find_analysis(self):
      analysis_in = bkgs + sfcs + traks + anasat
      return list(dict.fromkeys(analysis_in))
 
+  def copy_merra2(self):
+    if not self.config['input']['shared']['MERRA-2']:
+      return
+    bkg = self.config['output']['analysis']['bkg']
+    if ( not bkg ): return
+
+    expid = self.config['input']['shared']['expid']
+    yyyymmddhh_ = str(self.config['input']['shared']['yyyymmddhh'])
+    yyyy_ = yyyymmddhh_[0:4]
+    mm_   = yyyymmddhh_[4:6]
+    dd_   = yyyymmddhh_[6:8]
+    hh_   = yyyymmddhh_[8:10]
+
+    merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/'
+    rst_dir = self.config['input']['shared']['rst_dir'] + '/'
+    os.makedirs(rst_dir, exist_ok = True)
+    print(' Copy MERRA-2 analysis files \n from \n    ' + merra_2_rst_dir + '\n to\n    '+ rst_dir +'\n')
+
+    rst_time = datetime(year=int(yyyy_), month=int(mm_), day=int(dd_), hour = int(hh_))
+
+    anafiles=[]
+    for h in [3,4,5,6,7,8,9]:
+       delt = timedelta(hours = h-3)
+       new_time = rst_time + delt
+       yyyy = "Y"+str(new_time.year)
+       mm   = 'M%02d'%new_time.month
+       ymd  = '%04d%02d%02d'%(new_time.year,new_time.month, new_time.day)
+       hh   = '%02d'%h
+       newhh= '%02d'%new_time.hour
+       m2_rst_dir = merra_2_rst_dir.replace('Y'+yyyy_,yyyy).replace('M'+mm_,mm)
+       # bkg files
+       for ftype in ['sfc', 'eta']:
+          fname = expid + '.bkg'+hh+'_'+ftype+'_rst.'+ymd+'_'+newhh+'z.nc4'
+          f = m2_rst_dir+'/'+fname
+          if(os.path.isfile(f)):
+             anafiles.append(f)
+          else:
+             print('Warning: Cannot find '+f)
+
+       # gaas_bkg_sfc files
+       if (h==6 or h==9):
+          fname = expid+'.gaas_bkg_sfc_rst.'+ymd+'_'+newhh+'z.nc4'
+          f = m2_rst_dir+'/'+fname
+          if (os.path.isfile(f)):
+            anafiles.append(f)
+          else:
+            print('Warning: Cannot find '+f)
+    # trak.GDA.rst file
+    delt = timedelta(hours = 3)
+    new_time = rst_time - delt
+    yyyy = "Y"+str(new_time.year)
+    mm   = 'M%02d'%new_time.month
+    ymdh = '%04d%02d%02d%02d'%(new_time.year, new_time.month, new_time.day, new_time.hour)
+    m2_rst_dir = merra_2_rst_dir.replace('Y'+yyyy_,yyyy).replace('M'+mm_,mm)
+    fname = expid+'.trak.GDA.rst.'+ymdh+'z.txt'
+    f = m2_rst_dir+'/'+fname
+    if (os.path.isfile(f)): anafiles.append(f)
+
+    for f in anafiles:
+      fname    = os.path.basename(f)
+      f_tmp = rst_dir+'/'+fname
+      print("Copy file "+f +" to " + rst_dir)
+      shutil.copy(f,f_tmp)
+
 if __name__ == '__main__' :
    ana = analysis(params_file='remap_params.yaml')
    ana.remap()

GEOS_Util/post/remap_restart/remap_base.py

@@ -3,6 +3,7 @@
 import os
 import ruamel.yaml
 import shutil
+import subprocess
 
 class remap_base(object):
   def __init__(self, **configs):
@@ -28,3 +29,7 @@ def __init__(self, **configs):
           out_dir    = self.config['output']['shared']['out_dir']
           if not os.path.exists(out_dir) : os.makedirs(out_dir)
         break
+  def remove_merra2(self):
+    if self.config['input']['shared']['MERRA-2']:
+      print(" remove temporary folder that contains MERRA-2 archived files ... \n")
+      subprocess.call(['/bin/rm', '-rf', self.config['input']['shared']['rst_dir']])

GEOS_Util/post/remap_restart/remap_catchANDcn.py

@@ -12,6 +12,7 @@
 class catchANDcn(remap_base):
   def __init__(self, **configs):
      super().__init__(**configs)
+     self.copy_merra2()
 
   def remap(self):
      if not self.config['output']['surface']['remap_catch']:
@@ -42,8 +43,12 @@ def remap(self):
      in_wemin   = config['input']['surface']['wemin']
      out_wemin  = config['output']['surface']['wemin']
      surflay    = config['output']['surface']['surflay']
-     in_tilefile  = glob.glob(in_bcsdir+ '/*-Pfafstetter.til')[0]
-     out_tilefile = glob.glob(out_bcsdir+ '/*-Pfafstetter.til')[0]
+     in_tilefile = config['input']['surface']['catch_tilefile']
+     if not in_tilefile :
+        in_tilefile  = glob.glob(in_bcsdir+ '/*-Pfafstetter.til')[0]
+     out_tilefile = config['output']['surface']['catch_tilefile']
+     if not out_tilefile :
+        out_tilefile = glob.glob(out_bcsdir+ '/*-Pfafstetter.til')[0]
      account    = config['slurm']['account']
      # even the input is binary, the output si nc4
      suffix     = time+'z.nc4'
@@ -129,6 +134,32 @@ def remap(self):
      print( "cd " + cwdir)
      os.chdir(cwdir)
 
+     self.remove_merra2()
+
+  def copy_merra2(self):
+    if not self.config['input']['shared']['MERRA-2']:
+      return
+
+    expid = self.config['input']['shared']['expid']
+    yyyymmddhh_ = str(self.config['input']['shared']['yyyymmddhh'])
+    yyyy_ = yyyymmddhh_[0:4]
+    mm_   = yyyymmddhh_[4:6]
+    dd_   = yyyymmddhh_[6:8]
+    hh_   = yyyymmddhh_[8:10]
+
+    suffix = yyyymmddhh_[0:8]+'_'+ hh_ + 'z.bin'
+    merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/'
+    rst_dir = self.config['input']['shared']['rst_dir'] + '/'
+    os.makedirs(rst_dir, exist_ok = True)
+    print(' Copy MERRA-2 catchment Restart \n from \n    ' + merra_2_rst_dir + '\n to\n    '+ rst_dir +'\n')
+
+    f =  merra_2_rst_dir +  expid+'.catch_internal_rst.'    + suffix
+
+    fname = os.path.basename(f)
+    dest = rst_dir + '/'+fname
+    print("Copy file "+f +" to " + rst_dir)
+    shutil.copy(f, dest)
+
 if __name__ == '__main__' :
    catch = catchANDcn(params_file='remap_params.yaml')
    catch.remap()

GEOS_Util/post/remap_restart/remap_lake_landice_saltwater.py

@@ -11,6 +11,7 @@
 class lake_landice_saltwater(remap_base):
   def __init__(self, **configs):
      super().__init__(**configs)
+     self.copy_merra2()
 
   def remap(self):
      if not self.config['output']['surface']['remap_water']:
@@ -138,6 +139,8 @@ def remap(self):
      print('cd ' + cwdir)
      os.chdir(cwdir)
 
+     self.remove_merra2()
+
   def find_rst(self):
      surf_restarts =[
                  "route_internal_rst"       ,
@@ -164,6 +167,33 @@ def find_rst(self):
 
      return restarts_in
 
+  def copy_merra2(self):
+    if not self.config['input']['shared']['MERRA-2']:
+      return
+
+    expid = self.config['input']['shared']['expid']
+    yyyymmddhh_ = str(self.config['input']['shared']['yyyymmddhh'])
+    yyyy_ = yyyymmddhh_[0:4]
+    mm_   = yyyymmddhh_[4:6]
+    dd_   = yyyymmddhh_[6:8]
+    hh_   = yyyymmddhh_[8:10]
+
+    suffix = yyyymmddhh_[0:8]+'_'+ hh_ + 'z.bin'
+    merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/'
+    rst_dir = self.config['input']['shared']['rst_dir'] + '/'
+    os.makedirs(rst_dir, exist_ok = True)
+    print(' Copy MERRA-2 surface restarts \n from \n    ' + merra_2_rst_dir + '\n to\n    '+ rst_dir +'\n')
+
+    surfin = [ merra_2_rst_dir +  expid+'.lake_internal_rst.'     + suffix,
+               merra_2_rst_dir +  expid+'.landice_internal_rst.'  + suffix,
+               merra_2_rst_dir +  expid+'.saltwater_internal_rst.'+ suffix]
+
+    for f in surfin :
+       fname = os.path.basename(f)
+       dest = rst_dir + '/'+fname
+       print("Copy file "+f +" to " + rst_dir)
+       shutil.copy(f, dest)
+
 if __name__ == '__main__' :
    lls = lake_landice_saltwater(params_file='remap_params.yaml')
    lls.remap()