From 2e1cbe985bafbcc52a21d422dac3989e2f595af0 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Wed, 6 Mar 2024 17:00:51 +0000 Subject: [PATCH 01/27] Now can read different model names --- ww3tools/modelBuoy_collocation.py | 21 +- ww3tools/wread.py | 820 +++++++++++------------------- 2 files changed, 290 insertions(+), 551 deletions(-) mode change 100755 => 100644 ww3tools/wread.py diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index 3002b46..a9f794f 100755 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -108,7 +108,7 @@ # Paths # ndbcp="/data/buoys/NDBC/wparam" -ndbcp="/work/noaa/marine/ricardo.campos/data/buoys/NDBC/ncformat/wparam" +ndbcp="/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" # Copernicus buoys # copernp="/data/buoys/Copernicus/wtimeseries" copernp="/work/noaa/marine/ricardo.campos/data/buoys/Copernicus/wtimeseries" @@ -122,7 +122,7 @@ # # import os; os.system("ls -d $PWD/*tab.nc > ww3list.txt &") wlist=np.atleast_1d(np.loadtxt(sys.argv[1],dtype=str)) ftag=str(sys.argv[1]).split('list')[1].split('.txt')[0] - print(' Reading ww3 list '+str(sys.argv[1])) + print(' Reading ww3 list '+ str(sys.argv[1])) print(' Tag '+ftag) if len(sys.argv) >= 3: forecastds=int(sys.argv[2]) @@ -168,6 +168,7 @@ if str(wlist[i]).split('/')[-1].split('.')[-1]=='station_tar': result = wread.bull_tar(wlist[i]) + print("Result keys:", result.keys()) at=result['time'] fcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') if i==0: @@ -282,14 +283,6 @@ stname=np.append(stname,astname); del astname - funits=f.variables['time'].units - if str(funits).split(' ')[0] == 'seconds': - tincr=1 - elif str(funits).split(' ')[0] == 'hours': - tincr=3600 - elif str(funits).split(' ')[0] == 'days': - tincr=24*3600 - ahs = np.array(f.variables['hs'][:,:]).T if 'th1m' in f.variables.keys(): @@ -320,9 +313,7 @@ else: atm = np.array(np.copy(ahs*nan)) - ftunits=str(f.variables['time'].units).split('since')[1][1::].replace('T',' ').replace('+00:00','') - at = np.array(f.variables['time'][:]*tincr + timegm( strptime(ftunits,'%Y-%m-%d %H:%M:%S') )).astype('double') - + at = np.array(f.variables['time'][:]*24*3600 + timegm( strptime(str(f.variables['time'].units).split(' ')[2][0:4]+'01010000', '%Y%m%d%H%M') )).astype('double') f.close(); del f fcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') if t==0: @@ -368,7 +359,7 @@ ahs=[] try: - ahs=[];atm=[];atp=[];adm=[];atime=[] + ahs=[];atm=[];adm=[];atime=[] for y in yrange: f=nc.Dataset(ndbcp+"/"+stname[b]+"h"+repr(y)+".nc") @@ -385,7 +376,7 @@ atm = np.array(np.copy(ahs*nan)) if 'dominant_wpd' in f.variables.keys(): - atp = np.append(atp,f.variables['dominant_wpd'][:,0,0]) + atp = np.append(atm,f.variables['dominant_wpd'][:,0,0]) else: atp = np.array(np.copy(ahs*nan)) diff --git a/ww3tools/wread.py b/ww3tools/wread.py old mode 100755 new mode 100644 index f581c6b..05551a3 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -7,10 +7,9 @@ VERSION AND LAST UPDATE: v1.0 04/04/2022 v1.1 01/06/2023 - v1.2 10/12/2023 PURPOSE: - Group of python functions to Read Wave data: + Group of python functions to Read Wave data: WAVEWATCHIII results, and NDBC and Copernicus buoys. Prefix meaning: tseriesnc = time series (table of integrated parameters versus time). @@ -24,12 +23,10 @@ USAGE: functions - readconfig mask cyclonemap tseriesnc_ndbc tseriesnc_copernicus - aodn_altimeter tseriesnc_ww3 bull bull_tar @@ -49,9 +46,6 @@ AUTHOR and DATE: 04/04/2022: Ricardo M. Campos, first version. 01/06/2023: Ricardo M. Campos, new file formats added. - 10/12/2023: Ricardo M. Campos & Maryam Mohammadpour, new function readconfig - to read the configuration file ww3tools.yaml. And a new function aodn_altimeter - to read AODN altimeter data. PERSON OF CONTACT: Ricardo M Campos: ricardo.campos@noaa.gov @@ -59,8 +53,8 @@ """ import matplotlib +# matplotlib.use('Agg') # for backend plots, not for rendering in a window import time -import timeit from time import strptime from calendar import timegm import pandas as pd @@ -68,62 +62,16 @@ import netCDF4 as nc import numpy as np from pylab import * -import yaml -import re -import os +import matplotlib.pyplot as plt import sys +import pandas as pd from matplotlib import ticker # import pickle import sys import warnings; warnings.filterwarnings("ignore") -def readconfig(fname): - """ - Reads the configuration file ww3tools.yaml and returns a dictionary - containing all the information in the file. - User can enter the file name 'ww3tools.yaml' or the name including - the full path '/home/user/ww3tools.yaml' - """ - - try: - with open(fname, 'r') as file: - wconfig = yaml.safe_load(file) - except: - raise ValueError("wproc.readconfig: ww3tools.yaml not found.") - else: - - # paths - if "path_out" in wconfig: - if str(wconfig['path_out']) != '/': - wconfig['path_out']=str(wconfig['path_out'])+"/" - else: - wconfig['path_out']=str(os.getcwd())+"/" - - if "path_alt" in wconfig: - if str(wconfig['path_alt']) != '/': - wconfig['path_alt']=str(wconfig['path_alt'])+"/" - else: - wconfig['path_alt']=str(os.getcwd())+"/" - print("Warning: path_alt not found, using local directory "+wconfig['path_alt']) - - if "path_ndbc" in wconfig: - if str(wconfig['path_ndbc']) != '/': - wconfig['path_ndbc']=str(wconfig['path_ndbc'])+"/" - else: - wconfig['path_ndbc']=str(os.getcwd())+"/" - print("Warning: path_ndbc not found, using local directory "+wconfig['path_ndbc']) - - if "path_copernicus" in wconfig: - if str(wconfig['path_copernicus']) != '/': - wconfig['path_copernicus']=str(wconfig['path_copernicus'])+"/" - else: - wconfig['path_copernicus']=str(os.getcwd())+"/" - print("Warning: path_copernicus not found, using local directory "+wconfig['path_copernicus']) - - # returns a dictionary containing the information given by ww3tools.yaml - return wconfig - +# FIELDS def mask(*args): ''' @@ -147,11 +95,11 @@ def mask(*args): if 'distcoast' in f.variables.keys(): result['distcoast'] = np.array(f.variables['distcoast'][:,:]) if 'depth' in f.variables.keys(): - result['depth'] = np.array(f.variables['depth'][:,:]) + result['depth'] = np.array(f.variables['depth'][:,:]) if 'GlobalOceansSeas' in f.variables.keys(): - result['GlobalOceansSeas'] = np.array(f.variables['GlobalOceansSeas'][:,:]) + result['GlobalOceansSeas'] = np.array(f.variables['GlobalOceansSeas'][:,:]) if 'HighSeasMarineZones' in f.variables.keys(): - result['HighSeasMarineZones'] = np.array(f.variables['HighSeasMarineZones'][:,:]) + result['HighSeasMarineZones'] = np.array(f.variables['HighSeasMarineZones'][:,:]) if 'names_GlobalOceansSeas' in f.variables.keys(): result['names_GlobalOceansSeas'] = f.variables['names_GlobalOceansSeas'][:] if 'names_HighSeasMarineZones' in f.variables.keys(): @@ -178,7 +126,7 @@ def cyclonemap(*args): f=nc.MFDataset(fname, aggdim='time') at=f.variables['time'][:]; adate=[] for j in range(0,at.shape[0]): - adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) + adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) # -------- # build dictionary result={'latitude':np.array(f.variables['lat'][:]),'longitude':np.array(f.variables['lon'][:]), @@ -193,18 +141,20 @@ def cyclonemap(*args): del result -# ================= OBSERVATIONS ================= -# --- Buoys --- +# TIME-SERIES + # Observations NDBC, netcdf format -def tseriesnc_ndbc(fname=None,anh=None): +def tseriesnc_ndbc(*args): ''' Observations NDBC, time series/table, netcdf format - Input: file name (example: 46047h2016.nc), and anemometer height (optional) - Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, - and arrays sst,mslp,dwp,tmp,gst(10-m height),wsp(10-m height),wdir,hs,tm,tp,dm + Input: file name (example: 46047h2016.nc) + Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, + and arrays sst,mslp,dwp,tmp,gst,wsp,wdir,hs,tm,tp,dm ''' - if fname==None: - raise ValueError("NDBC file name must be informed.") + if len(args) == 1: + fname=str(args[0]) + elif len(args) > 1: + sys.exit(' Too many inputs') try: ds = xr.open_dataset(fname); f=nc.Dataset(fname) @@ -218,29 +168,27 @@ def tseriesnc_ndbc(fname=None,anh=None): bmslp = ds['air_pressure'].values[:,0,0] bdwp = ds['dewpt_temperature'].values[:,0,0] btmp = ds['air_temperature'].values[:,0,0] - bgst = ds['gust'].values[:,0,0] + bgst = ds['gust'].values[:,0,0] if 'wind_spd' in ds.keys(): bwsp = ds['wind_spd'].values[:,0,0] - - if anh==None: - try: - from urllib.request import urlopen - url = "https://www.ndbc.noaa.gov/station_page.php?station="+str(fname).split('/')[-1].split('h')[0] - page = urlopen(url) - html_bytes = page.read() - html = html_bytes.decode("utf-8") - except: - anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. - print('Information about the Anemometer height, for wind speed conversion to 10m, could not be obtained.') + try: + from urllib.request import urlopen + url = "https://www.ndbc.noaa.gov/station_page.php?station="+str(fname).split('/')[-1].split('h')[0] + page = urlopen(url) + html_bytes = page.read() + html = html_bytes.decode("utf-8") + except: + anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. + print('Information about the Anemometer height, for wind speed conversion to 10m, could not be obtained.') + else: + if "Anemometer height" in html: + anh=float(html.split('Anemometer height')[1][0:15].split(':')[1].split('m')[0]) else: - if "Anemometer height" in html: - anh=np.float(html.split('Anemometer height')[1][0:15].split(':')[1].split('m')[0]) - else: - print('Information about the Anemometer height, for wind speed conversion to 10m, could not be found.') - anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. + print('Information about the Anemometer height, for wind speed conversion to 10m, could not be found.') + anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. - del url,page,html_bytes,html + del url,page,html_bytes,html # convert wind speed to 10 meters (DNVGL C-205 Table 2-1, confirmed by https://onlinelibrary.wiley.com/doi/pdf/10.1002/er.6382) bwsp = np.copy(((10./anh)**(0.12)) * bwsp) @@ -266,8 +214,8 @@ def tseriesnc_ndbc(fname=None,anh=None): result={'latitude':np.array(ds['latitude'].values[:]),'longitude':np.array(ds['longitude'].values[:]), 'time':btime,'date':ds['time'].values[:], 'sst':bsst, 'mslp':bmslp, 'dewpt_temp':bdwp, - 'air_temp':btmp, 'gust':bgst, 'wind_spd':bwsp, - 'wind_dir':bwdir, 'hs':bhs, 'tm':btm, + 'air_temp':btmp, 'gust':bgst, 'wind_spd':bwsp, + 'wind_dir':bwdir, 'hs':bhs, 'tm':btm, 'tp':btp, 'dm':bdm, 'tm':btm} return result @@ -275,15 +223,17 @@ def tseriesnc_ndbc(fname=None,anh=None): del ds,btime,bsst,bmslp,bdwp,btmp,bgst,bwsp,bwdir,bhs,btm,btp,bdm # Observations NDBC, text format -def tseriestxt_ndbc(fname=None,anh=None): +def tseriestxt_ndbc(*args): ''' Observations NDBC, time series/table, stdmet format - Input: file name (example: NDBC_historical_stdmet_41004.txt), and anemometer height (optional) - Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, + Input: file name (example: NDBC_historical_stdmet_41004.txt) + Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, and arrays sst,mslp,dwp,tmp,gst,wsp,wdir,hs,tm,tp,dm ''' - if fname==None: - raise ValueError("NDBC file name must be informed.") + if len(args) == 1: + fname=str(args[0]) + elif len(args) > 1: + sys.exit(' Too many inputs') try: ds = pd.read_csv(fname,comment='#',delimiter=r"\s+") @@ -296,7 +246,7 @@ def tseriestxt_ndbc(fname=None,anh=None): ds['date']=pd.to_datetime(ds['date'],format='%Y %m %d %H') for i in range(0,btime.shape[0]): - btime[i]=double(ds['date'][i].timestamp()) + btime[i]=double(ds['date'][i].timestamp()) except: sys.exit(" Cannot open "+fname) @@ -315,7 +265,6 @@ def tseriestxt_ndbc(fname=None,anh=None): if 'WSPD' in ds.keys(): bwsp=np.array(ds['WSPD'].values[:]).astype('float') - try: from urllib.request import urlopen url = "https://www.ndbc.noaa.gov/station_page.php?station="+str(fname).split('/')[-1].split('h')[0].split('_')[-1] @@ -324,29 +273,26 @@ def tseriestxt_ndbc(fname=None,anh=None): html = html_bytes.decode("utf-8") auxlatlon=html.split('payload')[1][16:33] if 'S' in auxlatlon: - blat=-np.float(auxlatlon[0:6]) + blat=-float(auxlatlon[0:6]) else: - blat=np.float(auxlatlon[0:6]) + blat=float(auxlatlon[0:6]) if 'W' in auxlatlon: - blon=-np.float(auxlatlon[8:16]) + blon=-float(auxlatlon[8:16]) else: - blon=np.float(auxlatlon[8:16]) + blon=float(auxlatlon[8:16]) except: - if anh==None: - anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. - print('Information about the Anemometer height, for wind speed conversion to 10m, could not be found. Assuming 4.0 meters.') - + anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. blat=np.nan; blon=np.nan - print('Information of Lat and Lon could not be obtained.') + print('Information of Lat, Lon, and Anemometer height could not be obtained.') else: - if anh==None: - if "Anemometer height" in html: - anh=np.float(html.split('Anemometer height')[1][0:15].split(':')[1].split('m')[0]) - else: - print('Information about the Anemometer height, for wind speed conversion to 10m, could not be found. Assuming 4.0 meters.') - anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. + + if "Anemometer height" in html: + anh=float(html.split('Anemometer height')[1][0:15].split(':')[1].split('m')[0]) + else: + print('Information about the Anemometer height, for wind speed conversion to 10m, could not be found.') + anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. del url,page,html_bytes,html @@ -370,19 +316,20 @@ def tseriestxt_ndbc(fname=None,anh=None): result={'latitude':blat,'longitude':blon, 'time':btime,'date':ds['date'].values[:], 'sst':bsst, 'mslp':bmslp, 'dewpt_temp':bdwp, - 'air_temp':btmp, 'gust':bgst, 'wind_spd':bwsp, - 'wind_dir':bwdir, 'hs':bhs, 'tm':btm, + 'air_temp':btmp, 'gust':bgst, 'wind_spd':bwsp, + 'wind_dir':bwdir, 'hs':bhs, 'tm':btm, 'tp':btp, 'dm':bdm, 'tm':btm} return result del ds,btime,blat,blon,bsst,bmslp,bdwp,btmp,bgst,bwsp,bwdir,bhs,btm,btp,bdm + # Observations Copernicus, netcdf format def tseriesnc_copernicus(*args): ''' Observations NDBC, time series/table, netcdf format Input: file name (example: 46047h2016.nc) - Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, + Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, and arrays with the environmental variables available. ''' if len(args) == 1: @@ -403,8 +350,8 @@ def tseriesnc_copernicus(*args): result={'latitude':np.array(blat),'longitude':np.array(blon), 'time':btime,'date':ds['TIME'].values[:]} - if 'DEPH' in ds.keys(): - bdepth = np.nanmean(ds['DEPH'].values[:,:],axis=1) # Depth + if 'DEPH' in ds.keys(): + bdepth = np.nanmean(ds['DEPH'].values[:,:],axis=1) # Depth result['depth']=np.array(bdepth) if 'VHM0' in ds.keys(): @@ -416,7 +363,7 @@ def tseriesnc_copernicus(*args): bhs[(bhs<0)|(bhs>30)]=np.nan result['hs']=np.array(bhs) - if 'VAVH' in ds.keys(): + if 'VAVH' in ds.keys(): bvavh = np.nanmean(ds['VAVH'].values[:,:],axis=1) # H 1/3 vavh bvavh[(bvavh<0)|(bvavh>30)]=np.nan result['hs_vavh']=np.array(bvavh) @@ -440,8 +387,8 @@ def tseriesnc_copernicus(*args): btp[(btp<0)|(btp>40)]=np.nan result['tp']=np.array(btp) - if 'TEMP' in ds.keys(): - bsst = np.nanmean(ds['TEMP'].values[:,:],axis=1) # SST + if 'TEMP' in ds.keys(): + bsst = np.nanmean(ds['TEMP'].values[:,:],axis=1) # SST bsst[np.abs(bsst)>70]=np.nan result['sst']=np.array(bsst) @@ -467,13 +414,13 @@ def tseriesnc_copernicus(*args): result['gust']=np.array(bgst) if 'WSPD' in ds.keys(): - bwsp = np.nanmean(ds['WSPD'].values[:,:],axis=1) # wind speed + bwsp = np.nanmean(ds['WSPD'].values[:,:],axis=1) # wind speed bwsp=np.copy(((10./4.0)**(0.12))*bwsp) # conversion to 10m, approximation DNVGL C-205 Table 2-1 bwsp[(bwsp<0)|(bwsp>150)]=np.nan result['wind_spd']=np.array(bwsp) if 'WDIR' in ds.keys(): - bwdir = np.nanmean(ds['WDIR'].values[:,:],axis=1) # wind direction + bwdir = np.nanmean(ds['WDIR'].values[:,:],axis=1) # wind direction bwdir[(bwdir<-180)|(bwdir>360)]=np.nan result['wind_dir']=np.array(bwdir) @@ -482,7 +429,7 @@ def tseriesnc_copernicus(*args): bhcmax[(bhcmax<0)|(bhcmax>40)]=np.nan result['hc_max']=np.array(bhcmax) - if 'VMDR' in ds.keys(): + if 'VMDR' in ds.keys(): bdm = np.nanmean(ds['VMDR'].values[:,:],axis=1) # Mean direction bdm[(bdm<-180)|(bdm>360)]=np.nan result['dm']=np.array(bdm) @@ -495,182 +442,20 @@ def tseriesnc_copernicus(*args): return result ds.close(); del ds -# --- Altimeter --- -# Satellite data from Integrated Marine Observing System (IMOS), Australian Ocean Data Network (AODN) -def aodn_altimeter(satname,wconfig,datemin,datemax): - ''' - Read AODN altimeter data - http://thredds.aodn.org.au/thredds/catalog/IMOS/SRS/Surface-Waves/Wave-Wind-Altimetry-DM00/catalog.html - https://portal.aodn.org.au/ - Altimeter information: https://doi.org/10.1038/s41597-019-0083-9 - Inputs: - (1) satellite mission name. Select only one: - JASON3,JASON2,CRYOSAT2,JASON1,HY2,SARAL,SENTINEL3A,ENVISAT,ERS1,ERS2,GEOSAT,GFO,TOPEX,SENTINEL3B,CFOSAT - (2) wconfig dictionary, from wread.readconfig('ww3tools.yaml') - (3) initial date ('YYYYMMDDHH') - (4) final date ('YYYYMMDDHH') - Output: pandas dataframe containing: TIME (seconds since 1970), LATITUDE, LONGITUDE, WDEPTH, DISTCOAST, - HS, HS_CAL, WSPD, WSPD_CAL - Maryam Mohammadpour & Ricardo M. Campos - ''' - - # start time - start = timeit.default_timer() - - # date interval in seconds since 1970, user selection - adatemin= np.double(timegm( time.strptime(datemin, '%Y%m%d%H'))) - adatemax= np.double(timegm( time.strptime(datemax, '%Y%m%d%H'))) - - # Satellite missions available at AODN dataset, select only one. - sdname=np.array(['JASON3','JASON2','CRYOSAT2','JASON1','HY2','SARAL','SENTINEL3A','ENVISAT','ERS1','ERS2','GEOSAT','GFO','TOPEX','SENTINEL3B','CFOSAT']) - # Individual mission-specific Quality Control parameters - min_swh_numval = np.array([17,17,17,17,17,17,17,17,17,17,-inf,3,7,17,-inf]) - - if satname in sdname: - s=int(np.where(sdname==satname)[0]) - wconfig['min_swh_numval'] = min_swh_numval[s] - else: - raise ValueError("wread.aodn_altimeter; "+aodn_altimeter+" not included in the satellite missions available: "+", ".join(sdname) ) - - # name format for AODN reading - nsatname = re.sub(r'(\D)(\d)', r'\1-\2', satname) - - # Sat files (squares) considering the domain (lat lon from ww3tools.yaml) of interest, for the AODN file names - auxlat=np.array(np.arange(wconfig['latmin'],wconfig['latmax']+1.,1)).astype('int') - auxlon=np.array(np.arange(wconfig['lonmin'],wconfig['lonmax']+1.,1)).astype('int') - - # Read and allocate satellite data into arrays - ast=np.double(np.zeros((10**wconfig['pia']),'d')); aslat=np.zeros((10**wconfig['pia']),'f'); aslon=np.zeros((10**wconfig['pia']),'f') - ahsk=np.zeros((10**wconfig['pia']),'f'); ahskcal=np.zeros((10**wconfig['pia']),'f') - awnd=np.zeros((10**wconfig['pia']),'f'); awndcal=np.zeros((10**wconfig['pia']),'f'); asig0knstd=np.zeros((10**wconfig['pia']),'f') - aswhknobs=np.zeros((10**wconfig['pia']),'f'); aswhknstd=np.zeros((10**wconfig['pia']),'f'); aswhkqc=np.zeros((10**wconfig['pia']),'f') - aswdepth=np.zeros((10**wconfig['pia']),'f'); asdistcoast=np.zeros((10**wconfig['pia']),'f') - ii=0 - for j in auxlat: - for k in auxlon: - - if j>=0: - hem='N' - else: - hem='S' - - try: - fu=nc.Dataset(wconfig['path_alt']+satname+'/IMOS_SRS-Surface-Waves_MW_'+nsatname+'_FV02_'+str(np.abs(j)).zfill(3)+hem+'-'+str(k).zfill(3)+'E-DM00.nc') - except: - print(' '+wconfig['path_alt']+satname+'/IMOS_SRS-Surface-Waves_MW_'+nsatname+'_FV02_'+str(np.abs(j)).zfill(3)+hem+'-'+str(k).zfill(3)+'E-DM00.nc does not exist') - else: - st=np.double(fu.variables['TIME'][:]*24.*3600.+float(timegm( time.strptime('1985010100', '%Y%m%d%H') ))) - indt=np.where((st>=adatemin-wconfig['maxti']) & (st<=adatemax+wconfig['maxti'])) - # check if there is valid records inside the time range of interest - if np.size(indt)>10: - indt=indt[0] - # it does not read using the indexes because it is much slower - slat=fu.variables['LATITUDE'][:] - slon=fu.variables['LONGITUDE'][:] - swdepth=fu.variables['BOT_DEPTH'][:] - sdistcoast=fu.variables['DIST2COAST'][:] - wnd=fu.variables['WSPD'][:] - wndcal=fu.variables['WSPD_CAL'][:] - try: - hsk=fu.variables['SWH_KU'][:] - hskcal=fu.variables['SWH_KU_CAL'][:] - sig0knstd=fu.variables['SIG0_KU_std_dev'][:] - swhknobs=fu.variables['SWH_KU_num_obs'][:] - swhknstd=fu.variables['SWH_KU_std_dev'][:] - swhkqc=fu.variables['SWH_KU_quality_control'][:] - except: - print(' error reading KU, pick KA') - hsk=fu.variables['SWH_KA'][:] - hskcal=fu.variables['SWH_KA_CAL'][:] - sig0knstd=fu.variables['SIG0_KA_std_dev'][:] - swhknobs=fu.variables['SWH_KA_num_obs'][:] - swhknstd=fu.variables['SWH_KA_std_dev'][:] - swhkqc=fu.variables['SWH_KA_quality_control'][:] - - if ii+len(indt) <= ast.shape[0] : - # check the file is correct - if (st.shape[0]==wnd.shape[0]) & (slat.shape[0]==slon.shape[0]) & (hsk.shape[0]==hskcal.shape[0]) : - ast[ii:ii+len(indt)]=np.array(st[indt]).astype('double') - aslat[ii:ii+len(indt)]=np.array(slat[indt]).astype('float') - aslon[ii:ii+len(indt)]=np.array(slon[indt]).astype('float') - aswdepth[ii:ii+len(indt)]=np.array(swdepth[indt]).astype('float') - asdistcoast[ii:ii+len(indt)]=np.array(sdistcoast[indt]).astype('float') - ahsk[ii:ii+len(indt)]=np.array(hsk[indt]).astype('float') - ahskcal[ii:ii+len(indt)]=np.array(hskcal[indt]).astype('float') - awnd[ii:ii+len(indt)]=np.array(wnd[indt]).astype('float') - awndcal[ii:ii+len(indt)]=np.array(wndcal[indt]).astype('float') - asig0knstd[ii:ii+len(indt)]=np.array(sig0knstd[indt]).astype('float') - aswhknobs[ii:ii+len(indt)]=np.array(swhknobs[indt]).astype('float') - aswhknstd[ii:ii+len(indt)]=np.array(swhknstd[indt]).astype('float') - aswhkqc[ii:ii+len(indt)]=np.array(swhkqc[indt]).astype('float') - ii=ii+len(indt) - - else: - raise ValueError("gridSat_Altimeter.py; Small array to allocate the satellite data! Increase the power of initial array in ww3tools.yaml (pia)") - - del indt,st,slat,slon,swdepth,sdistcoast,hsk,hskcal,wnd,wndcal,sig0knstd,swhknobs,swhknstd,swhkqc - fu.close(); del fu - - # print(repr(j)+" "+repr(k)) - - # print(' Done reading and allocating satellite data '+satname) - del ii - - # water depth is positive by definition - aswdepth=aswdepth*-1. - - # Quality Control Check (optional) ---- - if wconfig['qc']==0: - indq = np.where( (ast>=adatemin) & (ast<=adatemax) ) - else: - indq = np.where( (aswdepth>=wconfig['mindepth']) & (asdistcoast>=wconfig['mindfc']) & (aswhknstd<=wconfig['max_swh_rms']) & - (asig0knstd<=wconfig['max_sig0_rms']) & (aswhknobs>=wconfig['min_swh_numval']) & (aswhkqc<=wconfig['max_swh_qc']) & - (ahsk>0.1) & (ahsk0.2) & (awnd0.1) & (ahskcal0.2) & (awndcal=adatemin) & (ast<=adatemax) ) - - del asig0knstd,aswhknobs,aswhknstd,aswhkqc,adatemin,adatemax - - if np.size(indq)>2: - indq=indq[0] - ast=np.double(np.copy(ast[indq])) - aslat=np.copy(aslat[indq]); aslon=np.copy(aslon[indq]) - aswdepth=np.copy(aswdepth[indq]); asdistcoast=np.copy(asdistcoast[indq]) - ahsk=np.copy(ahsk[indq]); ahskcal=np.copy(ahskcal[indq]) - awnd=np.copy(awnd[indq]); awndcal=np.copy(awndcal[indq]) - # dictionary - daodn = {'TIME': ast, 'LATITUDE': aslat, 'LONGITUDE': aslon, - 'WDEPTH': aswdepth, 'DISTCOAST': asdistcoast, - 'HS': ahsk, 'HS_CAL': ahskcal, - 'WSPD': awnd, 'WSPD_CAL': awndcal} - - else: - daodn = {'TIME': [], 'LATITUDE': [], 'LONGITUDE': [], - 'WDEPTH': [], 'DISTCOAST': [], - 'HS': [], 'HS_CAL': [], - 'WSPD': [], 'WSPD_CAL': []} - - AODN = pd.DataFrame(daodn) - stop = timeit.default_timer() - print('wread.aodn_altimeter successfully completed in '+repr(int(round(stop - start,0)))+' seconds. '+satname) - return AODN - - -# ========== MODEL ====================== -# --- WAVEWATCH III --- # WAVEWATCH III point output, netcdf format + def tseriestxt_ww3(*args): ''' WAVEWATCH III, time series/table, text tab format This file format has all point outputs (results) in the same file (not divided by point/buoy). Input: file name (example: tab50.ww3), and number of point ouputs (example: 4) - Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, + Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, and arrays with the wave variables available. Inside the dictionary, the arrays of wave variables have dimension (point_outputs, time). ''' if len(args) == 2: - fname=str(args[0]); tnb=int(args[1]) + fname=str(args[0]); tnb=np.int(args[1]) elif len(args) < 2 : sys.exit(' Two inputs are required: file name and station name') elif len(args) > 2: @@ -682,18 +467,18 @@ def tseriestxt_ww3(*args): sys.exit(" Cannot open "+fname) else: - tt = int(np.size(mcontent)/(7+tnb)+1) + tt = np.int(np.size(mcontent)/(7+tnb)+1) myear = []; mmonth = [] ; mday = [] ; mhour = []; mmin = [] - mlon = np.zeros((tnb,tt),'f'); mlat = np.zeros((tnb,tt),'f'); mhs = np.zeros((tnb,tt),'f'); mL = np.zeros((tnb,tt),'f') + mlon = np.zeros((tnb,tt),'f'); mlat = np.zeros((tnb,tt),'f'); mhs = np.zeros((tnb,tt),'f'); mL = np.zeros((tnb,tt),'f') mtm = np.zeros((tnb,tt),'f'); mdm = np.zeros((tnb,tt),'f'); mspr = np.zeros((tnb,tt),'f') atp = np.zeros((tnb,tt),'f'); mdp = np.zeros((tnb,tt),'f'); mpspr = np.zeros((tnb,tt),'f') for i in range(0,tt): j = i*(7+tnb) - myear = np.append(myear, int(mcontent[j].split(':')[1].split(' ')[1].split('/')[0]) ) - mmonth = np.append(mmonth, int(mcontent[j].split(':')[1].split(' ')[1].split('/')[1]) ) - mday = np.append(mday, int(mcontent[j].split(':')[1].split(' ')[1].split('/')[2]) ) - mhour = np.append(mhour, int(mcontent[j].split(':')[1].split(' ')[2]) ) - mmin = np.append(mmin, int(mcontent[j].split(':')[2]) ) + myear = np.append(myear, np.int(mcontent[j].split(':')[1].split(' ')[1].split('/')[0]) ) + mmonth = np.append(mmonth, np.int(mcontent[j].split(':')[1].split(' ')[1].split('/')[1]) ) + mday = np.append(mday, np.int(mcontent[j].split(':')[1].split(' ')[1].split('/')[2]) ) + mhour = np.append(mhour, np.int(mcontent[j].split(':')[1].split(' ')[2]) ) + mmin = np.append(mmin, np.int(mcontent[j].split(':')[2]) ) for k in range(0,tnb): mlon[k,i] = mcontent[j+tnb+1+k].strip().split()[0] mlat[k,i] = mcontent[j+tnb+1+k].strip().split()[1] @@ -707,10 +492,10 @@ def tseriestxt_ww3(*args): mpspr[k,i] = mcontent[j+tnb+1+k].strip().split()[9] mtp = np.zeros((atp.shape[0],atp.shape[1]),'f')*np.nan - for i in range(0,mtp.shape[0]): + for i in range(0,mtp.shape[0]): #mtp[i,atp[i,:]>0.0] = 1./atp[i,atp[i,:]>0.0] indtp=np.where(atp[i,:]>0.0) - if np.size(indtp)>0: + if size(indtp)>0: mtp[i,indtp] = np.copy(1./atp[i,indtp]) del indtp @@ -731,7 +516,7 @@ def tseriesnc_ww3(*args): ''' WAVEWATCH III, time series/table, netcdf format Input: file name (example: ww3gefs.20160928_tab.nc), and station name (example: 41002) - Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, + Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, and arrays with the wave variables available. ''' if len(args) == 2: @@ -748,14 +533,14 @@ def tseriesnc_ww3(*args): else: mtime = np.array(f.variables['time'][:]*24*3600 + timegm( strptime(str(f.variables['time'].units).split(' ')[2][0:4]+'01010000', '%Y%m%d%H%M') )).astype('double') f.close(); del f - + auxstationname=ds['station_name'].values[:,:]; stationname=[] for i in range(0,auxstationname.shape[0]): stationname=np.append(stationname,"".join(np.array(auxstationname[i,:]).astype('str'))) inds=np.where(stationname[:]==stname) - if np.size(inds)>0: - inds=int(inds[0][0]); stname=str(stationname[inds]) + if size(inds)>0: + inds=np.int(inds[0][0]); stname=str(stationname[inds]) else: sys.exit(' Station '+stname+' not included in the ww3 output file, or wrong station ID') @@ -763,7 +548,7 @@ def tseriesnc_ww3(*args): mlon = np.nanmean(ds['longitude'].values[:,inds]) # dictionary result={'latitude':np.array(mlat),'longitude':np.array(mlon), - 'time':mtime,'date':ds['time'].values[:]} + 'time':mtime,'date':ds['time'].values[:]} if 'hs' in ds.keys(): mhs = ds['hs'].values[:,inds] @@ -777,7 +562,7 @@ def tseriesnc_ww3(*args): if 'fp' in ds.keys(): mtp = np.zeros(mhs.shape[0],'f')*np.nan indtp=np.where(ds['fp'].values[:,inds]>0.0) - if np.size(indtp)>0: + if size(indtp)>0: mtp[indtp] = np.copy(1./ds['fp'].values[indtp,inds]) del indtp mtp[(mtp<0)|(mtp>40)]=np.nan @@ -838,22 +623,22 @@ def bull(*args): # lat / lon auxpos=str(lines[0]).replace("b'","").split('(')[1] if auxpos[5]=='N': - alat=np.float(auxpos[0:5]) + alat=float(auxpos[0:5]) else: - alat=-1.*np.float(auxpos[0:5]) + alat=-1.*float(auxpos[0:5]) if auxpos[13]=='E': - alon=np.float(auxpos[7:13]) + alon=float(auxpos[7:13]) else: - alon=-1.*np.float(auxpos[7:13]) + alon=-1.*float(auxpos[7:13]) # time ---- auxdate = str(lines[2]).split(':')[1].split('UTC')[0][1::] auxt = np.double(timegm( strptime( auxdate[0:8]+' '+auxdate[9:11]+'00', '%Y%m%d %H%M') )) year = int(time.gmtime(auxt)[0]); month = int(time.gmtime(auxt)[1]) pday=0 - for j in range(7,np.size(lines)-8): - day=int(lines[j][3:5]); hour=int(lines[j][6:8]) + for j in range(7,size(lines)-8): + day=np.int(lines[j][3:5]); hour=np.int(lines[j][6:8]) if day0: - ahs=np.append(ahs,np.float(lines[j][10:15])) - auxhs=np.array([np.float(lines[j][iauxhs[0]:iauxhs[1]])]) - for k in range(1,4): + ahs=np.append(ahs,float(lines[j][10:15])) + auxhs=np.array([float(lines[j][iauxhs[0]:iauxhs[1]])]) + for k in range(1,4): if len(str(lines[j][int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)]).replace(' ', '')): - auxhs=np.append(auxhs,np.float(lines[j][int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)])) + auxhs=np.append(auxhs,float(lines[j][int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)])) - auxtp=np.array([np.float(lines[j][iauxtp[0]:iauxtp[1]])]) - for k in range(1,4): + auxtp=np.array([float(lines[j][iauxtp[0]:iauxtp[1]])]) + for k in range(1,4): if len(str(lines[j][int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)]).replace(' ', '')): - auxtp=np.append(auxtp,np.float(lines[j][int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)])) + auxtp=np.append(auxtp,float(lines[j][int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)])) - auxdp=np.array([np.float(lines[j][iauxdp[0]:iauxdp[1]])]) - for k in range(1,4): + auxdp=np.array([float(lines[j][iauxdp[0]:iauxdp[1]])]) + for k in range(1,4): if len(str(lines[j][int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)]).replace(' ', '')): - auxdp=np.append(auxdp,np.float(lines[j][int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)])) + auxdp=np.append(auxdp,float(lines[j][int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)])) indaux=np.nanmin(np.where(auxhs==np.nanmax(auxhs))[0]) - atp=np.append(atp,np.float(auxtp[indaux])) - adp=np.append(adp,np.float(auxdp[indaux])) + atp=np.append(atp,float(auxtp[indaux])) + adp=np.append(adp,float(auxdp[indaux])) del indaux,auxhs,auxtp,auxdp else: ahs=np.append(ahs,np.nan) @@ -907,16 +692,16 @@ def bull(*args): # lat / lon auxpos=str(lines[1]).split('(')[1].split('N') - alat=np.float(auxpos[0]) - alon=np.float(auxpos[1].split('W')[0]) + alat=float(auxpos[0]) + alon=float(auxpos[1].split('W')[0]) # time ---- auxdate = str(lines[3]).split(':')[1].split('UTC')[0][1::] auxt = np.double(timegm( strptime( auxdate[0:8]+' '+auxdate[10:12]+'00', '%Y%m%d %H%M') )) year = int(time.gmtime(auxt)[0]); month = int(time.gmtime(auxt)[1]) pday=0 - for j in range(9,np.size(lines)-8): - day=int(lines[j][2:4]); hour=int(lines[j][5:7]) + for j in range(9,size(lines)-8): + day=np.int(lines[j][2:4]); hour=np.int(lines[j][5:7]) if day0: - ahs=np.append(ahs,np.float(lines[j][iauxhs[0]:iauxhs[1]])) - atp=np.append(atp,np.float(lines[j][iauxtp[0]:iauxtp[1]])) + ahs=np.append(ahs,float(lines[j][iauxhs[0]:iauxhs[1]])) + atp=np.append(atp,float(lines[j][iauxtp[0]:iauxtp[1]])) else: ahs=np.append(ahs,np.nan) atp=np.append(atp,np.nan) # build dictionary result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), - 'latitude':alat,'longitude':alon,'station_name':stname, + 'latitude':alat,'longitude':alon,'station_name':stname, 'hs':np.array(ahs),'tp':np.array(atp)} print(" Model data read, "+fname+", bull format.") @@ -959,194 +744,157 @@ def bull_tar(*args): Output: dictionary containing: time(seconds since 1970),time(datetime64),lat,lon,station names; Arrays: hs, tp, and dp (gfs only) ''' + result = {} + if len(args) == 1: - fname=str(args[0]) + fname = str(args[0]) elif len(args) > 1: sys.exit(' Too many inputs') # confirm file format - if fname.split('/')[-1].split('.')[-1]=='bull_tar': + if fname.split('/')[-1].split('.')[-1] == 'bull_tar': print(" reading ww3 bull_tar file ...") import tarfile - stname=[] + stname = [] try: tar = tarfile.open(fname) except: - sys.exit(' Cannot open '+fname) + sys.exit(' Cannot open ' + fname) else: - at=[]; adate=[]; alat=[]; alon=[] + at = [] + adate = [] + alat = [] + alon = [] + + # Determine model type + if 'gfs' in fname.split('/')[-1]: + model_type = 'gfs' + iauxhs = [24, 30] + iauxtp = [30, 34] + iauxdp = [35, 38] + else: + model_type = 'gfs' # Treat any other name as gfs for simplicity + iauxhs = [24, 30] + iauxtp = [30, 34] + iauxdp = [35, 38] - if 'gfs' in str(fname).split('/')[-1]: - iauxhs=[24,30];iauxtp=[30,34];iauxdp=[35,38] + for t in range(0, len(tar.getmembers())): + # station names + stname.append(str(tar.getmembers()[t].name).split('/')[-1].split('.')[-2]) - for t in range(0,np.size(tar.getmembers())): - # station names - stname=np.append(stname,str(str(tar.getmembers()[t].name).split('/')[-1]).split('/')[-1].split('.')[-2]) + try: + tfile = tar.extractfile(tar.getmembers()[t]) + lines = tfile.readlines() + except: + print(" Cannot open " + tar.getmembers()[t].name) + else: + # lat / lon + auxpos = str(lines[0]).replace("b'", "").split('(')[1] + if auxpos[5] == 'N': + alat.append(float(auxpos[0:5])) + else: + alat.append(-1. * float(auxpos[0:5])) - try: - tfile=tar.extractfile(tar.getmembers()[t]); lines = tfile.readlines() - except: - print(" Cannot open "+tar.getmembers()[t].name) + if auxpos[13] == 'E': + alon.append(float(auxpos[7:13])) else: + alon.append(-1. * float(auxpos[7:13])) - # lat / lon - auxpos=str(lines[0]).replace("b'","").split('(')[1] - if auxpos[5]=='N': - alat=np.append(alat,np.float(auxpos[0:5])) - else: - alat=np.append(alat,-1.*np.float(auxpos[0:5])) + if t == 0: + # time array ---- + auxdate = str(lines[2]).split(':')[1].split('UTC')[0][1::] + auxt = np.double(timegm(strptime(auxdate[0:8] + ' ' + auxdate[9:11] + '00', '%Y%m%d %H%M'))) + year = int(time.gmtime(auxt)[0]) + month = int(time.gmtime(auxt)[1]) + pday = 0 + for j in range(7, len(lines) - 8): + auxlines = str(lines[j]).replace("b'", "") + day = int(auxlines[3:5]) + hour = int(auxlines[6:8]) + if day < pday: + if month < 12: + month = month + 1 + else: + month = 1 + year = year + 1 + + at.append(np.double(timegm(strptime(repr(year) + str(month).zfill(2) + + str(day).zfill(2) + ' ' + str(hour).zfill(2) + + '00', '%Y%m%d %H%M')))) + pday = np.copy(day) + + for j in range(len(at)): + adate.append(datetime.datetime.utcfromtimestamp(at[j])) - if auxpos[13]=='E': - alon=np.append(alon,np.float(auxpos[7:13])) - else: - alon=np.append(alon,-1.*np.float(auxpos[7:13])) - - if t==0: - # time array ---- - auxdate = str(lines[2]).split(':')[1].split('UTC')[0][1::] - auxt = np.double(timegm( strptime( auxdate[0:8]+' '+auxdate[9:11]+'00', '%Y%m%d %H%M') )) - year = int(time.gmtime(auxt)[0]); month = int(time.gmtime(auxt)[1]) - pday=0 - for j in range(7,np.size(lines)-8): - auxlines = str(lines[j]).replace("b'","") - day=int(auxlines[3:5]); hour=int(auxlines[6:8]); del auxlines - if day0: - auxhs=np.append(auxhs,np.float(auxlines[10:15])) - fuxhs=np.array([np.float(auxlines[iauxhs[0]:iauxhs[1]])]) - for k in range(1,4): - if len(str(auxlines[int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)]).replace(' ', '')): - fuxhs=np.append(fuxhs,np.float(auxlines[int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)])) - - fuxtp=np.array([np.float(auxlines[iauxtp[0]:iauxtp[1]])]) - for k in range(1,4): - if len(str(auxlines[int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)]).replace(' ', '')): - fuxtp=np.append(fuxtp,np.float(auxlines[int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)])) - - fuxdp=np.array([np.float(auxlines[iauxdp[0]:iauxdp[1]])]) - for k in range(1,4): - if len(str(auxlines[int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)]).replace(' ', '')): - fuxdp=np.append(fuxdp,np.float(auxlines[int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)])) - - indaux=np.nanmin(np.where(fuxhs==np.nanmax(fuxhs))[0]) - auxtp=np.append(auxtp,np.float(fuxtp[indaux])) - auxdp=np.append(auxdp,np.float(fuxdp[indaux])) - del indaux,fuxhs,fuxtp,fuxdp - else: - auxhs=np.append(auxhs,np.nan) - auxtp=np.append(auxtp,np.nan) - auxdp=np.append(auxdp,np.nan) - - if ahs.shape[1]==auxhs.shape[0]: - ahs[t,:]=np.array(auxhs) - atp[t,:]=np.array(auxtp) - adp[t,:]=np.array(auxdp) + # -------- + ahs = np.zeros((len(tar.getmembers()), len(at)), 'f') * np.nan + atp = np.zeros((len(tar.getmembers()), len(at)), 'f') * np.nan + if model_type == 'gfs': + adp = np.zeros((len(tar.getmembers()), len(at)), 'f') * np.nan + + auxhs = [] + auxtp = [] + auxdp = [] + for j in range(7, len(lines) - 8): + auxlines = str(lines[j]).replace("b'", "") + if len(auxlines[iauxhs[0]:iauxhs[1]].replace(' ', '')) > 0: + auxhs.append(float(auxlines[10:15])) + fuxhs = np.array([float(auxlines[iauxhs[0]:iauxhs[1]])]) + for k in range(1, 4): + if len(str(auxlines[int(iauxhs[0] + 18 * k):int(iauxhs[1] + 18 * k)]).replace(' ', '')): + fuxhs = np.append(fuxhs, float(auxlines[int(iauxhs[0] + 18 * k):int(iauxhs[1] + 18 * k)])) + + fuxtp = np.array([float(auxlines[iauxtp[0]:iauxtp[1]])]) + for k in range(1, 4): + if len(str(auxlines[int(iauxtp[0] + 18 * k):int(iauxtp[1] + 18 * k)]).replace(' ', '')): + fuxtp = np.append(fuxtp, float(auxlines[int(iauxtp[0] + 18 * k):int(iauxtp[1] + 18 * k)])) + + if model_type == 'gfs': + fuxdp = np.array([float(auxlines[iauxdp[0]:iauxdp[1]])]) + for k in range(1, 4): + if len(str(auxlines[int(iauxdp[0] + 18 * k):int(iauxdp[1] + 18 * k)]).replace(' ', '')): + fuxdp = np.append(fuxdp, float(auxlines[int(iauxdp[0] + 18 * k):int(iauxdp[1] + 18 * k)])) + + indaux = np.nanargmax(fuxhs) + auxdp.append(fuxdp[indaux]) + + indaux = np.nanargmax(fuxhs) + auxtp.append(fuxtp[indaux]) else: - print(" Time duration of "+tar.getmembers()[t]+" (in "+fname+") do not match the other stations. Mantained NaN.") - - del auxhs,auxtp,auxdp,tfile,lines + auxhs.append(np.nan) + auxtp.append(np.nan) + if model_type == 'gfs': + auxdp.append(np.nan) + + if ahs.shape[1] == len(auxhs): + ahs[t, :] = np.array(auxhs) + atp[t, :] = np.array(auxtp) + if model_type == 'gfs': + adp[t, :] = np.array(auxdp) + else: + print(" Time duration of " + tar.getmembers()[t] + " (in " + fname + ") do not match the other stations. Maintained NaN.") - # build dictionary - result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), - 'latitude':np.array(alat),'longitude':np.array(alon),'station_name':np.array(stname), - 'hs':np.array(ahs),'tp':np.array(atp),'dp':np.array(adp)} + del auxhs, auxtp, auxdp, tfile, lines - del adp - - elif 'gefs' in str(fname).split('/')[-1]: - iauxhs=[10,15];iauxtp=[28,33] - - for t in range(0,np.size(tar.getmembers())): - # station names - stname=np.append(stname,str(str(tar.getmembers()[t].name).split('/')[-1]).split('/')[-1].split('.')[-2]) - - try: - tfile=tar.extractfile(tar.getmembers()[t]); lines = tfile.readlines() - except: - print(" Cannot open "+tar.getmembers()[t].name) - else: - # lat / lon - auxpos=str(lines[1]).replace("b'","").split('(')[1].split('N') - alat=np.append(alat,np.float(auxpos[0])) - alon=np.append(alon,np.float(auxpos[1].split('W')[0])) - - if t==0: - # time array ---- - auxdate = str(lines[3]).split(':')[1].split('UTC')[0][1::] - auxt = np.double(timegm( strptime( auxdate[0:8]+' '+auxdate[10:12]+'00', '%Y%m%d %H%M') )) - year = int(time.gmtime(auxt)[0]); month = int(time.gmtime(auxt)[1]) - pday=0 - for j in range(9,np.size(lines)-8): - auxlines = str(lines[j]).replace("b'","") - day=int(auxlines[2:4]); hour=int(auxlines[5:7]); del auxlines - if day0: - auxhs=np.append(auxhs,np.float(auxlines[iauxhs[0]:iauxhs[1]])) - auxtp=np.append(auxtp,np.float(auxlines[iauxtp[0]:iauxtp[1]])) - else: - auxhs=np.append(auxhs,np.nan) - auxtp=np.append(auxtp,np.nan) - - del auxlines - - if ahs.shape[1]==auxhs.shape[0]: - ahs[t,:]=np.array(auxhs) - atp[t,:]=np.array(auxtp) - else: - print(" Time duration of "+tar.getmembers()[t]+" (in "+fname+") do not match the other stations. Mantained NaN.") - - del auxhs,auxtp,tfile,lines + # build dictionary + result = {'time': np.array(at).astype('double'), 'date': [t.timestamp() for t in adate], + 'latitude': np.array(alat), 'longitude': np.array(alon), 'station_name': np.array(stname), + 'hs': np.array(ahs), 'tp': np.array(atp)} - # build dictionary - result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), - 'latitude':np.array(alat),'longitude':np.array(alon),'station_name':np.array(stname), - 'hs':np.array(ahs),'tp':np.array(atp)} + if model_type == 'gfs': + result['dp'] = np.array(adp) - print(" Model data read, "+fname+", bull_tar format.") + print(" Model data read, " + fname + ", bull_tar format.") return result - del result,tar,alat,alon,ahs,atp,at,adate + else: - sys.exit(" Skipped file "+fname+" Not bull_tar format.") + sys.exit(" Skipped file " + fname + " Not bull_tar format.") + + +# Example usage: +# data = bull_tar('path_to_your_file/multi_1.t00z.bull_tar') +# print(data) def ts(*args): @@ -1174,20 +922,20 @@ def ts(*args): if 'gefs' in str(fname).split('/')[-1]: # gefs lakes ww3 format at=[]; adate=[]; ahs=[]; ahspr=[]; atp=[] - for j in range(0,np.size(lines)): + for j in range(0,size(lines)): at=np.append(at,np.double(timegm( strptime( lines[j][1:12]+'00', '%Y%m%d %H%M') ))) adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) if len(lines[j])>0: - ahs=np.append(ahs,np.float(lines[j][13:18])) - ahspr=np.append(ahspr,np.float(lines[j][19:25])) - atp=np.append(atp,np.float(lines[j][27:32])) + ahs=np.append(ahs,float(lines[j][13:18])) + ahspr=np.append(ahspr,float(lines[j][19:25])) + atp=np.append(atp,float(lines[j][27:32])) else: ahs=np.append(ahs,np.nan) ahspr=np.append(ahspr,np.nan) atp=np.append(atp,np.nan) - # build dictionary + # build dictionary result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), 'station_name':np.array(stname),'hs':np.array(ahs),'hs_spr':np.array(ahspr),'tp':np.array(atp)} @@ -1198,19 +946,19 @@ def ts(*args): elif 'glwu' in str(fname).split('/')[-1]: # great lakes ww3 format at=[];adate=[];ahs=[];al=[];atr=[];adir=[];aspr=[];atp=[];ap_dir=[];ap_spr=[] - for j in range(0,np.size(lines)): + for j in range(0,size(lines)): at=np.append(at,np.double(timegm( strptime( lines[j][2:13]+'00', '%Y%m%d %H%M') ))) adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) if len(lines[j])>0: - ahs=np.append(ahs,np.float(lines[j][22:28])) - al=np.append(al,np.float(lines[j][31:35])) - atr=np.append(atr,np.float(lines[j][37:42])) - adir=np.append(adir,np.float(lines[j][44:49])) - aspr=np.append(aspr,np.float(lines[j][50:56])) - atp=np.append(atp,np.float(lines[j][57:64])) - ap_dir=np.append(ap_dir,np.float(lines[j][66:71])) - ap_spr=np.append(ap_spr,np.float(lines[j][72:78])) + ahs=np.append(ahs,float(lines[j][22:28])) + al=np.append(al,float(lines[j][31:35])) + atr=np.append(atr,float(lines[j][37:42])) + adir=np.append(adir,float(lines[j][44:49])) + aspr=np.append(aspr,float(lines[j][50:56])) + atp=np.append(atp,float(lines[j][57:64])) + ap_dir=np.append(ap_dir,float(lines[j][66:71])) + ap_spr=np.append(ap_spr,float(lines[j][72:78])) else: ahs=np.append(ahs,np.nan) al=np.append(al,np.nan) @@ -1223,7 +971,7 @@ def ts(*args): atp[atp<0.01]=np.nan; atp=1./atp - # build dictionary + # build dictionary result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), 'station_name':np.array(stname), 'hs':np.array(ahs),'l':np.array(al), @@ -1262,7 +1010,7 @@ def station_tar(*args): except: sys.exit(' Cannot open '+fname) else: - for t in range(0,np.size(tar.getmembers())): + for t in range(0,size(tar.getmembers())): # station names stname=np.append(stname,str(str(tar.getmembers()[t].name).split('/')[-1]).split('/')[-1].split('.')[-2]) @@ -1274,22 +1022,22 @@ def station_tar(*args): if t==0: # time array ---- at=[]; adate=[] - for j in range(0,np.size(lines)): + for j in range(0,size(lines)): at=np.append(at,np.double(timegm( strptime( str(lines[j])[3:14]+'00', '%Y%m%d %H%M') ))) adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) # -------- - ahs=np.zeros((np.size(tar.getmembers()),at.shape[0]),'f')*np.nan - ahspr=np.zeros((np.size(tar.getmembers()),at.shape[0]),'f')*np.nan - atp=np.zeros((np.size(tar.getmembers()),at.shape[0]),'f')*np.nan + ahs=np.zeros((size(tar.getmembers()),at.shape[0]),'f')*np.nan + ahspr=np.zeros((size(tar.getmembers()),at.shape[0]),'f')*np.nan + atp=np.zeros((size(tar.getmembers()),at.shape[0]),'f')*np.nan auxhs=[]; auxhspr=[]; auxtp=[] - for j in range(0,np.size(lines)): + for j in range(0,size(lines)): auxlines = str(lines[j]).replace("b'","") if len(lines[j])>0: - auxhs=np.append(auxhs,np.float(auxlines[13:18])) - auxhspr=np.append(auxhspr,np.float(auxlines[19:25])) - auxtp=np.append(auxtp,np.float(auxlines[27:32])) + auxhs=np.append(auxhs,float(auxlines[13:18])) + auxhspr=np.append(auxhspr,float(auxlines[19:25])) + auxtp=np.append(auxtp,float(auxlines[27:32])) else: auxhs=np.append(auxhs,np.nan) auxhspr=np.append(auxhspr,np.nan) @@ -1304,7 +1052,7 @@ def station_tar(*args): del auxhs,auxhspr,auxtp,tfile,lines - # build dictionary + # build dictionary result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), 'station_name':np.array(stname),'hs':np.array(ahs),'hs_spr':np.array(ahspr),'tp':np.array(atp)} @@ -1317,7 +1065,7 @@ def station_tar(*args): sys.exit(" Skipped file "+fname+" Not station_tar format.") -# SPECTRA +# SPECTRA # Observations NDBC, netcdf format def spec_ndbc(*args): @@ -1327,13 +1075,13 @@ def spec_ndbc(*args): Output: dictionary containing: time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pspec,dmspec,dpspec,dirspec ''' - sk=1; deltatheta=int(10) + sk=1; deltatheta=np.int(10) if len(args) >= 1: fname=str(args[0]) if len(args) >= 2: - sk=int(args[1]) + sk=np.int(args[1]) if len(args) >= 3: - deltatheta=int(args[3]) + deltatheta=np.int(args[3]) if len(args) > 3: sys.exit(' Too many inputs') @@ -1350,12 +1098,12 @@ def spec_ndbc(*args): freq = ds['frequency'].values[:] pspec = ds['spectral_wave_density'].values[::sk,:,0,0] dmspec = ds['mean_wave_dir'][::sk,:,0,0] - dpspec = ds['principal_wave_dir'][::sk,:,0,0] + dpspec = ds['principal_wave_dir'][::sk,:,0,0] r1spec = ds['wave_spectrum_r1'][::sk,:,0,0] r2spec = ds['wave_spectrum_r2'][::sk,:,0,0] ds.close(); del ds # DF in frequency (dfreq), https://www.ndbc.noaa.gov/wavespectra.shtml - if int(freq.shape[0])==47: + if np.int(freq.shape[0])==47: dfreq=np.zeros(47,'f') dfreq[0]=0.010; dfreq[1:14]=0.005; dfreq[14:40]=0.010; dfreq[40::]=0.020 else: @@ -1367,10 +1115,10 @@ def spec_ndbc(*args): # final directional wave spectrum (frequency X direction) dirspec = np.zeros((btime.shape[0],freq.shape[0],theta.shape[0]),'f') for t in range(0,btime.shape[0]): - dirspec[t,:,:] = np.array([pspec[t,:]]).T * (1/pi)*(0.5+ np.array([r1spec[t,:]]).T * cos(np.array( np.array([theta])-np.array([dmspec[t,:]]).T )*(pi/180)) + dirspec[t,:,:] = np.array([pspec[t,:]]).T * (1/pi)*(0.5+ np.array([r1spec[t,:]]).T * cos(np.array( np.array([theta])-np.array([dmspec[t,:]]).T )*(pi/180)) + np.array([r2spec[t,:]]).T*cos(2*np.array( np.array([theta]) - np.array([dpspec[t,:]]).T )*(pi/180))) - # build dictionary + # build dictionary result={'time':btime,'date':bdate,'latitude':blat,'longitude':blon, 'freq':freq,'deltafreq':dfreq,'pspec':pspec,'dmspec':dmspec,'dpspec':dpspec, 'theta':theta,'dirspec':dirspec} @@ -1393,7 +1141,7 @@ def spec_ww3(*args): if len(args) >= 2 : fname=str(args[0]); stname=str(args[1]) if len(args) > 2 : - sk=int(args[2]) + sk=np.int(args[2]) if len(args) > 3 : sys.exit(' Too many inputs') @@ -1411,8 +1159,8 @@ def spec_ww3(*args): stationname=np.append(stationname,"".join(np.array(auxstationname[i,:]).astype('str'))) inds=np.where(stationname[:]==stname) - if np.size(inds)>0: - inds=int(inds[0][0]); stname=str(stationname[inds]) + if size(inds)>0: + inds=np.int(inds[0][0]); stname=str(stationname[inds]) else: sys.exit(' Station '+stname+' not included in the output file, or wrong station ID') @@ -1438,8 +1186,8 @@ def spec_ww3(*args): # water depth (constant in time) depth=np.nanmean(ds['dpt'].values[::sk,inds],axis=0) lon=np.array(np.nanmean(ds['longitude'].values[::sk,inds],axis=0)) - lat=np.array(np.nanmean(ds['latitude'].values[::sk,inds],axis=0)) - + lat=np.array(np.nanmean(ds['latitude'].values[::sk,inds],axis=0)) + ds.close(); del ds, auxstationname, inds, stationname # ------------------ # 1D power spectrum @@ -1467,17 +1215,17 @@ def spec_ww3(*args): # 1D directional spectrum d1sp=np.zeros((dspec.shape[0],nf),'f') for t in range(0,dspec.shape[0]): - for il in range(0,nf): + for il in range(0,nf): a = np.sum(dspec[t,il,:] * np.array(np.sin((pi*dire)/180.)/np.sum(dspec[t,il,:])) ) b = np.sum(dspec[t,il,:] * np.array(np.cos((pi*dire)/180.)/np.sum(dspec[t,il,:])) ) aux = math.atan2(a,b)*(180./pi) if aux<0: aux=aux+360. - - d1sp[t,il]=np.float(aux) + + d1sp[t,il]=float(aux) del a,b,aux - # build dictionary + # build dictionary result={'time':mtime,'date':mdate,'latitude':lat,'longitude':lon, 'wind_spd':wnds,'wind_dir':wndd,'freq':freq,'freq1':freq1,'freq2':freq2, 'deltafreq':dfreq,'pspec':pwst,'theta':dire,'dmspec':d1sp,'dirspec':dspec} From 1dcfcb199f2db7c3b408c0323e09e2065be81479 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Wed, 6 Mar 2024 18:03:38 +0000 Subject: [PATCH 02/27] wind value added to the NDBC and the Copernp removed until the reliable wind values provided --- ww3tools/modelBuoy_collocation.py | 293 ++++++++++++------------------ 1 file changed, 115 insertions(+), 178 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index a9f794f..fc6725c 100755 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -108,10 +108,10 @@ # Paths # ndbcp="/data/buoys/NDBC/wparam" -ndbcp="/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" +ndbcp="/work/noaa/marine/ricardo.campos/data/buoys/NDBC/ncformat/wparam" # Copernicus buoys # copernp="/data/buoys/Copernicus/wtimeseries" -copernp="/work/noaa/marine/ricardo.campos/data/buoys/Copernicus/wtimeseries" +#copernp="/work/noaa/marine/ricardo.campos/data/buoys/Copernicus/wtimeseries" print(' ') # Options of including grid and cyclone information @@ -122,7 +122,7 @@ # # import os; os.system("ls -d $PWD/*tab.nc > ww3list.txt &") wlist=np.atleast_1d(np.loadtxt(sys.argv[1],dtype=str)) ftag=str(sys.argv[1]).split('list')[1].split('.txt')[0] - print(' Reading ww3 list '+ str(sys.argv[1])) + print(' Reading ww3 list '+str(sys.argv[1])) print(' Tag '+ftag) if len(sys.argv) >= 3: forecastds=int(sys.argv[2]) @@ -168,7 +168,6 @@ if str(wlist[i]).split('/')[-1].split('.')[-1]=='station_tar': result = wread.bull_tar(wlist[i]) - print("Result keys:", result.keys()) at=result['time'] fcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') if i==0: @@ -283,6 +282,14 @@ stname=np.append(stname,astname); del astname + funits=f.variables['time'].units + if str(funits).split(' ')[0] == 'seconds': + tincr=1 + elif str(funits).split(' ')[0] == 'hours': + tincr=3600 + elif str(funits).split(' ')[0] == 'days': + tincr=24*3600 + ahs = np.array(f.variables['hs'][:,:]).T if 'th1m' in f.variables.keys(): @@ -313,7 +320,9 @@ else: atm = np.array(np.copy(ahs*nan)) - at = np.array(f.variables['time'][:]*24*3600 + timegm( strptime(str(f.variables['time'].units).split(' ')[2][0:4]+'01010000', '%Y%m%d%H%M') )).astype('double') + ftunits=str(f.variables['time'].units).split('since')[1][1::].replace('T',' ').replace('+00:00','') + at = np.array(f.variables['time'][:]*tincr + timegm( strptime(ftunits,'%Y-%m-%d %H:%M:%S') )).astype('double') + f.close(); del f fcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') if t==0: @@ -351,184 +360,112 @@ bdm=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan bdp=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan lat=np.zeros(np.size(stname),'f')*np.nan; lon=np.zeros(np.size(stname),'f')*np.nan +bwind_spd = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan + # help reading NDBC buoys, divided by year yrange=np.array(np.arange(time.gmtime(mtime.min())[0],time.gmtime(mtime.min())[0]+1,1)).astype('int') # loop buoys for b in range(0,np.size(stname)): + ahs=[] + try: + ahs=[];atm=[];adm=[];atime=[] + for y in yrange: + f=nc.Dataset(ndbcp+"/"+stname[b]+"h"+repr(y)+".nc") + if 'wave_height' in f.variables.keys(): + ahs = np.append(ahs,f.variables['wave_height'][:,0,0]) + elif 'hs' in f.variables.keys(): + ahs = np.append(ahs,f.variables['hs'][:,0,0]) + elif 'swh' in f.variables.keys(): + ahs = np.append(ahs,f.variables['swh'][:,0,0]) + + if 'average_wpd' in f.variables.keys(): + atm = np.append(atm,f.variables['average_wpd'][:,0,0]) + else: + atm = np.array(np.copy(ahs*nan)) + + if 'dominant_wpd' in f.variables.keys(): + atp = np.append(atm,f.variables['dominant_wpd'][:,0,0]) + else: + atp = np.array(np.copy(ahs*nan)) + + if 'mean_wave_dir' in f.variables.keys(): + adm = np.append(adm,f.variables['mean_wave_dir'][:,0,0]) + else: + adm = np.array(np.copy(ahs*nan)) + + if 'latitude' in f.variables.keys(): + lat[b] = f.variables['latitude'][:] + elif 'LATITUDE' in f.variables.keys(): + lat[b] = f.variables['LATITUDE'][:] + else: + lat[b] = nan + + if 'longitude' in f.variables.keys(): + lon[b] = f.variables['longitude'][:] + elif 'LONGITUDE' in f.variables.keys(): + lon[b] = f.variables['LONGITUDE'][:] + else: + lon[b] = nan + # This block is no longer directly under an 'else', so it's a separate check. + if 'wind_spd' in f.variables.keys(): + awind_spd = np.append(awind_spd, f.variables['wind_spd'][:, 0, 0]) + else: + awind_spd = nan + + atime = np.append(atime,np.array(f.variables['time'][:]).astype('double')) + + f.close(); del f + + adp = adm*np.nan # no peak direction available in this format + + except: + ahs=[] + + if np.size(ahs)>0: + # First layer of simple quality-control + indq=np.where((ahs>30.)|(ahs<0.0)) + if np.size(indq)>0: + ahs[indq]=np.nan; del indq + + indq=np.where((atm>40.)|(atm<0.0)) + if np.size(indq)>0: + atm[indq]=np.nan; del indq + + indq=np.where((atp>40.)|(atp<0.0)) + if np.size(indq)>0: + atp[indq]=np.nan; del indq + + indq=np.where((adm>360.)|(adm<-180.)) + if np.size(indq)>0: + adm[indq]=np.nan; del indq + + indq=np.where((adp>360.)|(adp<-180.)) + if np.size(indq)>0: + adp[indq]=np.nan; del indq + + c=0 + for t in range(0,np.size(mtime)): + indt=np.where(np.abs(atime-mtime[t])<1800.) + if np.size(indt)>0: + if np.any(ahs[indt[0]].mask==False): + bhs[b,t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask==False]) + c=c+1 + if np.any(atm[indt[0]].mask==False): + btm[b,t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask==False]) + if np.any(atp[indt[0]].mask==False): + btp[b,t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask==False]) + if np.any(adm[indt[0]].mask==False): + bdm[b,t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask==False]) + if np.any(adp[indt[0]].mask==False): + bdp[b,t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask==False]) + if np.any(awind_spd[indt[0]].mask == False): + bwind_spd[b, t] = np.nanmean(awind_spd[indt[0]][awind_spd[indt[0]].mask == False]) + + del indt + + print(" station "+stname[b]+" ok") + del ahs - ahs=[] - try: - - ahs=[];atm=[];adm=[];atime=[] - for y in yrange: - - f=nc.Dataset(ndbcp+"/"+stname[b]+"h"+repr(y)+".nc") - if 'wave_height' in f.variables.keys(): - ahs = np.append(ahs,f.variables['wave_height'][:,0,0]) - elif 'hs' in f.variables.keys(): - ahs = np.append(ahs,f.variables['hs'][:,0,0]) - elif 'swh' in f.variables.keys(): - ahs = np.append(ahs,f.variables['swh'][:,0,0]) - - if 'average_wpd' in f.variables.keys(): - atm = np.append(atm,f.variables['average_wpd'][:,0,0]) - else: - atm = np.array(np.copy(ahs*nan)) - - if 'dominant_wpd' in f.variables.keys(): - atp = np.append(atm,f.variables['dominant_wpd'][:,0,0]) - else: - atp = np.array(np.copy(ahs*nan)) - - if 'mean_wave_dir' in f.variables.keys(): - adm = np.append(adm,f.variables['mean_wave_dir'][:,0,0]) - else: - adm = np.array(np.copy(ahs*nan)) - - if 'latitude' in f.variables.keys(): - lat[b] = f.variables['latitude'][:] - elif 'LATITUDE' in f.variables.keys(): - lat[b] = f.variables['LATITUDE'][:] - else: - lat[b] = nan - - if 'longitude' in f.variables.keys(): - lon[b] = f.variables['longitude'][:] - elif 'LONGITUDE' in f.variables.keys(): - lon[b] = f.variables['LONGITUDE'][:] - else: - lon[b] = nan - - atime = np.append(atime,np.array(f.variables['time'][:]).astype('double')) - - f.close(); del f - - adp = adm*np.nan # no peak direction available in this format - - except: - try: - f=nc.Dataset(copernp+"/GL_TS_MO_"+stname[b]+".nc") - if 'VHM0' in f.variables.keys(): - ahs = np.nanmean(f.variables['VHM0'][:,:],axis=1) - elif 'VAVH' in f.variables.keys(): - ahs = np.nanmean(f.variables['VAVH'][:,:],axis=1) - elif 'VGHS' in f.variables.keys(): - ahs = np.nanmean(f.variables['VGHS'][:,:],axis=1) - elif 'significant_swell_wave_height' in f.variables.keys(): - ahs = np.nanmean(f.variables['significant_swell_wave_height'][:,:],axis=1) - elif 'sea_surface_significant_wave_height' in f.variables.keys(): - ahs = np.nanmean(f.variables['sea_surface_significant_wave_height'][:,:],axis=1) - elif 'SWHT' in f.variables.keys(): - ahs = np.nanmean(f.variables['SWHT'][:,:],axis=1) - elif 'wave_height_h1d3' in f.variables.keys(): - ahs = np.nanmean(f.variables['wave_height_h1d3'][:,:],axis=1) - elif 'spectral_significant_wave_height' in f.variables.keys(): - ahs = np.nanmean(f.variables['spectral_significant_wave_height'][:,:],axis=1) - - if 'VTM02' in f.variables.keys(): - atm = np.nanmean(f.variables['VTM02'][:,:],axis=1) - elif 'VGTA' in f.variables.keys(): - atm = np.nanmean(f.variables['VGTA'][:,:],axis=1) - else: - atm = ahs*nan - - if 'VTPK' in f.variables.keys(): - atp = np.nanmean(f.variables['VTPK'][:,:],axis=1) - elif 'dominant_wave_period' in f.variables.keys(): - atp = np.nanmean(f.variables['dominant_wave_period'][:,:],axis=1) - elif 'sea_surface_wave_period_at_spectral_density_maximum' in f.variables.keys(): - atp = np.nanmean(f.variables['sea_surface_wave_period_at_spectral_density_maximum'][:,:],axis=1) - else: - atp = ahs*nan - - if 'VMDR' in f.variables.keys(): - adm = np.nanmean(f.variables['VMDR'][:,:],axis=1) - else: - adm = ahs*nan - - if 'LATITUDE' in f.variables.keys(): - lat[b] = np.nanmean(f.variables['LATITUDE'][:]) - elif 'latitude' in f.variables.keys(): - lat[b] = np.nanmean(f.variables['latitude'][:]) - elif 'LAT' in f.variables.keys(): - lat[b] = np.nanmean(f.variables['LAT'][:]) - elif 'lat' in f.variables.keys(): - lat[b] = np.nanmean(f.variables['lat'][:]) - else: - lat[b] = nan - - if 'LONGITUDE' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['LONGITUDE'][:]) - elif 'LON' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['LON'][:]) - elif 'LONG' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['LONG'][:]) - elif 'longitude' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['longitude'][:]) - elif 'lon' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['lon'][:]) - elif 'long' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['long'][:]) - else: - lon[b] = nan - - adp = ahs*nan # no peak direction available in this format - - if 'TIME' in f.variables.keys(): - atime = np.array(f.variables['TIME'][:]*24*3600 + timegm( strptime('195001010000', '%Y%m%d%H%M') )).astype('double') - elif 'time' in f.variables.keys(): - atime = np.array(f.variables['time'][:]*24*3600 + timegm( strptime('195001010000', '%Y%m%d%H%M') )).astype('double') - - f.close(); del f - except: - ahs=[] - - - if np.size(ahs)>0: - - # First layer of simple quality-control - indq=np.where((ahs>30.)|(ahs<0.0)) - if np.size(indq)>0: - ahs[indq]=np.nan; del indq - - indq=np.where((atm>40.)|(atm<0.0)) - if np.size(indq)>0: - atm[indq]=np.nan; del indq - - indq=np.where((atp>40.)|(atp<0.0)) - if np.size(indq)>0: - atp[indq]=np.nan; del indq - - indq=np.where((adm>360.)|(adm<-180.)) - if np.size(indq)>0: - adm[indq]=np.nan; del indq - - indq=np.where((adp>360.)|(adp<-180.)) - if np.size(indq)>0: - adp[indq]=np.nan; del indq - - c=0 - for t in range(0,np.size(mtime)): - indt=np.where(np.abs(atime-mtime[t])<1800.) - if np.size(indt)>0: - if np.any(ahs[indt[0]].mask==False): - bhs[b,t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask==False]) - c=c+1 - if np.any(atm[indt[0]].mask==False): - btm[b,t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask==False]) - if np.any(atp[indt[0]].mask==False): - btp[b,t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask==False]) - if np.any(adm[indt[0]].mask==False): - bdm[b,t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask==False]) - if np.any(adp[indt[0]].mask==False): - bdp[b,t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask==False]) - - del indt - - # print("counted "+repr(c)+" at "+stname[b]) - - print(" station "+stname[b]+" ok") - del ahs print(' ') # Simple quality-control (range) From 0085a9138a1f4c66399a044222ca885c59fe485b Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Thu, 21 Mar 2024 14:05:47 +0000 Subject: [PATCH 03/27] added .spec reader to the buoy valodation code --- ww3tools/modelBuoy_collocation.py | 350 +++++++++++++++++++++--------- ww3tools/wread.py | 152 +++++++++++++ 2 files changed, 396 insertions(+), 106 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index fc6725c..9702472 100755 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -103,15 +103,16 @@ from time import strptime from calendar import timegm import wread +import sys # netcdf format fnetcdf="NETCDF4" # Paths # ndbcp="/data/buoys/NDBC/wparam" -ndbcp="/work/noaa/marine/ricardo.campos/data/buoys/NDBC/ncformat/wparam" +ndbcp="/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" # Copernicus buoys # copernp="/data/buoys/Copernicus/wtimeseries" -#copernp="/work/noaa/marine/ricardo.campos/data/buoys/Copernicus/wtimeseries" +copernp="/work/noaa/marine/ricardo.campos/data/buoys/Copernicus/wtimeseries" print(' ') # Options of including grid and cyclone information @@ -120,9 +121,9 @@ sys.exit(' At least one argument (list of ww3 files) must be informed.') if len(sys.argv) >= 2 : # # import os; os.system("ls -d $PWD/*tab.nc > ww3list.txt &") - wlist=np.atleast_1d(np.loadtxt(sys.argv[1],dtype=str)) - ftag=str(sys.argv[1]).split('list')[1].split('.txt')[0] - print(' Reading ww3 list '+str(sys.argv[1])) + wlist=np.atleast_1d(np.loadtxt(sys.argv[1],dtype=str,usecols=(0,))) + ftag=str(sys.argv[1]).split('list')[1].split('.spec')[0] + print(' Reading ww3 list '+ str(sys.argv[1])) print(' Tag '+ftag) if len(sys.argv) >= 3: forecastds=int(sys.argv[2]) @@ -168,6 +169,7 @@ if str(wlist[i]).split('/')[-1].split('.')[-1]=='station_tar': result = wread.bull_tar(wlist[i]) + print("Result keys:", result.keys()) at=result['time'] fcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') if i==0: @@ -282,14 +284,6 @@ stname=np.append(stname,astname); del astname - funits=f.variables['time'].units - if str(funits).split(' ')[0] == 'seconds': - tincr=1 - elif str(funits).split(' ')[0] == 'hours': - tincr=3600 - elif str(funits).split(' ')[0] == 'days': - tincr=24*3600 - ahs = np.array(f.variables['hs'][:,:]).T if 'th1m' in f.variables.keys(): @@ -320,9 +314,7 @@ else: atm = np.array(np.copy(ahs*nan)) - ftunits=str(f.variables['time'].units).split('since')[1][1::].replace('T',' ').replace('+00:00','') - at = np.array(f.variables['time'][:]*tincr + timegm( strptime(ftunits,'%Y-%m-%d %H:%M:%S') )).astype('double') - + at = np.array(f.variables['time'][:]*24*3600 + timegm( strptime(str(f.variables['time'].units).split(' ')[2][0:4]+'01010000', '%Y%m%d%H%M') )).astype('double') f.close(); del f fcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') if t==0: @@ -346,12 +338,76 @@ print(" Read WW3 data OK."); print(' ') -else: - sys.exit(' Point output file format not recognized: only bull, bull_tar, and .nc implemented.') - # include other text formats: tab50, .ts +elif (str(wlist[0]).split('/')[-1].split('.')[-1] == 'spec'): + if forecastds > 0: + forecastds = 0 # no 2D time array possible. + print(" Warning: no 2D time array possible. \ + Use multiple tar files (one per forecast cycle), station_tar or bull_tar, containing text files for each station.") + + for i in range(0, np.size(wlist)): + # re-check each file in the list respects the same format + if str(wlist[i]).split('/')[-1].split('.')[-1] == 'spec': + # Check if the required number of command-line arguments is provided +# if len(sys.argv) < 3: +# sys.exit('Two inputs must be provided: fileName and StationName') + + # Call read_text_file with command-line arguments + result = wread.read_text_file(sys.argv[1], sys.argv[2]) + + ntime = result['ntime'] + print("time:", ntime) + + if i == 0: + # Initialize variables on the first iteration + mfcycle = np.array(np.zeros((ntime.shape[0]), 'd') + ntime[0]).astype('double') + stname = np.atleast_1d(np.array(result['station_name'])) + mtime = np.copy(ntime) + mhs = np.copy([result['wind_spd']]) # wind + mtp = np.copy([result['wind_dir']]) # wndd + if 'dp' in result.keys(): + mdp = np.copy([result['spec']]) # dspec + else: + mdp = np.copy(mhs) * np.nan + + if 'dm' in result.keys(): + mdm = np.copy([result['deltafreq']]) # dfreq + else: + mdm = np.copy(mhs) * np.nan + + if 'tm' in result.keys(): + mtm = np.copy([result['freq']]) # freq + else: + mtm = np.copy(mhs) * np.nan + else: + if (mhs.shape[1] == result['hs'].shape[0]): + stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) + mtime = np.append(mtime, ntime) + mhs = np.append(mhs, [result['wind_spd']], axis=0) + mtp = np.append(mtp, [result['wind_dir']], axis=0) + if 'dp' in result.keys(): + mdp = np.append(mdp, [result['spec']], axis=0) + else: + mdp = np.append(mdp, [np.copy(result['wind_spd']) * np.nan], axis=0) + + if 'dm' in result.keys(): + mdm = np.append(mdp, [result['deltafreq']], axis=0) + else: + mdm = np.append(mdm, [np.copy(result['wind_spd']) * np.nan], axis=0) + + if 'tm' in result.keys(): + mtm = np.append(mtm, [result['freq']], axis=0) + else: + mtm = np.append(mtm, [np.copy(result['wind_spd']) * np.nan], axis=0) + else: + print(" Stations in " + wlist[i] + " do not match the other spec files. Skipped " + wlist[i]) + + del result + print(" ww3 file " + wlist[i] + " OK") -print(" Start building the matchups model/buoy ..."); print(' ') +#else: + #sys.exit(' Point output file format not recognized: only bull, bull_tar, and .nc implemented.') + # include other text formats: tab50, .ts # BUOYS ------------------ bhs=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan @@ -360,112 +416,194 @@ bdm=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan bdp=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan lat=np.zeros(np.size(stname),'f')*np.nan; lon=np.zeros(np.size(stname),'f')*np.nan -bwind_spd = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan - # help reading NDBC buoys, divided by year yrange=np.array(np.arange(time.gmtime(mtime.min())[0],time.gmtime(mtime.min())[0]+1,1)).astype('int') # loop buoys for b in range(0,np.size(stname)): - ahs=[] - try: - ahs=[];atm=[];adm=[];atime=[] - for y in yrange: - f=nc.Dataset(ndbcp+"/"+stname[b]+"h"+repr(y)+".nc") - if 'wave_height' in f.variables.keys(): - ahs = np.append(ahs,f.variables['wave_height'][:,0,0]) - elif 'hs' in f.variables.keys(): - ahs = np.append(ahs,f.variables['hs'][:,0,0]) - elif 'swh' in f.variables.keys(): - ahs = np.append(ahs,f.variables['swh'][:,0,0]) - - if 'average_wpd' in f.variables.keys(): - atm = np.append(atm,f.variables['average_wpd'][:,0,0]) - else: - atm = np.array(np.copy(ahs*nan)) - if 'dominant_wpd' in f.variables.keys(): - atp = np.append(atm,f.variables['dominant_wpd'][:,0,0]) - else: - atp = np.array(np.copy(ahs*nan)) + ahs=[] + try: - if 'mean_wave_dir' in f.variables.keys(): - adm = np.append(adm,f.variables['mean_wave_dir'][:,0,0]) - else: - adm = np.array(np.copy(ahs*nan)) + ahs=[];atm=[];adm=[];atime=[] + for y in yrange: - if 'latitude' in f.variables.keys(): - lat[b] = f.variables['latitude'][:] - elif 'LATITUDE' in f.variables.keys(): - lat[b] = f.variables['LATITUDE'][:] - else: - lat[b] = nan +# f=nc.Dataset(ndbcp+"/"+stname[b]+"h"+repr(y)+".nc") +# if 'wave_height' in f.variables.keys(): +# ahs = np.append(ahs,f.variables['wave_height'][:,0,0]) +# elif 'hs' in f.variables.keys(): +# ahs = np.append(ahs,f.variables['hs'][:,0,0]) +# elif 'swh' in f.variables.keys(): +# ahs = np.append(ahs,f.variables['swh'][:,0,0]) - if 'longitude' in f.variables.keys(): - lon[b] = f.variables['longitude'][:] - elif 'LONGITUDE' in f.variables.keys(): - lon[b] = f.variables['LONGITUDE'][:] - else: - lon[b] = nan - # This block is no longer directly under an 'else', so it's a separate check. - if 'wind_spd' in f.variables.keys(): - awind_spd = np.append(awind_spd, f.variables['wind_spd'][:, 0, 0]) - else: - awind_spd = nan - atime = np.append(atime,np.array(f.variables['time'][:]).astype('double')) + f=nc.Dataset(ndbcp+"/"+stname[b]+"h"+repr(y)+".nc") + if 'wind_spd' in f.variables.keys(): + ahs = np.append(ahs,f.variables['wind_spd'][:,0,0]) + elif 'wsp' in f.variables.keys(): + ahs = np.append(ahs,f.variables['wsp'][:,0,0]) + elif 'swhp' in f.variables.keys(): + ahs = np.append(ahs,f.variables['swhp'][:,0,0]) + + + if 'average_wpd' in f.variables.keys(): + atm = np.append(atm,f.variables['average_wpd'][:,0,0]) + else: + atm = np.array(np.copy(ahs*nan)) + + if 'dominant_wpd' in f.variables.keys(): + atp = np.append(atm,f.variables['dominant_wpd'][:,0,0]) + else: + atp = np.array(np.copy(ahs*nan)) + + if 'mean_wave_dir' in f.variables.keys(): + adm = np.append(adm,f.variables['mean_wave_dir'][:,0,0]) + else: + adm = np.array(np.copy(ahs*nan)) + + if 'latitude' in f.variables.keys(): + lat[b] = f.variables['latitude'][:] + elif 'LATITUDE' in f.variables.keys(): + lat[b] = f.variables['LATITUDE'][:] + else: + lat[b] = nan - f.close(); del f + if 'longitude' in f.variables.keys(): + lon[b] = f.variables['longitude'][:] + elif 'LONGITUDE' in f.variables.keys(): + lon[b] = f.variables['LONGITUDE'][:] + else: + lon[b] = nan - adp = adm*np.nan # no peak direction available in this format + atime = np.append(atime,np.array(f.variables['time'][:]).astype('double')) - except: - ahs=[] + f.close(); del f - if np.size(ahs)>0: - # First layer of simple quality-control - indq=np.where((ahs>30.)|(ahs<0.0)) - if np.size(indq)>0: - ahs[indq]=np.nan; del indq + adp = adm*np.nan # no peak direction available in this format + + except: +# try: +# f=nc.Dataset(copernp+"/GL_TS_MO_"+stname[b]+".nc") +# if 'VHM0' in f.variables.keys(): +# ahs = np.nanmean(f.variables['VHM0'][:,:],axis=1) +# elif 'VAVH' in f.variables.keys(): +# ahs = np.nanmean(f.variables['VAVH'][:,:],axis=1) +# elif 'VGHS' in f.variables.keys(): +# ahs = np.nanmean(f.variables['VGHS'][:,:],axis=1) +# elif 'significant_swell_wave_height' in f.variables.keys(): +# ahs = np.nanmean(f.variables['significant_swell_wave_height'][:,:],axis=1) +# elif 'sea_surface_significant_wave_height' in f.variables.keys(): +# ahs = np.nanmean(f.variables['sea_surface_significant_wave_height'][:,:],axis=1) +# elif 'SWHT' in f.variables.keys(): +# ahs = np.nanmean(f.variables['SWHT'][:,:],axis=1) +# elif 'wave_height_h1d3' in f.variables.keys(): +# ahs = np.nanmean(f.variables['wave_height_h1d3'][:,:],axis=1) +# elif 'spectral_significant_wave_height' in f.variables.keys(): +# ahs = np.nanmean(f.variables['spectral_significant_wave_height'][:,:],axis=1) +# +# if 'VTM02' in f.variables.keys(): +# atm = np.nanmean(f.variables['VTM02'][:,:],axis=1) +# elif 'VGTA' in f.variables.keys(): +# atm = np.nanmean(f.variables['VGTA'][:,:],axis=1) +# else: +# atm = ahs*nan +# +# if 'VTPK' in f.variables.keys(): +# atp = np.nanmean(f.variables['VTPK'][:,:],axis=1) +# elif 'dominant_wave_period' in f.variables.keys(): +# atp = np.nanmean(f.variables['dominant_wave_period'][:,:],axis=1) +# elif 'sea_surface_wave_period_at_spectral_density_maximum' in f.variables.keys(): +# atp = np.nanmean(f.variables['sea_surface_wave_period_at_spectral_density_maximum'][:,:],axis=1) +# else: +# atp = ahs*nan +# +# if 'VMDR' in f.variables.keys(): +# adm = np.nanmean(f.variables['VMDR'][:,:],axis=1) +# else: +# adm = ahs*nan +# +# if 'LATITUDE' in f.variables.keys(): +# lat[b] = np.nanmean(f.variables['LATITUDE'][:]) +# elif 'latitude' in f.variables.keys(): +# lat[b] = np.nanmean(f.variables['latitude'][:]) +# elif 'LAT' in f.variables.keys(): +# lat[b] = np.nanmean(f.variables['LAT'][:]) +# elif 'lat' in f.variables.keys(): +# lat[b] = np.nanmean(f.variables['lat'][:]) +# else: +# lat[b] = nan +# +# if 'LONGITUDE' in f.variables.keys(): +# lon[b] = np.nanmean(f.variables['LONGITUDE'][:]) +# elif 'LON' in f.variables.keys(): +# lon[b] = np.nanmean(f.variables['LON'][:]) +# elif 'LONG' in f.variables.keys(): +# lon[b] = np.nanmean(f.variables['LONG'][:]) +# elif 'longitude' in f.variables.keys(): +# lon[b] = np.nanmean(f.variables['longitude'][:]) +# elif 'lon' in f.variables.keys(): +# lon[b] = np.nanmean(f.variables['lon'][:]) +# elif 'long' in f.variables.keys(): +# lon[b] = np.nanmean(f.variables['long'][:]) +# else: +# lon[b] = nan +# +# adp = ahs*nan # no peak direction available in this format +# +# if 'TIME' in f.variables.keys(): +# atime = np.array(f.variables['TIME'][:]*24*3600 + timegm( strptime('195001010000', '%Y%m%d%H%M') )).astype('double') +# elif 'time' in f.variables.keys(): +# atime = np.array(f.variables['time'][:]*24*3600 + timegm( strptime('195001010000', '%Y%m%d%H%M') )).astype('double') +# +# f.close(); del f +# except: +# ahs=[] + pass + + if np.size(ahs)>0: - indq=np.where((atm>40.)|(atm<0.0)) - if np.size(indq)>0: - atm[indq]=np.nan; del indq + # First layer of simple quality-control + indq=np.where((ahs>30.)|(ahs<0.0)) + if np.size(indq)>0: + ahs[indq]=np.nan; del indq - indq=np.where((atp>40.)|(atp<0.0)) - if np.size(indq)>0: - atp[indq]=np.nan; del indq + indq=np.where((atm>40.)|(atm<0.0)) + if np.size(indq)>0: + atm[indq]=np.nan; del indq - indq=np.where((adm>360.)|(adm<-180.)) - if np.size(indq)>0: - adm[indq]=np.nan; del indq + indq=np.where((atp>40.)|(atp<0.0)) + if np.size(indq)>0: + atp[indq]=np.nan; del indq - indq=np.where((adp>360.)|(adp<-180.)) - if np.size(indq)>0: - adp[indq]=np.nan; del indq + indq=np.where((adm>360.)|(adm<-180.)) + if np.size(indq)>0: + adm[indq]=np.nan; del indq - c=0 - for t in range(0,np.size(mtime)): - indt=np.where(np.abs(atime-mtime[t])<1800.) - if np.size(indt)>0: - if np.any(ahs[indt[0]].mask==False): - bhs[b,t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask==False]) - c=c+1 - if np.any(atm[indt[0]].mask==False): - btm[b,t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask==False]) - if np.any(atp[indt[0]].mask==False): - btp[b,t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask==False]) - if np.any(adm[indt[0]].mask==False): - bdm[b,t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask==False]) - if np.any(adp[indt[0]].mask==False): - bdp[b,t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask==False]) - if np.any(awind_spd[indt[0]].mask == False): - bwind_spd[b, t] = np.nanmean(awind_spd[indt[0]][awind_spd[indt[0]].mask == False]) + indq=np.where((adp>360.)|(adp<-180.)) + if np.size(indq)>0: + adp[indq]=np.nan; del indq - del indt + c=0 + for t in range(0,np.size(mtime)): + indt=np.where(np.abs(atime-mtime[t])<1800.) + if np.size(indt)>0: + if np.any(ahs[indt[0]].mask==False): + bhs[b,t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask==False]) + c=c+1 + if np.any(atm[indt[0]].mask==False): + btm[b,t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask==False]) + if np.any(atp[indt[0]].mask==False): + btp[b,t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask==False]) + if np.any(adm[indt[0]].mask==False): + bdm[b,t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask==False]) + if np.any(adp[indt[0]].mask==False): + bdp[b,t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask==False]) + + del indt - print(" station "+stname[b]+" ok") - del ahs + # print("counted "+repr(c)+" at "+stname[b]) + print(" station "+stname[b]+" ok") + del ahs print(' ') # Simple quality-control (range) @@ -674,7 +812,7 @@ vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbhs = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) diff --git a/ww3tools/wread.py b/ww3tools/wread.py index 05551a3..d23e7a6 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1233,3 +1233,155 @@ def spec_ww3(*args): return result del mtime,mdate,lat,lon,wnds,wndd,freq,freq1,freq2,dfreq,pwst,dire,d1sp,dspec + +#added a function to read the txt files + +def read_text_file(fname_txtfile, stname): + sk = 1 + try: + # Attempt to open and read the file name from the txt file + with open(fname_txtfile, 'r') as f: + lines = f.readlines() + if len(lines) != 1: + raise ValueError("The txt file should contain only one line with the file name.") + fname = lines[0].strip() + except FileNotFoundError: + sys.exit('Text file not found.') + except Exception as e: + sys.exit(f'Error reading txt file: {str(e)}') + + print("Filename:", fname) + print("Station Name:", stname) + print("sk:", sk) + + results = {} + + fp = open(fname) + nt = fp.read().count(stname) + fp.close() + + if nt >= 1: + # Open file and read the first parameters + fp = open(fname) + cabc = fp.readline().strip().split() + nf = int(cabc[3]) # number of frequencies + nd = int(cabc[4]) # number of directions + npo = int(cabc[5]) # number of point outputs + + freq = np.zeros(nf, 'f') + dire = np.zeros(nd, 'f') + dspec = np.zeros((nt, nf, nd), 'f') + adire = np.zeros(dire.shape) + adspec = np.zeros(dspec.shape) + ntime = np.zeros((nt), 'd') + + # Frequencies -------------------- + ncf = int(np.floor(nf/8)) + rncf = int(np.round(8*((float(nf)/8)-ncf))) + k = 0 + for i in range(0, ncf): + line = fp.readline() + line = line.strip().split() + for j in range(0, 8): + freq[k] = float(line[j]) + k = k+1 + + if rncf > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rncf): + freq[k] = float(line[i]) + k = k+1 + + # DF in frequency (dfreq) + dfreq = np.zeros(freq.shape[0], 'f') + for i in range(0, freq.shape[0]): + if i == 0 or i == (freq.shape[0]-1): + dfreq[i] = freq[i]*(1 + (((freq[-1]/freq[-2])-1)/2)) - freq[i] + else: + dfreq[i] = freq[i]*(freq[-1]/freq[-2]) - freq[i] + + # Directions --------------------- + ncd = int(np.floor(nd/7)) + rncd = int(np.round(7*((float(nd)/7)-ncd))) + k = 0 + for i in range(0, ncd): + line = fp.readline() + line = line.strip().split() + for j in range(0, 7): + dire[k] = float(line[j])*180/np.pi + k = k+1 + + if rncd > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rncd): + dire[k] = float(line[i])*180/np.pi + k = k+1 + + nl = int(np.floor((nf*nd)/7.)) + rnl = int(np.round(7*((float(nf*nd)/7)-nl))) + auxs = np.zeros((nf*nd), 'f') + wnds = np.zeros((nt), 'f') + wndd = np.zeros((nt), 'f') + for t in range(0, nt): + + cabc = fp.readline().strip().split()[0] + ntime[t] = np.double(timegm(strptime(cabc.strip().split()[0], '%Y%m%d%H'))) + cabc = fp.readline().strip().split() + if t == 0: + namep = cabc[0][1:] + lat_str, lon_str = cabc[2].split('-') + lat = float(lat_str) + lon = -float(lon_str) + depth = float(cabc[4]) + + wnds[t] = float(cabc[5]) + wndd[t] = float(cabc[6]) + + k = 0 + for i in range(0, nl): + line = fp.readline() + line = line.strip().split() + for j in range(0, 7): + auxs[k] = float(line[j]) + k = k+1 + + if rncd > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rnl): + auxs[k] = float(line[i]) + k = k+1 + + for ic in range(0, nf): + for il in range(0, nd): + dspec[t, ic, il] = auxs[il*nf+ic] + + fp.close() + + results['ntime'] = ntime + results['latitude'] = lat + results['longitude'] = lon + results['wind_spd'] = wnds + results['wind_dir'] = wndd + results['freq'] = freq + results['deltafreq'] = dfreq + results['dir'] = dire + results['spec'] = dspec + results['station_name'] = stname + # Print some values for verification + print("Number of time steps:", nt) + print("Latitude:", lat) + print("Longitude:", lon) + print("Depth:", depth) + + else: + sys.exit(' Station '+stname+' not included in the output file') + + return results + + +# Example usage: +# results = read_text_file('txtfile_containing_filename.txt', 'station_name') + From 5c304ea629142114663a32e74f4270401334f965 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 25 Mar 2024 14:19:58 +0000 Subject: [PATCH 04/27] added .spec and also is able to to the hs calculations in the code --- ww3tools/wread.py | 299 ++++++++++++++++++++++++++-------------------- 1 file changed, 171 insertions(+), 128 deletions(-) diff --git a/ww3tools/wread.py b/ww3tools/wread.py index d23e7a6..bcfb0bc 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1236,8 +1236,7 @@ def spec_ww3(*args): #added a function to read the txt files -def read_text_file(fname_txtfile, stname): - sk = 1 +def read_text_file(fname_txtfile): try: # Attempt to open and read the file name from the txt file with open(fname_txtfile, 'r') as f: @@ -1250,138 +1249,182 @@ def read_text_file(fname_txtfile, stname): except Exception as e: sys.exit(f'Error reading txt file: {str(e)}') - print("Filename:", fname) - print("Station Name:", stname) - print("sk:", sk) - results = {} + stname = [] + + try: + tar = tarfile.open(fname, "r:gz") # Open the tar file - fp = open(fname) - nt = fp.read().count(stname) - fp.close() - - if nt >= 1: - # Open file and read the first parameters - fp = open(fname) - cabc = fp.readline().strip().split() - nf = int(cabc[3]) # number of frequencies - nd = int(cabc[4]) # number of directions - npo = int(cabc[5]) # number of point outputs - - freq = np.zeros(nf, 'f') - dire = np.zeros(nd, 'f') - dspec = np.zeros((nt, nf, nd), 'f') - adire = np.zeros(dire.shape) - adspec = np.zeros(dspec.shape) - ntime = np.zeros((nt), 'd') - - # Frequencies -------------------- - ncf = int(np.floor(nf/8)) - rncf = int(np.round(8*((float(nf)/8)-ncf))) - k = 0 - for i in range(0, ncf): - line = fp.readline() - line = line.strip().split() - for j in range(0, 8): - freq[k] = float(line[j]) - k = k+1 - - if rncf > 0: - line = fp.readline() - line = line.strip().split() - for i in range(0, rncf): - freq[k] = float(line[i]) - k = k+1 + for t in range(0, len(tar.getmembers())): + # Station names - # DF in frequency (dfreq) - dfreq = np.zeros(freq.shape[0], 'f') - for i in range(0, freq.shape[0]): - if i == 0 or i == (freq.shape[0]-1): - dfreq[i] = freq[i]*(1 + (((freq[-1]/freq[-2])-1)/2)) - freq[i] + stname.append(str(tar.getmembers()[t].name).split('/')[-1].split('.')[-2]) + + try: + fp = tar.extractfile(tar.getmembers()[t]) + if fp is None: + raise ValueError("File is empty or cannot be extracted.") + lines = fp.readlines() + nt=len(lines) + except Exception as e: + print("Cannot open " + tar.getmembers()[t].name) + print("Error:", str(e)) + continue else: - dfreq[i] = freq[i]*(freq[-1]/freq[-2]) - freq[i] - - # Directions --------------------- - ncd = int(np.floor(nd/7)) - rncd = int(np.round(7*((float(nd)/7)-ncd))) - k = 0 - for i in range(0, ncd): - line = fp.readline() - line = line.strip().split() - for j in range(0, 7): - dire[k] = float(line[j])*180/np.pi - k = k+1 - - if rncd > 0: - line = fp.readline() - line = line.strip().split() - for i in range(0, rncd): - dire[k] = float(line[i])*180/np.pi - k = k+1 - - nl = int(np.floor((nf*nd)/7.)) - rnl = int(np.round(7*((float(nf*nd)/7)-nl))) - auxs = np.zeros((nf*nd), 'f') - wnds = np.zeros((nt), 'f') - wndd = np.zeros((nt), 'f') - for t in range(0, nt): - - cabc = fp.readline().strip().split()[0] - ntime[t] = np.double(timegm(strptime(cabc.strip().split()[0], '%Y%m%d%H'))) - cabc = fp.readline().strip().split() - if t == 0: - namep = cabc[0][1:] - lat_str, lon_str = cabc[2].split('-') - lat = float(lat_str) - lon = -float(lon_str) - depth = float(cabc[4]) - - wnds[t] = float(cabc[5]) - wndd[t] = float(cabc[6]) - - k = 0 - for i in range(0, nl): - line = fp.readline() - line = line.strip().split() - for j in range(0, 7): - auxs[k] = float(line[j]) - k = k+1 - - if rncd > 0: - line = fp.readline() - line = line.strip().split() - for i in range(0, rnl): - auxs[k] = float(line[i]) - k = k+1 - - for ic in range(0, nf): - for il in range(0, nd): - dspec[t, ic, il] = auxs[il*nf+ic] - - fp.close() - - results['ntime'] = ntime - results['latitude'] = lat - results['longitude'] = lon - results['wind_spd'] = wnds - results['wind_dir'] = wndd - results['freq'] = freq - results['deltafreq'] = dfreq - results['dir'] = dire - results['spec'] = dspec - results['station_name'] = stname - # Print some values for verification - print("Number of time steps:", nt) - print("Latitude:", lat) - print("Longitude:", lon) - print("Depth:", depth) + if nt == 0: + print("No lines to read in file:", tar.getmembers()[t].name) + continue + for line in lines: + line = line.strip().decode() + + if nt >= 1: + # Open file and read the first parameters + fp = tar.extractfile(tar.getmembers()[t]) + cabc = fp.readline().strip().split() + nf = int(cabc[3]) # number of frequencies + nd = int(cabc[4]) # number of directions + npo = int(cabc[5]) # number of point outputs + + freq = np.zeros(nf, 'f') + dire = np.zeros(nd, 'f') + dspec = np.zeros((nt, nf, nd), 'f') + adire = np.zeros(dire.shape) + adspec = np.zeros(dspec.shape) + ntime = np.zeros((nt), 'd') + + # Frequencies -------------------- + ncf = int(np.floor(nf/8)) + rncf = int(np.round(8*((float(nf)/8)-ncf))) + k = 0 + for i in range(0, ncf): + line = fp.readline() + line = line.strip().split() + for j in range(0, 8): + freq[k] = float(line[j]) + k = k+1 + + if rncf > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rncf): + freq[k] = float(line[i]) + k = k+1 + + # DF in frequency (dfreq) + dfreq = np.zeros(freq.shape[0], 'f') + for i in range(0, freq.shape[0]): + if i == 0 or i == (freq.shape[0]-1): + dfreq[i] = freq[i]*(1 + (((freq[-1]/freq[-2])-1)/2)) - freq[i] + else: + dfreq[i] = freq[i]*(freq[-1]/freq[-2]) - freq[i] + + # Directions --------------------- + ncd = int(np.floor(nd/7)) + rncd = int(np.round(7*((float(nd)/7)-ncd))) + k = 0 + for i in range(0, ncd): + line = fp.readline() + line = line.strip().split() + for j in range(0, 7): + dire[k] = float(line[j])*180/np.pi + k = k+1 + + if rncd > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rncd): + dire[k] = float(line[i])*180/np.pi + k = k+1 + + nl = int(np.floor((nf*nd)/7.)) + rnl = int(np.round(7*((float(nf*nd)/7)-nl))) + auxs = np.zeros((nf*nd), 'f') + wnds = np.zeros((nt), 'f') + wndd = np.zeros((nt), 'f') + hs = np.zeros(nt) # Initialize significant wave height array + + for t in range(0, nt): + + cabc = [item.decode() for item in fp.readline().strip().split()] + + if not cabc: + continue + ntime[t] = np.double(timegm( strptime(cabc[0]+cabc[1][0:2], '%Y%m%d%H') )) + cabc = [item.decode() for item in fp.readline().strip().split()] + + if not cabc: + continue + if t == 0: - else: - sys.exit(' Station '+stname+' not included in the output file') + if len(cabc) >= 8: + namep = cabc[0][1:] + lat_str = cabc[3] + lon_str = cabc[4] + lat = -float(lat_str) if lat_str.startswith('-') else float(lat_str) + lon = -float(lon_str) if lon_str.startswith('-') else float(lon_str) + depth = float(cabc[5]) + wnds[t] = float(cabc[6]) + wndd[t] = float(cabc[7]) + + + + elif len(cabc) == 7: + namep = cabc[0][1:-1] if cabc[0].startswith("'") and cabc[0].endswith("'") else cabc[0][1:] + print("Station Name:", namep) + lat_lon = cabc[1].split('-') + print("lat-lon:",lat_lon) + lat_str = lat_lon[0] + print("lat_str:",lat_str) + lon_str = lat_lon[1] + print("lon_str:",lon_str) + lat = -float(lat_str) if lat_str.startswith('-') else float(lat_str) + lon = -float(lon_str) if lon_str.startswith('-') else float(lon_str) + depth = float(cabc[3]) + wnds[t] = float(cabc[4]) + wndd[t] = float(cabc[5]) + + k = 0 + for i in range(0, nl): + line = fp.readline() + line = line.strip().split() + for j in range(0, 7): + auxs[k] = float(line[j]) + k = k+1 + + if rncd > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rnl): + auxs[k] = float(line[i]) + k = k+1 + + for ic in range(0, nf): + for il in range(0, nd): + dspec[t, ic, il] = auxs[il*nf+ic] + + # Calculate significant wave height + sp1d = np.sum(dspec[t], axis=1) * (np.abs(dire[1] - dire[0])) # Calculate 1D spectrum + hs[t] = 4 * np.sqrt(np.trapz(sp1d, x=freq)) # Calculate significant wave height + + fp.close() + + results['ntime'] = ntime + results['latitude'] = lat + results['longitude'] = lon + results['wind_spd'] = wnds + results['wind_dir'] = wndd + results['freq'] = freq + results['deltafreq'] = dfreq + results['dir'] = dire + results['spec'] = dspec + results['station_name'] = stname + results['hs'] = hs # Add significant wave height to results - return results + else: + sys.exit(f'Station {stname} not included in') + except Exception as e: + sys.exit(f'Error reading spectral file: {str(e)}') + return results -# Example usage: -# results = read_text_file('txtfile_containing_filename.txt', 'station_name') From 12a8a9d4479656122e149063a802eb3699159d81 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 8 Apr 2024 19:04:08 +0000 Subject: [PATCH 05/27] wind variable added to the .spec in a list of buoy nams and boy IDs --- ww3tools/model5.py | 689 ++++++++++++++++++ ww3tools/openfile2.py | 51 ++ ww3tools/wread.py | 1582 +++++------------------------------------ 3 files changed, 937 insertions(+), 1385 deletions(-) create mode 100644 ww3tools/model5.py create mode 100644 ww3tools/openfile2.py diff --git a/ww3tools/model5.py b/ww3tools/model5.py new file mode 100644 index 0000000..33646cc --- /dev/null +++ b/ww3tools/model5.py @@ -0,0 +1,689 @@ +import warnings +warnings.filterwarnings("ignore") +import numpy as np +from matplotlib.mlab import * +from pylab import * +import xarray as xr +import netCDF4 as nc +import time +from time import strptime +from calendar import timegm +import wread +# netcdf format +fnetcdf="NETCDF4" + +# Paths +# ndbcp="/data/buoys/NDBC/wparam" +ndbcp="/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" +# Copernicus buoys +# copernp="/data/buoys/Copernicus/wtimeseries" +copernp="/work/noaa/marine/ricardo.campos/data/buoys/Copernicus/wtimeseries" +print(' ') + + +# Options of including grid and cyclone information +gridinfo=int(0); cyclonemap=int(0); wlist=[]; ftag=''; forecastds=0 + + + +if len(sys.argv) >= 4: + gridinfo=str(sys.argv[3]) + print(' Using gridInfo '+gridinfo) + + + + +# READ DATA +print(" ") +if gridinfo!=0: + # Grid Information + gridmask = wread.mask(gridinfo) + mlat=gridmask['latitude']; mlon=gridmask['longitude'] + mask=gridmask['mask']; distcoast=gridmask['distcoast']; depth=gridmask['depth'] + oni=gridmask['GlobalOceansSeas']; ocnames=gridmask['names_GlobalOceansSeas'] + hsmz=gridmask['HighSeasMarineZones']; hsmznames=gridmask['names_HighSeasMarineZones'] + print(" GridInfo Ok. "+gridinfo) + + # Cyclone Information + if cyclonemap!=0: + cycloneinfo = wread.cyclonemap(cyclonemap) + clat=cycloneinfo['latitude']; clon=cycloneinfo['longitude'] + cmap=cycloneinfo['cmap']; ctime=cycloneinfo['time'] + cinfo=np.array(cycloneinfo['info'].split(':')[1].split(';')) + if np.array_equal(clat,mlat)==True & np.array_equal(clon,mlon)==True: + print(" CycloneMap Ok. "+cyclonemap) + else: + sys.exit(' Error: Cyclone grid and Mask grid are different.') + + + + + + + + + + +from wread import spec_ww3 + +# Check if the correct number of arguments is provided +if len(sys.argv) != 3: + print("Usage: python run_spec_ww3.py file_names.txt station_names.txt") + sys.exit(1) + +# Read file names from the first argument +file_names = [] +with open(sys.argv[1], 'r') as file: + for line in file: + file_names.append(line.strip()) + +# Read station names from the second argument +station_names = [] +with open(sys.argv[2], 'r') as file: + for line in file: + station_names.append(line.strip()) + +# Call the spec_ww3 function +results = spec_ww3(file_names, station_names) + +# Additional processing +mfcycle = None +stname = None +mtime = None +mhs = None +mtp = None +mwn = None +mwd = None +mfreq = None +mtm=None +mdm=None + + +for i, result in enumerate(results): + at = result['time'] + if i == 0: + mfcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') + stname = np.atleast_1d(np.array(result['station_name'])) + mtime = np.copy(at) + mhs = np.copy([result['Hs']]) + mtp = np.copy([result['Tp']]) + if 'wind_spd' in result.keys(): + mwn = np.copy([result['wind_spd']]) + else: + mwn = np.copy(mhs) * np.nan + + if 'wind_dir' in result.keys(): + mwd = np.copy([result['wind_dir']]) + else: + mwd = np.copy(mhs) * np.nan + + if 'freq' in result.keys(): + mfreq = np.copy([result['freq']]) + else: + mfreq = np.copy(mhs) * np.nan + + if 'tm' in result.keys(): + mtm = np.append(mtm,[result['tm']],axis=0) + else: + mtm = np.copy(mhs) * np.nan + + if 'dm' in result.keys(): + mdm=np.copy([result['dm']]) + else: + mdm=np.copy(mhs)*np.nan + + if 'dp' in result.keys(): + mdp=np.copy([result['dp']]) + else: + mdp=np.copy(mhs)*np.nan + + else: + if mhs.shape[1] == result['Hs'].shape[0]: + stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) + mtime = np.append(mtime, at) + mhs = np.append(mhs, [result['Hs']], axis=0) + mtp = np.append(mtp, [result['Tp']], axis=0) + if 'wind_spd' in result.keys(): + mwn = np.append(mwn, [result['wind_spd']], axis=0) + else: + mwn = np.append(mwn, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'wind_dir' in result.keys(): + mwd = np.append(mwd, [result['wind_dir']], axis=0) + else: + mwd = np.append(mwd, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'freq' in result.keys(): + mfreq = np.append(mfreq, [result['freq']], axis=0) + else: + mfreq = np.append(mfreq, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'tm' in result.keys(): + mtm=np.append(mtm,[result['tm']],axis=0) + else: + mtm=np.append(mtm,[np.copy(result['Hs'])*np.nan],axis=0) + + if 'dm' in result.keys(): + mdm=np.append(mdm,[result['dm']],axis=0) + else: + mdm=np.append(mdm,[np.copy(result['Hs'])*np.nan],axis=0) + + if 'dp' in result.keys(): + mdp=np.append(mdp,[result['dp']],axis=0) + else: + mdp=np.append(mdp,[np.copy(result['Hs'])*np.nan],axis=0) + + else: + print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) + +# Save proc +print('HSS:', mhs) +print('WIND:', mwn) + + +# BUOYS ------------------ +bwind = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bhs = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +btm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +btp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bdm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bdp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +lat = np.zeros(np.size(stname), 'f') * np.nan +lon = np.zeros(np.size(stname), 'f') * np.nan +# help reading NDBC buoys, divided by year +yrange = np.array(np.arange(time.gmtime(mtime.min())[0], time.gmtime(mtime.min())[0] + 1, 1)).astype('int') +# loop buoys +for b in range(0, np.size(stname)): + + ahs = [] + try: + awm = [] + ahs = [] + atm = [] + atp = [] + adm = [] + atime = [] + for y in yrange: + + f = nc.Dataset(ndbcp + "/" + stname[b] + "h" + repr(y) + ".nc") + if 'wave_height' in f.variables.keys(): + ahs = np.append(ahs, f.variables['wave_height'][:, 0, 0]) + elif 'hs' in f.variables.keys(): + ahs = np.append(ahs, f.variables['hs'][:, 0, 0]) + elif 'swh' in f.variables.keys(): + ahs = np.append(ahs, f.variables['swh'][:, 0, 0]) + + if 'wind_spd' in f.variables.keys(): + awm = np.append(awm, f.variables['wind_spd'][:, 0, 0]) + else: + awm = np.array(np.copy(ahs * nan)) + + if 'average_wpd' in f.variables.keys(): + atm = np.append(atm, f.variables['average_wpd'][:, 0, 0]) + else: + atm = np.array(np.copy(ahs * nan)) + + if 'dominant_wpd' in f.variables.keys(): + atp = np.append(atp, f.variables['dominant_wpd'][:, 0, 0]) + else: + atp = np.array(np.copy(ahs * nan)) + + if 'mean_wave_dir' in f.variables.keys(): + adm = np.append(adm, f.variables['mean_wave_dir'][:, 0, 0]) + else: + adm = np.array(np.copy(ahs * nan)) + + if 'latitude' in f.variables.keys(): + lat[b] = f.variables['latitude'][:] + elif 'LATITUDE' in f.variables.keys(): + lat[b] = f.variables['LATITUDE'][:] + else: + lat[b] = nan + + if 'longitude' in f.variables.keys(): + lon[b] = f.variables['longitude'][:] + elif 'LONGITUDE' in f.variables.keys(): + lon[b] = f.variables['LONGITUDE'][:] + else: + lon[b] = nan + + atime = np.append(atime, np.array(f.variables['time'][:]).astype('double')) + + f.close() + del f + + adp = adm * np.nan # no peak direction available in this format + + if np.size(ahs) > 0: + + # First layer of simple quality-control + indq = np.where((ahs > 30.) | (ahs < 0.0)) + if np.size(indq) > 0: + ahs[indq] = np.nan + del indq + + indq = np.where((atm > 40.) | (atm < 0.0)) + if np.size(indq) > 0: + atm[indq] = np.nan + del indq + + indq = np.where((atp > 40.) | (atp < 0.0)) + if np.size(indq) > 0: + atp[indq] = np.nan + del indq + + indq = np.where((adm > 360.) | (adm < -180.)) + if np.size(indq) > 0: + adm[indq] = np.nan + del indq + + indq = np.where((adp > 360.) | (adp < -180.)) + if np.size(indq) > 0: + adp[indq] = np.nan + del indq + + indq = np.where((awm > 50.) | (awm < 0.0)) + if np.size(indq) > 0: + awm[indq] = np.nan + del indq + + c = 0 + for t in range(0, np.size(mtime)): + indt = np.where(np.abs(atime - mtime[t]) < 1800.) + if np.size(indt) > 0: + if np.any(ahs[indt[0]].mask == False): + bhs[b, t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask == False]) + c = c + 1 + if np.any(atm[indt[0]].mask == False): + btm[b, t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask == False]) + if np.any(atp[indt[0]].mask == False): + btp[b, t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask == False]) + if np.any(adm[indt[0]].mask == False): + bdm[b, t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask == False]) + if np.any(adp[indt[0]].mask == False): + bdp[b, t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask == False]) + if np.any(awm[indt[0]].mask == False): + bwind[b, t] = np.nanmean(awm[indt[0]][awm[indt[0]].mask == False]) + + del indt + + # print("counted "+repr(c)+" at "+stname[b]) + + print(" station " + stname[b] + " ok") +# del ahs + except Exception as e: + print("Error occurred while processing station", stname[b]) + print(e) + +print('bwind:',bwind) +print('bhs:',bhs) + + +print(' ') +# Simple quality-control (range) +ind=np.where((bhs>30.)|(bhs<0.0)) +if np.size(ind)>0: + bhs[ind]=np.nan; del ind + +ind=np.where((btm>40.)|(btm<0.0)) +if np.size(ind)>0: + btm[ind]=np.nan; del ind + +ind=np.where((btp>40.)|(btp<0.0)) +if np.size(ind)>0: + btp[ind]=np.nan; del ind + +ind=np.where((bdm>360.)|(bdm<-180.)) +if np.size(ind)>0: + bdm[ind]=np.nan; del ind + +ind=np.where((bdp>360.)|(bdp<-180.)) +if np.size(ind)>0: + bdp[ind]=np.nan; del ind + +ind=np.where((bwind>50.0)|(bwind<0.0)) +if np.size(ind)>0: + bwind[ind]=np.nan; del ind + +ind=np.where((mhs>30.)|(mhs<0.0)) +if np.size(ind)>0: + mhs[ind]=np.nan; del ind + +ind=np.where((mtm>40.)|(mtm<0.0)) +if np.size(ind)>0: + mtm[ind]=np.nan; del ind + +ind=np.where((mtp>40.)|(mtp<0.0)) +if np.size(ind)>0: + mtp[ind]=np.nan; del ind + +ind=np.where((mdm>360.)|(mdm<-180.)) +if np.size(ind)>0: + mdm[ind]=np.nan; del ind + +ind=np.where((mdp>360.)|(mdp<-180.)) +if np.size(ind)>0: + mdp[ind]=np.nan; del ind + +ind=np.where((mwn>50.)|(mwn<0.0)) +if np.size(ind)>0: + mwn[ind]=np.nan; del ind + +# Clean data excluding some stations. Select matchups only when model and buoy are available. +ind=np.where( (np.isnan(lat)==False) & (np.isnan(lon)==False) & (np.isnan(np.nanmean(mhs,axis=1))==False) & (np.isnan(np.nanmean(bhs,axis=1))==False) ) +if np.size(ind)>0: + stname=np.array(stname[ind[0]]) + lat=np.array(lat[ind[0]]) + lon=np.array(lon[ind[0]]) + mhs=np.array(mhs[ind[0],:]) + mtm=np.array(mtm[ind[0],:]) + mtp=np.array(mtp[ind[0],:]) + mdm=np.array(mdm[ind[0],:]) + mdp=np.array(mdp[ind[0],:]) + mwn=np.array(mwn[ind[0],:]) + bhs=np.array(bhs[ind[0],:]) + btm=np.array(btm[ind[0],:]) + btp=np.array(btp[ind[0],:]) + bdm=np.array(bdm[ind[0],:]) + bdp=np.array(bdp[ind[0],:]) + bwind=np.array(bwind[ind[0],:]) +else: + sys.exit(' Error: No matchups Model/Buoy available.') + + +print(" Matchups model/buoy complete. Total of "+repr(np.size(ind))+" stations/buoys avaliable."); del ind + +# Processing grid and/or cyclone information +if gridinfo!=0: + print(" Adding extra information ... ") + alon=np.copy(lon); alon[alon<0]=alon[alon<0]+360. + indgplat=[]; indgplon=[] + for i in range(0,lat.shape[0]): + # indexes nearest point. + indgplat = np.append(indgplat,np.where( abs(mlat-lat[i])==abs(mlat-lat[i]).min() )[0][0]) + indgplon = np.append(indgplon,np.where( abs(mlon-alon[i])==abs(mlon-alon[i]).min() )[0][0]) + + indgplat=np.array(indgplat).astype('int'); indgplon=np.array(indgplon).astype('int') + pdistcoast=np.zeros(lat.shape[0],'f')*np.nan + pdepth=np.zeros(lat.shape[0],'f')*np.nan + poni=np.zeros(lat.shape[0],'f')*np.nan + phsmz=np.zeros(lat.shape[0],'f')*np.nan + for i in range(0,lat.shape[0]): + pdistcoast[i]=distcoast[indgplat[i],indgplon[i]] + pdepth[i]=depth[indgplat[i],indgplon[i]] + poni[i]=oni[indgplat[i],indgplon[i]] + phsmz[i]=hsmz[indgplat[i],indgplon[i]] + + print(" Grid Information Included.") + + # Excluding shallow water points too close to the coast (mask information not accurate) + ind=np.where( (np.isnan(pdistcoast)==False) & (np.isnan(pdepth)==False) ) + if np.size(ind)>0: + stname=np.array(stname[ind[0]]) + lat=np.array(lat[ind[0]]) + lon=np.array(lon[ind[0]]) + mhs=np.array(mhs[ind[0],:]) + mtm=np.array(mtm[ind[0],:]) + mtp=np.array(mtp[ind[0],:]) + mdm=np.array(mdm[ind[0],:]) + mdp=np.array(mdp[ind[0],:]) + mwn=np.array(mwn[ind[0],:]) + bhs=np.array(bhs[ind[0],:]) + btm=np.array(btm[ind[0],:]) + btp=np.array(btp[ind[0],:]) + bdm=np.array(bdm[ind[0],:]) + bdp=np.array(bdp[ind[0],:]) + bwind=np.array(bwind[ind[0],:]) + pdistcoast=np.array(pdistcoast[ind[0]]) + pdepth=np.array(pdepth[ind[0]]) + poni=np.array(poni[ind[0]]) + phsmz=np.array(phsmz[ind[0]]) + else: + sys.exit(' Error: No matchups Model/Buoy available after using grid mask.') + + del ind + + if cyclonemap!=0: + fcmap=np.zeros((lat.shape[0],mtime.shape[0]),'f')*np.nan + for t in range(0,np.size(mtime)): + # search for cyclone time index and cyclone map + indt=np.where(np.abs(ctime-mtime[t])<5400.) + if np.size(indt)>0: + for i in range(0,lat.shape[0]): + fcmap[i,t] = np.array(cmap[indt[0][0],indgplat[i],indgplon[i]]) + + del indt + else: + print(' - No cyclone information for this time step: '+repr(t)) + + # print(' Done cyclone analysis at step: '+repr(t)) + + ind=np.where(fcmap<0) + if np.size(ind)>0: + fcmap[ind]=np.nan + + print(" Cyclone Information Included.") + +# Edit format if this is forecast model data. Reshape and allocate + + +if forecastds>0: + unt=np.unique(mfcycle); mxsz=1 + for i in range(0,unt.shape[0]): + ind=np.where(mfcycle==unt[i])[0] + mxsz=np.max([mxsz,np.size(ind)]) + + for i in range(0,unt.shape[0]): + ind=np.where(mfcycle==unt[i])[0] + if i==0: + nmhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmwn=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbwind=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmtime=np.zeros((unt.shape[0],mxsz),'double')*np.nan + if cyclonemap!=0: + nfcmap=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + + nmtime[i,0:np.size(ind)]=np.array(mtime[ind]).astype('double') + nmhs[:,i,:][:,0:np.size(ind)]=np.array(mhs[:,ind]) + nmtm[:,i,:][:,0:np.size(ind)]=np.array(mtm[:,ind]) + nmtp[:,i,:][:,0:np.size(ind)]=np.array(mtp[:,ind]) + nmdm[:,i,:][:,0:np.size(ind)]=np.array(mdm[:,ind]) + nmdp[:,i,:][:,0:np.size(ind)]=np.array(mdp[:,ind]) + nmwn[:,i,:][:,0:np.size(ind)]=np.array(mwn[:,ind]) + nbhs[:,i,:][:,0:np.size(ind)]=np.array(bhs[:,ind]) + nbtm[:,i,:][:,0:np.size(ind)]=np.array(btm[:,ind]) + nbtp[:,i,:][:,0:np.size(ind)]=np.array(btp[:,ind]) + nbdm[:,i,:][:,0:np.size(ind)]=np.array(bdm[:,ind]) + nbdp[:,i,:][:,0:np.size(ind)]=np.array(bdp[:,ind]) + nwind[:,i,:][:,0:np.size(ind)]=np.array(bwind[:,ind]) + + if cyclonemap!=0: + nfcmap[:,i,:][:,0:np.size(ind)]=np.array(fcmap[:,ind]) + + + ind=np.where( (nmhs>0.0) & (nbhs>0.0) ) + +else: + + larger_shape = max(mhs.shape, bhs.shape) + padded_mhs = np.zeros(larger_shape) + padded_bhs = np.zeros(larger_shape) + padded_mhs[:mhs.shape[0], :mhs.shape[1]] = mhs + padded_bhs[:bhs.shape[0], :bhs.shape[1]] = bhs + + + # Create padded arrays for other variables + padded_mtm = np.zeros(larger_shape) # Assuming mtm has the same shape as mhs + padded_mtp = np.zeros(larger_shape) # Assuming mtp has the same shape as mhs + padded_mdm = np.zeros(larger_shape) # Assuming mdm has the same shape as mhs + padded_mdp = np.zeros(larger_shape) # Assuming mdp has the same shape as mhs + padded_mwn = np.zeros(larger_shape) # Assuming mwn has the same shape as mhs + + padded_btm = np.zeros(larger_shape) # Assuming btm has the same shape as bhs + padded_btp = np.zeros(larger_shape) # Assuming btp has the same shape as bhs + padded_bdm = np.zeros(larger_shape) # Assuming bdm has the same shape as bhs + padded_bdp = np.zeros(larger_shape) # Assuming bdp has the same shape as bhs + padded_bwind = np.zeros(larger_shape) # Assuming bwind has the same shape as bhs + + # Copy values from original arrays to padded arrays for other variables + padded_mtm[:mtm.shape[0], :mtm.shape[1]] = mtm + padded_mtp[:mtp.shape[0], :mtp.shape[1]] = mtp + padded_mdm[:mdm.shape[0], :mdm.shape[1]] = mdm + padded_mdp[:mdp.shape[0], :mdp.shape[1]] = mdp + padded_mwn[:mwn.shape[0], :mwn.shape[1]] = mwn + + padded_btm[:btm.shape[0], :btm.shape[1]] = btm + padded_btp[:btp.shape[0], :btp.shape[1]] = btp + padded_bdm[:bdm.shape[0], :bdm.shape[1]] = bdm + padded_bdp[:bdp.shape[0], :bdp.shape[1]] = bdp + padded_bwind[:bwind.shape[0], :bwind.shape[1]] = bwind + + + +# ind=np.where( (mhs>0.0) & (bhs>0.0) ) + ind = np.where((padded_mhs > 0.0) & (padded_bhs > 0.0)) +if np.size(ind)>0: + print(' Total amount of matchups model/buoy: '+repr(np.size(ind))) + + # Save netcdf output file + lon[lon>180.]=lon[lon>180.]-360. + initime=str(time.gmtime(mtime.min())[0])+str(time.gmtime(mtime.min())[1]).zfill(2)+str(time.gmtime(mtime.min())[2]).zfill(2)+str(time.gmtime(mtime.min())[3]).zfill(2) + fintime=str(time.gmtime(mtime.max())[0])+str(time.gmtime(mtime.max())[1]).zfill(2)+str(time.gmtime(mtime.max())[2]).zfill(2)+str(time.gmtime(mtime.max())[3]).zfill(2) + ncfile = nc.Dataset('WW3.Buoy'+ftag+'_'+initime+'to'+fintime+'.nc', "w", format=fnetcdf) + ncfile.history="Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of "+repr(bhs[bhs>0.].shape[0])+" observations or pairs model/observation." + # create dimensions + ncfile.createDimension('buoypoints', bhs.shape[0] ) + if gridinfo!=0: + ncfile.createDimension('GlobalOceansSeas', ocnames.shape[0] ) + ncfile.createDimension('HighSeasMarineZones', hsmznames.shape[0] ) + if cyclonemap!=0: + ncfile.createDimension('cycloneinfo', cinfo.shape[0] ) + vcinfo = ncfile.createVariable('cycloneinfo',dtype('a25'),('cycloneinfo')) + # create variables. + vstname = ncfile.createVariable('buoyID',dtype('a25'),('buoypoints')) + vlat = ncfile.createVariable('latitude',np.dtype('float32').char,('buoypoints')) + vlon = ncfile.createVariable('longitude',np.dtype('float32').char,('buoypoints')) + + if forecastds>0: + ncfile.createDimension('time', nmhs.shape[2] ) + ncfile.createDimension('fcycle', unt.shape[0] ) + vt = ncfile.createVariable('time',np.dtype('float64').char,('fcycle','time')) + vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmwn = ncfile.createVariable('model_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) + + vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbwind = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) + + else: + ncfile.createDimension('time', bhs.shape[1] ) + vt = ncfile.createVariable('time',np.dtype('float64').char,('time')) + vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','time')) + vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','time')) + vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','time')) + vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','time')) + vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','time')) + vmwn = ncfile.createVariable('model_wind',np.dtype('float32').char,('buoypoints','time')) + + vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','time')) + vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','time')) + vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','time')) + vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','time')) + vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','time')) + vbwind = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','time')) + + + + + if gridinfo!=0: + vpdistcoast = ncfile.createVariable('distcoast',np.dtype('float32').char,('buoypoints')) + vpdepth = ncfile.createVariable('depth',np.dtype('float32').char,('buoypoints')) + vponi = ncfile.createVariable('GlobalOceansSeas',np.dtype('float32').char,('buoypoints')) + vocnames = ncfile.createVariable('names_GlobalOceansSeas',dtype('a25'),('GlobalOceansSeas')) + vphsmz = ncfile.createVariable('HighSeasMarineZones',np.dtype('float32').char,('buoypoints')) + vhsmznames = ncfile.createVariable('names_HighSeasMarineZones',dtype('a25'),('HighSeasMarineZones')) + if cyclonemap!=0: + if forecastds>0: + vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','fcycle','time')) + else: + vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','time')) + + # Assign units + vlat.units = 'degrees_north' ; vlon.units = 'degrees_east' + vt.units = 'seconds since 1970-01-01T00:00:00+00:00' + vmhs.units='m'; vbhs.units='m' + vmtm.units='s'; vbtm.units='s' + vmtp.units='s'; vbtp.units='s' + vmdm.units='degrees'; vbdm.units='degrees' + vmdp.units='degrees'; vbdp.units='degrees' + vmwn.unit='m/s';vbwind.unit='m/s' + + if gridinfo!=0: + vpdepth.units='m'; vpdistcoast.units='km' + + # Allocate Data + vstname[:]=stname[:]; vlat[:] = lat[:]; vlon[:] = lon[:] + if forecastds>0: + vt[:,:]=nmtime[:,:] + vmhs[:,:,:]=nmhs[:,:,:] + vmtm[:,:,:]=nmtm[:,:,:] + vmtp[:,:,:]=nmtp[:,:,:] + vmdm[:,:,:]=nmdm[:,:,:] + vmdp[:,:,:]=nmdp[:,:,:] + vmwn[:,:,:]=nmwn[:,:,:] + vbhs[:,:,:]=nbhs[:,:,:] + vbtm[:,:,:]=nbtm[:,:,:] + vbtp[:,:,:]=nbtp[:,:,:] + vbdm[:,:,:]=nbdm[:,:,:] + vbdp[:,:,:]=nbdp[:,:,:] + vbwind[:,:,:]=nbwind[:,:,:] + + else: + vt[:]=mtime[:] + vmhs[:,:]=padded_mhs[:,:] + vmtm[:,:]=padded_mtm[:,:] + vmtp[:,:]=padded_mtp[:,:] + vmdm[:,:]=padded_mdm[:,:] + vmdp[:,:]=padded_mdp[:,:] + vmwn[:,:]=padded_mwn[:,:] + vbhs[:,:]=padded_bhs[:,:] + vbtm[:,:]=btm[:,:] + vbtp[:,:]=btp[:,:] + vbdm[:,:]=bdm[:,:] + vbdp[:,:]=bdp[:,:] + vbwind[:,:]=bwind[:,:] + + + + + if gridinfo!=0: + vpdistcoast[:]=pdistcoast[:] + vpdepth[:]=pdepth[:] + vponi[:]=poni[:]; vocnames[:] = ocnames[:] + vphsmz[:]=phsmz[:]; vhsmznames[:] = hsmznames[:] + if cyclonemap!=0: + vcinfo[:] = cinfo[:] + if forecastds>0: + vcmap[:,:,:]=nfcmap[:,:,:] + else: + vcmap[:,:]=fcmap[:,:] + + ncfile.close() + print(' ') + print('Done. Netcdf ok. New file saved: WW3.Buoy'+ftag+'_'+initime+'to'+fintime+'.nc') diff --git a/ww3tools/openfile2.py b/ww3tools/openfile2.py new file mode 100644 index 0000000..f57c6ea --- /dev/null +++ b/ww3tools/openfile2.py @@ -0,0 +1,51 @@ +import gzip +import tarfile +import os + +def unzip_and_untar(gz_file, output_dir): + # Extracting the filename without extension + file_name = os.path.splitext(os.path.basename(gz_file))[0] + print(file_name) + # Get the base name of the extracted folder + base_name = file_name.rsplit('.', 1)[0] # Get everything before the last dot + + # Setting the output path for the extracted file + output_file = os.path.join(output_dir, file_name) + + # Unzipping the .gz file + with gzip.open(gz_file, 'rb') as f_in: + with open(output_file, 'wb') as f_out: + f_out.write(f_in.read()) + + # Untaring the extracted file + with tarfile.open(output_file, 'r') as tar: + tar.extractall(output_dir) + + # Get the base name of the extracted folder + extracted_folder = os.path.join(output_dir, base_name) + + print(extracted_folder) + # Creating a list of the extracted files + list_file = os.path.join(output_dir, f'{base_name}_contents.txt') + with open(list_file, 'w') as f: + for root, dirs, files in os.walk(extracted_folder): + for file in files: + f.write(os.path.join(root, file) + '\n') + + # Creating a list of ids between two dots in the filenames + id_file = os.path.join(output_dir, f'{base_name}_id.txt') + with open(id_file, 'w') as f: + for root, dirs, files in os.walk(extracted_folder): + for file in files: + file_parts = file.split('.') + if len(file_parts) >= 3: + id_between_dots = file_parts[1] + f.write(id_between_dots + '\n') + +# Define the input .gz file and the output directory +gz_file = 'multi_1.t00z.spec_tar.gz' +output_dir = '.' # You can specify any output directory here + +# Call the function to unzip and untar the file +unzip_and_untar(gz_file, output_dir) + diff --git a/ww3tools/wread.py b/ww3tools/wread.py index bcfb0bc..f607cd3 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1,57 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -wread.py - -VERSION AND LAST UPDATE: - v1.0 04/04/2022 - v1.1 01/06/2023 - -PURPOSE: - Group of python functions to Read Wave data: - WAVEWATCHIII results, and NDBC and Copernicus buoys. - Prefix meaning: - tseriesnc = time series (table of integrated parameters versus time). - spec = wave spectrum. - Users can import as a standard python function, and use it accordingly: - For example: - import wread - wread.tseriesnc_ww3(filename.nc,stationID) - Users can help() each function to obtain information about inputs/outputs - help(wread.tseriesnc_ww3) - -USAGE: - functions - mask - cyclonemap - tseriesnc_ndbc - tseriesnc_copernicus - tseriesnc_ww3 - bull - bull_tar - ts - station_tar - spec_ndbc - spec_ww3 - Explanation for each function is contained in the headers - -OUTPUT: - Dictionary containing arrays and info. - Description of variables is contained in the header of each function. - -DEPENDENCIES: - See setup.py and the imports below. - -AUTHOR and DATE: - 04/04/2022: Ricardo M. Campos, first version. - 01/06/2023: Ricardo M. Campos, new file formats added. - -PERSON OF CONTACT: - Ricardo M Campos: ricardo.campos@noaa.gov - -""" - import matplotlib # matplotlib.use('Agg') # for backend plots, not for rendering in a window import time @@ -69,1362 +15,228 @@ # import pickle import sys import warnings; warnings.filterwarnings("ignore") +import tarfile +import math - -# FIELDS - -def mask(*args): - ''' - Read gridmask netcdf file generated with prepGridMask.py - Input: file name (example: gridInfo_GEFSv12.nc) - Output: dictionary containing the arrays and string names - ''' - if len(args) == 1: - fname=str(args[0]) - else: - sys.exit(' Too many inputs') - - print(" reading ww3_tools mask ...") - try: - f=nc.Dataset(fname) - # build dictionary - result={'latitude':np.array(f.variables['latitude'][:]),'longitude':np.array(f.variables['longitude'][:]),'mask':np.array(f.variables['mask'][:,:])} - except: - sys.exit(" Cannot open "+fname) - else: - if 'distcoast' in f.variables.keys(): - result['distcoast'] = np.array(f.variables['distcoast'][:,:]) - if 'depth' in f.variables.keys(): - result['depth'] = np.array(f.variables['depth'][:,:]) - if 'GlobalOceansSeas' in f.variables.keys(): - result['GlobalOceansSeas'] = np.array(f.variables['GlobalOceansSeas'][:,:]) - if 'HighSeasMarineZones' in f.variables.keys(): - result['HighSeasMarineZones'] = np.array(f.variables['HighSeasMarineZones'][:,:]) - if 'names_GlobalOceansSeas' in f.variables.keys(): - result['names_GlobalOceansSeas'] = f.variables['names_GlobalOceansSeas'][:] - if 'names_HighSeasMarineZones' in f.variables.keys(): - result['names_HighSeasMarineZones'] = f.variables['names_HighSeasMarineZones'][:] - - f.close(); del f - print(" GridMask Read. "+fname) - return result - del result - -def cyclonemap(*args): - ''' - Read cyclonemap netcdf file generated with procyclmap.py - Input: file name (example: CycloneMap2020.nc) - Output: dictionary containing the arrays and string names - ''' - if len(args) == 1: - fname=str(args[0]) - else: - sys.exit(' Too many inputs') - - print(" reading ww3_tools cyclonemap ...") - try: - f=nc.MFDataset(fname, aggdim='time') - at=f.variables['time'][:]; adate=[] - for j in range(0,at.shape[0]): - adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) - # -------- - # build dictionary - result={'latitude':np.array(f.variables['lat'][:]),'longitude':np.array(f.variables['lon'][:]), - 'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), - 'cmap':f.variables['cmap'], 'info':str(f.info), 'netcdf':f} - # it does not allocate the data of cmap using [:,:,:] yet as it can take a lot of data/memory and time. - except: - sys.exit(" Cannot open "+fname) - else: - print(" CycloneInfo Read. "+fname) - return result - del result - - -# TIME-SERIES - -# Observations NDBC, netcdf format -def tseriesnc_ndbc(*args): - ''' - Observations NDBC, time series/table, netcdf format - Input: file name (example: 46047h2016.nc) - Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, - and arrays sst,mslp,dwp,tmp,gst,wsp,wdir,hs,tm,tp,dm - ''' - if len(args) == 1: - fname=str(args[0]) - elif len(args) > 1: - sys.exit(' Too many inputs') - - try: - ds = xr.open_dataset(fname); f=nc.Dataset(fname) - except: - sys.exit(" Cannot open "+fname) - else: - btm = f.variables['average_wpd'][:,0,0]; btp = f.variables['dominant_wpd'][:,0,0] - btime = np.array(f.variables['time'][:]).astype('double') - f.close(); del f - bsst = ds['sea_surface_temperature'].values[:,0,0] - bmslp = ds['air_pressure'].values[:,0,0] - bdwp = ds['dewpt_temperature'].values[:,0,0] - btmp = ds['air_temperature'].values[:,0,0] - bgst = ds['gust'].values[:,0,0] - - if 'wind_spd' in ds.keys(): - bwsp = ds['wind_spd'].values[:,0,0] - try: - from urllib.request import urlopen - url = "https://www.ndbc.noaa.gov/station_page.php?station="+str(fname).split('/')[-1].split('h')[0] - page = urlopen(url) - html_bytes = page.read() - html = html_bytes.decode("utf-8") - except: - anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. - print('Information about the Anemometer height, for wind speed conversion to 10m, could not be obtained.') - else: - if "Anemometer height" in html: - anh=float(html.split('Anemometer height')[1][0:15].split(':')[1].split('m')[0]) - else: - print('Information about the Anemometer height, for wind speed conversion to 10m, could not be found.') - anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. - - del url,page,html_bytes,html - - # convert wind speed to 10 meters (DNVGL C-205 Table 2-1, confirmed by https://onlinelibrary.wiley.com/doi/pdf/10.1002/er.6382) - bwsp = np.copy(((10./anh)**(0.12)) * bwsp) - bgst = np.copy(((10./anh)**(0.12)) * bgst) - - bwdir = ds['wind_dir'].values[:,0,0] - bhs = ds['wave_height'].values[:,0,0] - bdm = ds['mean_wave_dir'].values[:,0,0] - - # Automatic and basic Quality Control - bsst[np.abs(bsst)>70]=np.nan - bmslp[(bmslp<500)|(bmslp>1500)]=np.nan - bdwp[np.abs(bdwp)>80]=np.nan - btmp[np.abs(btmp)>80]=np.nan - bgst[(bgst<0)|(bgst>200)]=np.nan - bwsp[(bwsp<0)|(bwsp>150)]=np.nan - bwdir[(bwdir<-180)|(bwdir>360)]=np.nan - bhs[(bhs<0)|(bhs>30)]=np.nan - btm[(btm<0)|(btm>40)]=np.nan - btp[(btp<0)|(btp>40)]=np.nan - bdm[(bdm<-180)|(bdm>360)]=np.nan - - result={'latitude':np.array(ds['latitude'].values[:]),'longitude':np.array(ds['longitude'].values[:]), - 'time':btime,'date':ds['time'].values[:], - 'sst':bsst, 'mslp':bmslp, 'dewpt_temp':bdwp, - 'air_temp':btmp, 'gust':bgst, 'wind_spd':bwsp, - 'wind_dir':bwdir, 'hs':bhs, 'tm':btm, - 'tp':btp, 'dm':bdm, 'tm':btm} - - return result - ds.close() - del ds,btime,bsst,bmslp,bdwp,btmp,bgst,bwsp,bwdir,bhs,btm,btp,bdm - -# Observations NDBC, text format -def tseriestxt_ndbc(*args): - ''' - Observations NDBC, time series/table, stdmet format - Input: file name (example: NDBC_historical_stdmet_41004.txt) - Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, - and arrays sst,mslp,dwp,tmp,gst,wsp,wdir,hs,tm,tp,dm - ''' - if len(args) == 1: - fname=str(args[0]) - elif len(args) > 1: - sys.exit(' Too many inputs') - - try: - ds = pd.read_csv(fname,comment='#',delimiter=r"\s+") - btime=np.zeros(ds.shape[0],'d') - if 'mm' in ds.keys(): - ds = pd.read_csv(fname,comment='#',delimiter=r"\s+",parse_dates= {"date" : ["YY","MM","DD","hh","mm"]}) - ds['date']=pd.to_datetime(ds['date'],format='%Y %m %d %H %M') - else: - ds = pd.read_csv(fname,comment='#',delimiter=r"\s+",parse_dates= {"date" : ["YY","MM","DD","hh"]}) - ds['date']=pd.to_datetime(ds['date'],format='%Y %m %d %H') - - for i in range(0,btime.shape[0]): - btime[i]=double(ds['date'][i].timestamp()) - - except: - sys.exit(" Cannot open "+fname) - else: - - bwdir=np.array(ds['WDIR'].values[:]).astype('float') - bgst=np.array(ds['GST'].values[:]).astype('float') - bhs=np.array(ds['WVHT'].values[:]).astype('float') - btp=np.array(ds['DPD'].values[:]).astype('float') - btm=np.array(ds['APD'].values[:]).astype('float') - bdm=np.array(ds['MWD'].values[:]).astype('float') - bmslp=np.array(ds['PRES'].values[:]).astype('float') - btmp=np.array(ds['ATMP'].values[:]).astype('float') - bsst=np.array(ds['WTMP'].values[:]).astype('float') - bdwp=np.array(ds['DEWP'].values[:]).astype('float') - - if 'WSPD' in ds.keys(): - bwsp=np.array(ds['WSPD'].values[:]).astype('float') - try: - from urllib.request import urlopen - url = "https://www.ndbc.noaa.gov/station_page.php?station="+str(fname).split('/')[-1].split('h')[0].split('_')[-1] - page = urlopen(url) - html_bytes = page.read() - html = html_bytes.decode("utf-8") - auxlatlon=html.split('payload')[1][16:33] - if 'S' in auxlatlon: - blat=-float(auxlatlon[0:6]) - else: - blat=float(auxlatlon[0:6]) - - if 'W' in auxlatlon: - blon=-float(auxlatlon[8:16]) - else: - blon=float(auxlatlon[8:16]) - - except: - anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. - blat=np.nan; blon=np.nan - print('Information of Lat, Lon, and Anemometer height could not be obtained.') - else: - - if "Anemometer height" in html: - anh=float(html.split('Anemometer height')[1][0:15].split(':')[1].split('m')[0]) - else: - print('Information about the Anemometer height, for wind speed conversion to 10m, could not be found.') - anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. - - del url,page,html_bytes,html - - # convert wind speed to 10 meters (DNVGL C-205 Table 2-1, confirmed by https://onlinelibrary.wiley.com/doi/pdf/10.1002/er.6382) - bwsp = np.copy(((10./anh)**(0.12)) * bwsp) - bgst = np.copy(((10./anh)**(0.12)) * bgst) - - # Automatic and basic Quality Control - bsst[np.abs(bsst)>70]=np.nan - bmslp[(bmslp<500)|(bmslp>1500)]=np.nan - bdwp[np.abs(bdwp)>80]=np.nan - btmp[np.abs(btmp)>80]=np.nan - bgst[(bgst<0)|(bgst>200)]=np.nan - bwsp[(bwsp<0)|(bwsp>150)]=np.nan - bwdir[(bwdir<-180)|(bwdir>360)]=np.nan - bhs[(bhs<0)|(bhs>30)]=np.nan - btm[(btm<0)|(btm>40)]=np.nan - btp[(btp<0)|(btp>40)]=np.nan - bdm[(bdm<-180)|(bdm>360)]=np.nan - - result={'latitude':blat,'longitude':blon, - 'time':btime,'date':ds['date'].values[:], - 'sst':bsst, 'mslp':bmslp, 'dewpt_temp':bdwp, - 'air_temp':btmp, 'gust':bgst, 'wind_spd':bwsp, - 'wind_dir':bwdir, 'hs':bhs, 'tm':btm, - 'tp':btp, 'dm':bdm, 'tm':btm} - - return result - del ds,btime,blat,blon,bsst,bmslp,bdwp,btmp,bgst,bwsp,bwdir,bhs,btm,btp,bdm - - -# Observations Copernicus, netcdf format -def tseriesnc_copernicus(*args): - ''' - Observations NDBC, time series/table, netcdf format - Input: file name (example: 46047h2016.nc) - Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, - and arrays with the environmental variables available. - ''' - if len(args) == 1: - fname=str(args[0]) - elif len(args) > 1: - sys.exit(' Too many inputs') - - try: - ds = xr.open_dataset(fname); f=nc.Dataset(fname) - except: - sys.exit(" Cannot open "+fname) - else: - btime = np.array(f.variables['TIME'][:]*24*3600 + timegm( strptime('195001010000', '%Y%m%d%H%M') )).astype('double') - f.close(); del f - blat = np.nanmean(ds['LATITUDE'].values[:]) - blon = np.nanmean(ds['LONGITUDE'].values[:]) - # dictionary - result={'latitude':np.array(blat),'longitude':np.array(blon), - 'time':btime,'date':ds['TIME'].values[:]} - - if 'DEPH' in ds.keys(): - bdepth = np.nanmean(ds['DEPH'].values[:,:],axis=1) # Depth - result['depth']=np.array(bdepth) - - if 'VHM0' in ds.keys(): - bhs = np.nanmean(ds['VHM0'].values[:,:],axis=1) # Hs - bhs[(bhs<0)|(bhs>30)]=np.nan - result['hs']=np.array(bhs) - elif 'VGHS' in ds.keys(): - bhs = np.nanmean(ds['VGHS'].values[:,:],axis=1) # Hs - bhs[(bhs<0)|(bhs>30)]=np.nan - result['hs']=np.array(bhs) - - if 'VAVH' in ds.keys(): - bvavh = np.nanmean(ds['VAVH'].values[:,:],axis=1) # H 1/3 vavh - bvavh[(bvavh<0)|(bvavh>30)]=np.nan - result['hs_vavh']=np.array(bvavh) - - if 'VZMX' in ds.keys(): - bhmax = np.nanmean(ds['VZMX'].values[:,:],axis=1) # Hmax - bhmax[(bhmax<0)|(bhmax>40)]=np.nan - result['hmax']=np.array(bhmax) - - if 'VTM02' in ds.keys(): - btm = np.nanmean(ds['VTM02'].values[:,:],axis=1) # Tm - btm[(btm<0)|(btm>40)]=np.nan - result['tm']=np.array(btm) - elif 'VGTA' in ds.keys(): - btm = np.nanmean(ds['VGTA'].values[:,:],axis=1) # Tm - btm[(btm<0)|(btm>40)]=np.nan - result['tm']=np.array(btm) - - if 'VTPK' in ds.keys(): - btp = np.nanmean(ds['VTPK'].values[:,:],axis=1) # Tp - btp[(btp<0)|(btp>40)]=np.nan - result['tp']=np.array(btp) - - if 'TEMP' in ds.keys(): - bsst = np.nanmean(ds['TEMP'].values[:,:],axis=1) # SST - bsst[np.abs(bsst)>70]=np.nan - result['sst']=np.array(bsst) - - if 'ATMS' in ds.keys(): - bmslp = np.nanmean(ds['ATMS'].values[:,:],axis=1) # Pressure - bmslp[(bmslp<500)|(bmslp>1500)]=np.nan - result['mslp']=np.array(bmslp) - - if 'DEWT' in ds.keys(): - bdwp = np.nanmean(ds['DEWT'].values[:,:],axis=1) # Dewpoint - bdwp[np.abs(bdwp)>80]=np.nan - result['dewpt_temp']=np.array(bdwp) - - if 'DRYT' in ds.keys(): - btmp = np.nanmean(ds['DRYT'].values[:,:],axis=1) # air temperature - btmp[np.abs(btmp)>80]=np.nan - result['air_temp']=np.array(btmp) - - if 'GSPD' in ds.keys(): - bgst = np.nanmean(ds['GSPD'].values[:,:],axis=1) # gust - bgst=np.copy(((10./4.0)**(0.12))*bgst) # conversion to 10m, approximation DNVGL C-205 Table 2-1 - bgst[(bgst<0)|(bgst>200)]=np.nan - result['gust']=np.array(bgst) - - if 'WSPD' in ds.keys(): - bwsp = np.nanmean(ds['WSPD'].values[:,:],axis=1) # wind speed - bwsp=np.copy(((10./4.0)**(0.12))*bwsp) # conversion to 10m, approximation DNVGL C-205 Table 2-1 - bwsp[(bwsp<0)|(bwsp>150)]=np.nan - result['wind_spd']=np.array(bwsp) - - if 'WDIR' in ds.keys(): - bwdir = np.nanmean(ds['WDIR'].values[:,:],axis=1) # wind direction - bwdir[(bwdir<-180)|(bwdir>360)]=np.nan - result['wind_dir']=np.array(bwdir) - - if 'VCMX' in ds.keys(): - bhcmax = np.nanmean(ds['VCMX'].values[:,:],axis=1) # Hcrest max - bhcmax[(bhcmax<0)|(bhcmax>40)]=np.nan - result['hc_max']=np.array(bhcmax) - - if 'VMDR' in ds.keys(): - bdm = np.nanmean(ds['VMDR'].values[:,:],axis=1) # Mean direction - bdm[(bdm<-180)|(bdm>360)]=np.nan - result['dm']=np.array(bdm) - - if 'VPED' in ds.keys(): - bdp = np.nanmean(ds['VPED'].values[:,:],axis=1) # Peak direction - bdp[(bdp<-180)|(bdp>360)]=np.nan - result['dp']=np.array(bdp) - - return result - ds.close(); del ds - - -# WAVEWATCH III point output, netcdf format - -def tseriestxt_ww3(*args): - ''' - WAVEWATCH III, time series/table, text tab format - This file format has all point outputs (results) in the same file (not divided by point/buoy). - Input: file name (example: tab50.ww3), and number of point ouputs (example: 4) - Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, - and arrays with the wave variables available. Inside the dictionary, the arrays of wave variables - have dimension (point_outputs, time). - ''' - if len(args) == 2: - fname=str(args[0]); tnb=np.int(args[1]) - elif len(args) < 2 : - sys.exit(' Two inputs are required: file name and station name') - elif len(args) > 2: - sys.exit(' Too many inputs') - - try: - mcontent = open(fname).readlines() - except: - sys.exit(" Cannot open "+fname) - else: - - tt = np.int(np.size(mcontent)/(7+tnb)+1) - myear = []; mmonth = [] ; mday = [] ; mhour = []; mmin = [] - mlon = np.zeros((tnb,tt),'f'); mlat = np.zeros((tnb,tt),'f'); mhs = np.zeros((tnb,tt),'f'); mL = np.zeros((tnb,tt),'f') - mtm = np.zeros((tnb,tt),'f'); mdm = np.zeros((tnb,tt),'f'); mspr = np.zeros((tnb,tt),'f') - atp = np.zeros((tnb,tt),'f'); mdp = np.zeros((tnb,tt),'f'); mpspr = np.zeros((tnb,tt),'f') - for i in range(0,tt): - j = i*(7+tnb) - myear = np.append(myear, np.int(mcontent[j].split(':')[1].split(' ')[1].split('/')[0]) ) - mmonth = np.append(mmonth, np.int(mcontent[j].split(':')[1].split(' ')[1].split('/')[1]) ) - mday = np.append(mday, np.int(mcontent[j].split(':')[1].split(' ')[1].split('/')[2]) ) - mhour = np.append(mhour, np.int(mcontent[j].split(':')[1].split(' ')[2]) ) - mmin = np.append(mmin, np.int(mcontent[j].split(':')[2]) ) - for k in range(0,tnb): - mlon[k,i] = mcontent[j+tnb+1+k].strip().split()[0] - mlat[k,i] = mcontent[j+tnb+1+k].strip().split()[1] - mhs[k,i] = mcontent[j+tnb+1+k].strip().split()[2] - mL[k,i] = mcontent[j+tnb+1+k].strip().split()[3] - mtm[k,i] = mcontent[j+tnb+1+k].strip().split()[4] - mdm[k,i] = mcontent[j+tnb+1+k].strip().split()[5] - mspr[k,i] = mcontent[j+tnb+1+k].strip().split()[6] - atp[k,i] = mcontent[j+tnb+1+k].strip().split()[7] - mdp[k,i] = mcontent[j+tnb+1+k].strip().split()[8] - mpspr[k,i] = mcontent[j+tnb+1+k].strip().split()[9] - - mtp = np.zeros((atp.shape[0],atp.shape[1]),'f')*np.nan - for i in range(0,mtp.shape[0]): - #mtp[i,atp[i,:]>0.0] = 1./atp[i,atp[i,:]>0.0] - indtp=np.where(atp[i,:]>0.0) - if size(indtp)>0: - mtp[i,indtp] = np.copy(1./atp[i,indtp]) - del indtp - - mdate = pd.to_datetime(dict(year=myear,month=mmonth,day=mday,hour=mhour,minute=mmin)) - mtime=np.zeros(mdate.shape[0],'d') - for i in range(0,mtime.shape[0]): - mtime[i]=double(mdate[i].timestamp()) - - result={'latitude':mlat,'longitude':mlon, - 'time':mtime,'date':mdate, - 'hs':mhs,'lm':mL,'tm':mtm,'dm':mdm, - 'spr':mspr,'tp':mtp,'dp':mdp,'spr_dp':mpspr} - - return result - del mdate,mtime,mlon,mlat,mhs,mL,mtm,mdm,mspr,atp,mtp,mdp,mpspr - -def tseriesnc_ww3(*args): - ''' - WAVEWATCH III, time series/table, netcdf format - Input: file name (example: ww3gefs.20160928_tab.nc), and station name (example: 41002) - Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, - and arrays with the wave variables available. - ''' - if len(args) == 2: - fname=str(args[0]); stname=str(args[1]) - elif len(args) < 2 : - sys.exit(' Two inputs are required: file name and station name') - elif len(args) > 2: - sys.exit(' Too many inputs') - - try: - ds = xr.open_dataset(fname); f=nc.Dataset(fname) - except: - sys.exit(" Cannot open "+fname) - else: - mtime = np.array(f.variables['time'][:]*24*3600 + timegm( strptime(str(f.variables['time'].units).split(' ')[2][0:4]+'01010000', '%Y%m%d%H%M') )).astype('double') - f.close(); del f - - auxstationname=ds['station_name'].values[:,:]; stationname=[] - for i in range(0,auxstationname.shape[0]): - stationname=np.append(stationname,"".join(np.array(auxstationname[i,:]).astype('str'))) - - inds=np.where(stationname[:]==stname) - if size(inds)>0: - inds=np.int(inds[0][0]); stname=str(stationname[inds]) - else: - sys.exit(' Station '+stname+' not included in the ww3 output file, or wrong station ID') - - mlat = np.nanmean(ds['latitude'].values[:,inds]) - mlon = np.nanmean(ds['longitude'].values[:,inds]) - # dictionary - result={'latitude':np.array(mlat),'longitude':np.array(mlon), - 'time':mtime,'date':ds['time'].values[:]} - - if 'hs' in ds.keys(): - mhs = ds['hs'].values[:,inds] - mhs[(mhs<0)|(mhs>30)]=np.nan - result['hs']=np.array(mhs) - elif 'swh' in ds.keys(): - mhs = ds['swh'].values[:,inds] - mhs[(mhs<0)|(mhs>30)]=np.nan - result['hs']=np.array(mhs) - - if 'fp' in ds.keys(): - mtp = np.zeros(mhs.shape[0],'f')*np.nan - indtp=np.where(ds['fp'].values[:,inds]>0.0) - if size(indtp)>0: - mtp[indtp] = np.copy(1./ds['fp'].values[indtp,inds]) - del indtp - mtp[(mtp<0)|(mtp>40)]=np.nan - - result['tp']=np.array(mtp) - if 'tr' in ds.keys(): - mtm = ds['tr'].values[:,inds] - mtm[(mtm<0)|(mtm>40)]=np.nan - result['tm']=np.array(mtm) - if 'th1p' in ds.keys(): - mdp = ds['th1p'].values[:,inds] - mdp[(mdp<-180)|(mdp>360)]=np.nan - result['dp']=np.array(mdp) - if 'th1m' in ds.keys(): - mdm = ds['th1m'].values[:,inds] - mdm[(mdm<-180)|(mdm>360)]=np.nan - result['dm']=np.array(mdm) - if 'sth1m' in ds.keys(): - result['spr']=np.array(ds['sth1m'].values[:,inds]) - if 'lm' in ds.keys(): - result['lm']=np.array(ds['lm'].values[:,inds]) - if 'sth1p' in ds.keys(): - result['spr_dp']=np.array(ds['sth1p'].values[:,inds]) - - return result - ds.close(); del ds - - -# Operational WW3 formats - -def bull(*args): - ''' - WAVEWATCH III, bull operational point output, see https://www.ftp.ncep.noaa.gov/data/nccf/com/ - Input: file name (example: gefs.wave.41004.bull) - Output: dictionary containing: - time(seconds since 1970),time(datetime64),lat,lon,station name; Arrays: hs, tp, and dp (gfs only) - ''' - if len(args) == 1: - fname=str(args[0]) - elif len(args) > 1: - sys.exit(' Too many inputs') - - # confirm format - if str(fname).split('/')[-1].split('.')[-1]=='bull': - print(" reading ww3 bull file ...") - at=[]; adate=[]; ahs=[]; atp=[]; adp=[] - stname=str(fname).split('/')[-1].split('.')[-2] - - try: - tfile = open(fname, 'r'); lines = tfile.readlines() - except: - sys.exit(' Cannot open '+fname) - else: - - if 'gfs' in str(fname).split('/')[-1]: - iauxhs=[24,30];iauxtp=[30,34];iauxdp=[35,38] - - # lat / lon - auxpos=str(lines[0]).replace("b'","").split('(')[1] - if auxpos[5]=='N': - alat=float(auxpos[0:5]) - else: - alat=-1.*float(auxpos[0:5]) - - if auxpos[13]=='E': - alon=float(auxpos[7:13]) - else: - alon=-1.*float(auxpos[7:13]) - - # time ---- - auxdate = str(lines[2]).split(':')[1].split('UTC')[0][1::] - auxt = np.double(timegm( strptime( auxdate[0:8]+' '+auxdate[9:11]+'00', '%Y%m%d %H%M') )) - year = int(time.gmtime(auxt)[0]); month = int(time.gmtime(auxt)[1]) - pday=0 - for j in range(7,size(lines)-8): - day=np.int(lines[j][3:5]); hour=np.int(lines[j][6:8]) - if day0: - ahs=np.append(ahs,float(lines[j][10:15])) - auxhs=np.array([float(lines[j][iauxhs[0]:iauxhs[1]])]) - for k in range(1,4): - if len(str(lines[j][int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)]).replace(' ', '')): - auxhs=np.append(auxhs,float(lines[j][int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)])) - - auxtp=np.array([float(lines[j][iauxtp[0]:iauxtp[1]])]) - for k in range(1,4): - if len(str(lines[j][int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)]).replace(' ', '')): - auxtp=np.append(auxtp,float(lines[j][int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)])) - - auxdp=np.array([float(lines[j][iauxdp[0]:iauxdp[1]])]) - for k in range(1,4): - if len(str(lines[j][int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)]).replace(' ', '')): - auxdp=np.append(auxdp,float(lines[j][int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)])) - - indaux=np.nanmin(np.where(auxhs==np.nanmax(auxhs))[0]) - atp=np.append(atp,float(auxtp[indaux])) - adp=np.append(adp,float(auxdp[indaux])) - del indaux,auxhs,auxtp,auxdp - else: - ahs=np.append(ahs,np.nan) - atp=np.append(atp,np.nan) - adp=np.append(adp,np.nan) - - # build dictionary - result={'latitude':alat,'longitude':alon,'station_name':stname, - 'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), - 'hs':np.array(ahs),'tp':np.array(atp),'dp':np.array(adp)} - - del adp - - elif 'gefs' in str(fname).split('/')[-1]: - iauxhs=[10,15];iauxtp=[28,33] - - # lat / lon - auxpos=str(lines[1]).split('(')[1].split('N') - alat=float(auxpos[0]) - alon=float(auxpos[1].split('W')[0]) - - # time ---- - auxdate = str(lines[3]).split(':')[1].split('UTC')[0][1::] - auxt = np.double(timegm( strptime( auxdate[0:8]+' '+auxdate[10:12]+'00', '%Y%m%d %H%M') )) - year = int(time.gmtime(auxt)[0]); month = int(time.gmtime(auxt)[1]) - pday=0 - for j in range(9,size(lines)-8): - day=np.int(lines[j][2:4]); hour=np.int(lines[j][5:7]) - if day0: - ahs=np.append(ahs,float(lines[j][iauxhs[0]:iauxhs[1]])) - atp=np.append(atp,float(lines[j][iauxtp[0]:iauxtp[1]])) - else: - ahs=np.append(ahs,np.nan) - atp=np.append(atp,np.nan) - - # build dictionary - result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), - 'latitude':alat,'longitude':alon,'station_name':stname, - 'hs':np.array(ahs),'tp':np.array(atp)} - - print(" Model data read, "+fname+", bull format.") - return result - del result,alat,alon,at,adate,ahs,atp,tfile,lines - else: - sys.exit(" Skipped file "+fname+" Not bull_tar format.") - - -def bull_tar(*args): - ''' - WAVEWATCH III, bull_tar operational point output, see https://www.ftp.ncep.noaa.gov/data/nccf/com/ - Input: file name (example: gfswave.t00z.bull_tar) - Output: dictionary containing: - time(seconds since 1970),time(datetime64),lat,lon,station names; Arrays: hs, tp, and dp (gfs only) - ''' - result = {} - - if len(args) == 1: - fname = str(args[0]) - elif len(args) > 1: - sys.exit(' Too many inputs') - - # confirm file format - if fname.split('/')[-1].split('.')[-1] == 'bull_tar': - print(" reading ww3 bull_tar file ...") - import tarfile - stname = [] - - try: - tar = tarfile.open(fname) - except: - sys.exit(' Cannot open ' + fname) - else: - at = [] - adate = [] - alat = [] - alon = [] - - # Determine model type - if 'gfs' in fname.split('/')[-1]: - model_type = 'gfs' - iauxhs = [24, 30] - iauxtp = [30, 34] - iauxdp = [35, 38] - else: - model_type = 'gfs' # Treat any other name as gfs for simplicity - iauxhs = [24, 30] - iauxtp = [30, 34] - iauxdp = [35, 38] - - for t in range(0, len(tar.getmembers())): - # station names - stname.append(str(tar.getmembers()[t].name).split('/')[-1].split('.')[-2]) - - try: - tfile = tar.extractfile(tar.getmembers()[t]) - lines = tfile.readlines() - except: - print(" Cannot open " + tar.getmembers()[t].name) - else: - # lat / lon - auxpos = str(lines[0]).replace("b'", "").split('(')[1] - if auxpos[5] == 'N': - alat.append(float(auxpos[0:5])) - else: - alat.append(-1. * float(auxpos[0:5])) - - if auxpos[13] == 'E': - alon.append(float(auxpos[7:13])) - else: - alon.append(-1. * float(auxpos[7:13])) - - if t == 0: - # time array ---- - auxdate = str(lines[2]).split(':')[1].split('UTC')[0][1::] - auxt = np.double(timegm(strptime(auxdate[0:8] + ' ' + auxdate[9:11] + '00', '%Y%m%d %H%M'))) - year = int(time.gmtime(auxt)[0]) - month = int(time.gmtime(auxt)[1]) - pday = 0 - for j in range(7, len(lines) - 8): - auxlines = str(lines[j]).replace("b'", "") - day = int(auxlines[3:5]) - hour = int(auxlines[6:8]) - if day < pday: - if month < 12: - month = month + 1 - else: - month = 1 - year = year + 1 - - at.append(np.double(timegm(strptime(repr(year) + str(month).zfill(2) + - str(day).zfill(2) + ' ' + str(hour).zfill(2) + - '00', '%Y%m%d %H%M')))) - pday = np.copy(day) - - for j in range(len(at)): - adate.append(datetime.datetime.utcfromtimestamp(at[j])) - - # -------- - ahs = np.zeros((len(tar.getmembers()), len(at)), 'f') * np.nan - atp = np.zeros((len(tar.getmembers()), len(at)), 'f') * np.nan - if model_type == 'gfs': - adp = np.zeros((len(tar.getmembers()), len(at)), 'f') * np.nan - - auxhs = [] - auxtp = [] - auxdp = [] - for j in range(7, len(lines) - 8): - auxlines = str(lines[j]).replace("b'", "") - if len(auxlines[iauxhs[0]:iauxhs[1]].replace(' ', '')) > 0: - auxhs.append(float(auxlines[10:15])) - fuxhs = np.array([float(auxlines[iauxhs[0]:iauxhs[1]])]) - for k in range(1, 4): - if len(str(auxlines[int(iauxhs[0] + 18 * k):int(iauxhs[1] + 18 * k)]).replace(' ', '')): - fuxhs = np.append(fuxhs, float(auxlines[int(iauxhs[0] + 18 * k):int(iauxhs[1] + 18 * k)])) - - fuxtp = np.array([float(auxlines[iauxtp[0]:iauxtp[1]])]) - for k in range(1, 4): - if len(str(auxlines[int(iauxtp[0] + 18 * k):int(iauxtp[1] + 18 * k)]).replace(' ', '')): - fuxtp = np.append(fuxtp, float(auxlines[int(iauxtp[0] + 18 * k):int(iauxtp[1] + 18 * k)])) - - if model_type == 'gfs': - fuxdp = np.array([float(auxlines[iauxdp[0]:iauxdp[1]])]) - for k in range(1, 4): - if len(str(auxlines[int(iauxdp[0] + 18 * k):int(iauxdp[1] + 18 * k)]).replace(' ', '')): - fuxdp = np.append(fuxdp, float(auxlines[int(iauxdp[0] + 18 * k):int(iauxdp[1] + 18 * k)])) - - indaux = np.nanargmax(fuxhs) - auxdp.append(fuxdp[indaux]) - - indaux = np.nanargmax(fuxhs) - auxtp.append(fuxtp[indaux]) - else: - auxhs.append(np.nan) - auxtp.append(np.nan) - if model_type == 'gfs': - auxdp.append(np.nan) - - if ahs.shape[1] == len(auxhs): - ahs[t, :] = np.array(auxhs) - atp[t, :] = np.array(auxtp) - if model_type == 'gfs': - adp[t, :] = np.array(auxdp) - else: - print(" Time duration of " + tar.getmembers()[t] + " (in " + fname + ") do not match the other stations. Maintained NaN.") - - del auxhs, auxtp, auxdp, tfile, lines - - # build dictionary - result = {'time': np.array(at).astype('double'), 'date': [t.timestamp() for t in adate], - 'latitude': np.array(alat), 'longitude': np.array(alon), 'station_name': np.array(stname), - 'hs': np.array(ahs), 'tp': np.array(atp)} - - if model_type == 'gfs': - result['dp'] = np.array(adp) - - print(" Model data read, " + fname + ", bull_tar format.") - return result - - else: - sys.exit(" Skipped file " + fname + " Not bull_tar format.") - - -# Example usage: -# data = bull_tar('path_to_your_file/multi_1.t00z.bull_tar') -# print(data) - - -def ts(*args): - ''' - WAVEWATCH III, ts operational point output, see https://www.ftp.ncep.noaa.gov/data/nccf/com/ - Input: file name (example: gefs.wave.41004.ts) - Output: dictionary containing: - time(seconds since 1970),time(datetime64),station name; Arrays: hs, hs_spr, tp (glwu or gefs) - ''' - if len(args) == 1: - fname=str(args[0]) - elif len(args) > 1: - sys.exit(' Too many inputs') - - # confirm format - if str(fname).split('/')[-1].split('.')[-1]=='ts': - print(" reading ww3 ts file ...") - stname=str(fname).split('/')[-1].split('.')[-2] - try: - tfile = pd.read_csv(fname,skiprows=2); lines = tfile.values[:,0] - except: - sys.exit(' Cannot open '+fname) - else: - - if 'gefs' in str(fname).split('/')[-1]: - # gefs lakes ww3 format - at=[]; adate=[]; ahs=[]; ahspr=[]; atp=[] - for j in range(0,size(lines)): - at=np.append(at,np.double(timegm( strptime( lines[j][1:12]+'00', '%Y%m%d %H%M') ))) - adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) - - if len(lines[j])>0: - ahs=np.append(ahs,float(lines[j][13:18])) - ahspr=np.append(ahspr,float(lines[j][19:25])) - atp=np.append(atp,float(lines[j][27:32])) - else: - ahs=np.append(ahs,np.nan) - ahspr=np.append(ahspr,np.nan) - atp=np.append(atp,np.nan) - - # build dictionary - result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), - 'station_name':np.array(stname),'hs':np.array(ahs),'hs_spr':np.array(ahspr),'tp':np.array(atp)} - - print(" Model data read, "+fname+", ts format.") - return result - del result,at,adate,ahs,ahspr,atp,tfile,lines - - elif 'glwu' in str(fname).split('/')[-1]: - # great lakes ww3 format - at=[];adate=[];ahs=[];al=[];atr=[];adir=[];aspr=[];atp=[];ap_dir=[];ap_spr=[] - for j in range(0,size(lines)): - at=np.append(at,np.double(timegm( strptime( lines[j][2:13]+'00', '%Y%m%d %H%M') ))) - adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) - - if len(lines[j])>0: - ahs=np.append(ahs,float(lines[j][22:28])) - al=np.append(al,float(lines[j][31:35])) - atr=np.append(atr,float(lines[j][37:42])) - adir=np.append(adir,float(lines[j][44:49])) - aspr=np.append(aspr,float(lines[j][50:56])) - atp=np.append(atp,float(lines[j][57:64])) - ap_dir=np.append(ap_dir,float(lines[j][66:71])) - ap_spr=np.append(ap_spr,float(lines[j][72:78])) - else: - ahs=np.append(ahs,np.nan) - al=np.append(al,np.nan) - atr=np.append(atr,np.nan) - adir=np.append(adir,np.nan) - aspr=np.append(aspr,np.nan) - atp=np.append(atp,np.nan) - ap_dir=np.append(ap_dir,np.nan) - ap_spr=np.append(ap_spr,np.nan) - - atp[atp<0.01]=np.nan; atp=1./atp - - # build dictionary - result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), - 'station_name':np.array(stname), - 'hs':np.array(ahs),'l':np.array(al), - 'tm':np.array(atr),'dm':np.array(adir), - 'spr':np.array(aspr),'tp':np.array(atp), - 'dp':np.array(ap_dir),'peak_spr':np.array(ap_spr)} - - print(" Model data read, "+fname+", ts format.") - return result - del result,at,adate,ahs,al,atr,adir,aspr,atp,ap_dir,ap_spr,tfile,lines - - else: - sys.exit(" Skipped file "+fname+" Not ts format.") - - -def station_tar(*args): - ''' - WAVEWATCH III, station_tar operational point output, see https://www.ftp.ncep.noaa.gov/data/nccf/com/ - Input: file name (example: gefs.wave.t00z.station_tar) - Output: dictionary containing: - time(seconds since 1970),time(datetime64),station name; Arrays: hs, hs_spr, tp (gefs only) - ''' - if len(args) == 1: - fname=str(args[0]) - elif len(args) > 1: - sys.exit(' Too many inputs') - - # confirm format - if str(fname).split('/')[-1].split('.')[-1]=='station_tar': - print(" reading ww3 station_tar file ...") - import tarfile - stname=[] - - try: - tar = tarfile.open(fname) - except: - sys.exit(' Cannot open '+fname) - else: - for t in range(0,size(tar.getmembers())): - # station names - stname=np.append(stname,str(str(tar.getmembers()[t].name).split('/')[-1]).split('/')[-1].split('.')[-2]) - - try: - tfile=tar.extractfile(tar.getmembers()[t]); lines = tfile.readlines()[3::] - except: - print(" Cannot open "+tar.getmembers()[t].name) - else: - if t==0: - # time array ---- - at=[]; adate=[] - for j in range(0,size(lines)): - at=np.append(at,np.double(timegm( strptime( str(lines[j])[3:14]+'00', '%Y%m%d %H%M') ))) - adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) - - # -------- - ahs=np.zeros((size(tar.getmembers()),at.shape[0]),'f')*np.nan - ahspr=np.zeros((size(tar.getmembers()),at.shape[0]),'f')*np.nan - atp=np.zeros((size(tar.getmembers()),at.shape[0]),'f')*np.nan - - auxhs=[]; auxhspr=[]; auxtp=[] - for j in range(0,size(lines)): - auxlines = str(lines[j]).replace("b'","") - if len(lines[j])>0: - auxhs=np.append(auxhs,float(auxlines[13:18])) - auxhspr=np.append(auxhspr,float(auxlines[19:25])) - auxtp=np.append(auxtp,float(auxlines[27:32])) - else: - auxhs=np.append(auxhs,np.nan) - auxhspr=np.append(auxhspr,np.nan) - auxtp=np.append(auxtp,np.nan) - - if ahs.shape[1]==auxhs.shape[0]: - ahs[t,:]=np.array(auxhs) - ahspr[t,:]=np.array(auxhspr) - atp[t,:]=np.array(auxtp) - else: - print(" Time duration of "+tar.getmembers()[t]+" (in "+fname+") do not match the other stations. Mantained NaN.") - - del auxhs,auxhspr,auxtp,tfile,lines - - # build dictionary - result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), - 'station_name':np.array(stname),'hs':np.array(ahs),'hs_spr':np.array(ahspr),'tp':np.array(atp)} - - return result - del result,tar,ahs,ahspr,atp,at,adate - - print(" Model data read, "+fname+", station_tar format.") - - else: - sys.exit(" Skipped file "+fname+" Not station_tar format.") - - -# SPECTRA - -# Observations NDBC, netcdf format -def spec_ndbc(*args): - ''' - Observations NDBC, wave spectrum, netcdf format - Input: file name (example: 46047w2016.nc) - Output: dictionary containing: - time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pspec,dmspec,dpspec,dirspec - ''' - sk=1; deltatheta=np.int(10) - if len(args) >= 1: - fname=str(args[0]) - if len(args) >= 2: - sk=np.int(args[1]) - if len(args) >= 3: - deltatheta=np.int(args[3]) - if len(args) > 3: - sys.exit(' Too many inputs') - - try: - ds = xr.open_dataset(fname); f=nc.Dataset(fname) - except: - sys.exit(" Cannot open "+fname) - else: - btime = np.array(f.variables['time'][::sk]).astype('double') - f.close(); del f - bdate = ds['time'].values[::sk] - blat = ds['latitude'].values[:] - blon = ds['longitude'].values[:] - freq = ds['frequency'].values[:] - pspec = ds['spectral_wave_density'].values[::sk,:,0,0] - dmspec = ds['mean_wave_dir'][::sk,:,0,0] - dpspec = ds['principal_wave_dir'][::sk,:,0,0] - r1spec = ds['wave_spectrum_r1'][::sk,:,0,0] - r2spec = ds['wave_spectrum_r2'][::sk,:,0,0] - ds.close(); del ds - # DF in frequency (dfreq), https://www.ndbc.noaa.gov/wavespectra.shtml - if np.int(freq.shape[0])==47: - dfreq=np.zeros(47,'f') - dfreq[0]=0.010; dfreq[1:14]=0.005; dfreq[14:40]=0.010; dfreq[40::]=0.020 - else: - dfreq=np.zeros(freq.shape[0],'f')+0.01 - - pspec=np.array(pspec*dfreq) - # Directional 2D Spectrum, https://www.ndbc.noaa.gov/measdes.shtml#swden , https://www.ndbc.noaa.gov/wavemeas.pdf - theta = np.array(np.arange(0,360+0.1,deltatheta)) - # final directional wave spectrum (frequency X direction) - dirspec = np.zeros((btime.shape[0],freq.shape[0],theta.shape[0]),'f') - for t in range(0,btime.shape[0]): - dirspec[t,:,:] = np.array([pspec[t,:]]).T * (1/pi)*(0.5+ np.array([r1spec[t,:]]).T * cos(np.array( np.array([theta])-np.array([dmspec[t,:]]).T )*(pi/180)) - + np.array([r2spec[t,:]]).T*cos(2*np.array( np.array([theta]) - np.array([dpspec[t,:]]).T )*(pi/180))) - - # build dictionary - result={'time':btime,'date':bdate,'latitude':blat,'longitude':blon, - 'freq':freq,'deltafreq':dfreq,'pspec':pspec,'dmspec':dmspec,'dpspec':dpspec, - 'theta':theta,'dirspec':dirspec} - - return result - del btime,bdate,blat,blon,freq,dfreq,pspec,dmspec,dpspec,theta,dirspec - - -# WAVEWATCH III spectra output, netcdf format def spec_ww3(*args): ''' - WAVEWATCH III, wave spectrum, netcdf format - Input: file name (example: ww3gefs.20160928_spec.nc), and station name (example: 41002) - Output: dictionary containing: + WAVEWATCH III, wave spectrum, netcdf (.nc) or text (.spec) format + Input: file names (list of file names), and station names (list of station names) + Output: list of dictionaries containing: time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pwst,d1sp,dire,dspec,wnds,wndd ''' - sk=1 - if len(args) < 2 : - sys.exit(' Two inputs are required: file name and station name') - if len(args) >= 2 : - fname=str(args[0]); stname=str(args[1]) - if len(args) > 2 : - sk=np.int(args[2]) - if len(args) > 3 : - sys.exit(' Too many inputs') - - try: - ds = xr.open_dataset(fname); f=nc.Dataset(fname) - except: - sys.exit(" Cannot open "+fname) - else: - - mtime = np.array(f.variables['time'][::sk]*24*3600 + timegm( strptime(str(f.variables['time'].units).split(' ')[2][0:4]+'01010000', '%Y%m%d%H%M') )).astype('double') - f.close(); del f - - auxstationname=ds['station_name'].values[:,:]; stationname=[] - for i in range(0,auxstationname.shape[0]): - stationname=np.append(stationname,"".join(np.array(auxstationname[i,:]).astype('str'))) - - inds=np.where(stationname[:]==stname) - if size(inds)>0: - inds=np.int(inds[0][0]); stname=str(stationname[inds]) - else: - sys.exit(' Station '+stname+' not included in the output file, or wrong station ID') - - # Spectrum - dspec=np.array(ds['efth'][::sk,inds,:,:]) - # number of directions - nd=dspec.shape[2] - # number of frequencies - nf=dspec.shape[1] - # directions - dire=np.array(ds['direction'].values[:]) - # frequencies - freq=np.array(ds['frequency'].values[:]) - freq1=np.array(ds['frequency1'].values[:]) - freq2=np.array(ds['frequency2'].values[:]) - # DF in frequency (dfreq) - dfreq=np.array(freq2 - freq1) - # wind intensity and wind direction - wnds=np.array(ds['wnd'].values[::sk,inds]) - wndd=np.array(ds['wnddir'].values[::sk,inds]) - # Time datetime64 array - mdate=np.array(ds['time'].values[::sk]) - # water depth (constant in time) - depth=np.nanmean(ds['dpt'].values[::sk,inds],axis=0) - lon=np.array(np.nanmean(ds['longitude'].values[::sk,inds],axis=0)) - lat=np.array(np.nanmean(ds['latitude'].values[::sk,inds],axis=0)) - - ds.close(); del ds, auxstationname, inds, stationname - # ------------------ - # 1D power spectrum - pwst=np.zeros((dspec.shape[0],nf),'f') - for t in range(0,dspec.shape[0]): - for il in range(0,nf): - pwst[t,il]=sum(dspec[t,il,:]*(2*np.pi)/nd) - - pwst[t,:]=pwst[t,:]*dfreq[:] - - # organizing directions ----- - adspec=np.copy(dspec); inddire=int(np.where(dire==min(dire))[0][0]) - for t in range(0,dspec.shape[0]): - adspec[t,:,0:nd-(inddire+1)]=dspec[t,:,(inddire+1):nd] - adspec[t,:,nd-(inddire+1):nd]=dspec[t,:,0:(inddire+1)] - for i in range(0,nd): - dspec[t,:,i]=adspec[t,:,nd-i-1] - adspec[t,:,0:int(nd/2)]=dspec[t,:,int(nd/2):nd] - adspec[t,:,int(nd/2):nd]=dspec[t,:,0:int(nd/2)] - dspec[t,:,:]=adspec[t,:,:] + if len(args) < 2: + sys.exit(' Two inputs are required: list of file names and list of station names') - dire=np.sort(dire) - - # 1D directional spectrum - d1sp=np.zeros((dspec.shape[0],nf),'f') - for t in range(0,dspec.shape[0]): - for il in range(0,nf): - a = np.sum(dspec[t,il,:] * np.array(np.sin((pi*dire)/180.)/np.sum(dspec[t,il,:])) ) - b = np.sum(dspec[t,il,:] * np.array(np.cos((pi*dire)/180.)/np.sum(dspec[t,il,:])) ) - aux = math.atan2(a,b)*(180./pi) - if aux<0: - aux=aux+360. - - d1sp[t,il]=float(aux) - del a,b,aux - - # build dictionary - result={'time':mtime,'date':mdate,'latitude':lat,'longitude':lon, - 'wind_spd':wnds,'wind_dir':wndd,'freq':freq,'freq1':freq1,'freq2':freq2, - 'deltafreq':dfreq,'pspec':pwst,'theta':dire,'dmspec':d1sp,'dirspec':dspec} - - return result - del mtime,mdate,lat,lon,wnds,wndd,freq,freq1,freq2,dfreq,pwst,dire,d1sp,dspec - - -#added a function to read the txt files - -def read_text_file(fname_txtfile): - try: - # Attempt to open and read the file name from the txt file - with open(fname_txtfile, 'r') as f: - lines = f.readlines() - if len(lines) != 1: - raise ValueError("The txt file should contain only one line with the file name.") - fname = lines[0].strip() - except FileNotFoundError: - sys.exit('Text file not found.') - except Exception as e: - sys.exit(f'Error reading txt file: {str(e)}') - - results = {} - stname = [] - - try: - tar = tarfile.open(fname, "r:gz") # Open the tar file - - for t in range(0, len(tar.getmembers())): - # Station names + fnames = args[0] + stnames = args[1] + sk = 1 + if len(args) > 2: + sk = int(args[2]) + if len(args) > 3: + sys.exit(' Too many inputs') - stname.append(str(tar.getmembers()[t].name).split('/')[-1].split('.')[-2]) + file_names = args[0] + station_names = args[1] + results = [] + for fname in fnames: + for stname in stnames: try: - fp = tar.extractfile(tar.getmembers()[t]) - if fp is None: - raise ValueError("File is empty or cannot be extracted.") - lines = fp.readlines() - nt=len(lines) - except Exception as e: - print("Cannot open " + tar.getmembers()[t].name) - print("Error:", str(e)) - continue - else: - if nt == 0: - print("No lines to read in file:", tar.getmembers()[t].name) - continue - for line in lines: - line = line.strip().decode() - - if nt >= 1: - # Open file and read the first parameters - fp = tar.extractfile(tar.getmembers()[t]) - cabc = fp.readline().strip().split() - nf = int(cabc[3]) # number of frequencies - nd = int(cabc[4]) # number of directions - npo = int(cabc[5]) # number of point outputs - - freq = np.zeros(nf, 'f') - dire = np.zeros(nd, 'f') - dspec = np.zeros((nt, nf, nd), 'f') - adire = np.zeros(dire.shape) - adspec = np.zeros(dspec.shape) - ntime = np.zeros((nt), 'd') - - # Frequencies -------------------- - ncf = int(np.floor(nf/8)) - rncf = int(np.round(8*((float(nf)/8)-ncf))) - k = 0 - for i in range(0, ncf): - line = fp.readline() - line = line.strip().split() - for j in range(0, 8): - freq[k] = float(line[j]) - k = k+1 - - if rncf > 0: - line = fp.readline() - line = line.strip().split() - for i in range(0, rncf): - freq[k] = float(line[i]) - k = k+1 - - # DF in frequency (dfreq) - dfreq = np.zeros(freq.shape[0], 'f') - for i in range(0, freq.shape[0]): - if i == 0 or i == (freq.shape[0]-1): - dfreq[i] = freq[i]*(1 + (((freq[-1]/freq[-2])-1)/2)) - freq[i] - else: - dfreq[i] = freq[i]*(freq[-1]/freq[-2]) - freq[i] - - # Directions --------------------- - ncd = int(np.floor(nd/7)) - rncd = int(np.round(7*((float(nd)/7)-ncd))) - k = 0 - for i in range(0, ncd): - line = fp.readline() - line = line.strip().split() - for j in range(0, 7): - dire[k] = float(line[j])*180/np.pi - k = k+1 - - if rncd > 0: - line = fp.readline() - line = line.strip().split() - for i in range(0, rncd): - dire[k] = float(line[i])*180/np.pi - k = k+1 - - nl = int(np.floor((nf*nd)/7.)) - rnl = int(np.round(7*((float(nf*nd)/7)-nl))) - auxs = np.zeros((nf*nd), 'f') - wnds = np.zeros((nt), 'f') - wndd = np.zeros((nt), 'f') - hs = np.zeros(nt) # Initialize significant wave height array - - for t in range(0, nt): - - cabc = [item.decode() for item in fp.readline().strip().split()] - - if not cabc: - continue - ntime[t] = np.double(timegm( strptime(cabc[0]+cabc[1][0:2], '%Y%m%d%H') )) - cabc = [item.decode() for item in fp.readline().strip().split()] - - if not cabc: - continue - if t == 0: - - if len(cabc) >= 8: - namep = cabc[0][1:] - lat_str = cabc[3] - lon_str = cabc[4] - lat = -float(lat_str) if lat_str.startswith('-') else float(lat_str) - lon = -float(lon_str) if lon_str.startswith('-') else float(lon_str) - depth = float(cabc[5]) - wnds[t] = float(cabc[6]) - wndd[t] = float(cabc[7]) - - + # Text format (only one point allowed here, same as WW3/NOAA operational) + fp = open(fname) + nt = fp.read().count(stname) + fp.close() + del fp + if nt >= 1: + # Open file and read the first parameters + fp = open(fname) + cabc = fp.readline() + cabc = cabc.strip().split() + nf = int(cabc[3]) # number of frequencies + nd = int(cabc[4]) # number of directions + npo = int(cabc[5]) # number of point outputs + + freq = zeros(nf, 'f') + dire = zeros(nd, 'f') + dspec = zeros((nt, nf, nd), 'f') + adire = zeros(dire.shape) + adspec = zeros(dspec.shape) + mtime = np.zeros((nt), 'd') + + # Frequencies -------------------- + ncf = int(np.floor(nf/8)) + rncf = int(np.round(8*((float(nf)/8)-ncf))) + k = 0 + for i in range(0, ncf): + line = fp.readline() + line = line.strip().split() + for j in range(0, 8): + freq[k] = float(line[j]) + k = k + 1 - elif len(cabc) == 7: - namep = cabc[0][1:-1] if cabc[0].startswith("'") and cabc[0].endswith("'") else cabc[0][1:] - print("Station Name:", namep) - lat_lon = cabc[1].split('-') - print("lat-lon:",lat_lon) - lat_str = lat_lon[0] - print("lat_str:",lat_str) - lon_str = lat_lon[1] - print("lon_str:",lon_str) - lat = -float(lat_str) if lat_str.startswith('-') else float(lat_str) - lon = -float(lon_str) if lon_str.startswith('-') else float(lon_str) - depth = float(cabc[3]) - wnds[t] = float(cabc[4]) - wndd[t] = float(cabc[5]) + if rncf > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rncf): + freq[k] = float(line[i]) + k = k + 1 + + # DF in frequency (dfreq) + dfreq = np.zeros(freq.shape[0], 'f') + for i in range(0, freq.shape[0]): + if i == 0 or i == (freq.shape[0]-1): + dfreq[i] = freq[i] * \ + (1 + (((freq[-1]/freq[-2])-1)/2)) - freq[i] + else: + dfreq[i] = freq[i] * \ + (freq[-1]/freq[-2]) - freq[i] + # Directions --------------------- + ncd = int(np.floor(nd/7)) + rncd = int(np.round(7*((float(nd)/7)-ncd))) k = 0 - for i in range(0, nl): + for i in range(0, ncd): line = fp.readline() line = line.strip().split() for j in range(0, 7): - auxs[k] = float(line[j]) + dire[k] = float(line[j])*180/pi k = k+1 if rncd > 0: line = fp.readline() line = line.strip().split() - for i in range(0, rnl): - auxs[k] = float(line[i]) + for i in range(0, rncd): + dire[k] = float(line[i])*180/pi k = k+1 - for ic in range(0, nf): - for il in range(0, nd): - dspec[t, ic, il] = auxs[il*nf+ic] - - # Calculate significant wave height - sp1d = np.sum(dspec[t], axis=1) * (np.abs(dire[1] - dire[0])) # Calculate 1D spectrum - hs[t] = 4 * np.sqrt(np.trapz(sp1d, x=freq)) # Calculate significant wave height - - fp.close() - - results['ntime'] = ntime - results['latitude'] = lat - results['longitude'] = lon - results['wind_spd'] = wnds - results['wind_dir'] = wndd - results['freq'] = freq - results['deltafreq'] = dfreq - results['dir'] = dire - results['spec'] = dspec - results['station_name'] = stname - results['hs'] = hs # Add significant wave height to results + nl = int(floor((nf*nd)/7.)) + rnl = int(np.round(7*((float(nf*nd)/7)-nl))) + auxs = np.zeros((nf*nd), 'f') + wnds = np.zeros((nt), 'f') + wndd = np.zeros((nt), 'f') + + for t in range(0, nt): + + cabc = fp.readline() + cabc.strip().split()[0] + mtime[t] = np.double(timegm(strptime( + cabc.strip().split()[0]+cabc.strip().split()[1][0:2], '%Y%m%d%H'))) + cabc = fp.readline() + cabc = cabc.strip().split() + print(cabc) + if len(cabc) >8: + # Format: ["'42085", "'", '17.86', '-66.52', '126.1', '1.12', '146.9'] + namep = cabc[0][1:] + lat = float(cabc[2]) + lon = float(cabc[3]) + depth = float(cabc[4]) + wnds_index = 5 + wndd_index = 6 + elif len(cabc) == 8: + # Format: ["'46021", "'", '57.70-160.00', '50.8', '5.47', '6.9', '0.00', '270.0'] + namep = cabc[0][1:] + lat_lon_str = cabc[2] + lat_lon_str = lat_lon_str.strip("'") + lat_lon_parts = lat_lon_str.split('-') + print(lat_lon_parts) + lat = float(lat_lon_parts[0]) + lon = -float(lat_lon_parts[1]) - else: - sys.exit(f'Station {stname} not included in') - except Exception as e: - sys.exit(f'Error reading spectral file: {str(e)}') + depth = float(cabc[3]) + wnds_index = 4 + wndd_index = 5 + else: + continue # Skip this file as it cannot be processed + # sys.exit('Unrecognized format of cabc') + + wnds[t] = float(cabc[wnds_index]) + wndd[t] = float(cabc[wndd_index]) + + k = 0 + for i in range(0, nl): + line = fp.readline() + line = line.strip().split() + for j in range(0, 7): + auxs[k] = float(line[j]) + k = k+1 + + if rncd > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rnl): + auxs[k] = float(line[i]) + k = k+1 + + for ic in range(0, nf): + for il in range(0, nd): + dspec[t, ic, il] = auxs[il*nf+ic] + + fp.close() + del fp + + mdate = pd.to_datetime(mtime, unit='s').strftime( + '%Y-%m-%dT%H:%M:%S.%f') + freq1 = freq*np.nan + freq2 = freq*np.nan + + # ------------------ + # 1D power spectrum + pwst = np.zeros((dspec.shape[0], nf), 'f') + for t in range(0, dspec.shape[0]): + for il in range(0, nf): + pwst[t, il] = sum( + dspec[t, il, :]*(2*np.pi)/nd) + + pwst[t, :] = pwst[t, :]*dfreq[:] + + # organizing directions ----- + adspec = np.copy(dspec) + inddire = int(np.where(dire == min(dire))[0][0]) + for t in range(0, dspec.shape[0]): + adspec[t, :, 0:nd-(inddire+1)] = dspec[t,:, (inddire+1):] + adspec[t, :, nd-(inddire+1):nd] = dspec[t,:, :(inddire+1)] + for i in range(0, nd): + dspec[t, :, i] = adspec[t, :, nd-i-1] + + adspec[t, :, :int(nd/2)] = dspec[t, :, int(nd/2):] + adspec[t, :, int(nd/2):] = dspec[t, :, :int(nd/2)] + dspec[t, :, :] = adspec[t, :, :] + + dire = np.sort(dire) + + # 1D directional spectrum + d1sp = np.zeros((dspec.shape[0], nf), 'f') + + for t in range(0, dspec.shape[0]): + for il in range(0, nf): + a = np.sum(dspec[t, il, :] * np.array( + np.sin((pi*dire)/180.)/np.sum(dspec[t, il, :]))) + b = np.sum(dspec[t, il, :] * np.array( + np.cos((pi*dire)/180.)/np.sum(dspec[t, il, :]))) + aux = math.atan2(a, b)*(180./pi) + if aux < 0: + aux = aux+360. + + d1sp[t, il] = float(aux) + del a, b, aux + + hs = np.sqrt(2 * np.trapz(np.trapz(dspec, x=dire, axis=-1), x=freq, axis=-1)) + tp = freq[np.argmax(np.max(dspec, axis=-1), axis=-1)] + + # build dictionary + result = {'time': mtime, 'date': mdate, 'latitude': lat, 'longitude': lon, 'depth': depth, + 'wind_spd': wnds, 'wind_dir': wndd, 'freq': freq, 'freq1': freq1, 'freq2': freq2, + 'deltafreq': dfreq, 'pspec': pwst, 'theta': dire, 'dmspec': d1sp, 'dirspec': dspec, 'Hs': hs, 'Tp': tp,'station_name': stname} + + results.append(result) + except Exception as e: + print(f"Skipping file {fname} for station {stname}: {str(e)}") + continue return results +# Example usage: +# spec_ww3(['file1.spec', 'file2.spec'], ['station1', 'station2']) From 4c29c4c9da0b628175224efc99a66d8f4b65bdf0 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Thu, 11 Apr 2024 23:05:32 +0000 Subject: [PATCH 06/27] hs and tp added to the .spec file format --- ww3tools/model5.py | 689 ------------ ww3tools/modelBuoy_collocation.py | 1642 ++++++++++++++++------------- ww3tools/wread.py | 1624 +++++++++++++++++++++++++++- 3 files changed, 2521 insertions(+), 1434 deletions(-) delete mode 100644 ww3tools/model5.py diff --git a/ww3tools/model5.py b/ww3tools/model5.py deleted file mode 100644 index 33646cc..0000000 --- a/ww3tools/model5.py +++ /dev/null @@ -1,689 +0,0 @@ -import warnings -warnings.filterwarnings("ignore") -import numpy as np -from matplotlib.mlab import * -from pylab import * -import xarray as xr -import netCDF4 as nc -import time -from time import strptime -from calendar import timegm -import wread -# netcdf format -fnetcdf="NETCDF4" - -# Paths -# ndbcp="/data/buoys/NDBC/wparam" -ndbcp="/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" -# Copernicus buoys -# copernp="/data/buoys/Copernicus/wtimeseries" -copernp="/work/noaa/marine/ricardo.campos/data/buoys/Copernicus/wtimeseries" -print(' ') - - -# Options of including grid and cyclone information -gridinfo=int(0); cyclonemap=int(0); wlist=[]; ftag=''; forecastds=0 - - - -if len(sys.argv) >= 4: - gridinfo=str(sys.argv[3]) - print(' Using gridInfo '+gridinfo) - - - - -# READ DATA -print(" ") -if gridinfo!=0: - # Grid Information - gridmask = wread.mask(gridinfo) - mlat=gridmask['latitude']; mlon=gridmask['longitude'] - mask=gridmask['mask']; distcoast=gridmask['distcoast']; depth=gridmask['depth'] - oni=gridmask['GlobalOceansSeas']; ocnames=gridmask['names_GlobalOceansSeas'] - hsmz=gridmask['HighSeasMarineZones']; hsmznames=gridmask['names_HighSeasMarineZones'] - print(" GridInfo Ok. "+gridinfo) - - # Cyclone Information - if cyclonemap!=0: - cycloneinfo = wread.cyclonemap(cyclonemap) - clat=cycloneinfo['latitude']; clon=cycloneinfo['longitude'] - cmap=cycloneinfo['cmap']; ctime=cycloneinfo['time'] - cinfo=np.array(cycloneinfo['info'].split(':')[1].split(';')) - if np.array_equal(clat,mlat)==True & np.array_equal(clon,mlon)==True: - print(" CycloneMap Ok. "+cyclonemap) - else: - sys.exit(' Error: Cyclone grid and Mask grid are different.') - - - - - - - - - - -from wread import spec_ww3 - -# Check if the correct number of arguments is provided -if len(sys.argv) != 3: - print("Usage: python run_spec_ww3.py file_names.txt station_names.txt") - sys.exit(1) - -# Read file names from the first argument -file_names = [] -with open(sys.argv[1], 'r') as file: - for line in file: - file_names.append(line.strip()) - -# Read station names from the second argument -station_names = [] -with open(sys.argv[2], 'r') as file: - for line in file: - station_names.append(line.strip()) - -# Call the spec_ww3 function -results = spec_ww3(file_names, station_names) - -# Additional processing -mfcycle = None -stname = None -mtime = None -mhs = None -mtp = None -mwn = None -mwd = None -mfreq = None -mtm=None -mdm=None - - -for i, result in enumerate(results): - at = result['time'] - if i == 0: - mfcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') - stname = np.atleast_1d(np.array(result['station_name'])) - mtime = np.copy(at) - mhs = np.copy([result['Hs']]) - mtp = np.copy([result['Tp']]) - if 'wind_spd' in result.keys(): - mwn = np.copy([result['wind_spd']]) - else: - mwn = np.copy(mhs) * np.nan - - if 'wind_dir' in result.keys(): - mwd = np.copy([result['wind_dir']]) - else: - mwd = np.copy(mhs) * np.nan - - if 'freq' in result.keys(): - mfreq = np.copy([result['freq']]) - else: - mfreq = np.copy(mhs) * np.nan - - if 'tm' in result.keys(): - mtm = np.append(mtm,[result['tm']],axis=0) - else: - mtm = np.copy(mhs) * np.nan - - if 'dm' in result.keys(): - mdm=np.copy([result['dm']]) - else: - mdm=np.copy(mhs)*np.nan - - if 'dp' in result.keys(): - mdp=np.copy([result['dp']]) - else: - mdp=np.copy(mhs)*np.nan - - else: - if mhs.shape[1] == result['Hs'].shape[0]: - stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) - mtime = np.append(mtime, at) - mhs = np.append(mhs, [result['Hs']], axis=0) - mtp = np.append(mtp, [result['Tp']], axis=0) - if 'wind_spd' in result.keys(): - mwn = np.append(mwn, [result['wind_spd']], axis=0) - else: - mwn = np.append(mwn, [np.copy(result['Hs']) * np.nan], axis=0) - - if 'wind_dir' in result.keys(): - mwd = np.append(mwd, [result['wind_dir']], axis=0) - else: - mwd = np.append(mwd, [np.copy(result['Hs']) * np.nan], axis=0) - - if 'freq' in result.keys(): - mfreq = np.append(mfreq, [result['freq']], axis=0) - else: - mfreq = np.append(mfreq, [np.copy(result['Hs']) * np.nan], axis=0) - - if 'tm' in result.keys(): - mtm=np.append(mtm,[result['tm']],axis=0) - else: - mtm=np.append(mtm,[np.copy(result['Hs'])*np.nan],axis=0) - - if 'dm' in result.keys(): - mdm=np.append(mdm,[result['dm']],axis=0) - else: - mdm=np.append(mdm,[np.copy(result['Hs'])*np.nan],axis=0) - - if 'dp' in result.keys(): - mdp=np.append(mdp,[result['dp']],axis=0) - else: - mdp=np.append(mdp,[np.copy(result['Hs'])*np.nan],axis=0) - - else: - print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) - -# Save proc -print('HSS:', mhs) -print('WIND:', mwn) - - -# BUOYS ------------------ -bwind = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -bhs = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -btm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -btp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -bdm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -bdp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -lat = np.zeros(np.size(stname), 'f') * np.nan -lon = np.zeros(np.size(stname), 'f') * np.nan -# help reading NDBC buoys, divided by year -yrange = np.array(np.arange(time.gmtime(mtime.min())[0], time.gmtime(mtime.min())[0] + 1, 1)).astype('int') -# loop buoys -for b in range(0, np.size(stname)): - - ahs = [] - try: - awm = [] - ahs = [] - atm = [] - atp = [] - adm = [] - atime = [] - for y in yrange: - - f = nc.Dataset(ndbcp + "/" + stname[b] + "h" + repr(y) + ".nc") - if 'wave_height' in f.variables.keys(): - ahs = np.append(ahs, f.variables['wave_height'][:, 0, 0]) - elif 'hs' in f.variables.keys(): - ahs = np.append(ahs, f.variables['hs'][:, 0, 0]) - elif 'swh' in f.variables.keys(): - ahs = np.append(ahs, f.variables['swh'][:, 0, 0]) - - if 'wind_spd' in f.variables.keys(): - awm = np.append(awm, f.variables['wind_spd'][:, 0, 0]) - else: - awm = np.array(np.copy(ahs * nan)) - - if 'average_wpd' in f.variables.keys(): - atm = np.append(atm, f.variables['average_wpd'][:, 0, 0]) - else: - atm = np.array(np.copy(ahs * nan)) - - if 'dominant_wpd' in f.variables.keys(): - atp = np.append(atp, f.variables['dominant_wpd'][:, 0, 0]) - else: - atp = np.array(np.copy(ahs * nan)) - - if 'mean_wave_dir' in f.variables.keys(): - adm = np.append(adm, f.variables['mean_wave_dir'][:, 0, 0]) - else: - adm = np.array(np.copy(ahs * nan)) - - if 'latitude' in f.variables.keys(): - lat[b] = f.variables['latitude'][:] - elif 'LATITUDE' in f.variables.keys(): - lat[b] = f.variables['LATITUDE'][:] - else: - lat[b] = nan - - if 'longitude' in f.variables.keys(): - lon[b] = f.variables['longitude'][:] - elif 'LONGITUDE' in f.variables.keys(): - lon[b] = f.variables['LONGITUDE'][:] - else: - lon[b] = nan - - atime = np.append(atime, np.array(f.variables['time'][:]).astype('double')) - - f.close() - del f - - adp = adm * np.nan # no peak direction available in this format - - if np.size(ahs) > 0: - - # First layer of simple quality-control - indq = np.where((ahs > 30.) | (ahs < 0.0)) - if np.size(indq) > 0: - ahs[indq] = np.nan - del indq - - indq = np.where((atm > 40.) | (atm < 0.0)) - if np.size(indq) > 0: - atm[indq] = np.nan - del indq - - indq = np.where((atp > 40.) | (atp < 0.0)) - if np.size(indq) > 0: - atp[indq] = np.nan - del indq - - indq = np.where((adm > 360.) | (adm < -180.)) - if np.size(indq) > 0: - adm[indq] = np.nan - del indq - - indq = np.where((adp > 360.) | (adp < -180.)) - if np.size(indq) > 0: - adp[indq] = np.nan - del indq - - indq = np.where((awm > 50.) | (awm < 0.0)) - if np.size(indq) > 0: - awm[indq] = np.nan - del indq - - c = 0 - for t in range(0, np.size(mtime)): - indt = np.where(np.abs(atime - mtime[t]) < 1800.) - if np.size(indt) > 0: - if np.any(ahs[indt[0]].mask == False): - bhs[b, t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask == False]) - c = c + 1 - if np.any(atm[indt[0]].mask == False): - btm[b, t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask == False]) - if np.any(atp[indt[0]].mask == False): - btp[b, t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask == False]) - if np.any(adm[indt[0]].mask == False): - bdm[b, t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask == False]) - if np.any(adp[indt[0]].mask == False): - bdp[b, t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask == False]) - if np.any(awm[indt[0]].mask == False): - bwind[b, t] = np.nanmean(awm[indt[0]][awm[indt[0]].mask == False]) - - del indt - - # print("counted "+repr(c)+" at "+stname[b]) - - print(" station " + stname[b] + " ok") -# del ahs - except Exception as e: - print("Error occurred while processing station", stname[b]) - print(e) - -print('bwind:',bwind) -print('bhs:',bhs) - - -print(' ') -# Simple quality-control (range) -ind=np.where((bhs>30.)|(bhs<0.0)) -if np.size(ind)>0: - bhs[ind]=np.nan; del ind - -ind=np.where((btm>40.)|(btm<0.0)) -if np.size(ind)>0: - btm[ind]=np.nan; del ind - -ind=np.where((btp>40.)|(btp<0.0)) -if np.size(ind)>0: - btp[ind]=np.nan; del ind - -ind=np.where((bdm>360.)|(bdm<-180.)) -if np.size(ind)>0: - bdm[ind]=np.nan; del ind - -ind=np.where((bdp>360.)|(bdp<-180.)) -if np.size(ind)>0: - bdp[ind]=np.nan; del ind - -ind=np.where((bwind>50.0)|(bwind<0.0)) -if np.size(ind)>0: - bwind[ind]=np.nan; del ind - -ind=np.where((mhs>30.)|(mhs<0.0)) -if np.size(ind)>0: - mhs[ind]=np.nan; del ind - -ind=np.where((mtm>40.)|(mtm<0.0)) -if np.size(ind)>0: - mtm[ind]=np.nan; del ind - -ind=np.where((mtp>40.)|(mtp<0.0)) -if np.size(ind)>0: - mtp[ind]=np.nan; del ind - -ind=np.where((mdm>360.)|(mdm<-180.)) -if np.size(ind)>0: - mdm[ind]=np.nan; del ind - -ind=np.where((mdp>360.)|(mdp<-180.)) -if np.size(ind)>0: - mdp[ind]=np.nan; del ind - -ind=np.where((mwn>50.)|(mwn<0.0)) -if np.size(ind)>0: - mwn[ind]=np.nan; del ind - -# Clean data excluding some stations. Select matchups only when model and buoy are available. -ind=np.where( (np.isnan(lat)==False) & (np.isnan(lon)==False) & (np.isnan(np.nanmean(mhs,axis=1))==False) & (np.isnan(np.nanmean(bhs,axis=1))==False) ) -if np.size(ind)>0: - stname=np.array(stname[ind[0]]) - lat=np.array(lat[ind[0]]) - lon=np.array(lon[ind[0]]) - mhs=np.array(mhs[ind[0],:]) - mtm=np.array(mtm[ind[0],:]) - mtp=np.array(mtp[ind[0],:]) - mdm=np.array(mdm[ind[0],:]) - mdp=np.array(mdp[ind[0],:]) - mwn=np.array(mwn[ind[0],:]) - bhs=np.array(bhs[ind[0],:]) - btm=np.array(btm[ind[0],:]) - btp=np.array(btp[ind[0],:]) - bdm=np.array(bdm[ind[0],:]) - bdp=np.array(bdp[ind[0],:]) - bwind=np.array(bwind[ind[0],:]) -else: - sys.exit(' Error: No matchups Model/Buoy available.') - - -print(" Matchups model/buoy complete. Total of "+repr(np.size(ind))+" stations/buoys avaliable."); del ind - -# Processing grid and/or cyclone information -if gridinfo!=0: - print(" Adding extra information ... ") - alon=np.copy(lon); alon[alon<0]=alon[alon<0]+360. - indgplat=[]; indgplon=[] - for i in range(0,lat.shape[0]): - # indexes nearest point. - indgplat = np.append(indgplat,np.where( abs(mlat-lat[i])==abs(mlat-lat[i]).min() )[0][0]) - indgplon = np.append(indgplon,np.where( abs(mlon-alon[i])==abs(mlon-alon[i]).min() )[0][0]) - - indgplat=np.array(indgplat).astype('int'); indgplon=np.array(indgplon).astype('int') - pdistcoast=np.zeros(lat.shape[0],'f')*np.nan - pdepth=np.zeros(lat.shape[0],'f')*np.nan - poni=np.zeros(lat.shape[0],'f')*np.nan - phsmz=np.zeros(lat.shape[0],'f')*np.nan - for i in range(0,lat.shape[0]): - pdistcoast[i]=distcoast[indgplat[i],indgplon[i]] - pdepth[i]=depth[indgplat[i],indgplon[i]] - poni[i]=oni[indgplat[i],indgplon[i]] - phsmz[i]=hsmz[indgplat[i],indgplon[i]] - - print(" Grid Information Included.") - - # Excluding shallow water points too close to the coast (mask information not accurate) - ind=np.where( (np.isnan(pdistcoast)==False) & (np.isnan(pdepth)==False) ) - if np.size(ind)>0: - stname=np.array(stname[ind[0]]) - lat=np.array(lat[ind[0]]) - lon=np.array(lon[ind[0]]) - mhs=np.array(mhs[ind[0],:]) - mtm=np.array(mtm[ind[0],:]) - mtp=np.array(mtp[ind[0],:]) - mdm=np.array(mdm[ind[0],:]) - mdp=np.array(mdp[ind[0],:]) - mwn=np.array(mwn[ind[0],:]) - bhs=np.array(bhs[ind[0],:]) - btm=np.array(btm[ind[0],:]) - btp=np.array(btp[ind[0],:]) - bdm=np.array(bdm[ind[0],:]) - bdp=np.array(bdp[ind[0],:]) - bwind=np.array(bwind[ind[0],:]) - pdistcoast=np.array(pdistcoast[ind[0]]) - pdepth=np.array(pdepth[ind[0]]) - poni=np.array(poni[ind[0]]) - phsmz=np.array(phsmz[ind[0]]) - else: - sys.exit(' Error: No matchups Model/Buoy available after using grid mask.') - - del ind - - if cyclonemap!=0: - fcmap=np.zeros((lat.shape[0],mtime.shape[0]),'f')*np.nan - for t in range(0,np.size(mtime)): - # search for cyclone time index and cyclone map - indt=np.where(np.abs(ctime-mtime[t])<5400.) - if np.size(indt)>0: - for i in range(0,lat.shape[0]): - fcmap[i,t] = np.array(cmap[indt[0][0],indgplat[i],indgplon[i]]) - - del indt - else: - print(' - No cyclone information for this time step: '+repr(t)) - - # print(' Done cyclone analysis at step: '+repr(t)) - - ind=np.where(fcmap<0) - if np.size(ind)>0: - fcmap[ind]=np.nan - - print(" Cyclone Information Included.") - -# Edit format if this is forecast model data. Reshape and allocate - - -if forecastds>0: - unt=np.unique(mfcycle); mxsz=1 - for i in range(0,unt.shape[0]): - ind=np.where(mfcycle==unt[i])[0] - mxsz=np.max([mxsz,np.size(ind)]) - - for i in range(0,unt.shape[0]): - ind=np.where(mfcycle==unt[i])[0] - if i==0: - nmhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmwn=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbwind=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtime=np.zeros((unt.shape[0],mxsz),'double')*np.nan - if cyclonemap!=0: - nfcmap=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - - nmtime[i,0:np.size(ind)]=np.array(mtime[ind]).astype('double') - nmhs[:,i,:][:,0:np.size(ind)]=np.array(mhs[:,ind]) - nmtm[:,i,:][:,0:np.size(ind)]=np.array(mtm[:,ind]) - nmtp[:,i,:][:,0:np.size(ind)]=np.array(mtp[:,ind]) - nmdm[:,i,:][:,0:np.size(ind)]=np.array(mdm[:,ind]) - nmdp[:,i,:][:,0:np.size(ind)]=np.array(mdp[:,ind]) - nmwn[:,i,:][:,0:np.size(ind)]=np.array(mwn[:,ind]) - nbhs[:,i,:][:,0:np.size(ind)]=np.array(bhs[:,ind]) - nbtm[:,i,:][:,0:np.size(ind)]=np.array(btm[:,ind]) - nbtp[:,i,:][:,0:np.size(ind)]=np.array(btp[:,ind]) - nbdm[:,i,:][:,0:np.size(ind)]=np.array(bdm[:,ind]) - nbdp[:,i,:][:,0:np.size(ind)]=np.array(bdp[:,ind]) - nwind[:,i,:][:,0:np.size(ind)]=np.array(bwind[:,ind]) - - if cyclonemap!=0: - nfcmap[:,i,:][:,0:np.size(ind)]=np.array(fcmap[:,ind]) - - - ind=np.where( (nmhs>0.0) & (nbhs>0.0) ) - -else: - - larger_shape = max(mhs.shape, bhs.shape) - padded_mhs = np.zeros(larger_shape) - padded_bhs = np.zeros(larger_shape) - padded_mhs[:mhs.shape[0], :mhs.shape[1]] = mhs - padded_bhs[:bhs.shape[0], :bhs.shape[1]] = bhs - - - # Create padded arrays for other variables - padded_mtm = np.zeros(larger_shape) # Assuming mtm has the same shape as mhs - padded_mtp = np.zeros(larger_shape) # Assuming mtp has the same shape as mhs - padded_mdm = np.zeros(larger_shape) # Assuming mdm has the same shape as mhs - padded_mdp = np.zeros(larger_shape) # Assuming mdp has the same shape as mhs - padded_mwn = np.zeros(larger_shape) # Assuming mwn has the same shape as mhs - - padded_btm = np.zeros(larger_shape) # Assuming btm has the same shape as bhs - padded_btp = np.zeros(larger_shape) # Assuming btp has the same shape as bhs - padded_bdm = np.zeros(larger_shape) # Assuming bdm has the same shape as bhs - padded_bdp = np.zeros(larger_shape) # Assuming bdp has the same shape as bhs - padded_bwind = np.zeros(larger_shape) # Assuming bwind has the same shape as bhs - - # Copy values from original arrays to padded arrays for other variables - padded_mtm[:mtm.shape[0], :mtm.shape[1]] = mtm - padded_mtp[:mtp.shape[0], :mtp.shape[1]] = mtp - padded_mdm[:mdm.shape[0], :mdm.shape[1]] = mdm - padded_mdp[:mdp.shape[0], :mdp.shape[1]] = mdp - padded_mwn[:mwn.shape[0], :mwn.shape[1]] = mwn - - padded_btm[:btm.shape[0], :btm.shape[1]] = btm - padded_btp[:btp.shape[0], :btp.shape[1]] = btp - padded_bdm[:bdm.shape[0], :bdm.shape[1]] = bdm - padded_bdp[:bdp.shape[0], :bdp.shape[1]] = bdp - padded_bwind[:bwind.shape[0], :bwind.shape[1]] = bwind - - - -# ind=np.where( (mhs>0.0) & (bhs>0.0) ) - ind = np.where((padded_mhs > 0.0) & (padded_bhs > 0.0)) -if np.size(ind)>0: - print(' Total amount of matchups model/buoy: '+repr(np.size(ind))) - - # Save netcdf output file - lon[lon>180.]=lon[lon>180.]-360. - initime=str(time.gmtime(mtime.min())[0])+str(time.gmtime(mtime.min())[1]).zfill(2)+str(time.gmtime(mtime.min())[2]).zfill(2)+str(time.gmtime(mtime.min())[3]).zfill(2) - fintime=str(time.gmtime(mtime.max())[0])+str(time.gmtime(mtime.max())[1]).zfill(2)+str(time.gmtime(mtime.max())[2]).zfill(2)+str(time.gmtime(mtime.max())[3]).zfill(2) - ncfile = nc.Dataset('WW3.Buoy'+ftag+'_'+initime+'to'+fintime+'.nc', "w", format=fnetcdf) - ncfile.history="Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of "+repr(bhs[bhs>0.].shape[0])+" observations or pairs model/observation." - # create dimensions - ncfile.createDimension('buoypoints', bhs.shape[0] ) - if gridinfo!=0: - ncfile.createDimension('GlobalOceansSeas', ocnames.shape[0] ) - ncfile.createDimension('HighSeasMarineZones', hsmznames.shape[0] ) - if cyclonemap!=0: - ncfile.createDimension('cycloneinfo', cinfo.shape[0] ) - vcinfo = ncfile.createVariable('cycloneinfo',dtype('a25'),('cycloneinfo')) - # create variables. - vstname = ncfile.createVariable('buoyID',dtype('a25'),('buoypoints')) - vlat = ncfile.createVariable('latitude',np.dtype('float32').char,('buoypoints')) - vlon = ncfile.createVariable('longitude',np.dtype('float32').char,('buoypoints')) - - if forecastds>0: - ncfile.createDimension('time', nmhs.shape[2] ) - ncfile.createDimension('fcycle', unt.shape[0] ) - vt = ncfile.createVariable('time',np.dtype('float64').char,('fcycle','time')) - vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmwn = ncfile.createVariable('model_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) - - vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbwind = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) - - else: - ncfile.createDimension('time', bhs.shape[1] ) - vt = ncfile.createVariable('time',np.dtype('float64').char,('time')) - vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','time')) - vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','time')) - vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','time')) - vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','time')) - vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','time')) - vmwn = ncfile.createVariable('model_wind',np.dtype('float32').char,('buoypoints','time')) - - vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','time')) - vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','time')) - vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','time')) - vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','time')) - vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','time')) - vbwind = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','time')) - - - - - if gridinfo!=0: - vpdistcoast = ncfile.createVariable('distcoast',np.dtype('float32').char,('buoypoints')) - vpdepth = ncfile.createVariable('depth',np.dtype('float32').char,('buoypoints')) - vponi = ncfile.createVariable('GlobalOceansSeas',np.dtype('float32').char,('buoypoints')) - vocnames = ncfile.createVariable('names_GlobalOceansSeas',dtype('a25'),('GlobalOceansSeas')) - vphsmz = ncfile.createVariable('HighSeasMarineZones',np.dtype('float32').char,('buoypoints')) - vhsmznames = ncfile.createVariable('names_HighSeasMarineZones',dtype('a25'),('HighSeasMarineZones')) - if cyclonemap!=0: - if forecastds>0: - vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','fcycle','time')) - else: - vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','time')) - - # Assign units - vlat.units = 'degrees_north' ; vlon.units = 'degrees_east' - vt.units = 'seconds since 1970-01-01T00:00:00+00:00' - vmhs.units='m'; vbhs.units='m' - vmtm.units='s'; vbtm.units='s' - vmtp.units='s'; vbtp.units='s' - vmdm.units='degrees'; vbdm.units='degrees' - vmdp.units='degrees'; vbdp.units='degrees' - vmwn.unit='m/s';vbwind.unit='m/s' - - if gridinfo!=0: - vpdepth.units='m'; vpdistcoast.units='km' - - # Allocate Data - vstname[:]=stname[:]; vlat[:] = lat[:]; vlon[:] = lon[:] - if forecastds>0: - vt[:,:]=nmtime[:,:] - vmhs[:,:,:]=nmhs[:,:,:] - vmtm[:,:,:]=nmtm[:,:,:] - vmtp[:,:,:]=nmtp[:,:,:] - vmdm[:,:,:]=nmdm[:,:,:] - vmdp[:,:,:]=nmdp[:,:,:] - vmwn[:,:,:]=nmwn[:,:,:] - vbhs[:,:,:]=nbhs[:,:,:] - vbtm[:,:,:]=nbtm[:,:,:] - vbtp[:,:,:]=nbtp[:,:,:] - vbdm[:,:,:]=nbdm[:,:,:] - vbdp[:,:,:]=nbdp[:,:,:] - vbwind[:,:,:]=nbwind[:,:,:] - - else: - vt[:]=mtime[:] - vmhs[:,:]=padded_mhs[:,:] - vmtm[:,:]=padded_mtm[:,:] - vmtp[:,:]=padded_mtp[:,:] - vmdm[:,:]=padded_mdm[:,:] - vmdp[:,:]=padded_mdp[:,:] - vmwn[:,:]=padded_mwn[:,:] - vbhs[:,:]=padded_bhs[:,:] - vbtm[:,:]=btm[:,:] - vbtp[:,:]=btp[:,:] - vbdm[:,:]=bdm[:,:] - vbdp[:,:]=bdp[:,:] - vbwind[:,:]=bwind[:,:] - - - - - if gridinfo!=0: - vpdistcoast[:]=pdistcoast[:] - vpdepth[:]=pdepth[:] - vponi[:]=poni[:]; vocnames[:] = ocnames[:] - vphsmz[:]=phsmz[:]; vhsmznames[:] = hsmznames[:] - if cyclonemap!=0: - vcinfo[:] = cinfo[:] - if forecastds>0: - vcmap[:,:,:]=nfcmap[:,:,:] - else: - vcmap[:,:]=fcmap[:,:] - - ncfile.close() - print(' ') - print('Done. Netcdf ok. New file saved: WW3.Buoy'+ftag+'_'+initime+'to'+fintime+'.nc') diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index 9702472..3295763 100755 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -93,808 +93,984 @@ """ -import warnings; warnings.filterwarnings("ignore") +import sys +import os +import gzip +import tarfile +import warnings import numpy as np -from matplotlib.mlab import * -from pylab import * -import xarray as xr import netCDF4 as nc import time from time import strptime from calendar import timegm import wread -import sys -# netcdf format -fnetcdf="NETCDF4" - -# Paths -# ndbcp="/data/buoys/NDBC/wparam" -ndbcp="/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" -# Copernicus buoys -# copernp="/data/buoys/Copernicus/wtimeseries" -copernp="/work/noaa/marine/ricardo.campos/data/buoys/Copernicus/wtimeseries" -print(' ') -# Options of including grid and cyclone information -gridinfo=int(0); cyclonemap=int(0); wlist=[]; ftag=''; forecastds=0 -if len(sys.argv) < 2 : - sys.exit(' At least one argument (list of ww3 files) must be informed.') -if len(sys.argv) >= 2 : - # # import os; os.system("ls -d $PWD/*tab.nc > ww3list.txt &") - wlist=np.atleast_1d(np.loadtxt(sys.argv[1],dtype=str,usecols=(0,))) - ftag=str(sys.argv[1]).split('list')[1].split('.spec')[0] - print(' Reading ww3 list '+ str(sys.argv[1])) - print(' Tag '+ftag) -if len(sys.argv) >= 3: - forecastds=int(sys.argv[2]) - if forecastds>0: - print(' Forecast-type data structure') -if len(sys.argv) >= 4: - gridinfo=str(sys.argv[3]) - print(' Using gridInfo '+gridinfo) -if len(sys.argv) >= 5: - cyclonemap=str(sys.argv[4]) - print(' Using cyclone map '+cyclonemap) -if len(sys.argv) > 5: - sys.exit(' Too many inputs') - -# READ DATA -print(" ") -if gridinfo!=0: - # Grid Information - gridmask = wread.mask(gridinfo) - mlat=gridmask['latitude']; mlon=gridmask['longitude'] - mask=gridmask['mask']; distcoast=gridmask['distcoast']; depth=gridmask['depth'] - oni=gridmask['GlobalOceansSeas']; ocnames=gridmask['names_GlobalOceansSeas'] - hsmz=gridmask['HighSeasMarineZones']; hsmznames=gridmask['names_HighSeasMarineZones'] - print(" GridInfo Ok. "+gridinfo) - - # Cyclone Information - if cyclonemap!=0: - cycloneinfo = wread.cyclonemap(cyclonemap) - clat=cycloneinfo['latitude']; clon=cycloneinfo['longitude'] - cmap=cycloneinfo['cmap']; ctime=cycloneinfo['time'] - cinfo=np.array(cycloneinfo['info'].split(':')[1].split(';')) - if np.array_equal(clat,mlat)==True & np.array_equal(clon,mlon)==True: - print(" CycloneMap Ok. "+cyclonemap) - else: - sys.exit(' Error: Cyclone grid and Mask grid are different.') - -# MODEL Point output, search among possible formats ------------------ - -if (str(wlist[0]).split('/')[-1].split('.')[-1]=='bull_tar') or (str(wlist[0]).split('/')[-1].split('.')[-1]=='station_tar'): - for i in range(0,np.size(wlist)): - if str(wlist[i]).split('/')[-1].split('.')[-1]=='bull_tar': - result = wread.bull_tar(wlist[i]) - if str(wlist[i]).split('/')[-1].split('.')[-1]=='station_tar': - result = wread.bull_tar(wlist[i]) - - print("Result keys:", result.keys()) - at=result['time'] - fcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') - if i==0: - stname=result['station_name'] - mtime=np.copy(at) - mfcycle=np.copy(fcycle) - mhs=np.copy(result['hs']) - mtp=np.copy(result['tp']) - if 'dp' in result.keys(): - mdp=np.copy(result['dp']) - else: - mdp=np.copy(mhs)*np.nan - - else: - if (mhs.shape[0]==result['hs'].shape[0]) and (np.size(stname)==np.size(result['station_name'])): - if (stname==result['station_name']).all(): - mtime=np.append(mtime,at) - mfcycle=np.append(mfcycle,fcycle) - mhs=np.append(mhs,result['hs'],axis=1) - mtp=np.append(mtp,result['tp'],axis=1) - if 'dp' in result.keys(): - mdp=np.append(mdp,result['dp'],axis=1) - else: - mdp=np.append(mdp,np.copy(result['hs'])*np.nan,axis=1) - - else: - print(" Stations in "+wlist[i]+" do not match the other tar files. Skipped "+wlist[i]) - - del result,at,fcycle - mdm=np.copy(mhs)*np.nan; mtm=np.copy(mhs)*np.nan # not saved in this file format - print(" ww3 file "+wlist[i]+" OK") - -elif (str(wlist[0]).split('/')[-1].split('.')[-1]=='bull') or (str(wlist[0]).split('/')[-1].split('.')[-1]=='ts'): - if forecastds>0: - forecastds=0 # no 2D time array possible. - print(" Warning: no 2D time array possible. \ - Use multiple tar files (one per forecast cycle), station_tar or bull_tar, containing text files for each station.") - - for i in range(0,np.size(wlist)): - # re-check each file in the list respects the same format - if str(wlist[i]).split('/')[-1].split('.')[-1]=='ts': - result = wread.ts(wlist[i]) - elif str(wlist[i]).split('/')[-1].split('.')[-1]=='bull': - result = wread.bull(wlist[i]) - - at=result['time'] - if i==0: - mfcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') - stname=np.atleast_1d(np.array(result['station_name'])) - mtime=np.copy(at) - mhs=np.copy([result['hs']]) - mtp=np.copy([result['tp']]) - if 'dp' in result.keys(): - mdp=np.copy([result['dp']]) - else: - mdp=np.copy(mhs)*np.nan - - if 'dm' in result.keys(): - mdm=np.copy([result['dm']]) - else: - mdm=np.copy(mhs)*np.nan - - if 'tm' in result.keys(): - mtm=np.copy([result['tm']]) - else: - mtm=np.copy(mhs)*np.nan - - else: - if (mhs.shape[1]==result['hs'].shape[0]): - stname=np.append(stname,np.atleast_1d(np.array(result['station_name']))) - mtime=np.append(mtime,at) - mhs=np.append(mhs,[result['hs']],axis=0) - mtp=np.append(mtp,[result['tp']],axis=0) - if 'dp' in result.keys(): - mdp=np.append(mdp,[result['dp']],axis=0) - else: - mdp=np.append(mdp,[np.copy(result['hs'])*np.nan],axis=0) - - if 'dm' in result.keys(): - mdm=np.append(mdp,[result['dm']],axis=0) - else: - mdm=np.append(mdm,[np.copy(result['hs'])*np.nan],axis=0) - - if 'tm' in result.keys(): - mtm=np.append(mtm,[result['tm']],axis=0) - else: - mtm=np.append(mtm,[np.copy(result['hs'])*np.nan],axis=0) - - else: - print(" Stations in "+wlist[i]+" do not match the other tar files. Skipped "+wlist[i]) - - del result,at - print(" ww3 file "+wlist[i]+" OK") - - -elif str(wlist[0]).split('/')[-1].split('.')[-1]=='nc': - print(" Using ww3 netcdf point output format") - # netcdf point output file - for t in range(0,np.size(wlist)): - try: - f=nc.Dataset(str(wlist[t])) - except: - print(" Cannot open "+wlist[t]) - else: - # list of station/buoy names - if t==0: - auxstationname=f.variables['station_name'][:,:]; stname=[] - for i in range(0,auxstationname.shape[0]): - astname="".join(np.array(auxstationname[i,:]).astype('str')) - if '\t' in astname: - astname=str(astname).replace("\t","") - - stname=np.append(stname,astname); del astname - - ahs = np.array(f.variables['hs'][:,:]).T - - if 'th1m' in f.variables.keys(): - adm = np.array(f.variables['th1m'][:,:]).T - else: - adm = np.array(np.copy(ahs*nan)) - - if 'th1p' in f.variables.keys(): - adp = np.array(f.variables['th1p'][:,:]).T - else: - adp = np.array(np.copy(ahs*nan)) - - if 'tr' in f.variables.keys(): - atm = np.array(f.variables['tr'][:,:]).T - else: - atm = np.array(np.copy(ahs*nan)) - - if 'fp' in f.variables.keys(): - auxtp = np.array(f.variables['fp'][:,:]).T - indtp=np.where(auxtp>0.) - if np.size(indtp)>0: - atp=np.copy(auxtp) - atp[indtp]=np.copy(1./atp[indtp]) - del indtp - - del auxtp - - else: - atm = np.array(np.copy(ahs*nan)) - - at = np.array(f.variables['time'][:]*24*3600 + timegm( strptime(str(f.variables['time'].units).split(' ')[2][0:4]+'01010000', '%Y%m%d%H%M') )).astype('double') - f.close(); del f - fcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') - if t==0: - mhs=np.copy(ahs) - mtm=np.copy(atm) - mtp=np.copy(atp) - mdm=np.copy(adm) - mdp=np.copy(adp) - mtime=np.copy(at) - mfcycle=np.copy(fcycle) - else: - mhs=np.append(mhs,ahs,axis=1) - mtm=np.append(mtm,atm,axis=1) - mtp=np.append(mtp,atp,axis=1) - mdm=np.append(mdm,adm,axis=1) - mdp=np.append(mdp,adp,axis=1) - mtime=np.append(mtime,at) - mfcycle=np.append(mfcycle,fcycle) - - del ahs,atm,atp,adm,adp,at,fcycle - - print(" Read WW3 data OK."); print(' ') - -elif (str(wlist[0]).split('/')[-1].split('.')[-1] == 'spec'): - if forecastds > 0: - forecastds = 0 # no 2D time array possible. - print(" Warning: no 2D time array possible. \ - Use multiple tar files (one per forecast cycle), station_tar or bull_tar, containing text files for each station.") - - for i in range(0, np.size(wlist)): - # re-check each file in the list respects the same format - if str(wlist[i]).split('/')[-1].split('.')[-1] == 'spec': - # Check if the required number of command-line arguments is provided -# if len(sys.argv) < 3: -# sys.exit('Two inputs must be provided: fileName and StationName') - - # Call read_text_file with command-line arguments - result = wread.read_text_file(sys.argv[1], sys.argv[2]) - - ntime = result['ntime'] - print("time:", ntime) - - if i == 0: - # Initialize variables on the first iteration - mfcycle = np.array(np.zeros((ntime.shape[0]), 'd') + ntime[0]).astype('double') - stname = np.atleast_1d(np.array(result['station_name'])) - mtime = np.copy(ntime) - mhs = np.copy([result['wind_spd']]) # wind - mtp = np.copy([result['wind_dir']]) # wndd - if 'dp' in result.keys(): - mdp = np.copy([result['spec']]) # dspec - else: - mdp = np.copy(mhs) * np.nan +# Suppressing warnings +warnings.filterwarnings("ignore") + +# netCDF format specification +fnetcdf = "NETCDF4" + +gridinfo = int(0) +cyclonemap = int(0) +wlist = [] +ftag = '' +forecastds = 0 + + +# Function to unzip and untar files +def unzip_and_untar(gz_file, output_dir): + file_name = os.path.splitext(os.path.basename(gz_file))[0] + base_name = file_name.rsplit('.', 1)[0] + output_file = os.path.join(output_dir, file_name) + + with gzip.open(gz_file, 'rb') as f_in: + with open(output_file, 'wb') as f_out: + f_out.write(f_in.read()) + + with tarfile.open(output_file, 'r') as tar: + tar.extractall(output_dir) + + extracted_folder = os.path.join(output_dir, base_name) + # Creating a list of the extracted files + list_file = os.path.join(output_dir, f'{base_name}_contents.txt') + with open(list_file, 'w') as f: + for root, dirs, files in os.walk(extracted_folder): + for file in files: + f.write(os.path.join(root, file) + '\n') + + # Creating a list of ids between two dots in the filenames + id_file = os.path.join(output_dir, f'{base_name}_id.txt') + with open(id_file, 'w') as f: + for root, dirs, files in os.walk(extracted_folder): + for file in files: + file_parts = file.split('.') + if len(file_parts) >= 3: + id_between_dots = file_parts[1] + f.write(id_between_dots + '\n') + + # Return the path to the extracted folder + return extracted_folder, list_file, id_file + + + +# Main part of the script +if __name__ == "__main__": + mode = sys.argv[1] if len(sys.argv) > 1 else "" + + if mode == "unzip": + if len(sys.argv) != 5: + sys.exit("Usage: python script.py unzip input_gz_file output_directory buoy_path") + + gz_file = sys.argv[2] + output_dir = sys.argv[3] + buoy_path = sys.argv[4] + ndbcp = buoy_path + extracted_folder, list_file, id_file = unzip_and_untar(gz_file, output_dir) + + print("Extraction completed. Extracted files are in:", extracted_folder) + + # Read file names from the extracted folder + file_names = [] + for root, dirs, files in os.walk(extracted_folder): + for file in files: + file_names.append(os.path.join(root, file)) + + # Read id_files (if needed) + id_files = [] + with open(id_file, 'r') as f: + for line in f: + id_files.append(line.strip()) + + # Call the spec_ww3 function with the file names and ID files + results = wread.spec1_ww3(file_names, id_files) + + # Additional processing (if needed) + mfcycle = None + stname = None + mtime = None + mhs = None + mtp = None + mwn = None + mwd = None + mfreq = None + mtm = None + mdm = None + mdp = None + + for i, result in enumerate(results): + at = result['time'] + + if i == 0: + mfcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') + stname = np.atleast_1d(np.array(result['station_name'])) + mtime = np.copy(at) + mhs = np.copy([result['Hs']]) + mtp = np.copy([result['Tp']]) + + if 'wind_spd' in result.keys(): + mwn = np.copy([result['wind_spd']]) + else: + mwn = np.copy(mhs) * np.nan - if 'dm' in result.keys(): - mdm = np.copy([result['deltafreq']]) # dfreq - else: - mdm = np.copy(mhs) * np.nan + if 'wind_dir' in result.keys(): + mwd = np.copy([result['wind_dir']]) + else: + mwd = np.copy(mhs) * np.nan + + if 'freq' in result.keys(): + mfreq = np.copy([result['freq']]) + else: + mfreq = np.copy(mhs) * np.nan + + if 'tm' in result.keys(): + mtm = np.copy([result['tm']]) + else: + mtm = np.copy(mhs) * np.nan - if 'tm' in result.keys(): - mtm = np.copy([result['freq']]) # freq - else: - mtm = np.copy(mhs) * np.nan - else: - if (mhs.shape[1] == result['hs'].shape[0]): - stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) - mtime = np.append(mtime, ntime) - mhs = np.append(mhs, [result['wind_spd']], axis=0) - mtp = np.append(mtp, [result['wind_dir']], axis=0) if 'dp' in result.keys(): - mdp = np.append(mdp, [result['spec']], axis=0) + mdp = np.copy([result['dp']]) else: - mdp = np.append(mdp, [np.copy(result['wind_spd']) * np.nan], axis=0) + mdp = np.copy(mhs) * np.nan if 'dm' in result.keys(): - mdm = np.append(mdp, [result['deltafreq']], axis=0) + mdm = np.copy([result['dm']]) else: - mdm = np.append(mdm, [np.copy(result['wind_spd']) * np.nan], axis=0) + mdm = np.copy(mhs) * np.nan - if 'tm' in result.keys(): - mtm = np.append(mtm, [result['freq']], axis=0) - else: - mtm = np.append(mtm, [np.copy(result['wind_spd']) * np.nan], axis=0) else: - print(" Stations in " + wlist[i] + " do not match the other spec files. Skipped " + wlist[i]) - del result - print(" ww3 file " + wlist[i] + " OK") + if mhs.shape[1] == result['Hs'].shape[0]: + stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) + mtime = np.append(mtime, at) + mhs = np.append(mhs, [result['Hs']], axis=0) + mtp = np.append(mtp, [result['Tp']], axis=0) + + if 'wind_spd' in result.keys(): + mwn = np.append(mwn, [result['wind_spd']], axis=0) + else: + mwn = np.append(mwn, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'wind_dir' in result.keys(): + mwd = np.append(mwd, [result['wind_dir']], axis=0) + else: + mwd = np.append(mwd, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'freq' in result.keys(): + mfreq = np.append(mfreq, [result['freq']], axis=0) + else: + mfreq = np.append(mfreq, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'tm' in result.keys(): + mtm = np.append(mtm, [result['tm']], axis=0) + else: + mtm = np.append(mtm, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'dp' in result.keys(): + mdp = np.append(mdp, [result['dp']], axis=0) + else: + mdp = np.append(mdp, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'dm' in result.keys(): + mdm = np.append(mdp, [result['dm']], axis=0) + else: + mdm = np.append(mdm, [np.copy(result['Hs']) * np.nan], axis=0) + else: + print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) + + else: + gridinfo = int(0) + cyclonemap = int(0) + wlist = [] + ftag = '' + forecastds = 0 + + if len(sys.argv) < 2: + sys.exit('At least one argument (list of ww3 files) must be informed.') + wlist = np.atleast_1d(np.loadtxt(sys.argv[1], dtype=str)) + ftag = str(sys.argv[1]).split('list')[1].split('.txt')[0] + print('Reading ww3 list ' + str(sys.argv[1])) + print('Tag ' + ftag) + + if len(sys.argv) >= 3: + forecastds = int(sys.argv[2]) + if forecastds > 0: + print('Forecast-type data structure') + + if len(sys.argv) >= 4: + gridinfo = str(sys.argv[3]) + print('Writing gridinfo ' + gridinfo) + + if len(sys.argv) >= 5: + cyclonemap = str(sys.argv[4]) + print('Writing cyclonemap ' + cyclonemap) + + # Paths + ndbcp = "/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" + + # READ DATA + print(" ") + if gridinfo != 0: + # Grid Information + gridmask = wread.mask(gridinfo) + mlat = gridmask['latitude'] + mlon = gridmask['longitude'] + mask = gridmask['mask'] + distcoast = gridmask['distcoast'] + depth = gridmask['depth'] + oni = gridmask['GlobalOceansSeas'] + ocnames = gridmask['names_GlobalOceansSeas'] + hsmz = gridmask['HighSeasMarineZones'] + hsmznames = gridmask['names_HighSeasMarineZones'] + print(" GridInfo Ok. " + gridinfo) + + # Cyclone Information + if cyclonemap != 0: + cycloneinfo = wread.cyclonemap(cyclonemap) + clat = cycloneinfo['latitude'] + clon = cycloneinfo['longitude'] + cmap = cycloneinfo['cmap'] + ctime = cycloneinfo['time'] + cinfo = np.array(cycloneinfo['info'].split(':')[1].split(';')) + if np.array_equal(clat, mlat) == True & np.array_equal(clon, mlon) == True: + print(" CycloneMap Ok. " + cyclonemap) + else: + sys.exit(' Error: Cyclone grid and Mask grid are different.') + + # MODEL Point output, search among possible formats ------------------ + if (str(wlist[0]).split('/')[-1].split('.')[-1] == 'bull_tar') or ( + str(wlist[0]).split('/')[-1].split('.')[-1] == 'station_tar'): + for i in range(0, np.size(wlist)): + if str(wlist[i]).split('/')[-1].split('.')[-1] == 'bull_tar': + result = wread.bull_tar(wlist[i]) + if str(wlist[i]).split('/')[-1].split('.')[-1] == 'station_tar': + result = wread.bull_tar(wlist[i]) + + at = result['time'] + fcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') + if i == 0: + stname = result['station_name'] + mtime = np.copy(at) + mfcycle = np.copy(fcycle) + mhs = np.copy(result['hs']) + mtp = np.copy(result['tp']) + if 'dp' in result.keys(): + mdp = np.copy(result['dp']) + else: + mdp = np.copy(mhs) * np.nan + + if 'wind_spd' in result.keys(): + mwn = np.copy(result['wind_spd']) + else: + mwn = np.copy(mhs) * np.nan + + if 'wind_dir' in result.keys(): + mwd = np.copy(result['wind_dir']) + else: + mwd = np.copy(mhs) * np.nan + + + else: + if (mhs.shape[0] == result['hs'].shape[0]) and ( + np.size(stname) == np.size(result['station_name'])): + if (stname == result['station_name']).all(): + mtime = np.append(mtime, at) + mfcycle = np.append(mfcycle, fcycle) + mhs = np.append(mhs, result['hs'], axis=1) + mtp = np.append(mtp, result['tp'], axis=1) + if 'dp' in result.keys(): + mdp = np.append(mdp, result['dp'], axis=1) + else: + mdp = np.append(mdp, np.copy(result['hs']) * np.nan, axis=1) + + if 'wind_spd' in result.keys(): + mwn = np.append(mwn, result['wind_spd'], axis=1) + else: + mwn = np.append(mwn, np.copy(result['hs']) * np.nan, axis=1) + + if 'wind_dir' in result.keys(): + mwd = np.append(mwd, result['wind_dir'], axis=1) + else: + mwd = np.append(mwd, np.copy(result['hs']) * np.nan, axis=1) + + + else: + print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[ + i]) + + del result, at, fcycle + mdm = np.copy(mhs) * np.nan; + mtm = np.copy(mhs) * np.nan # not saved in this file format + print(" ww3 file " + wlist[i] + " OK") + + elif (str(wlist[0]).split('/')[-1].split('.')[-1] == 'bull') or ( + str(wlist[0]).split('/')[-1].split('.')[-1] == 'ts'): + if forecastds > 0: + forecastds = 0 # no 2D time array possible. + print( + " Warning: no 2D time array possible. Use multiple tar files (one per forecast cycle), station_tar or bull_tar, containing text files for each station.") + + for i in range(0, np.size(wlist)): + # re-check each file in the list respects the same format + if str(wlist[i]).split('/')[-1].split('.')[-1] == 'ts': + result = wread.ts(wlist[i]) + elif str(wlist[i]).split('/')[-1].split('.')[-1] == 'bull': + result = wread.bull(wlist[i]) + + at = result['time'] + if i == 0: + mfcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') + stname = np.atleast_1d(np.array(result['station_name'])) + mtime = np.copy(at) + mhs = np.copy([result['hs']]) + mtp = np.copy([result['tp']]) + if 'dp' in result.keys(): + mdp = np.copy([result['dp']]) + else: + mdp = np.copy(mhs) * np.nan + + if 'dm' in result.keys(): + mdm = np.copy([result['dm']]) + else: + mdm = np.copy(mhs) * np.nan + + if 'tm' in result.keys(): + mtm = np.copy([result['tm']]) + else: + mtm = np.copy(mhs) * np.nan -#else: - #sys.exit(' Point output file format not recognized: only bull, bull_tar, and .nc implemented.') - # include other text formats: tab50, .ts + else: + if (mhs.shape[1] == result['hs'].shape[0]): + stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) + mtime = np.append(mtime, at) + mhs = np.append(mhs, [result['hs']], axis=0) + mtp = np.append(mtp, [result['tp']], axis=0) + if 'dp' in result.keys(): + mdp = np.append(mdp, [result['dp']], axis=0) + else: + mdp = np.append(mdp, [np.copy(result['hs']) * np.nan], axis=0) + + if 'dm' in result.keys(): + mdm = np.append(mdp, [result['dm']], axis=0) + else: + mdm = np.append(mdm, [np.copy(result['hs']) * np.nan], axis=0) + + if 'tm' in result.keys(): + mtm = np.append(mtm, [result['tm']], axis=0) + else: + mtm = np.append(mtm, [np.copy(result['hs']) * np.nan], axis=0) + + else: + print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) + + del result, at + print(" ww3 file " + wlist[i] + " OK") + + elif str(wlist[0]).split('/')[-1].split('.')[-1] == 'nc': + print(" Using ww3 netcdf point output format") + # netcdf point output file + for t in range(0, np.size(wlist)): + try: + f = nc.Dataset(str(wlist[t])) + except: + print(" Cannot open " + wlist[t]) + else: + # list of station/buoy names + if t == 0: + auxstationname = f.variables['station_name'][:, :]; + stname = [] + for i in range(0, auxstationname.shape[0]): + astname = "".join(np.array(auxstationname[i, :]).astype('str')) + if '\t' in astname: + astname = str(astname).replace("\t", "") + + stname = np.append(stname, astname); + del astname + + funits = f.variables['time'].units + if str(funits).split(' ')[0] == 'seconds': + tincr = 1 + elif str(funits).split(' ')[0] == 'hours': + tincr = 3600 + elif str(funits).split(' ')[0] == 'days': + tincr = 24 * 3600 + + ahs = np.array(f.variables['hs'][:, :]).T + + if 'th1m' in f.variables.keys(): + adm = np.array(f.variables['th1m'][:, :]).T + else: + adm = np.array(np.copy(ahs * nan)) + + if 'th1p' in f.variables.keys(): + adp = np.array(f.variables['th1p'][:, :]).T + else: + adp = np.array(np.copy(ahs * nan)) + + if 'tr' in f.variables.keys(): + atm = np.array(f.variables['tr'][:, :]).T + else: + atm = np.array(np.copy(ahs * nan)) + + if 'fp' in f.variables.keys(): + auxtp = np.array(f.variables['fp'][:, :]).T + indtp = np.where(auxtp > 0.) + if np.size(indtp) > 0: + atp = np.copy(auxtp) + atp[indtp] = np.copy(1. / atp[indtp]) + del indtp + + del auxtp + + else: + atm = np.array(np.copy(ahs * nan)) + + ftunits = str(f.variables['time'].units).split('since')[1][1::].replace('T', ' ').replace('+00:00', + '') + at = np.array( + f.variables['time'][:] * tincr + timegm(strptime(ftunits, '%Y-%m-%d %H:%M:%S'))).astype( + 'double') + + f.close(); + del f + fcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') + if t == 0: + mhs = np.copy(ahs) + mtm = np.copy(atm) + mtp = np.copy(atp) + mdm = np.copy(adm) + mdp = np.copy(adp) + mtime = np.copy(at) + mfcycle = np.copy(fcycle) + else: + mhs = np.append(mhs, ahs, axis=1) + mtm = np.append(mtm, atm, axis=1) + mtp = np.append(mtp, atp, axis=1) + mdm = np.append(mdm, adm, axis=1) + mdp = np.append(mdp, adp, axis=1) + mtime = np.append(mtime, at) + mfcycle = np.append(mfcycle, fcycle) + + del ahs, atm, atp, adm, adp, at, fcycle + + print(" Read WW3 data OK."); + print(' ') -# BUOYS ------------------ -bhs=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan -btm=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan -btp=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan -bdm=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan -bdp=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan -lat=np.zeros(np.size(stname),'f')*np.nan; lon=np.zeros(np.size(stname),'f')*np.nan + else: + sys.exit(' Point output file format not recognized: only bull, bull_tar, and .nc implemented.') + # include other text formats: tab50, .ts + + print(" Start building the matchups model/buoy ..."); + print(' ') + + +# BUOYS ------------------ +bwind = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bhs = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +btm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +btp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bdm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bdp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +lat = np.zeros(np.size(stname), 'f') * np.nan +lon = np.zeros(np.size(stname), 'f') * np.nan # help reading NDBC buoys, divided by year -yrange=np.array(np.arange(time.gmtime(mtime.min())[0],time.gmtime(mtime.min())[0]+1,1)).astype('int') +yrange = np.array(np.arange(time.gmtime(mtime.min())[0], time.gmtime(mtime.min())[0] + 1, 1)).astype('int') # loop buoys -for b in range(0,np.size(stname)): - - ahs=[] - try: - - ahs=[];atm=[];adm=[];atime=[] - for y in yrange: - -# f=nc.Dataset(ndbcp+"/"+stname[b]+"h"+repr(y)+".nc") -# if 'wave_height' in f.variables.keys(): -# ahs = np.append(ahs,f.variables['wave_height'][:,0,0]) -# elif 'hs' in f.variables.keys(): -# ahs = np.append(ahs,f.variables['hs'][:,0,0]) -# elif 'swh' in f.variables.keys(): -# ahs = np.append(ahs,f.variables['swh'][:,0,0]) - - - f=nc.Dataset(ndbcp+"/"+stname[b]+"h"+repr(y)+".nc") - if 'wind_spd' in f.variables.keys(): - ahs = np.append(ahs,f.variables['wind_spd'][:,0,0]) - elif 'wsp' in f.variables.keys(): - ahs = np.append(ahs,f.variables['wsp'][:,0,0]) - elif 'swhp' in f.variables.keys(): - ahs = np.append(ahs,f.variables['swhp'][:,0,0]) - - - if 'average_wpd' in f.variables.keys(): - atm = np.append(atm,f.variables['average_wpd'][:,0,0]) - else: - atm = np.array(np.copy(ahs*nan)) - - if 'dominant_wpd' in f.variables.keys(): - atp = np.append(atm,f.variables['dominant_wpd'][:,0,0]) - else: - atp = np.array(np.copy(ahs*nan)) - - if 'mean_wave_dir' in f.variables.keys(): - adm = np.append(adm,f.variables['mean_wave_dir'][:,0,0]) - else: - adm = np.array(np.copy(ahs*nan)) - - if 'latitude' in f.variables.keys(): - lat[b] = f.variables['latitude'][:] - elif 'LATITUDE' in f.variables.keys(): - lat[b] = f.variables['LATITUDE'][:] - else: - lat[b] = nan - - if 'longitude' in f.variables.keys(): - lon[b] = f.variables['longitude'][:] - elif 'LONGITUDE' in f.variables.keys(): - lon[b] = f.variables['LONGITUDE'][:] - else: - lon[b] = nan - - atime = np.append(atime,np.array(f.variables['time'][:]).astype('double')) - - f.close(); del f - - adp = adm*np.nan # no peak direction available in this format - - except: -# try: -# f=nc.Dataset(copernp+"/GL_TS_MO_"+stname[b]+".nc") -# if 'VHM0' in f.variables.keys(): -# ahs = np.nanmean(f.variables['VHM0'][:,:],axis=1) -# elif 'VAVH' in f.variables.keys(): -# ahs = np.nanmean(f.variables['VAVH'][:,:],axis=1) -# elif 'VGHS' in f.variables.keys(): -# ahs = np.nanmean(f.variables['VGHS'][:,:],axis=1) -# elif 'significant_swell_wave_height' in f.variables.keys(): -# ahs = np.nanmean(f.variables['significant_swell_wave_height'][:,:],axis=1) -# elif 'sea_surface_significant_wave_height' in f.variables.keys(): -# ahs = np.nanmean(f.variables['sea_surface_significant_wave_height'][:,:],axis=1) -# elif 'SWHT' in f.variables.keys(): -# ahs = np.nanmean(f.variables['SWHT'][:,:],axis=1) -# elif 'wave_height_h1d3' in f.variables.keys(): -# ahs = np.nanmean(f.variables['wave_height_h1d3'][:,:],axis=1) -# elif 'spectral_significant_wave_height' in f.variables.keys(): -# ahs = np.nanmean(f.variables['spectral_significant_wave_height'][:,:],axis=1) -# -# if 'VTM02' in f.variables.keys(): -# atm = np.nanmean(f.variables['VTM02'][:,:],axis=1) -# elif 'VGTA' in f.variables.keys(): -# atm = np.nanmean(f.variables['VGTA'][:,:],axis=1) -# else: -# atm = ahs*nan -# -# if 'VTPK' in f.variables.keys(): -# atp = np.nanmean(f.variables['VTPK'][:,:],axis=1) -# elif 'dominant_wave_period' in f.variables.keys(): -# atp = np.nanmean(f.variables['dominant_wave_period'][:,:],axis=1) -# elif 'sea_surface_wave_period_at_spectral_density_maximum' in f.variables.keys(): -# atp = np.nanmean(f.variables['sea_surface_wave_period_at_spectral_density_maximum'][:,:],axis=1) -# else: -# atp = ahs*nan -# -# if 'VMDR' in f.variables.keys(): -# adm = np.nanmean(f.variables['VMDR'][:,:],axis=1) -# else: -# adm = ahs*nan -# -# if 'LATITUDE' in f.variables.keys(): -# lat[b] = np.nanmean(f.variables['LATITUDE'][:]) -# elif 'latitude' in f.variables.keys(): -# lat[b] = np.nanmean(f.variables['latitude'][:]) -# elif 'LAT' in f.variables.keys(): -# lat[b] = np.nanmean(f.variables['LAT'][:]) -# elif 'lat' in f.variables.keys(): -# lat[b] = np.nanmean(f.variables['lat'][:]) -# else: -# lat[b] = nan -# -# if 'LONGITUDE' in f.variables.keys(): -# lon[b] = np.nanmean(f.variables['LONGITUDE'][:]) -# elif 'LON' in f.variables.keys(): -# lon[b] = np.nanmean(f.variables['LON'][:]) -# elif 'LONG' in f.variables.keys(): -# lon[b] = np.nanmean(f.variables['LONG'][:]) -# elif 'longitude' in f.variables.keys(): -# lon[b] = np.nanmean(f.variables['longitude'][:]) -# elif 'lon' in f.variables.keys(): -# lon[b] = np.nanmean(f.variables['lon'][:]) -# elif 'long' in f.variables.keys(): -# lon[b] = np.nanmean(f.variables['long'][:]) -# else: -# lon[b] = nan -# -# adp = ahs*nan # no peak direction available in this format -# -# if 'TIME' in f.variables.keys(): -# atime = np.array(f.variables['TIME'][:]*24*3600 + timegm( strptime('195001010000', '%Y%m%d%H%M') )).astype('double') -# elif 'time' in f.variables.keys(): -# atime = np.array(f.variables['time'][:]*24*3600 + timegm( strptime('195001010000', '%Y%m%d%H%M') )).astype('double') -# -# f.close(); del f -# except: -# ahs=[] - pass - - if np.size(ahs)>0: - - # First layer of simple quality-control - indq=np.where((ahs>30.)|(ahs<0.0)) - if np.size(indq)>0: - ahs[indq]=np.nan; del indq - - indq=np.where((atm>40.)|(atm<0.0)) - if np.size(indq)>0: - atm[indq]=np.nan; del indq - - indq=np.where((atp>40.)|(atp<0.0)) - if np.size(indq)>0: - atp[indq]=np.nan; del indq - - indq=np.where((adm>360.)|(adm<-180.)) - if np.size(indq)>0: - adm[indq]=np.nan; del indq - - indq=np.where((adp>360.)|(adp<-180.)) - if np.size(indq)>0: - adp[indq]=np.nan; del indq - - c=0 - for t in range(0,np.size(mtime)): - indt=np.where(np.abs(atime-mtime[t])<1800.) - if np.size(indt)>0: - if np.any(ahs[indt[0]].mask==False): - bhs[b,t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask==False]) - c=c+1 - if np.any(atm[indt[0]].mask==False): - btm[b,t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask==False]) - if np.any(atp[indt[0]].mask==False): - btp[b,t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask==False]) - if np.any(adm[indt[0]].mask==False): - bdm[b,t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask==False]) - if np.any(adp[indt[0]].mask==False): - bdp[b,t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask==False]) - - del indt - - # print("counted "+repr(c)+" at "+stname[b]) - - print(" station "+stname[b]+" ok") - del ahs +for b in range(0, np.size(stname)): + + ahs = [] + try: + awm = [] + ahs = [] + atm = [] + atp = [] + adm = [] + atime = [] + for y in yrange: + + f = nc.Dataset(ndbcp + "/" + stname[b] + "h" + repr(y) + ".nc") + if 'wave_height' in f.variables.keys(): + ahs = np.append(ahs, f.variables['wave_height'][:, 0, 0]) + elif 'hs' in f.variables.keys(): + ahs = np.append(ahs, f.variables['hs'][:, 0, 0]) + elif 'swh' in f.variables.keys(): + ahs = np.append(ahs, f.variables['swh'][:, 0, 0]) + + if 'wind_spd' in f.variables.keys(): + awm = np.append(awm, f.variables['wind_spd'][:, 0, 0]) + else: + awm = np.array(np.copy(ahs * nan)) + + if 'average_wpd' in f.variables.keys(): + atm = np.append(atm, f.variables['average_wpd'][:, 0, 0]) + else: + atm = np.array(np.copy(ahs * nan)) + + if 'dominant_wpd' in f.variables.keys(): + atp = np.append(atp, f.variables['dominant_wpd'][:, 0, 0]) + else: + atp = np.array(np.copy(ahs * nan)) + + if 'mean_wave_dir' in f.variables.keys(): + adm = np.append(adm, f.variables['mean_wave_dir'][:, 0, 0]) + else: + adm = np.array(np.copy(ahs * nan)) + + if 'latitude' in f.variables.keys(): + lat[b] = f.variables['latitude'][:] + elif 'LATITUDE' in f.variables.keys(): + lat[b] = f.variables['LATITUDE'][:] + else: + lat[b] = nan + + if 'longitude' in f.variables.keys(): + lon[b] = f.variables['longitude'][:] + elif 'LONGITUDE' in f.variables.keys(): + lon[b] = f.variables['LONGITUDE'][:] + else: + lon[b] = nan + + atime = np.append(atime, np.array(f.variables['time'][:]).astype('double')) + + f.close() + del f + + adp = adm * np.nan # no peak direction available in this format + + if np.size(ahs) > 0: + + # First layer of simple quality-control + indq = np.where((ahs > 30.) | (ahs < 0.0)) + if np.size(indq) > 0: + ahs[indq] = np.nan + del indq + + indq = np.where((atm > 40.) | (atm < 0.0)) + if np.size(indq) > 0: + atm[indq] = np.nan + del indq + + indq = np.where((atp > 40.) | (atp < 0.0)) + if np.size(indq) > 0: + atp[indq] = np.nan + del indq + + indq = np.where((adm > 360.) | (adm < -180.)) + if np.size(indq) > 0: + adm[indq] = np.nan + del indq + + indq = np.where((adp > 360.) | (adp < -180.)) + if np.size(indq) > 0: + adp[indq] = np.nan + del indq + + indq = np.where((awm > 50.) | (awm < 0.0)) + if np.size(indq) > 0: + awm[indq] = np.nan + del indq + + c = 0 + for t in range(0, np.size(mtime)): + indt = np.where(np.abs(atime - mtime[t]) < 1800.) + if np.size(indt) > 0: + if np.any(ahs[indt[0]].mask == False): + bhs[b, t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask == False]) + c = c + 1 + if np.any(atm[indt[0]].mask == False): + btm[b, t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask == False]) + if np.any(atp[indt[0]].mask == False): + btp[b, t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask == False]) + if np.any(adm[indt[0]].mask == False): + bdm[b, t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask == False]) + if np.any(adp[indt[0]].mask == False): + bdp[b, t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask == False]) + if np.any(awm[indt[0]].mask == False): + bwind[b, t] = np.nanmean(awm[indt[0]][awm[indt[0]].mask == False]) + + del indt + + # print("counted "+repr(c)+" at "+stname[b]) + + print(" station " + stname[b] + " ok") +# del ahs + except Exception as e: + print("Error occurred while processing station", stname[b]) + print(e) + +print('bwind:',bwind) +print('bhs:',bhs) + print(' ') # Simple quality-control (range) ind=np.where((bhs>30.)|(bhs<0.0)) if np.size(ind)>0: - bhs[ind]=np.nan; del ind + bhs[ind]=np.nan; del ind ind=np.where((btm>40.)|(btm<0.0)) if np.size(ind)>0: - btm[ind]=np.nan; del ind + btm[ind]=np.nan; del ind ind=np.where((btp>40.)|(btp<0.0)) if np.size(ind)>0: - btp[ind]=np.nan; del ind + btp[ind]=np.nan; del ind ind=np.where((bdm>360.)|(bdm<-180.)) if np.size(ind)>0: - bdm[ind]=np.nan; del ind + bdm[ind]=np.nan; del ind ind=np.where((bdp>360.)|(bdp<-180.)) if np.size(ind)>0: - bdp[ind]=np.nan; del ind + bdp[ind]=np.nan; del ind + +ind=np.where((bwind>50.0)|(bwind<0.0)) +if np.size(ind)>0: + bwind[ind]=np.nan; del ind ind=np.where((mhs>30.)|(mhs<0.0)) if np.size(ind)>0: - mhs[ind]=np.nan; del ind + mhs[ind]=np.nan; del ind ind=np.where((mtm>40.)|(mtm<0.0)) if np.size(ind)>0: - mtm[ind]=np.nan; del ind - + mtm[ind]=np.nan; del ind + ind=np.where((mtp>40.)|(mtp<0.0)) if np.size(ind)>0: - mtp[ind]=np.nan; del ind + mtp[ind]=np.nan; del ind + ind=np.where((mdm>360.)|(mdm<-180.)) if np.size(ind)>0: - mdm[ind]=np.nan; del ind + mdm[ind]=np.nan; del ind ind=np.where((mdp>360.)|(mdp<-180.)) if np.size(ind)>0: - mdp[ind]=np.nan; del ind + mdp[ind]=np.nan; del ind + +ind=np.where((mwn>50.)|(mwn<0.0)) +if np.size(ind)>0: + mwn[ind]=np.nan; del ind # Clean data excluding some stations. Select matchups only when model and buoy are available. ind=np.where( (np.isnan(lat)==False) & (np.isnan(lon)==False) & (np.isnan(np.nanmean(mhs,axis=1))==False) & (np.isnan(np.nanmean(bhs,axis=1))==False) ) if np.size(ind)>0: - stname=np.array(stname[ind[0]]) - lat=np.array(lat[ind[0]]) - lon=np.array(lon[ind[0]]) - mhs=np.array(mhs[ind[0],:]) - mtm=np.array(mtm[ind[0],:]) - mtp=np.array(mtp[ind[0],:]) - mdm=np.array(mdm[ind[0],:]) - mdp=np.array(mdp[ind[0],:]) - bhs=np.array(bhs[ind[0],:]) - btm=np.array(btm[ind[0],:]) - btp=np.array(btp[ind[0],:]) - bdm=np.array(bdm[ind[0],:]) - bdp=np.array(bdp[ind[0],:]) + stname=np.array(stname[ind[0]]) + lat=np.array(lat[ind[0]]) + lon=np.array(lon[ind[0]]) + mhs=np.array(mhs[ind[0],:]) + mtm=np.array(mtm[ind[0],:]) + mtp=np.array(mtp[ind[0],:]) + mdm=np.array(mdm[ind[0],:]) + # mdp=np.array(mdp[ind[0],:]) + mdp = np.array(mdp[ind[0], :]) + mwn=np.array(mwn[ind[0],:]) + bhs=np.array(bhs[ind[0],:]) + btm=np.array(btm[ind[0],:]) + btp=np.array(btp[ind[0],:]) + bdm=np.array(bdm[ind[0],:]) + bdp=np.array(bdp[ind[0],:]) + bwind=np.array(bwind[ind[0],:]) else: - sys.exit(' Error: No matchups Model/Buoy available.') + sys.exit(' Error: No matchups Model/Buoy available.') + print(" Matchups model/buoy complete. Total of "+repr(np.size(ind))+" stations/buoys avaliable."); del ind # Processing grid and/or cyclone information if gridinfo!=0: - print(" Adding extra information ... ") - alon=np.copy(lon); alon[alon<0]=alon[alon<0]+360. - indgplat=[]; indgplon=[] - for i in range(0,lat.shape[0]): - # indexes nearest point. - indgplat = np.append(indgplat,np.where( abs(mlat-lat[i])==abs(mlat-lat[i]).min() )[0][0]) - indgplon = np.append(indgplon,np.where( abs(mlon-alon[i])==abs(mlon-alon[i]).min() )[0][0]) - - indgplat=np.array(indgplat).astype('int'); indgplon=np.array(indgplon).astype('int') - pdistcoast=np.zeros(lat.shape[0],'f')*np.nan - pdepth=np.zeros(lat.shape[0],'f')*np.nan - poni=np.zeros(lat.shape[0],'f')*np.nan - phsmz=np.zeros(lat.shape[0],'f')*np.nan - for i in range(0,lat.shape[0]): - pdistcoast[i]=distcoast[indgplat[i],indgplon[i]] - pdepth[i]=depth[indgplat[i],indgplon[i]] - poni[i]=oni[indgplat[i],indgplon[i]] - phsmz[i]=hsmz[indgplat[i],indgplon[i]] - - print(" Grid Information Included.") - - # Excluding shallow water points too close to the coast (mask information not accurate) - ind=np.where( (np.isnan(pdistcoast)==False) & (np.isnan(pdepth)==False) ) - if np.size(ind)>0: - stname=np.array(stname[ind[0]]) - lat=np.array(lat[ind[0]]) - lon=np.array(lon[ind[0]]) - mhs=np.array(mhs[ind[0],:]) - mtm=np.array(mtm[ind[0],:]) - mtp=np.array(mtp[ind[0],:]) - mdm=np.array(mdm[ind[0],:]) - mdp=np.array(mdp[ind[0],:]) - bhs=np.array(bhs[ind[0],:]) - btm=np.array(btm[ind[0],:]) - btp=np.array(btp[ind[0],:]) - bdm=np.array(bdm[ind[0],:]) - bdp=np.array(bdp[ind[0],:]) - pdistcoast=np.array(pdistcoast[ind[0]]) - pdepth=np.array(pdepth[ind[0]]) - poni=np.array(poni[ind[0]]) - phsmz=np.array(phsmz[ind[0]]) - else: - sys.exit(' Error: No matchups Model/Buoy available after using grid mask.') - - del ind - - if cyclonemap!=0: - fcmap=np.zeros((lat.shape[0],mtime.shape[0]),'f')*np.nan - for t in range(0,np.size(mtime)): - # search for cyclone time index and cyclone map - indt=np.where(np.abs(ctime-mtime[t])<5400.) - if np.size(indt)>0: - for i in range(0,lat.shape[0]): - fcmap[i,t] = np.array(cmap[indt[0][0],indgplat[i],indgplon[i]]) - - del indt - else: - print(' - No cyclone information for this time step: '+repr(t)) - - # print(' Done cyclone analysis at step: '+repr(t)) - - ind=np.where(fcmap<0) - if np.size(ind)>0: - fcmap[ind]=np.nan - - print(" Cyclone Information Included.") + print(" Adding extra information ... ") + alon=np.copy(lon); alon[alon<0]=alon[alon<0]+360. + indgplat=[]; indgplon=[] + for i in range(0,lat.shape[0]): + # indexes nearest point. + indgplat = np.append(indgplat,np.where( abs(mlat-lat[i])==abs(mlat-lat[i]).min() )[0][0]) + indgplon = np.append(indgplon,np.where( abs(mlon-alon[i])==abs(mlon-alon[i]).min() )[0][0]) + + indgplat=np.array(indgplat).astype('int'); indgplon=np.array(indgplon).astype('int') + pdistcoast=np.zeros(lat.shape[0],'f')*np.nan + pdepth=np.zeros(lat.shape[0],'f')*np.nan + poni=np.zeros(lat.shape[0],'f')*np.nan + phsmz=np.zeros(lat.shape[0],'f')*np.nan + for i in range(0,lat.shape[0]): + pdistcoast[i]=distcoast[indgplat[i],indgplon[i]] + pdepth[i]=depth[indgplat[i],indgplon[i]] + poni[i]=oni[indgplat[i],indgplon[i]] + phsmz[i]=hsmz[indgplat[i],indgplon[i]] + + print(" Grid Information Included.") + + # Excluding shallow water points too close to the coast (mask information not accurate) + ind=np.where( (np.isnan(pdistcoast)==False) & (np.isnan(pdepth)==False) ) + if np.size(ind)>0: + stname=np.array(stname[ind[0]]) + lat=np.array(lat[ind[0]]) + lon=np.array(lon[ind[0]]) + mhs=np.array(mhs[ind[0],:]) + mtm=np.array(mtm[ind[0],:]) + mtp=np.array(mtp[ind[0],:]) + mdm=np.array(mdm[ind[0],:]) + mdp=np.array(mdp[ind[0],:]) + mwn=np.array(mwn[ind[0],:]) + bhs=np.array(bhs[ind[0],:]) + btm=np.array(btm[ind[0],:]) + btp=np.array(btp[ind[0],:]) + bdm=np.array(bdm[ind[0],:]) + bdp=np.array(bdp[ind[0],:]) + bwind=np.array(bwind[ind[0],:]) + pdistcoast=np.array(pdistcoast[ind[0]]) + pdepth=np.array(pdepth[ind[0]]) + poni=np.array(poni[ind[0]]) + phsmz=np.array(phsmz[ind[0]]) + else: + sys.exit(' Error: No matchups Model/Buoy available after using grid mask.') + + del ind + + if cyclonemap!=0: + fcmap=np.zeros((lat.shape[0],mtime.shape[0]),'f')*np.nan + for t in range(0,np.size(mtime)): + # search for cyclone time index and cyclone map + indt=np.where(np.abs(ctime-mtime[t])<5400.) + if np.size(indt)>0: + for i in range(0,lat.shape[0]): + fcmap[i,t] = np.array(cmap[indt[0][0],indgplat[i],indgplon[i]]) + + del indt + else: + print(' - No cyclone information for this time step: '+repr(t)) + + # print(' Done cyclone analysis at step: '+repr(t)) + + ind=np.where(fcmap<0) + if np.size(ind)>0: + fcmap[ind]=np.nan + + print(" Cyclone Information Included.") # Edit format if this is forecast model data. Reshape and allocate + + if forecastds>0: - unt=np.unique(mfcycle); mxsz=1 - for i in range(0,unt.shape[0]): - ind=np.where(mfcycle==unt[i])[0] - mxsz=np.max([mxsz,np.size(ind)]) - - for i in range(0,unt.shape[0]): - ind=np.where(mfcycle==unt[i])[0] - if i==0: - nmhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtime=np.zeros((unt.shape[0],mxsz),'double')*np.nan - if cyclonemap!=0: - nfcmap=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - - nmtime[i,0:np.size(ind)]=np.array(mtime[ind]).astype('double') - nmhs[:,i,:][:,0:np.size(ind)]=np.array(mhs[:,ind]) - nmtm[:,i,:][:,0:np.size(ind)]=np.array(mtm[:,ind]) - nmtp[:,i,:][:,0:np.size(ind)]=np.array(mtp[:,ind]) - nmdm[:,i,:][:,0:np.size(ind)]=np.array(mdm[:,ind]) - nmdp[:,i,:][:,0:np.size(ind)]=np.array(mdp[:,ind]) - nbhs[:,i,:][:,0:np.size(ind)]=np.array(bhs[:,ind]) - nbtm[:,i,:][:,0:np.size(ind)]=np.array(btm[:,ind]) - nbtp[:,i,:][:,0:np.size(ind)]=np.array(btp[:,ind]) - nbdm[:,i,:][:,0:np.size(ind)]=np.array(bdm[:,ind]) - nbdp[:,i,:][:,0:np.size(ind)]=np.array(bdp[:,ind]) - if cyclonemap!=0: - nfcmap[:,i,:][:,0:np.size(ind)]=np.array(fcmap[:,ind]) - - ind=np.where( (nmhs>0.0) & (nbhs>0.0) ) + unt=np.unique(mfcycle); mxsz=1 + for i in range(0,unt.shape[0]): + ind=np.where(mfcycle==unt[i])[0] + mxsz=np.max([mxsz,np.size(ind)]) + + for i in range(0,unt.shape[0]): + ind=np.where(mfcycle==unt[i])[0] + if i==0: + nmhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmwn=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbwind=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmtime=np.zeros((unt.shape[0],mxsz),'double')*np.nan + if cyclonemap!=0: + nfcmap=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + + nmtime[i,0:np.size(ind)]=np.array(mtime[ind]).astype('double') + nmhs[:,i,:][:,0:np.size(ind)]=np.array(mhs[:,ind]) + nmtm[:,i,:][:,0:np.size(ind)]=np.array(mtm[:,ind]) + nmtp[:,i,:][:,0:np.size(ind)]=np.array(mtp[:,ind]) + nmdm[:,i,:][:,0:np.size(ind)]=np.array(mdm[:,ind]) + nmdp[:,i,:][:,0:np.size(ind)]=np.array(mdp[:,ind]) + nmwn[:,i,:][:,0:np.size(ind)]=np.array(mwn[:,ind]) + nbhs[:,i,:][:,0:np.size(ind)]=np.array(bhs[:,ind]) + nbtm[:,i,:][:,0:np.size(ind)]=np.array(btm[:,ind]) + nbtp[:,i,:][:,0:np.size(ind)]=np.array(btp[:,ind]) + nbdm[:,i,:][:,0:np.size(ind)]=np.array(bdm[:,ind]) + nbdp[:,i,:][:,0:np.size(ind)]=np.array(bdp[:,ind]) + nbwind[:,i,:][:,0:np.size(ind)]=np.array(bwind[:,ind]) + + if cyclonemap!=0: + nfcmap[:,i,:][:,0:np.size(ind)]=np.array(fcmap[:,ind]) + + + ind=np.where( (nmhs>0.0) & (nbhs>0.0) ) else: - ind=np.where( (mhs>0.0) & (bhs>0.0) ) + + larger_shape = max(mhs.shape, bhs.shape) + padded_mhs = np.zeros(larger_shape) + padded_bhs = np.zeros(larger_shape) + padded_mhs[:mhs.shape[0], :mhs.shape[1]] = mhs + padded_bhs[:bhs.shape[0], :bhs.shape[1]] = bhs + + + # Create padded arrays for other variables + padded_mtm = np.zeros(larger_shape) # Assuming mtm has the same shape as mhs + padded_mtp = np.zeros(larger_shape) # Assuming mtp has the same shape as mhs + padded_mdm = np.zeros(larger_shape) # Assuming mdm has the same shape as mhs + padded_mdp = np.zeros(larger_shape) # Assuming mdp has the same shape as mhs + padded_mwn = np.zeros(larger_shape) # Assuming mwn has the same shape as mhs + + padded_btm = np.zeros(larger_shape) # Assuming btm has the same shape as bhs + padded_btp = np.zeros(larger_shape) # Assuming btp has the same shape as bhs + padded_bdm = np.zeros(larger_shape) # Assuming bdm has the same shape as bhs + padded_bdp = np.zeros(larger_shape) # Assuming bdp has the same shape as bhs + padded_bwind = np.zeros(larger_shape) # Assuming bwind has the same shape as bhs + + # Copy values from original arrays to padded arrays for other variables + padded_mtm[:mtm.shape[0], :mtm.shape[1]] = mtm + padded_mtp[:mtp.shape[0], :mtp.shape[1]] = mtp + padded_mdm[:mdm.shape[0], :mdm.shape[1]] = mdm + padded_mdp[:mdp.shape[0], :mdp.shape[1]] = mdp + padded_mwn[:mwn.shape[0], :mwn.shape[1]] = mwn + + padded_btm[:btm.shape[0], :btm.shape[1]] = btm + padded_btp[:btp.shape[0], :btp.shape[1]] = btp + padded_bdm[:bdm.shape[0], :bdm.shape[1]] = bdm + padded_bdp[:bdp.shape[0], :bdp.shape[1]] = bdp + padded_bwind[:bwind.shape[0], :bwind.shape[1]] = bwind + +# ind=np.where( (mhs>0.0) & (bhs>0.0) ) + ind = np.where((padded_mhs > 0.0) & (padded_bhs > 0.0)) if np.size(ind)>0: - print(' Total amount of matchups model/buoy: '+repr(np.size(ind))) - - # Save netcdf output file - lon[lon>180.]=lon[lon>180.]-360. - initime=str(time.gmtime(mtime.min())[0])+str(time.gmtime(mtime.min())[1]).zfill(2)+str(time.gmtime(mtime.min())[2]).zfill(2)+str(time.gmtime(mtime.min())[3]).zfill(2) - fintime=str(time.gmtime(mtime.max())[0])+str(time.gmtime(mtime.max())[1]).zfill(2)+str(time.gmtime(mtime.max())[2]).zfill(2)+str(time.gmtime(mtime.max())[3]).zfill(2) - ncfile = nc.Dataset('WW3.Buoy'+ftag+'_'+initime+'to'+fintime+'.nc', "w", format=fnetcdf) - ncfile.history="Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of "+repr(bhs[bhs>0.].shape[0])+" observations or pairs model/observation." - # create dimensions - ncfile.createDimension('buoypoints', bhs.shape[0] ) - if gridinfo!=0: - ncfile.createDimension('GlobalOceansSeas', ocnames.shape[0] ) - ncfile.createDimension('HighSeasMarineZones', hsmznames.shape[0] ) - if cyclonemap!=0: - ncfile.createDimension('cycloneinfo', cinfo.shape[0] ) - vcinfo = ncfile.createVariable('cycloneinfo',dtype('a25'),('cycloneinfo')) - # create variables. - vstname = ncfile.createVariable('buoyID',dtype('a25'),('buoypoints')) - vlat = ncfile.createVariable('latitude',np.dtype('float32').char,('buoypoints')) - vlon = ncfile.createVariable('longitude',np.dtype('float32').char,('buoypoints')) - - if forecastds>0: - ncfile.createDimension('time', nmhs.shape[2] ) - ncfile.createDimension('fcycle', unt.shape[0] ) - vt = ncfile.createVariable('time',np.dtype('float64').char,('fcycle','time')) - vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbhs = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) - else: - ncfile.createDimension('time', bhs.shape[1] ) - vt = ncfile.createVariable('time',np.dtype('float64').char,('time')) - vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','time')) - vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','time')) - vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','time')) - vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','time')) - vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','time')) - vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','time')) - vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','time')) - vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','time')) - vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','time')) - vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','time')) - - if gridinfo!=0: - vpdistcoast = ncfile.createVariable('distcoast',np.dtype('float32').char,('buoypoints')) - vpdepth = ncfile.createVariable('depth',np.dtype('float32').char,('buoypoints')) - vponi = ncfile.createVariable('GlobalOceansSeas',np.dtype('float32').char,('buoypoints')) - vocnames = ncfile.createVariable('names_GlobalOceansSeas',dtype('a25'),('GlobalOceansSeas')) - vphsmz = ncfile.createVariable('HighSeasMarineZones',np.dtype('float32').char,('buoypoints')) - vhsmznames = ncfile.createVariable('names_HighSeasMarineZones',dtype('a25'),('HighSeasMarineZones')) - if cyclonemap!=0: - if forecastds>0: - vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','fcycle','time')) - else: - vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','time')) - - # Assign units - vlat.units = 'degrees_north' ; vlon.units = 'degrees_east' - vt.units = 'seconds since 1970-01-01T00:00:00+00:00' - vmhs.units='m'; vbhs.units='m' - vmtm.units='s'; vbtm.units='s' - vmtp.units='s'; vbtp.units='s' - vmdm.units='degrees'; vbdm.units='degrees' - vmdp.units='degrees'; vbdp.units='degrees' - if gridinfo!=0: - vpdepth.units='m'; vpdistcoast.units='km' - - # Allocate Data - vstname[:]=stname[:]; vlat[:] = lat[:]; vlon[:] = lon[:] - if forecastds>0: - vt[:,:]=nmtime[:,:] - vmhs[:,:,:]=nmhs[:,:,:] - vmtm[:,:,:]=nmtm[:,:,:] - vmtp[:,:,:]=nmtp[:,:,:] - vmdm[:,:,:]=nmdm[:,:,:] - vmdp[:,:,:]=nmdp[:,:,:] - vbhs[:,:,:]=nbhs[:,:,:] - vbtm[:,:,:]=nbtm[:,:,:] - vbtp[:,:,:]=nbtp[:,:,:] - vbdm[:,:,:]=nbdm[:,:,:] - vbdp[:,:,:]=nbdp[:,:,:] - else: - vt[:]=mtime[:] - vmhs[:,:]=mhs[:,:] - vmtm[:,:]=mtm[:,:] - vmtp[:,:]=mtp[:,:] - vmdm[:,:]=mdm[:,:] - vmdp[:,:]=mdp[:,:] - vbhs[:,:]=bhs[:,:] - vbtm[:,:]=btm[:,:] - vbtp[:,:]=btp[:,:] - vbdm[:,:]=bdm[:,:] - vbdp[:,:]=bdp[:,:] - - if gridinfo!=0: - vpdistcoast[:]=pdistcoast[:] - vpdepth[:]=pdepth[:] - vponi[:]=poni[:]; vocnames[:] = ocnames[:] - vphsmz[:]=phsmz[:]; vhsmznames[:] = hsmznames[:] - if cyclonemap!=0: - vcinfo[:] = cinfo[:] - if forecastds>0: - vcmap[:,:,:]=nfcmap[:,:,:] - else: - vcmap[:,:]=fcmap[:,:] - - ncfile.close() - print(' ') - print('Done. Netcdf ok. New file saved: WW3.Buoy'+ftag+'_'+initime+'to'+fintime+'.nc') + print(' Total amount of matchups model/buoy: '+repr(np.size(ind))) + + # Save netcdf output file + lon[lon>180.]=lon[lon>180.]-360. + initime=str(time.gmtime(mtime.min())[0])+str(time.gmtime(mtime.min())[1]).zfill(2)+str(time.gmtime(mtime.min())[2]).zfill(2)+str(time.gmtime(mtime.min())[3]).zfill(2) + fintime=str(time.gmtime(mtime.max())[0])+str(time.gmtime(mtime.max())[1]).zfill(2)+str(time.gmtime(mtime.max())[2]).zfill(2)+str(time.gmtime(mtime.max())[3]).zfill(2) + ncfile = nc.Dataset('WW3.Buoy'+str(ftag)+'_'+initime+'to'+fintime+'.nc', "w", format=fnetcdf) + ncfile.history="Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of "+repr(bhs[bhs>0.].shape[0])+" observations or pairs model/observation." + # create dimensions + ncfile.createDimension('buoypoints', bhs.shape[0] ) + if gridinfo!=0: + ncfile.createDimension('GlobalOceansSeas', ocnames.shape[0] ) + ncfile.createDimension('HighSeasMarineZones', hsmznames.shape[0] ) + if cyclonemap!=0: + ncfile.createDimension('cycloneinfo', cinfo.shape[0] ) + vcinfo = ncfile.createVariable('cycloneinfo',dtype('a25'),('cycloneinfo')) + # create variables. + vstname = ncfile.createVariable('buoyID',type('a25'),('buoypoints')) + vlat = ncfile.createVariable('latitude',np.dtype('float32').char,('buoypoints')) + vlon = ncfile.createVariable('longitude',np.dtype('float32').char,('buoypoints')) + + if forecastds>0: + ncfile.createDimension('time', nmhs.shape[2] ) + ncfile.createDimension('fcycle', unt.shape[0] ) + vt = ncfile.createVariable('time',np.dtype('float64').char,('fcycle','time')) + vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmwn = ncfile.createVariable('model_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) + + vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbwind = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) + + else: + ncfile.createDimension('time', bhs.shape[1] ) + vt = ncfile.createVariable('time',np.dtype('float64').char,('time')) + vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','time')) + vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','time')) + vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','time')) + vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','time')) + vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','time')) + vmwn = ncfile.createVariable('model_wind',np.dtype('float32').char,('buoypoints','time')) + + vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','time')) + vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','time')) + vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','time')) + vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','time')) + vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','time')) + vbwind = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','time')) + + + + + if gridinfo!=0: + vpdistcoast = ncfile.createVariable('distcoast',np.dtype('float32').char,('buoypoints')) + vpdepth = ncfile.createVariable('depth',np.dtype('float32').char,('buoypoints')) + vponi = ncfile.createVariable('GlobalOceansSeas',np.dtype('float32').char,('buoypoints')) + vocnames = ncfile.createVariable('names_GlobalOceansSeas',dtype('a25'),('GlobalOceansSeas')) + vphsmz = ncfile.createVariable('HighSeasMarineZones',np.dtype('float32').char,('buoypoints')) + vhsmznames = ncfile.createVariable('names_HighSeasMarineZones',dtype('a25'),('HighSeasMarineZones')) + if cyclonemap!=0: + if forecastds>0: + vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','fcycle','time')) + else: + vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','time')) + + # Assign units + vlat.units = 'degrees_north' ; vlon.units = 'degrees_east' + vt.units = 'seconds since 1970-01-01T00:00:00+00:00' + vmhs.units='m'; vbhs.units='m' + vmtm.units='s'; vbtm.units='s' + vmtp.units='s'; vbtp.units='s' + vmdm.units='degrees'; vbdm.units='degrees' + vmdp.units='degrees'; vbdp.units='degrees' + vmwn.unit='m/s';vbwind.unit='m/s' + + if gridinfo!=0: + vpdepth.units='m'; vpdistcoast.units='km' + + # Allocate Data + vstname[:]=stname[:]; vlat[:] = lat[:]; vlon[:] = lon[:] + if forecastds>0: + vt[:,:]=nmtime[:,:] + vmhs[:,:,:]=nmhs[:,:,:] + vmtm[:,:,:]=nmtm[:,:,:] + vmtp[:,:,:]=nmtp[:,:,:] + vmdm[:,:,:]=nmdm[:,:,:] + vmdp[:,:,:]=nmdp[:,:,:] + vmwn[:,:,:]=nmwn[:,:,:] + vbhs[:,:,:]=nbhs[:,:,:] + vbtm[:,:,:]=nbtm[:,:,:] + vbtp[:,:,:]=nbtp[:,:,:] + vbdm[:,:,:]=nbdm[:,:,:] + vbdp[:,:,:]=nbdp[:,:,:] + vbwind[:,:,:]=nbwind[:,:,:] + + else: + vt[:]=mtime[:] + vmhs[:,:]=padded_mhs[:,:] + vmtm[:,:]=padded_mtm[:,:] + vmtp[:,:]=padded_mtp[:,:] + vmdm[:,:]=padded_mdm[:,:] + vmdp[:,:]=padded_mdp[:,:] + vmwn[:,:]=padded_mwn[:,:] + vbhs[:,:]=padded_bhs[:,:] + vbtm[:,:]=btm[:,:] + vbtp[:,:]=btp[:,:] + vbdm[:,:]=bdm[:,:] + vbdp[:,:]=bdp[:,:] + vbwind[:,:]=bwind[:,:] + + + + + if gridinfo!=0: + vpdistcoast[:]=pdistcoast[:] + vpdepth[:]=pdepth[:] + vponi[:]=poni[:]; vocnames[:] = ocnames[:] + vphsmz[:]=phsmz[:]; vhsmznames[:] = hsmznames[:] + if cyclonemap!=0: + vcinfo[:] = cinfo[:] + if forecastds>0: + vcmap[:,:,:]=nfcmap[:,:,:] + else: + vcmap[:,:]=fcmap[:,:] + + ncfile.close() + print(' ') + print('Done. Netcdf ok. New file saved: WW3.Buoy'+str(ftag)+'_'+initime+'to'+fintime+'.nc') diff --git a/ww3tools/wread.py b/ww3tools/wread.py index f607cd3..f833aa4 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1,5 +1,64 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +wread.py + +VERSION AND LAST UPDATE: + v1.0 04/04/2022 + v1.1 01/06/2023 + v1.2 10/12/2023 + +PURPOSE: + Group of python functions to Read Wave data: + WAVEWATCHIII results, and NDBC and Copernicus buoys. + Prefix meaning: + tseriesnc = time series (table of integrated parameters versus time). + spec = wave spectrum. + Users can import as a standard python function, and use it accordingly: + For example: + import wread + wread.tseriesnc_ww3(filename.nc,stationID) + Users can help() each function to obtain information about inputs/outputs + help(wread.tseriesnc_ww3) + +USAGE: + functions + readconfig + mask + cyclonemap + tseriesnc_ndbc + tseriesnc_copernicus + aodn_altimeter + tseriesnc_ww3 + bull + bull_tar + ts + station_tar + spec_ndbc + spec_ww3 + Explanation for each function is contained in the headers + +OUTPUT: + Dictionary containing arrays and info. + Description of variables is contained in the header of each function. + +DEPENDENCIES: + See setup.py and the imports below. + +AUTHOR and DATE: + 04/04/2022: Ricardo M. Campos, first version. + 01/06/2023: Ricardo M. Campos, new file formats added. + 10/12/2023: Ricardo M. Campos & Maryam Mohammadpour, new function readconfig + to read the configuration file ww3tools.yaml. And a new function aodn_altimeter + to read AODN altimeter data. + +PERSON OF CONTACT: + Ricardo M Campos: ricardo.campos@noaa.gov + +""" + import matplotlib -# matplotlib.use('Agg') # for backend plots, not for rendering in a window import time from time import strptime from calendar import timegm @@ -18,7 +77,1549 @@ import tarfile import math + +def readconfig(fname): + """ + Reads the configuration file ww3tools.yaml and returns a dictionary + containing all the information in the file. + User can enter the file name 'ww3tools.yaml' or the name including + the full path '/home/user/ww3tools.yaml' + """ + + try: + with open(fname, 'r') as file: + wconfig = yaml.safe_load(file) + except: + raise ValueError("wproc.readconfig: ww3tools.yaml not found.") + else: + + # paths + if "path_out" in wconfig: + if str(wconfig['path_out']) != '/': + wconfig['path_out']=str(wconfig['path_out'])+"/" + else: + wconfig['path_out']=str(os.getcwd())+"/" + + if "path_alt" in wconfig: + if str(wconfig['path_alt']) != '/': + wconfig['path_alt']=str(wconfig['path_alt'])+"/" + else: + wconfig['path_alt']=str(os.getcwd())+"/" + print("Warning: path_alt not found, using local directory "+wconfig['path_alt']) + + if "path_ndbc" in wconfig: + if str(wconfig['path_ndbc']) != '/': + wconfig['path_ndbc']=str(wconfig['path_ndbc'])+"/" + else: + wconfig['path_ndbc']=str(os.getcwd())+"/" + print("Warning: path_ndbc not found, using local directory "+wconfig['path_ndbc']) + + if "path_copernicus" in wconfig: + if str(wconfig['path_copernicus']) != '/': + wconfig['path_copernicus']=str(wconfig['path_copernicus'])+"/" + else: + wconfig['path_copernicus']=str(os.getcwd())+"/" + print("Warning: path_copernicus not found, using local directory "+wconfig['path_copernicus']) + + # returns a dictionary containing the information given by ww3tools.yaml + return wconfig + + +def mask(*args): + ''' + Read gridmask netcdf file generated with prepGridMask.py + Input: file name (example: gridInfo_GEFSv12.nc) + Output: dictionary containing the arrays and string names + ''' + if len(args) == 1: + fname=str(args[0]) + else: + sys.exit(' Too many inputs') + + print(" reading ww3_tools mask ...") + try: + f=nc.Dataset(fname) + # build dictionary + result={'latitude':np.array(f.variables['latitude'][:]),'longitude':np.array(f.variables['longitude'][:]),'mask':np.array(f.variables['mask'][:,:])} + except: + sys.exit(" Cannot open "+fname) + else: + if 'distcoast' in f.variables.keys(): + result['distcoast'] = np.array(f.variables['distcoast'][:,:]) + if 'depth' in f.variables.keys(): + result['depth'] = np.array(f.variables['depth'][:,:]) + if 'GlobalOceansSeas' in f.variables.keys(): + result['GlobalOceansSeas'] = np.array(f.variables['GlobalOceansSeas'][:,:]) + if 'HighSeasMarineZones' in f.variables.keys(): + result['HighSeasMarineZones'] = np.array(f.variables['HighSeasMarineZones'][:,:]) + if 'names_GlobalOceansSeas' in f.variables.keys(): + result['names_GlobalOceansSeas'] = f.variables['names_GlobalOceansSeas'][:] + if 'names_HighSeasMarineZones' in f.variables.keys(): + result['names_HighSeasMarineZones'] = f.variables['names_HighSeasMarineZones'][:] + + f.close(); del f + print(" GridMask Read. "+fname) + return result + del result + +def cyclonemap(*args): + ''' + Read cyclonemap netcdf file generated with procyclmap.py + Input: file name (example: CycloneMap2020.nc) + Output: dictionary containing the arrays and string names + ''' + if len(args) == 1: + fname=str(args[0]) + else: + sys.exit(' Too many inputs') + + print(" reading ww3_tools cyclonemap ...") + try: + f=nc.MFDataset(fname, aggdim='time') + at=f.variables['time'][:]; adate=[] + for j in range(0,at.shape[0]): + adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) + # -------- + # build dictionary + result={'latitude':np.array(f.variables['lat'][:]),'longitude':np.array(f.variables['lon'][:]), + 'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), + 'cmap':f.variables['cmap'], 'info':str(f.info), 'netcdf':f} + # it does not allocate the data of cmap using [:,:,:] yet as it can take a lot of data/memory and time. + except: + sys.exit(" Cannot open "+fname) + else: + print(" CycloneInfo Read. "+fname) + return result + del result + + +# ================= OBSERVATIONS ================= +# --- Buoys --- +# Observations NDBC, netcdf format +def tseriesnc_ndbc(fname=None,anh=None): + ''' + Observations NDBC, time series/table, netcdf format + Input: file name (example: 46047h2016.nc), and anemometer height (optional) + Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, + and arrays sst,mslp,dwp,tmp,gst(10-m height),wsp(10-m height),wdir,hs,tm,tp,dm + ''' + if fname==None: + raise ValueError("NDBC file name must be informed.") + + try: + ds = xr.open_dataset(fname); f=nc.Dataset(fname) + except: + sys.exit(" Cannot open "+fname) + else: + btm = f.variables['average_wpd'][:,0,0]; btp = f.variables['dominant_wpd'][:,0,0] + btime = np.array(f.variables['time'][:]).astype('double') + f.close(); del f + bsst = ds['sea_surface_temperature'].values[:,0,0] + bmslp = ds['air_pressure'].values[:,0,0] + bdwp = ds['dewpt_temperature'].values[:,0,0] + btmp = ds['air_temperature'].values[:,0,0] + bgst = ds['gust'].values[:,0,0] + + if 'wind_spd' in ds.keys(): + bwsp = ds['wind_spd'].values[:,0,0] + + if anh==None: + try: + from urllib.request import urlopen + url = "https://www.ndbc.noaa.gov/station_page.php?station="+str(fname).split('/')[-1].split('h')[0] + page = urlopen(url) + html_bytes = page.read() + html = html_bytes.decode("utf-8") + except: + anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. + print('Information about the Anemometer height, for wind speed conversion to 10m, could not be obtained.') + else: + if "Anemometer height" in html: + anh=float(html.split('Anemometer height')[1][0:15].split(':')[1].split('m')[0]) + else: + print('Information about the Anemometer height, for wind speed conversion to 10m, could not be found.') + anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. + + del url,page,html_bytes,html + + # convert wind speed to 10 meters (DNVGL C-205 Table 2-1, confirmed by https://onlinelibrary.wiley.com/doi/pdf/10.1002/er.6382) + bwsp = np.copy(((10./anh)**(0.12)) * bwsp) + bgst = np.copy(((10./anh)**(0.12)) * bgst) + + bwdir = ds['wind_dir'].values[:,0,0] + bhs = ds['wave_height'].values[:,0,0] + bdm = ds['mean_wave_dir'].values[:,0,0] + + # Automatic and basic Quality Control + bsst[np.abs(bsst)>70]=np.nan + bmslp[(bmslp<500)|(bmslp>1500)]=np.nan + bdwp[np.abs(bdwp)>80]=np.nan + btmp[np.abs(btmp)>80]=np.nan + bgst[(bgst<0)|(bgst>200)]=np.nan + bwsp[(bwsp<0)|(bwsp>150)]=np.nan + bwdir[(bwdir<-180)|(bwdir>360)]=np.nan + bhs[(bhs<0)|(bhs>30)]=np.nan + btm[(btm<0)|(btm>40)]=np.nan + btp[(btp<0)|(btp>40)]=np.nan + bdm[(bdm<-180)|(bdm>360)]=np.nan + + result={'latitude':np.array(ds['latitude'].values[:]),'longitude':np.array(ds['longitude'].values[:]), + 'time':btime,'date':ds['time'].values[:], + 'sst':bsst, 'mslp':bmslp, 'dewpt_temp':bdwp, + 'air_temp':btmp, 'gust':bgst, 'wind_spd':bwsp, + 'wind_dir':bwdir, 'hs':bhs, 'tm':btm, + 'tp':btp, 'dm':bdm, 'tm':btm} + + return result + ds.close() + del ds,btime,bsst,bmslp,bdwp,btmp,bgst,bwsp,bwdir,bhs,btm,btp,bdm + +# Observations NDBC, text format +def tseriestxt_ndbc(fname=None,anh=None): + ''' + Observations NDBC, time series/table, stdmet format + Input: file name (example: NDBC_historical_stdmet_41004.txt), and anemometer height (optional) + Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, + and arrays sst,mslp,dwp,tmp,gst,wsp,wdir,hs,tm,tp,dm + ''' + if fname==None: + raise ValueError("NDBC file name must be informed.") + + try: + ds = pd.read_csv(fname,comment='#',delimiter=r"\s+") + btime=np.zeros(ds.shape[0],'d') + if 'mm' in ds.keys(): + ds = pd.read_csv(fname,comment='#',delimiter=r"\s+",parse_dates= {"date" : ["YY","MM","DD","hh","mm"]}) + ds['date']=pd.to_datetime(ds['date'],format='%Y %m %d %H %M') + else: + ds = pd.read_csv(fname,comment='#',delimiter=r"\s+",parse_dates= {"date" : ["YY","MM","DD","hh"]}) + ds['date']=pd.to_datetime(ds['date'],format='%Y %m %d %H') + + for i in range(0,btime.shape[0]): + btime[i]=double(ds['date'][i].timestamp()) + + except: + sys.exit(" Cannot open "+fname) + else: + + bwdir=np.array(ds['WDIR'].values[:]).astype('float') + bgst=np.array(ds['GST'].values[:]).astype('float') + bhs=np.array(ds['WVHT'].values[:]).astype('float') + btp=np.array(ds['DPD'].values[:]).astype('float') + btm=np.array(ds['APD'].values[:]).astype('float') + bdm=np.array(ds['MWD'].values[:]).astype('float') + bmslp=np.array(ds['PRES'].values[:]).astype('float') + btmp=np.array(ds['ATMP'].values[:]).astype('float') + bsst=np.array(ds['WTMP'].values[:]).astype('float') + bdwp=np.array(ds['DEWP'].values[:]).astype('float') + + if 'WSPD' in ds.keys(): + bwsp=np.array(ds['WSPD'].values[:]).astype('float') + + try: + from urllib.request import urlopen + url = "https://www.ndbc.noaa.gov/station_page.php?station="+str(fname).split('/')[-1].split('h')[0].split('_')[-1] + page = urlopen(url) + html_bytes = page.read() + html = html_bytes.decode("utf-8") + auxlatlon=html.split('payload')[1][16:33] + if 'S' in auxlatlon: + blat=-float(auxlatlon[0:6]) + else: + blat=float(auxlatlon[0:6]) + + if 'W' in auxlatlon: + blon=-float(auxlatlon[8:16]) + else: + blon=float(auxlatlon[8:16]) + + except: + if anh==None: + anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. + print('Information about the Anemometer height, for wind speed conversion to 10m, could not be found. Assuming 4.0 meters.') + + blat=np.nan; blon=np.nan + print('Information of Lat and Lon could not be obtained.') + else: + if anh==None: + if "Anemometer height" in html: + anh=float(html.split('Anemometer height')[1][0:15].split(':')[1].split('m')[0]) + else: + print('Information about the Anemometer height, for wind speed conversion to 10m, could not be found. Assuming 4.0 meters.') + anh=4.0 # assuming most of anemometer heights are between 3.7 to 4.1. + + del url,page,html_bytes,html + + # convert wind speed to 10 meters (DNVGL C-205 Table 2-1, confirmed by https://onlinelibrary.wiley.com/doi/pdf/10.1002/er.6382) + bwsp = np.copy(((10./anh)**(0.12)) * bwsp) + bgst = np.copy(((10./anh)**(0.12)) * bgst) + + # Automatic and basic Quality Control + bsst[np.abs(bsst)>70]=np.nan + bmslp[(bmslp<500)|(bmslp>1500)]=np.nan + bdwp[np.abs(bdwp)>80]=np.nan + btmp[np.abs(btmp)>80]=np.nan + bgst[(bgst<0)|(bgst>200)]=np.nan + bwsp[(bwsp<0)|(bwsp>150)]=np.nan + bwdir[(bwdir<-180)|(bwdir>360)]=np.nan + bhs[(bhs<0)|(bhs>30)]=np.nan + btm[(btm<0)|(btm>40)]=np.nan + btp[(btp<0)|(btp>40)]=np.nan + bdm[(bdm<-180)|(bdm>360)]=np.nan + + result={'latitude':blat,'longitude':blon, + 'time':btime,'date':ds['date'].values[:], + 'sst':bsst, 'mslp':bmslp, 'dewpt_temp':bdwp, + 'air_temp':btmp, 'gust':bgst, 'wind_spd':bwsp, + 'wind_dir':bwdir, 'hs':bhs, 'tm':btm, + 'tp':btp, 'dm':bdm, 'tm':btm} + + return result + del ds,btime,blat,blon,bsst,bmslp,bdwp,btmp,bgst,bwsp,bwdir,bhs,btm,btp,bdm + +# Observations Copernicus, netcdf format +def tseriesnc_copernicus(*args): + ''' + Observations NDBC, time series/table, netcdf format + Input: file name (example: 46047h2016.nc) + Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, + and arrays with the environmental variables available. + ''' + if len(args) == 1: + fname=str(args[0]) + elif len(args) > 1: + sys.exit(' Too many inputs') + + try: + ds = xr.open_dataset(fname); f=nc.Dataset(fname) + except: + sys.exit(" Cannot open "+fname) + else: + btime = np.array(f.variables['TIME'][:]*24*3600 + timegm( strptime('195001010000', '%Y%m%d%H%M') )).astype('double') + f.close(); del f + blat = np.nanmean(ds['LATITUDE'].values[:]) + blon = np.nanmean(ds['LONGITUDE'].values[:]) + # dictionary + result={'latitude':np.array(blat),'longitude':np.array(blon), + 'time':btime,'date':ds['TIME'].values[:]} + + if 'DEPH' in ds.keys(): + bdepth = np.nanmean(ds['DEPH'].values[:,:],axis=1) # Depth + result['depth']=np.array(bdepth) + + if 'VHM0' in ds.keys(): + bhs = np.nanmean(ds['VHM0'].values[:,:],axis=1) # Hs + bhs[(bhs<0)|(bhs>30)]=np.nan + result['hs']=np.array(bhs) + elif 'VGHS' in ds.keys(): + bhs = np.nanmean(ds['VGHS'].values[:,:],axis=1) # Hs + bhs[(bhs<0)|(bhs>30)]=np.nan + result['hs']=np.array(bhs) + + if 'VAVH' in ds.keys(): + bvavh = np.nanmean(ds['VAVH'].values[:,:],axis=1) # H 1/3 vavh + bvavh[(bvavh<0)|(bvavh>30)]=np.nan + result['hs_vavh']=np.array(bvavh) + + if 'VZMX' in ds.keys(): + bhmax = np.nanmean(ds['VZMX'].values[:,:],axis=1) # Hmax + bhmax[(bhmax<0)|(bhmax>40)]=np.nan + result['hmax']=np.array(bhmax) + + if 'VTM02' in ds.keys(): + btm = np.nanmean(ds['VTM02'].values[:,:],axis=1) # Tm + btm[(btm<0)|(btm>40)]=np.nan + result['tm']=np.array(btm) + elif 'VGTA' in ds.keys(): + btm = np.nanmean(ds['VGTA'].values[:,:],axis=1) # Tm + btm[(btm<0)|(btm>40)]=np.nan + result['tm']=np.array(btm) + + if 'VTPK' in ds.keys(): + btp = np.nanmean(ds['VTPK'].values[:,:],axis=1) # Tp + btp[(btp<0)|(btp>40)]=np.nan + result['tp']=np.array(btp) + + if 'TEMP' in ds.keys(): + bsst = np.nanmean(ds['TEMP'].values[:,:],axis=1) # SST + bsst[np.abs(bsst)>70]=np.nan + result['sst']=np.array(bsst) + + if 'ATMS' in ds.keys(): + bmslp = np.nanmean(ds['ATMS'].values[:,:],axis=1) # Pressure + bmslp[(bmslp<500)|(bmslp>1500)]=np.nan + result['mslp']=np.array(bmslp) + + if 'DEWT' in ds.keys(): + bdwp = np.nanmean(ds['DEWT'].values[:,:],axis=1) # Dewpoint + bdwp[np.abs(bdwp)>80]=np.nan + result['dewpt_temp']=np.array(bdwp) + + if 'DRYT' in ds.keys(): + btmp = np.nanmean(ds['DRYT'].values[:,:],axis=1) # air temperature + btmp[np.abs(btmp)>80]=np.nan + result['air_temp']=np.array(btmp) + + if 'GSPD' in ds.keys(): + bgst = np.nanmean(ds['GSPD'].values[:,:],axis=1) # gust + bgst=np.copy(((10./4.0)**(0.12))*bgst) # conversion to 10m, approximation DNVGL C-205 Table 2-1 + bgst[(bgst<0)|(bgst>200)]=np.nan + result['gust']=np.array(bgst) + + if 'WSPD' in ds.keys(): + bwsp = np.nanmean(ds['WSPD'].values[:,:],axis=1) # wind speed + bwsp=np.copy(((10./4.0)**(0.12))*bwsp) # conversion to 10m, approximation DNVGL C-205 Table 2-1 + bwsp[(bwsp<0)|(bwsp>150)]=np.nan + result['wind_spd']=np.array(bwsp) + + if 'WDIR' in ds.keys(): + bwdir = np.nanmean(ds['WDIR'].values[:,:],axis=1) # wind direction + bwdir[(bwdir<-180)|(bwdir>360)]=np.nan + result['wind_dir']=np.array(bwdir) + + if 'VCMX' in ds.keys(): + bhcmax = np.nanmean(ds['VCMX'].values[:,:],axis=1) # Hcrest max + bhcmax[(bhcmax<0)|(bhcmax>40)]=np.nan + result['hc_max']=np.array(bhcmax) + + if 'VMDR' in ds.keys(): + bdm = np.nanmean(ds['VMDR'].values[:,:],axis=1) # Mean direction + bdm[(bdm<-180)|(bdm>360)]=np.nan + result['dm']=np.array(bdm) + + if 'VPED' in ds.keys(): + bdp = np.nanmean(ds['VPED'].values[:,:],axis=1) # Peak direction + bdp[(bdp<-180)|(bdp>360)]=np.nan + result['dp']=np.array(bdp) + + return result + ds.close(); del ds + +# --- Altimeter --- +# Satellite data from Integrated Marine Observing System (IMOS), Australian Ocean Data Network (AODN) +def aodn_altimeter(satname,wconfig,datemin,datemax): + ''' + Read AODN altimeter data + http://thredds.aodn.org.au/thredds/catalog/IMOS/SRS/Surface-Waves/Wave-Wind-Altimetry-DM00/catalog.html + https://portal.aodn.org.au/ + Altimeter information: https://doi.org/10.1038/s41597-019-0083-9 + Inputs: + (1) satellite mission name. Select only one: + JASON3,JASON2,CRYOSAT2,JASON1,HY2,SARAL,SENTINEL3A,ENVISAT,ERS1,ERS2,GEOSAT,GFO,TOPEX,SENTINEL3B,CFOSAT + (2) wconfig dictionary, from wread.readconfig('ww3tools.yaml') + (3) initial date ('YYYYMMDDHH') + (4) final date ('YYYYMMDDHH') + Output: pandas dataframe containing: TIME (seconds since 1970), LATITUDE, LONGITUDE, WDEPTH, DISTCOAST, + HS, HS_CAL, WSPD, WSPD_CAL + Maryam Mohammadpour & Ricardo M. Campos + ''' + + # start time + start = timeit.default_timer() + + # date interval in seconds since 1970, user selection + adatemin= np.double(timegm( time.strptime(datemin, '%Y%m%d%H'))) + adatemax= np.double(timegm( time.strptime(datemax, '%Y%m%d%H'))) + + # Satellite missions available at AODN dataset, select only one. + sdname=np.array(['JASON3','JASON2','CRYOSAT2','JASON1','HY2','SARAL','SENTINEL3A','ENVISAT','ERS1','ERS2','GEOSAT','GFO','TOPEX','SENTINEL3B','CFOSAT']) + # Individual mission-specific Quality Control parameters + min_swh_numval = np.array([17,17,17,17,17,17,17,17,17,17,-inf,3,7,17,-inf]) + + if satname in sdname: + s=int(np.where(sdname==satname)[0]) + wconfig['min_swh_numval'] = min_swh_numval[s] + else: + raise ValueError("wread.aodn_altimeter; "+aodn_altimeter+" not included in the satellite missions available: "+", ".join(sdname) ) + + # name format for AODN reading + nsatname = re.sub(r'(\D)(\d)', r'\1-\2', satname) + + # Sat files (squares) considering the domain (lat lon from ww3tools.yaml) of interest, for the AODN file names + auxlat=np.array(np.arange(wconfig['latmin'],wconfig['latmax']+1.,1)).astype('int') + auxlon=np.array(np.arange(wconfig['lonmin'],wconfig['lonmax']+1.,1)).astype('int') + + # Read and allocate satellite data into arrays + ast=np.double(np.zeros((10**wconfig['pia']),'d')); aslat=np.zeros((10**wconfig['pia']),'f'); aslon=np.zeros((10**wconfig['pia']),'f') + ahsk=np.zeros((10**wconfig['pia']),'f'); ahskcal=np.zeros((10**wconfig['pia']),'f') + awnd=np.zeros((10**wconfig['pia']),'f'); awndcal=np.zeros((10**wconfig['pia']),'f'); asig0knstd=np.zeros((10**wconfig['pia']),'f') + aswhknobs=np.zeros((10**wconfig['pia']),'f'); aswhknstd=np.zeros((10**wconfig['pia']),'f'); aswhkqc=np.zeros((10**wconfig['pia']),'f') + aswdepth=np.zeros((10**wconfig['pia']),'f'); asdistcoast=np.zeros((10**wconfig['pia']),'f') + ii=0 + for j in auxlat: + for k in auxlon: + + if j>=0: + hem='N' + else: + hem='S' + + try: + fu=nc.Dataset(wconfig['path_alt']+satname+'/IMOS_SRS-Surface-Waves_MW_'+nsatname+'_FV02_'+str(np.abs(j)).zfill(3)+hem+'-'+str(k).zfill(3)+'E-DM00.nc') + except: + print(' '+wconfig['path_alt']+satname+'/IMOS_SRS-Surface-Waves_MW_'+nsatname+'_FV02_'+str(np.abs(j)).zfill(3)+hem+'-'+str(k).zfill(3)+'E-DM00.nc does not exist') + else: + st=np.double(fu.variables['TIME'][:]*24.*3600.+float(timegm( time.strptime('1985010100', '%Y%m%d%H') ))) + indt=np.where((st>=adatemin-wconfig['maxti']) & (st<=adatemax+wconfig['maxti'])) + # check if there is valid records inside the time range of interest + if size(indt)>10: + indt=indt[0] + # it does not read using the indexes because it is much slower + slat=fu.variables['LATITUDE'][:] + slon=fu.variables['LONGITUDE'][:] + swdepth=fu.variables['BOT_DEPTH'][:] + sdistcoast=fu.variables['DIST2COAST'][:] + wnd=fu.variables['WSPD'][:] + wndcal=fu.variables['WSPD_CAL'][:] + try: + hsk=fu.variables['SWH_KU'][:] + hskcal=fu.variables['SWH_KU_CAL'][:] + sig0knstd=fu.variables['SIG0_KU_std_dev'][:] + swhknobs=fu.variables['SWH_KU_num_obs'][:] + swhknstd=fu.variables['SWH_KU_std_dev'][:] + swhkqc=fu.variables['SWH_KU_quality_control'][:] + except: + print(' error reading KU, pick KA') + hsk=fu.variables['SWH_KA'][:] + hskcal=fu.variables['SWH_KA_CAL'][:] + sig0knstd=fu.variables['SIG0_KA_std_dev'][:] + swhknobs=fu.variables['SWH_KA_num_obs'][:] + swhknstd=fu.variables['SWH_KA_std_dev'][:] + swhkqc=fu.variables['SWH_KA_quality_control'][:] + + if ii+len(indt) <= ast.shape[0] : + # check the file is correct + if (st.shape[0]==wnd.shape[0]) & (slat.shape[0]==slon.shape[0]) & (hsk.shape[0]==hskcal.shape[0]) : + ast[ii:ii+len(indt)]=np.array(st[indt]).astype('double') + aslat[ii:ii+len(indt)]=np.array(slat[indt]).astype('float') + aslon[ii:ii+len(indt)]=np.array(slon[indt]).astype('float') + aswdepth[ii:ii+len(indt)]=np.array(swdepth[indt]).astype('float') + asdistcoast[ii:ii+len(indt)]=np.array(sdistcoast[indt]).astype('float') + ahsk[ii:ii+len(indt)]=np.array(hsk[indt]).astype('float') + ahskcal[ii:ii+len(indt)]=np.array(hskcal[indt]).astype('float') + awnd[ii:ii+len(indt)]=np.array(wnd[indt]).astype('float') + awndcal[ii:ii+len(indt)]=np.array(wndcal[indt]).astype('float') + asig0knstd[ii:ii+len(indt)]=np.array(sig0knstd[indt]).astype('float') + aswhknobs[ii:ii+len(indt)]=np.array(swhknobs[indt]).astype('float') + aswhknstd[ii:ii+len(indt)]=np.array(swhknstd[indt]).astype('float') + aswhkqc[ii:ii+len(indt)]=np.array(swhkqc[indt]).astype('float') + ii=ii+len(indt) + + else: + raise ValueError("gridSat_Altimeter.py; Small array to allocate the satellite data! Increase the power of initial array in ww3tools.yaml (pia)") + + del indt,st,slat,slon,swdepth,sdistcoast,hsk,hskcal,wnd,wndcal,sig0knstd,swhknobs,swhknstd,swhkqc + fu.close(); del fu + + # print(repr(j)+" "+repr(k)) + + # print(' Done reading and allocating satellite data '+satname) + del ii + + # water depth is positive by definition + aswdepth=aswdepth*-1. + + # Quality Control Check (optional) ---- + if wconfig['qc']==0: + indq = np.where( (ast>=adatemin) & (ast<=adatemax) ) + else: + indq = np.where( (aswdepth>=wconfig['mindepth']) & (asdistcoast>=wconfig['mindfc']) & (aswhknstd<=wconfig['max_swh_rms']) & + (asig0knstd<=wconfig['max_sig0_rms']) & (aswhknobs>=wconfig['min_swh_numval']) & (aswhkqc<=wconfig['max_swh_qc']) & + (ahsk>0.1) & (ahsk0.2) & (awnd0.1) & (ahskcal0.2) & (awndcal=adatemin) & (ast<=adatemax) ) + + del asig0knstd,aswhknobs,aswhknstd,aswhkqc,adatemin,adatemax + + if size(indq)>2: + indq=indq[0] + ast=np.double(np.copy(ast[indq])) + aslat=np.copy(aslat[indq]); aslon=np.copy(aslon[indq]) + aswdepth=np.copy(aswdepth[indq]); asdistcoast=np.copy(asdistcoast[indq]) + ahsk=np.copy(ahsk[indq]); ahskcal=np.copy(ahskcal[indq]) + awnd=np.copy(awnd[indq]); awndcal=np.copy(awndcal[indq]) + # dictionary + daodn = {'TIME': ast, 'LATITUDE': aslat, 'LONGITUDE': aslon, + 'WDEPTH': aswdepth, 'DISTCOAST': asdistcoast, + 'HS': ahsk, 'HS_CAL': ahskcal, + 'WSPD': awnd, 'WSPD_CAL': awndcal} + + else: + daodn = {'TIME': [], 'LATITUDE': [], 'LONGITUDE': [], + 'WDEPTH': [], 'DISTCOAST': [], + 'HS': [], 'HS_CAL': [], + 'WSPD': [], 'WSPD_CAL': []} + + AODN = pd.DataFrame(daodn) + stop = timeit.default_timer() + print('wread.aodn_altimeter successfully completed in '+repr(int(round(stop - start,0)))+' seconds. '+satname) + return AODN + + +# ========== MODEL ====================== +# --- WAVEWATCH III --- + +# WAVEWATCH III point output, netcdf format +def tseriestxt_ww3(*args): + ''' + WAVEWATCH III, time series/table, text tab format + This file format has all point outputs (results) in the same file (not divided by point/buoy). + Input: file name (example: tab50.ww3), and number of point ouputs (example: 4) + Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, + and arrays with the wave variables available. Inside the dictionary, the arrays of wave variables + have dimension (point_outputs, time). + ''' + if len(args) == 2: + fname=str(args[0]); tnb=int(args[1]) + elif len(args) < 2 : + sys.exit(' Two inputs are required: file name and station name') + elif len(args) > 2: + sys.exit(' Too many inputs') + + try: + mcontent = open(fname).readlines() + except: + sys.exit(" Cannot open "+fname) + else: + + tt = int(size(mcontent)/(7+tnb)+1) + myear = []; mmonth = [] ; mday = [] ; mhour = []; mmin = [] + mlon = np.zeros((tnb,tt),'f'); mlat = np.zeros((tnb,tt),'f'); mhs = np.zeros((tnb,tt),'f'); mL = np.zeros((tnb,tt),'f') + mtm = np.zeros((tnb,tt),'f'); mdm = np.zeros((tnb,tt),'f'); mspr = np.zeros((tnb,tt),'f') + atp = np.zeros((tnb,tt),'f'); mdp = np.zeros((tnb,tt),'f'); mpspr = np.zeros((tnb,tt),'f') + for i in range(0,tt): + j = i*(7+tnb) + myear = np.append(myear, int(mcontent[j].split(':')[1].split(' ')[1].split('/')[0]) ) + mmonth = np.append(mmonth, int(mcontent[j].split(':')[1].split(' ')[1].split('/')[1]) ) + mday = np.append(mday, int(mcontent[j].split(':')[1].split(' ')[1].split('/')[2]) ) + mhour = np.append(mhour, int(mcontent[j].split(':')[1].split(' ')[2]) ) + mmin = np.append(mmin, int(mcontent[j].split(':')[2]) ) + for k in range(0,tnb): + mlon[k,i] = mcontent[j+tnb+1+k].strip().split()[0] + mlat[k,i] = mcontent[j+tnb+1+k].strip().split()[1] + mhs[k,i] = mcontent[j+tnb+1+k].strip().split()[2] + mL[k,i] = mcontent[j+tnb+1+k].strip().split()[3] + mtm[k,i] = mcontent[j+tnb+1+k].strip().split()[4] + mdm[k,i] = mcontent[j+tnb+1+k].strip().split()[5] + mspr[k,i] = mcontent[j+tnb+1+k].strip().split()[6] + atp[k,i] = mcontent[j+tnb+1+k].strip().split()[7] + mdp[k,i] = mcontent[j+tnb+1+k].strip().split()[8] + mpspr[k,i] = mcontent[j+tnb+1+k].strip().split()[9] + + mtp = np.zeros((atp.shape[0],atp.shape[1]),'f')*np.nan + for i in range(0,mtp.shape[0]): + #mtp[i,atp[i,:]>0.0] = 1./atp[i,atp[i,:]>0.0] + indtp=np.where(atp[i,:]>0.0) + if size(indtp)>0: + mtp[i,indtp] = np.copy(1./atp[i,indtp]) + del indtp + + mdate = pd.to_datetime(dict(year=myear,month=mmonth,day=mday,hour=mhour,minute=mmin)) + mtime=np.zeros(mdate.shape[0],'d') + for i in range(0,mtime.shape[0]): + mtime[i]=double(mdate[i].timestamp()) + + result={'latitude':mlat,'longitude':mlon, + 'time':mtime,'date':mdate, + 'hs':mhs,'lm':mL,'tm':mtm,'dm':mdm, + 'spr':mspr,'tp':mtp,'dp':mdp,'spr_dp':mpspr} + + return result + del mdate,mtime,mlon,mlat,mhs,mL,mtm,mdm,mspr,atp,mtp,mdp,mpspr + +def tseriesnc_ww3(*args): + ''' + WAVEWATCH III, time series/table, netcdf format + Input: file name (example: ww3gefs.20160928_tab.nc), and station name (example: 41002) + Output: dictionary containing the arrays: time(seconds since 1970),time(datetime64),lat,lon, + and arrays with the wave variables available. + ''' + if len(args) == 2: + fname=str(args[0]); stname=str(args[1]) + elif len(args) < 2 : + sys.exit(' Two inputs are required: file name and station name') + elif len(args) > 2: + sys.exit(' Too many inputs') + + try: + ds = xr.open_dataset(fname); f=nc.Dataset(fname) + except: + sys.exit(" Cannot open "+fname) + else: + mtime = np.array(f.variables['time'][:]*24*3600 + timegm( strptime(str(f.variables['time'].units).split(' ')[2][0:4]+'01010000', '%Y%m%d%H%M') )).astype('double') + f.close(); del f + + auxstationname=ds['station_name'].values[:,:]; stationname=[] + for i in range(0,auxstationname.shape[0]): + stationname=np.append(stationname,"".join(np.array(auxstationname[i,:]).astype('str'))) + + inds=np.where(stationname[:]==stname) + if size(inds)>0: + inds=int(inds[0][0]); stname=str(stationname[inds]) + else: + sys.exit(' Station '+stname+' not included in the ww3 output file, or wrong station ID') + + mlat = np.nanmean(ds['latitude'].values[:,inds]) + mlon = np.nanmean(ds['longitude'].values[:,inds]) + # dictionary + result={'latitude':np.array(mlat),'longitude':np.array(mlon), + 'time':mtime,'date':ds['time'].values[:]} + + if 'hs' in ds.keys(): + mhs = ds['hs'].values[:,inds] + mhs[(mhs<0)|(mhs>30)]=np.nan + result['hs']=np.array(mhs) + elif 'swh' in ds.keys(): + mhs = ds['swh'].values[:,inds] + mhs[(mhs<0)|(mhs>30)]=np.nan + result['hs']=np.array(mhs) + + if 'fp' in ds.keys(): + mtp = np.zeros(mhs.shape[0],'f')*np.nan + indtp=np.where(ds['fp'].values[:,inds]>0.0) + if size(indtp)>0: + mtp[indtp] = np.copy(1./ds['fp'].values[indtp,inds]) + del indtp + mtp[(mtp<0)|(mtp>40)]=np.nan + + result['tp']=np.array(mtp) + if 'tr' in ds.keys(): + mtm = ds['tr'].values[:,inds] + mtm[(mtm<0)|(mtm>40)]=np.nan + result['tm']=np.array(mtm) + if 'th1p' in ds.keys(): + mdp = ds['th1p'].values[:,inds] + mdp[(mdp<-180)|(mdp>360)]=np.nan + result['dp']=np.array(mdp) + if 'th1m' in ds.keys(): + mdm = ds['th1m'].values[:,inds] + mdm[(mdm<-180)|(mdm>360)]=np.nan + result['dm']=np.array(mdm) + if 'sth1m' in ds.keys(): + result['spr']=np.array(ds['sth1m'].values[:,inds]) + if 'lm' in ds.keys(): + result['lm']=np.array(ds['lm'].values[:,inds]) + if 'sth1p' in ds.keys(): + result['spr_dp']=np.array(ds['sth1p'].values[:,inds]) + + return result + ds.close(); del ds + + +# Operational WW3 formats + +def bull(*args): + ''' + WAVEWATCH III, bull operational point output, see https://www.ftp.ncep.noaa.gov/data/nccf/com/ + Input: file name (example: gefs.wave.41004.bull) + Output: dictionary containing: + time(seconds since 1970),time(datetime64),lat,lon,station name; Arrays: hs, tp, and dp (gfs only) + ''' + if len(args) == 1: + fname=str(args[0]) + elif len(args) > 1: + sys.exit(' Too many inputs') + + # confirm format + if 'bull' in str(fname).split('/')[-1]: + + print(" reading ww3 bull file ...") + at=[]; adate=[]; ahs=[]; atp=[]; adp=[] + stname=str(fname).split('/')[-1].split('.')[-2] + + try: + tfile = open(fname, 'r'); lines = tfile.readlines() + except: + sys.exit(' Cannot open '+fname) + else: + # GEFS specific format + if 'gefs' in str(fname).split('/')[-1]: + iauxhs=[10,15];iauxtp=[28,33] + + # lat / lon + auxpos=str(lines[1]).split('(')[1].split('N') + alat=float(auxpos[0]) + alon=float(auxpos[1].split('W')[0]) + + # time ---- + auxdate = str(lines[3]).split(':')[1].split('UTC')[0][1::] + auxt = np.double(timegm( strptime( auxdate[0:8]+' '+auxdate[10:12]+'00', '%Y%m%d %H%M') )) + year = int(time.gmtime(auxt)[0]); month = int(time.gmtime(auxt)[1]) + pday=0 + for j in range(9,size(lines)-8): + day=int(lines[j][2:4]); hour=int(lines[j][5:7]) + if day0: + ahs=np.append(ahs,float(lines[j][iauxhs[0]:iauxhs[1]])) + atp=np.append(atp,float(lines[j][iauxtp[0]:iauxtp[1]])) + else: + ahs=np.append(ahs,np.nan) + atp=np.append(atp,np.nan) + + # build dictionary + result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), + 'latitude':alat,'longitude':alon,'station_name':stname, + 'hs':np.array(ahs),'tp':np.array(atp)} + + # GFS, HAFS, and other formats + else: + iauxhs=[24,30];iauxtp=[30,34];iauxdp=[35,38] + + # lat / lon + auxpos=str(lines[0]).replace("b'","").split('(')[1] + if auxpos[5]=='N': + alat=float(auxpos[0:5]) + else: + alat=-1.*float(auxpos[0:5]) + + if auxpos[13]=='E': + alon=float(auxpos[7:13]) + else: + alon=-1.*float(auxpos[7:13]) + + # time ---- + auxdate = str(lines[2]).split(':')[1].split('UTC')[0][1::] + auxt = np.double(timegm( strptime( auxdate[0:8]+' '+auxdate[9:11]+'00', '%Y%m%d %H%M') )) + year = int(time.gmtime(auxt)[0]); month = int(time.gmtime(auxt)[1]) + pday=0 + for j in range(7,size(lines)-8): + day=int(lines[j][3:5]); hour=int(lines[j][6:8]) + if day0: + ahs=np.append(ahs,float(lines[j][10:15])) + + # aux... is organizing the partitions. Ready for future versions (not included yet) + auxhs=[] + for k in range(0,4): + if len(str(lines[j][int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)]).replace(' ', '')): + auxhs=np.append(auxhs,float(lines[j][int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)])) + else: + auxhs=np.append(auxhs,np.nan) + + auxtp=[] + for k in range(0,4): + if len(str(lines[j][int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)]).replace(' ', '')): + auxtp=np.append(auxtp,float(lines[j][int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)])) + else: + auxtp=np.append(auxtp,np.nan) + + auxdp=[] + for k in range(0,4): + if len(str(lines[j][int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)]).replace(' ', '')): + auxdp=np.append(auxdp,float(lines[j][int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)])) + else: + auxdp=np.append(auxdp,np.nan) + + if np.nanmin(auxhs)>-999: + indaux=np.nanmin(np.where(auxhs==np.nanmax(auxhs))[0]) + else: + indaux=0 + + atp=np.append(atp,float(auxtp[indaux])) + adp=np.append(adp,float(auxdp[indaux])) + del indaux,auxhs,auxtp,auxdp + else: + ahs=np.append(ahs,np.nan) + atp=np.append(atp,np.nan) + adp=np.append(adp,np.nan) + + # build dictionary + result={'latitude':alat,'longitude':alon,'station_name':stname, + 'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), + 'hs':np.array(ahs),'tp':np.array(atp),'dp':np.array(adp)} + + del adp + + print(" Model data read, "+fname+", bull format.") + return result + del result,alat,alon,at,adate,ahs,atp,tfile,lines + else: + sys.exit(" Skipped file "+fname+" Not bull_tar format.") + + +def bull_tar(*args): + ''' + WAVEWATCH III, bull_tar operational point output, see https://www.ftp.ncep.noaa.gov/data/nccf/com/ + Input: file name (example: gfswave.t00z.bull_tar) + Output: dictionary containing: + time(seconds since 1970),time(datetime64),lat,lon,station names; Arrays: hs, tp, and dp (gfs only) + ''' + if len(args) == 1: + fname=str(args[0]) + elif len(args) > 1: + sys.exit(' Too many inputs') + + # confirm file format + if ('bull' in str(fname).split('/')[-1]) and ('tar' in str(fname).split('/')[-1]): + + print(" reading ww3 bull_tar file ...") + import tarfile + stname=[] + + try: + tar = tarfile.open(fname) + except: + sys.exit(' Cannot open '+fname) + else: + at=[]; adate=[]; alat=[]; alon=[] + + # GEFS specific format + if 'gefs' in str(fname).split('/')[-1]: + iauxhs=[10,15];iauxtp=[28,33] + + for t in range(0,size(tar.getmembers())): + # station names + stname=np.append(stname,str(str(tar.getmembers()[t].name).split('/')[-1]).split('/')[-1].split('.')[-2]) + + try: + tfile=tar.extractfile(tar.getmembers()[t]); lines = tfile.readlines() + except: + print(" Cannot open "+tar.getmembers()[t].name) + else: + # lat / lon + auxpos=str(lines[1]).replace("b'","").split('(')[1].split('N') + alat=np.append(alat,float(auxpos[0])) + alon=np.append(alon,float(auxpos[1].split('W')[0])) + + if t==0: + # time array ---- + auxdate = str(lines[3]).split(':')[1].split('UTC')[0][1::] + auxt = np.double(timegm( strptime( auxdate[0:8]+' '+auxdate[10:12]+'00', '%Y%m%d %H%M') )) + year = int(time.gmtime(auxt)[0]); month = int(time.gmtime(auxt)[1]) + pday=0 + for j in range(9,size(lines)-8): + auxlines = str(lines[j]).replace("b'","") + day=int(auxlines[2:4]); hour=int(auxlines[5:7]); del auxlines + if day0: + auxhs=np.append(auxhs,float(auxlines[iauxhs[0]:iauxhs[1]])) + auxtp=np.append(auxtp,float(auxlines[iauxtp[0]:iauxtp[1]])) + else: + auxhs=np.append(auxhs,np.nan) + auxtp=np.append(auxtp,np.nan) + + del auxlines + + if ahs.shape[1]==auxhs.shape[0]: + ahs[t,:]=np.array(auxhs) + atp[t,:]=np.array(auxtp) + else: + print(" Time duration of "+tar.getmembers()[t]+" (in "+fname+") do not match the other stations. Mantained NaN.") + + del auxhs,auxtp,tfile,lines + + # build dictionary + result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), + 'latitude':np.array(alat),'longitude':np.array(alon),'station_name':np.array(stname), + 'hs':np.array(ahs),'tp':np.array(atp)} + + # GFS, HAFS, and other formats + else: + iauxhs=[24,30];iauxtp=[30,34];iauxdp=[35,38] + + for t in range(0,size(tar.getmembers())): + # station names + stname=np.append(stname,str(str(tar.getmembers()[t].name).split('/')[-1]).split('/')[-1].split('.')[-2]) + + try: + tfile=tar.extractfile(tar.getmembers()[t]); lines = tfile.readlines() + except: + print(" Cannot open "+tar.getmembers()[t].name) + else: + + # lat / lon + auxpos=str(lines[0]).replace("b'","").split('(')[1] + if auxpos[5]=='N': + alat=np.append(alat,float(auxpos[0:5])) + else: + alat=np.append(alat,-1.*float(auxpos[0:5])) + + if auxpos[13]=='E': + alon=np.append(alon,float(auxpos[7:13])) + else: + alon=np.append(alon,-1.*float(auxpos[7:13])) + + if t==0: + # time array ---- + auxdate = str(lines[2]).split(':')[1].split('UTC')[0][1::] + auxt = np.double(timegm( strptime( auxdate[0:8]+' '+auxdate[9:11]+'00', '%Y%m%d %H%M') )) + year = int(time.gmtime(auxt)[0]); month = int(time.gmtime(auxt)[1]) + pday=0 + for j in range(7,size(lines)-8): + auxlines = str(lines[j]).replace("b'","") + day=int(auxlines[3:5]); hour=int(auxlines[6:8]); del auxlines + if day0: + auxhs=np.append(auxhs,float(auxlines[10:15])) + fuxhs=[] + for k in range(0,4): + if len(str(auxlines[int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)]).replace(' ', '')): + fuxhs=np.append(fuxhs,float(auxlines[int(iauxhs[0]+18*k):int(iauxhs[1]+18*k)])) + else: + fuxhs=np.append(fuxhs,np.nan) + + fuxtp=[] + for k in range(0,4): + if len(str(auxlines[int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)]).replace(' ', '')): + fuxtp=np.append(fuxtp,float(auxlines[int(iauxtp[0]+18*k):int(iauxtp[1]+18*k)])) + else: + fuxtp=np.append(fuxtp,np.nan) + + fuxdp=[] + for k in range(0,4): + if len(str(auxlines[int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)]).replace(' ', '')): + fuxdp=np.append(fuxdp,float(auxlines[int(iauxdp[0]+18*k):int(iauxdp[1]+18*k)])) + else: + fuxdp=np.append(fuxdp,np.nan) + + if np.nanmin(fuxhs)>-999: + indaux=np.nanmin(np.where(fuxhs==np.nanmax(fuxhs))[0]) + else: + indaux=0 + + auxtp=np.append(auxtp,float(fuxtp[indaux])) + auxdp=np.append(auxdp,float(fuxdp[indaux])) + del indaux,fuxhs,fuxtp,fuxdp + else: + auxhs=np.append(auxhs,np.nan) + auxtp=np.append(auxtp,np.nan) + auxdp=np.append(auxdp,np.nan) + + if ahs.shape[1]==auxhs.shape[0]: + ahs[t,:]=np.array(auxhs) + atp[t,:]=np.array(auxtp) + adp[t,:]=np.array(auxdp) + else: + print(" Time duration of "+tar.getmembers()[t]+" (in "+fname+") do not match the other stations. Mantained NaN.") + + del auxhs,auxtp,auxdp,tfile,lines + + # build dictionary + result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), + 'latitude':np.array(alat),'longitude':np.array(alon),'station_name':np.array(stname), + 'hs':np.array(ahs),'tp':np.array(atp),'dp':np.array(adp)} + + del adp + + print(" Model data read, "+fname+", bull_tar format.") + return result + del result,tar,alat,alon,ahs,atp,at,adate + else: + sys.exit(" Skipped file "+fname+" Not bull_tar format.") + + +def ts(*args): + ''' + WAVEWATCH III, ts operational point output, see https://www.ftp.ncep.noaa.gov/data/nccf/com/ + Input: file name (example: gefs.wave.41004.ts) + Output: dictionary containing: + time(seconds since 1970),time(datetime64),station name; Arrays: hs, hs_spr, tp (glwu or gefs) + ''' + if len(args) == 1: + fname=str(args[0]) + elif len(args) > 1: + sys.exit(' Too many inputs') + + # confirm format + if str(fname).split('/')[-1].split('.')[-1]=='ts': + print(" reading ww3 ts file ...") + stname=str(fname).split('/')[-1].split('.')[-2] + try: + tfile = pd.read_csv(fname,skiprows=2); lines = tfile.values[:,0] + except: + sys.exit(' Cannot open '+fname) + else: + + if 'gefs' in str(fname).split('/')[-1]: + # gefs lakes ww3 format + at=[]; adate=[]; ahs=[]; ahspr=[]; atp=[] + for j in range(0,size(lines)): + at=np.append(at,np.double(timegm( strptime( lines[j][1:12]+'00', '%Y%m%d %H%M') ))) + adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) + + if len(lines[j])>0: + ahs=np.append(ahs,float(lines[j][13:18])) + ahspr=np.append(ahspr,float(lines[j][19:25])) + atp=np.append(atp,float(lines[j][27:32])) + else: + ahs=np.append(ahs,np.nan) + ahspr=np.append(ahspr,np.nan) + atp=np.append(atp,np.nan) + + # build dictionary + result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), + 'station_name':np.array(stname),'hs':np.array(ahs),'hs_spr':np.array(ahspr),'tp':np.array(atp)} + + print(" Model data read, "+fname+", ts format.") + return result + del result,at,adate,ahs,ahspr,atp,tfile,lines + + elif 'glwu' in str(fname).split('/')[-1]: + # great lakes ww3 format + at=[];adate=[];ahs=[];al=[];atr=[];adir=[];aspr=[];atp=[];ap_dir=[];ap_spr=[] + for j in range(0,size(lines)): + at=np.append(at,np.double(timegm( strptime( lines[j][2:13]+'00', '%Y%m%d %H%M') ))) + adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) + + if len(lines[j])>0: + ahs=np.append(ahs,float(lines[j][22:28])) + al=np.append(al,float(lines[j][31:35])) + atr=np.append(atr,float(lines[j][37:42])) + adir=np.append(adir,float(lines[j][44:49])) + aspr=np.append(aspr,float(lines[j][50:56])) + atp=np.append(atp,float(lines[j][57:64])) + ap_dir=np.append(ap_dir,float(lines[j][66:71])) + ap_spr=np.append(ap_spr,float(lines[j][72:78])) + else: + ahs=np.append(ahs,np.nan) + al=np.append(al,np.nan) + atr=np.append(atr,np.nan) + adir=np.append(adir,np.nan) + aspr=np.append(aspr,np.nan) + atp=np.append(atp,np.nan) + ap_dir=np.append(ap_dir,np.nan) + ap_spr=np.append(ap_spr,np.nan) + + atp[atp<0.01]=np.nan; atp=1./atp + + # build dictionary + result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), + 'station_name':np.array(stname), + 'hs':np.array(ahs),'l':np.array(al), + 'tm':np.array(atr),'dm':np.array(adir), + 'spr':np.array(aspr),'tp':np.array(atp), + 'dp':np.array(ap_dir),'peak_spr':np.array(ap_spr)} + + print(" Model data read, "+fname+", ts format.") + return result + del result,at,adate,ahs,al,atr,adir,aspr,atp,ap_dir,ap_spr,tfile,lines + + else: + sys.exit(" Skipped file "+fname+" Not ts format.") + + +def station_tar(*args): + ''' + WAVEWATCH III, station_tar operational point output, see https://www.ftp.ncep.noaa.gov/data/nccf/com/ + Input: file name (example: gefs.wave.t00z.station_tar) + Output: dictionary containing: + time(seconds since 1970),time(datetime64),station name; Arrays: hs, hs_spr, tp (gefs only) + ''' + if len(args) == 1: + fname=str(args[0]) + elif len(args) > 1: + sys.exit(' Too many inputs') + + # confirm format + if str(fname).split('/')[-1].split('.')[-1]=='station_tar': + print(" reading ww3 station_tar file ...") + import tarfile + stname=[] + + try: + tar = tarfile.open(fname) + except: + sys.exit(' Cannot open '+fname) + else: + for t in range(0,size(tar.getmembers())): + # station names + stname=np.append(stname,str(str(tar.getmembers()[t].name).split('/')[-1]).split('/')[-1].split('.')[-2]) + + try: + tfile=tar.extractfile(tar.getmembers()[t]); lines = tfile.readlines()[3::] + except: + print(" Cannot open "+tar.getmembers()[t].name) + else: + if t==0: + # time array ---- + at=[]; adate=[] + for j in range(0,size(lines)): + at=np.append(at,np.double(timegm( strptime( str(lines[j])[3:14]+'00', '%Y%m%d %H%M') ))) + adate=np.append(adate,date2num(datetime.datetime(time.gmtime(at[j])[0],time.gmtime(at[j])[1],time.gmtime(at[j])[2],time.gmtime(at[j])[3],time.gmtime(at[j])[4]))) + + # -------- + ahs=np.zeros((size(tar.getmembers()),at.shape[0]),'f')*np.nan + ahspr=np.zeros((size(tar.getmembers()),at.shape[0]),'f')*np.nan + atp=np.zeros((size(tar.getmembers()),at.shape[0]),'f')*np.nan + + auxhs=[]; auxhspr=[]; auxtp=[] + for j in range(0,size(lines)): + auxlines = str(lines[j]).replace("b'","") + if len(lines[j])>0: + auxhs=np.append(auxhs,float(auxlines[13:18])) + auxhspr=np.append(auxhspr,float(auxlines[19:25])) + auxtp=np.append(auxtp,float(auxlines[27:32])) + else: + auxhs=np.append(auxhs,np.nan) + auxhspr=np.append(auxhspr,np.nan) + auxtp=np.append(auxtp,np.nan) + + if ahs.shape[1]==auxhs.shape[0]: + ahs[t,:]=np.array(auxhs) + ahspr[t,:]=np.array(auxhspr) + atp[t,:]=np.array(auxtp) + else: + print(" Time duration of "+tar.getmembers()[t]+" (in "+fname+") do not match the other stations. Mantained NaN.") + + del auxhs,auxhspr,auxtp,tfile,lines + + # build dictionary + result={'time':np.array(at).astype('double'),'date':np.array(adate).astype('double'), + 'station_name':np.array(stname),'hs':np.array(ahs),'hs_spr':np.array(ahspr),'tp':np.array(atp)} + + return result + del result,tar,ahs,ahspr,atp,at,adate + + print(" Model data read, "+fname+", station_tar format.") + + else: + sys.exit(" Skipped file "+fname+" Not station_tar format.") + + +# SPECTRA + +# Observations NDBC, netcdf format +def spec_ndbc(*args): + ''' + Observations NDBC, wave spectrum, netcdf format + Input: file name (example: 46047w2016.nc) + Output: dictionary containing: + time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pspec,dmspec,dpspec,dirspec + ''' + sk=1; deltatheta=int(10) + if len(args) >= 1: + fname=str(args[0]) + if len(args) >= 2: + sk=int(args[1]) + if len(args) >= 3: + deltatheta=int(args[3]) + if len(args) > 3: + sys.exit(' Too many inputs') + + try: + ds = xr.open_dataset(fname); f=nc.Dataset(fname) + except: + sys.exit(" Cannot open "+fname) + else: + btime = np.array(f.variables['time'][::sk]).astype('double') + f.close(); del f + bdate = ds['time'].values[::sk] + blat = ds['latitude'].values[:] + blon = ds['longitude'].values[:] + freq = ds['frequency'].values[:] + pspec = ds['spectral_wave_density'].values[::sk,:,0,0] + dmspec = ds['mean_wave_dir'][::sk,:,0,0] + dpspec = ds['principal_wave_dir'][::sk,:,0,0] + r1spec = ds['wave_spectrum_r1'][::sk,:,0,0] + r2spec = ds['wave_spectrum_r2'][::sk,:,0,0] + ds.close(); del ds + # DF in frequency (dfreq), https://www.ndbc.noaa.gov/wavespectra.shtml + if int(freq.shape[0])==47: + dfreq=np.zeros(47,'f') + dfreq[0]=0.010; dfreq[1:14]=0.005; dfreq[14:40]=0.010; dfreq[40::]=0.020 + else: + dfreq=np.zeros(freq.shape[0],'f')+0.01 + + pspec=np.array(pspec*dfreq) + # Directional 2D Spectrum, https://www.ndbc.noaa.gov/measdes.shtml#swden , https://www.ndbc.noaa.gov/wavemeas.pdf + theta = np.array(np.arange(0,360+0.1,deltatheta)) + # final directional wave spectrum (frequency X direction) + dirspec = np.zeros((btime.shape[0],freq.shape[0],theta.shape[0]),'f') + for t in range(0,btime.shape[0]): + dirspec[t,:,:] = np.array([pspec[t,:]]).T * (1/pi)*(0.5+ np.array([r1spec[t,:]]).T * cos(np.array( np.array([theta])-np.array([dmspec[t,:]]).T )*(pi/180)) + + np.array([r2spec[t,:]]).T*cos(2*np.array( np.array([theta]) - np.array([dpspec[t,:]]).T )*(pi/180))) + + # build dictionary + result={'time':btime,'date':bdate,'latitude':blat,'longitude':blon, + 'freq':freq,'deltafreq':dfreq,'pspec':pspec,'dmspec':dmspec,'dpspec':dpspec, + 'theta':theta,'dirspec':dirspec} + + return result + del btime,bdate,blat,blon,freq,dfreq,pspec,dmspec,dpspec,theta,dirspec + + +# WAVEWATCH III spectra output def spec_ww3(*args): + ''' + WAVEWATCH III, wave spectrum, netcdf (.nc) or text (.spec) format + Input: file name (example: ww3gefs.20160928_spec.nc), and station name (example: 41002) + Output: dictionary containing: + time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pwst,d1sp,dire,dspec,wnds,wndd + ''' + sk=1 + if len(args) < 2 : + sys.exit(' Two inputs are required: file name and station name') + if len(args) >= 2 : + fname=str(args[0]); stname=str(args[1]) + if len(args) > 2 : + sk=int(args[2]) + if len(args) > 3 : + sys.exit(' Too many inputs') + + if str(fname).split('/')[-1].split('.')[-1]=='nc': + + try: + ds = xr.open_dataset(fname); f=nc.Dataset(fname) + except: + sys.exit(" Cannot open "+fname) + else: + + mtime = np.array(f.variables['time'][::sk]*24*3600 + timegm( strptime(str(f.variables['time'].units).split(' ')[2][0:4]+'01010000', '%Y%m%d%H%M') )).astype('double') + f.close(); del f + + auxstationname=ds['station_name'].values[:,:]; stationname=[] + for i in range(0,auxstationname.shape[0]): + stationname=np.append(stationname,"".join(np.array(auxstationname[i,:]).astype('str'))) + + inds=np.where(stationname[:]==stname) + if size(inds)>0: + inds=int(inds[0][0]); stname=str(stationname[inds]) + else: + sys.exit(' Station '+stname+' not included in the output file, or wrong station ID') + + # Spectrum + dspec=np.array(ds['efth'][::sk,inds,:,:]) + # number of directions + nd=dspec.shape[2] + # number of frequencies + nf=dspec.shape[1] + # directions + dire=np.array(ds['direction'].values[:]) + # frequencies + freq=np.array(ds['frequency'].values[:]) + freq1=np.array(ds['frequency1'].values[:]) + freq2=np.array(ds['frequency2'].values[:]) + # DF in frequency (dfreq) + dfreq=np.array(freq2 - freq1) + # wind intensity and wind direction + wnds=np.array(ds['wnd'].values[::sk,inds]) + wndd=np.array(ds['wnddir'].values[::sk,inds]) + # Time datetime64 array + mdate=np.array(ds['time'].values[::sk]) + # water depth (constant in time) + depth=np.nanmean(ds['dpt'].values[::sk,inds],axis=0) + lon=np.array(np.nanmean(ds['longitude'].values[::sk,inds],axis=0)) + lat=np.array(np.nanmean(ds['latitude'].values[::sk,inds],axis=0)) + + ds.close(); del ds, auxstationname, inds, stationname + freq1=freq; freq2=freq + + else: + + # Text format (only one point allowed here, same as WW3/NOAA operational) + fp = open(fname); nt = fp.read().count(stname); fp.close(); del fp + if nt>=1: + # Open file and read the first parameters + fp = open(fname) + cabc=fp.readline(); cabc=cabc.strip().split() + nf=int(cabc[3]) # number of frequencies + nd=int(cabc[4]) # number of directions + npo=int(cabc[5]) # number of point outputs + + freq=zeros(nf,'f');dire=zeros(nd,'f') + dspec=zeros((nt,nf,nd),'f') + adire=zeros(dire.shape) + adspec=zeros(dspec.shape) + mtime=np.zeros((nt),'d') + + # Frequencies -------------------- + ncf=int(np.floor(nf/8));rncf=int(np.round(8*((float(nf)/8)-ncf))) + k=0 + for i in range(0,ncf): + line=fp.readline() + line=line.strip().split() + for j in range(0,8): + freq[k]=float(line[j]) + k=k+1 + + if rncf>0: + line=fp.readline() + line=line.strip().split() + for i in range(0,rncf): + freq[k]=float(line[i]) + k=k+1 + + # DF in frequency (dfreq) + dfreq=np.zeros(freq.shape[0],'f') + for i in range(0,freq.shape[0]): + if i==0 or i==(freq.shape[0]-1): + dfreq[i] = freq[i]*(1+ ( ((freq[-1]/freq[-2])-1)/2 )) - freq[i] + else: + dfreq[i] = freq[i]*(freq[-1]/freq[-2]) - freq[i] + + # Directions --------------------- + ncd=int(np.floor(nd/7));rncd=int(np.round(7*((float(nd)/7)-ncd))) + k=0 + for i in range(0,ncd): + line=fp.readline() + line=line.strip().split() + for j in range(0,7): + dire[k]=float(line[j])*180/pi + k=k+1 + + if rncd>0: + line=fp.readline() + line=line.strip().split() + for i in range(0,rncd): + dire[k]=float(line[i])*180/pi + k=k+1 + + nl=int(floor((nf*nd)/7.)); rnl=int(np.round(7*((float(nf*nd)/7)-nl))) + auxs=np.zeros((nf*nd),'f') + wnds=np.zeros((nt),'f');wndd=np.zeros((nt),'f') + + for t in range(0,nt): + + cabc=fp.readline(); cabc.strip().split()[0] + mtime[t] = np.double(timegm( strptime(cabc.strip().split()[0]+cabc.strip().split()[1][0:2], '%Y%m%d%H') )) + cabc=fp.readline(); cabc=cabc.strip().split() + if t==0: + namep=cabc[0][1:] + lat=float(cabc[2]);lon=float(cabc[3]) + depth=float(cabc[4]) + + wnds[t]=float(cabc[5]);wndd[t]=float(cabc[6]) + + k=0 + for i in range(0,nl): + line=fp.readline() + line=line.strip().split() + for j in range(0,7): + auxs[k]=float(line[j]) + k=k+1 + + if rncd>0: + line=fp.readline() + line=line.strip().split() + for i in range(0,rnl): + auxs[k]=float(line[i]) + k=k+1 + + for ic in range(0,nf): + for il in range(0,nd): + dspec[t,ic,il]=auxs[il*nf+ic] + + fp.close(); del fp + + # mdate = [date2num(datetime.datetime.utcfromtimestamp(time_stamp)) for time_stamp in mtime] + mdate = pd.to_datetime(mtime, unit='s').strftime('%Y-%m-%dT%H:%M:%S.%f') + freq1=freq*np.nan; freq2=freq*np.nan + + + # ------------------ + # 1D power spectrum + pwst=np.zeros((dspec.shape[0],nf),'f') + for t in range(0,dspec.shape[0]): + for il in range(0,nf): + pwst[t,il]=sum(dspec[t,il,:]*(2*np.pi)/nd) + + pwst[t,:]=pwst[t,:]*dfreq[:] + + # organizing directions ----- + adspec=np.copy(dspec); inddire=int(np.where(dire==min(dire))[0][0]) + for t in range(0,dspec.shape[0]): + adspec[t,:,0:nd-(inddire+1)]=dspec[t,:,(inddire+1):nd] + adspec[t,:,nd-(inddire+1):nd]=dspec[t,:,0:(inddire+1)] + for i in range(0,nd): + dspec[t,:,i]=adspec[t,:,nd-i-1] + + adspec[t,:,0:int(nd/2)]=dspec[t,:,int(nd/2):nd] + adspec[t,:,int(nd/2):nd]=dspec[t,:,0:int(nd/2)] + dspec[t,:,:]=adspec[t,:,:] + + dire=np.sort(dire) + + # 1D directional spectrum + d1sp=np.zeros((dspec.shape[0],nf),'f') + for t in range(0,dspec.shape[0]): + for il in range(0,nf): + a = np.sum(dspec[t,il,:] * np.array(np.sin((pi*dire)/180.)/np.sum(dspec[t,il,:])) ) + b = np.sum(dspec[t,il,:] * np.array(np.cos((pi*dire)/180.)/np.sum(dspec[t,il,:])) ) + aux = math.atan2(a,b)*(180./pi) + if aux<0: + aux=aux+360. + + d1sp[t,il]=float(aux) + del a,b,aux + + # build dictionary + result={'time':mtime,'date':mdate,'latitude':lat,'longitude':lon,'depth':depth, + 'wind_spd':wnds,'wind_dir':wndd,'freq':freq,'freq1':freq1,'freq2':freq2, + 'deltafreq':dfreq,'pspec':pwst,'theta':dire,'dmspec':d1sp,'dirspec':dspec} + + return result + del mtime,mdate,lat,lon,wnds,wndd,freq,freq1,freq2,dfreq,pwst,dire,d1sp,dspec + + +def spec1_ww3(*args): ''' WAVEWATCH III, wave spectrum, netcdf (.nc) or text (.spec) format Input: file names (list of file names), and station names (list of station names) @@ -125,9 +1726,9 @@ def spec_ww3(*args): cabc.strip().split()[0]+cabc.strip().split()[1][0:2], '%Y%m%d%H'))) cabc = fp.readline() cabc = cabc.strip().split() - print(cabc) + if len(cabc) >8: - # Format: ["'42085", "'", '17.86', '-66.52', '126.1', '1.12', '146.9'] + namep = cabc[0][1:] lat = float(cabc[2]) lon = float(cabc[3]) @@ -135,12 +1736,11 @@ def spec_ww3(*args): wnds_index = 5 wndd_index = 6 elif len(cabc) == 8: - # Format: ["'46021", "'", '57.70-160.00', '50.8', '5.47', '6.9', '0.00', '270.0'] + namep = cabc[0][1:] lat_lon_str = cabc[2] lat_lon_str = lat_lon_str.strip("'") lat_lon_parts = lat_lon_str.split('-') - print(lat_lon_parts) lat = float(lat_lon_parts[0]) lon = -float(lat_lon_parts[1]) @@ -148,8 +1748,8 @@ def spec_ww3(*args): wnds_index = 4 wndd_index = 5 else: - continue # Skip this file as it cannot be processed - # sys.exit('Unrecognized format of cabc') + continue + wnds[t] = float(cabc[wnds_index]) wndd[t] = float(cabc[wndd_index]) @@ -222,8 +1822,11 @@ def spec_ww3(*args): d1sp[t, il] = float(aux) del a, b, aux - hs = np.sqrt(2 * np.trapz(np.trapz(dspec, x=dire, axis=-1), x=freq, axis=-1)) - tp = freq[np.argmax(np.max(dspec, axis=-1), axis=-1)] + m0 = np.sum(pwst , axis=1) + hs = 4 * np.sqrt(m0) + max_index = np.argmax(pwst, axis=1) + f_max = freq[max_index] + tp = 1 / f_max # build dictionary result = {'time': mtime, 'date': mdate, 'latitude': lat, 'longitude': lon, 'depth': depth, @@ -237,6 +1840,3 @@ def spec_ww3(*args): return results -# Example usage: -# spec_ww3(['file1.spec', 'file2.spec'], ['station1', 'station2']) - From 9c09915d8eb269bab76d7b6f1720208fc8f2caa5 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Thu, 11 Apr 2024 23:07:18 +0000 Subject: [PATCH 07/27] removed --- ww3tools/openfile2.py | 51 ------------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 ww3tools/openfile2.py diff --git a/ww3tools/openfile2.py b/ww3tools/openfile2.py deleted file mode 100644 index f57c6ea..0000000 --- a/ww3tools/openfile2.py +++ /dev/null @@ -1,51 +0,0 @@ -import gzip -import tarfile -import os - -def unzip_and_untar(gz_file, output_dir): - # Extracting the filename without extension - file_name = os.path.splitext(os.path.basename(gz_file))[0] - print(file_name) - # Get the base name of the extracted folder - base_name = file_name.rsplit('.', 1)[0] # Get everything before the last dot - - # Setting the output path for the extracted file - output_file = os.path.join(output_dir, file_name) - - # Unzipping the .gz file - with gzip.open(gz_file, 'rb') as f_in: - with open(output_file, 'wb') as f_out: - f_out.write(f_in.read()) - - # Untaring the extracted file - with tarfile.open(output_file, 'r') as tar: - tar.extractall(output_dir) - - # Get the base name of the extracted folder - extracted_folder = os.path.join(output_dir, base_name) - - print(extracted_folder) - # Creating a list of the extracted files - list_file = os.path.join(output_dir, f'{base_name}_contents.txt') - with open(list_file, 'w') as f: - for root, dirs, files in os.walk(extracted_folder): - for file in files: - f.write(os.path.join(root, file) + '\n') - - # Creating a list of ids between two dots in the filenames - id_file = os.path.join(output_dir, f'{base_name}_id.txt') - with open(id_file, 'w') as f: - for root, dirs, files in os.walk(extracted_folder): - for file in files: - file_parts = file.split('.') - if len(file_parts) >= 3: - id_between_dots = file_parts[1] - f.write(id_between_dots + '\n') - -# Define the input .gz file and the output directory -gz_file = 'multi_1.t00z.spec_tar.gz' -output_dir = '.' # You can specify any output directory here - -# Call the function to unzip and untar the file -unzip_and_untar(gz_file, output_dir) - From c22d3c52aa7e95fc9d3ac72c05d383f3459b5bac Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Thu, 18 Apr 2024 13:39:41 +0000 Subject: [PATCH 08/27] removed an extra discription --- ww3tools/wread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ww3tools/wread.py b/ww3tools/wread.py index f833aa4..936a81c 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1645,7 +1645,7 @@ def spec1_ww3(*args): for fname in fnames: for stname in stnames: try: - # Text format (only one point allowed here, same as WW3/NOAA operational) + fp = open(fname) nt = fp.read().count(stname) fp.close() From e1f3616568dd7f4705d47fb0c80c3bab3da00ce1 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 25 Mar 2024 14:19:58 +0000 Subject: [PATCH 09/27] added .spec and also is able to to the hs calculations in the code --- ww3tools/wread.py | 197 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/ww3tools/wread.py b/ww3tools/wread.py index 74ce53c..babd39c 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1619,3 +1619,200 @@ def spec_ww3(*args): del mtime,mdate,lat,lon,wnds,wndd,freq,freq1,freq2,dfreq,pwst,dire,d1sp,dspec +<<<<<<< HEAD +======= +#added a function to read the txt files + +def read_text_file(fname_txtfile): + try: + # Attempt to open and read the file name from the txt file + with open(fname_txtfile, 'r') as f: + lines = f.readlines() + if len(lines) != 1: + raise ValueError("The txt file should contain only one line with the file name.") + fname = lines[0].strip() + except FileNotFoundError: + sys.exit('Text file not found.') + except Exception as e: + sys.exit(f'Error reading txt file: {str(e)}') + + results = {} + stname = [] + + try: + tar = tarfile.open(fname, "r:gz") # Open the tar file + + for t in range(0, len(tar.getmembers())): + # Station names + + stname.append(str(tar.getmembers()[t].name).split('/')[-1].split('.')[-2]) + + try: + fp = tar.extractfile(tar.getmembers()[t]) + if fp is None: + raise ValueError("File is empty or cannot be extracted.") + lines = fp.readlines() + nt=len(lines) + except Exception as e: + print("Cannot open " + tar.getmembers()[t].name) + print("Error:", str(e)) + continue + else: + if nt == 0: + print("No lines to read in file:", tar.getmembers()[t].name) + continue + for line in lines: + line = line.strip().decode() + + if nt >= 1: + # Open file and read the first parameters + fp = tar.extractfile(tar.getmembers()[t]) + cabc = fp.readline().strip().split() + nf = int(cabc[3]) # number of frequencies + nd = int(cabc[4]) # number of directions + npo = int(cabc[5]) # number of point outputs + + freq = np.zeros(nf, 'f') + dire = np.zeros(nd, 'f') + dspec = np.zeros((nt, nf, nd), 'f') + adire = np.zeros(dire.shape) + adspec = np.zeros(dspec.shape) + ntime = np.zeros((nt), 'd') + + # Frequencies -------------------- + ncf = int(np.floor(nf/8)) + rncf = int(np.round(8*((float(nf)/8)-ncf))) + k = 0 + for i in range(0, ncf): + line = fp.readline() + line = line.strip().split() + for j in range(0, 8): + freq[k] = float(line[j]) + k = k+1 + + if rncf > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rncf): + freq[k] = float(line[i]) + k = k+1 + + # DF in frequency (dfreq) + dfreq = np.zeros(freq.shape[0], 'f') + for i in range(0, freq.shape[0]): + if i == 0 or i == (freq.shape[0]-1): + dfreq[i] = freq[i]*(1 + (((freq[-1]/freq[-2])-1)/2)) - freq[i] + else: + dfreq[i] = freq[i]*(freq[-1]/freq[-2]) - freq[i] + + # Directions --------------------- + ncd = int(np.floor(nd/7)) + rncd = int(np.round(7*((float(nd)/7)-ncd))) + k = 0 + for i in range(0, ncd): + line = fp.readline() + line = line.strip().split() + for j in range(0, 7): + dire[k] = float(line[j])*180/np.pi + k = k+1 + + if rncd > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rncd): + dire[k] = float(line[i])*180/np.pi + k = k+1 + + nl = int(np.floor((nf*nd)/7.)) + rnl = int(np.round(7*((float(nf*nd)/7)-nl))) + auxs = np.zeros((nf*nd), 'f') + wnds = np.zeros((nt), 'f') + wndd = np.zeros((nt), 'f') + hs = np.zeros(nt) # Initialize significant wave height array + + for t in range(0, nt): + + cabc = [item.decode() for item in fp.readline().strip().split()] + + if not cabc: + continue + ntime[t] = np.double(timegm( strptime(cabc[0]+cabc[1][0:2], '%Y%m%d%H') )) + cabc = [item.decode() for item in fp.readline().strip().split()] + + if not cabc: + continue + if t == 0: + + if len(cabc) >= 8: + namep = cabc[0][1:] + lat_str = cabc[3] + lon_str = cabc[4] + lat = -float(lat_str) if lat_str.startswith('-') else float(lat_str) + lon = -float(lon_str) if lon_str.startswith('-') else float(lon_str) + depth = float(cabc[5]) + wnds[t] = float(cabc[6]) + wndd[t] = float(cabc[7]) + + + + elif len(cabc) == 7: + namep = cabc[0][1:-1] if cabc[0].startswith("'") and cabc[0].endswith("'") else cabc[0][1:] + print("Station Name:", namep) + lat_lon = cabc[1].split('-') + print("lat-lon:",lat_lon) + lat_str = lat_lon[0] + print("lat_str:",lat_str) + lon_str = lat_lon[1] + print("lon_str:",lon_str) + lat = -float(lat_str) if lat_str.startswith('-') else float(lat_str) + lon = -float(lon_str) if lon_str.startswith('-') else float(lon_str) + depth = float(cabc[3]) + wnds[t] = float(cabc[4]) + wndd[t] = float(cabc[5]) + + k = 0 + for i in range(0, nl): + line = fp.readline() + line = line.strip().split() + for j in range(0, 7): + auxs[k] = float(line[j]) + k = k+1 + + if rncd > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rnl): + auxs[k] = float(line[i]) + k = k+1 + + for ic in range(0, nf): + for il in range(0, nd): + dspec[t, ic, il] = auxs[il*nf+ic] + + # Calculate significant wave height + sp1d = np.sum(dspec[t], axis=1) * (np.abs(dire[1] - dire[0])) # Calculate 1D spectrum + hs[t] = 4 * np.sqrt(np.trapz(sp1d, x=freq)) # Calculate significant wave height + + fp.close() + + results['ntime'] = ntime + results['latitude'] = lat + results['longitude'] = lon + results['wind_spd'] = wnds + results['wind_dir'] = wndd + results['freq'] = freq + results['deltafreq'] = dfreq + results['dir'] = dire + results['spec'] = dspec + results['station_name'] = stname + results['hs'] = hs # Add significant wave height to results + + else: + sys.exit(f'Station {stname} not included in') + except Exception as e: + sys.exit(f'Error reading spectral file: {str(e)}') + + return results + + +>>>>>>> 5c304ea (added .spec and also is able to to the hs calculations in the code) From 6592b9682ceaf950a444bd0e205c6779249277ae Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 22 Apr 2024 07:25:18 +0000 Subject: [PATCH 10/27] modified --- ww3tools/wread.py | 367 ++++++++++++++++++++++++++-------------------- 1 file changed, 209 insertions(+), 158 deletions(-) diff --git a/ww3tools/wread.py b/ww3tools/wread.py index babd39c..527866c 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1,3 +1,4 @@ +<<<<<<< HEAD #!/usr/bin/env python3 # -*- coding: utf-8 -*- @@ -58,6 +59,8 @@ """ +======= +>>>>>>> 12a8a9d (wind variable added to the .spec in a list of buoy nams and boy IDs) import matplotlib import time import timeit @@ -76,7 +79,10 @@ # import pickle import sys import warnings; warnings.filterwarnings("ignore") +import tarfile +import math +<<<<<<< HEAD def readconfig(fname): """ @@ -126,16 +132,17 @@ def readconfig(fname): def mask(*args): +======= +def spec_ww3(*args): +>>>>>>> 12a8a9d (wind variable added to the .spec in a list of buoy nams and boy IDs) ''' - Read gridmask netcdf file generated with prepGridMask.py - Input: file name (example: gridInfo_GEFSv12.nc) - Output: dictionary containing the arrays and string names + WAVEWATCH III, wave spectrum, netcdf (.nc) or text (.spec) format + Input: file names (list of file names), and station names (list of station names) + Output: list of dictionaries containing: + time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pwst,d1sp,dire,dspec,wnds,wndd ''' - if len(args) == 1: - fname=str(args[0]) - else: - sys.exit(' Too many inputs') +<<<<<<< HEAD print(" reading ww3_tools mask ...") try: f=nc.Dataset(fname) @@ -1646,173 +1653,217 @@ def read_text_file(fname_txtfile): # Station names stname.append(str(tar.getmembers()[t].name).split('/')[-1].split('.')[-2]) +======= + if len(args) < 2: + sys.exit(' Two inputs are required: list of file names and list of station names') + + fnames = args[0] + stnames = args[1] + sk = 1 + if len(args) > 2: + sk = int(args[2]) + if len(args) > 3: + sys.exit(' Too many inputs') + + file_names = args[0] + station_names = args[1] + results = [] +>>>>>>> 12a8a9d (wind variable added to the .spec in a list of buoy nams and boy IDs) + for fname in fnames: + for stname in stnames: try: - fp = tar.extractfile(tar.getmembers()[t]) - if fp is None: - raise ValueError("File is empty or cannot be extracted.") - lines = fp.readlines() - nt=len(lines) - except Exception as e: - print("Cannot open " + tar.getmembers()[t].name) - print("Error:", str(e)) - continue - else: - if nt == 0: - print("No lines to read in file:", tar.getmembers()[t].name) - continue - for line in lines: - line = line.strip().decode() - - if nt >= 1: - # Open file and read the first parameters - fp = tar.extractfile(tar.getmembers()[t]) - cabc = fp.readline().strip().split() - nf = int(cabc[3]) # number of frequencies - nd = int(cabc[4]) # number of directions - npo = int(cabc[5]) # number of point outputs - - freq = np.zeros(nf, 'f') - dire = np.zeros(nd, 'f') - dspec = np.zeros((nt, nf, nd), 'f') - adire = np.zeros(dire.shape) - adspec = np.zeros(dspec.shape) - ntime = np.zeros((nt), 'd') - - # Frequencies -------------------- - ncf = int(np.floor(nf/8)) - rncf = int(np.round(8*((float(nf)/8)-ncf))) - k = 0 - for i in range(0, ncf): - line = fp.readline() - line = line.strip().split() - for j in range(0, 8): - freq[k] = float(line[j]) - k = k+1 - - if rncf > 0: - line = fp.readline() - line = line.strip().split() - for i in range(0, rncf): - freq[k] = float(line[i]) - k = k+1 - - # DF in frequency (dfreq) - dfreq = np.zeros(freq.shape[0], 'f') - for i in range(0, freq.shape[0]): - if i == 0 or i == (freq.shape[0]-1): - dfreq[i] = freq[i]*(1 + (((freq[-1]/freq[-2])-1)/2)) - freq[i] - else: - dfreq[i] = freq[i]*(freq[-1]/freq[-2]) - freq[i] - - # Directions --------------------- - ncd = int(np.floor(nd/7)) - rncd = int(np.round(7*((float(nd)/7)-ncd))) - k = 0 - for i in range(0, ncd): - line = fp.readline() - line = line.strip().split() - for j in range(0, 7): - dire[k] = float(line[j])*180/np.pi - k = k+1 - - if rncd > 0: - line = fp.readline() - line = line.strip().split() - for i in range(0, rncd): - dire[k] = float(line[i])*180/np.pi - k = k+1 - - nl = int(np.floor((nf*nd)/7.)) - rnl = int(np.round(7*((float(nf*nd)/7)-nl))) - auxs = np.zeros((nf*nd), 'f') - wnds = np.zeros((nt), 'f') - wndd = np.zeros((nt), 'f') - hs = np.zeros(nt) # Initialize significant wave height array - - for t in range(0, nt): - - cabc = [item.decode() for item in fp.readline().strip().split()] - - if not cabc: - continue - ntime[t] = np.double(timegm( strptime(cabc[0]+cabc[1][0:2], '%Y%m%d%H') )) - cabc = [item.decode() for item in fp.readline().strip().split()] - - if not cabc: - continue - if t == 0: - - if len(cabc) >= 8: - namep = cabc[0][1:] - lat_str = cabc[3] - lon_str = cabc[4] - lat = -float(lat_str) if lat_str.startswith('-') else float(lat_str) - lon = -float(lon_str) if lon_str.startswith('-') else float(lon_str) - depth = float(cabc[5]) - wnds[t] = float(cabc[6]) - wndd[t] = float(cabc[7]) - - - - elif len(cabc) == 7: - namep = cabc[0][1:-1] if cabc[0].startswith("'") and cabc[0].endswith("'") else cabc[0][1:] - print("Station Name:", namep) - lat_lon = cabc[1].split('-') - print("lat-lon:",lat_lon) - lat_str = lat_lon[0] - print("lat_str:",lat_str) - lon_str = lat_lon[1] - print("lon_str:",lon_str) - lat = -float(lat_str) if lat_str.startswith('-') else float(lat_str) - lon = -float(lon_str) if lon_str.startswith('-') else float(lon_str) - depth = float(cabc[3]) - wnds[t] = float(cabc[4]) - wndd[t] = float(cabc[5]) + # Text format (only one point allowed here, same as WW3/NOAA operational) + fp = open(fname) + nt = fp.read().count(stname) + fp.close() + del fp + if nt >= 1: + # Open file and read the first parameters + fp = open(fname) + cabc = fp.readline() + cabc = cabc.strip().split() + nf = int(cabc[3]) # number of frequencies + nd = int(cabc[4]) # number of directions + npo = int(cabc[5]) # number of point outputs + + freq = zeros(nf, 'f') + dire = zeros(nd, 'f') + dspec = zeros((nt, nf, nd), 'f') + adire = zeros(dire.shape) + adspec = zeros(dspec.shape) + mtime = np.zeros((nt), 'd') + + # Frequencies -------------------- + ncf = int(np.floor(nf/8)) + rncf = int(np.round(8*((float(nf)/8)-ncf))) + k = 0 + for i in range(0, ncf): + line = fp.readline() + line = line.strip().split() + for j in range(0, 8): + freq[k] = float(line[j]) + k = k + 1 + if rncf > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rncf): + freq[k] = float(line[i]) + k = k + 1 + + # DF in frequency (dfreq) + dfreq = np.zeros(freq.shape[0], 'f') + for i in range(0, freq.shape[0]): + if i == 0 or i == (freq.shape[0]-1): + dfreq[i] = freq[i] * \ + (1 + (((freq[-1]/freq[-2])-1)/2)) - freq[i] + else: + dfreq[i] = freq[i] * \ + (freq[-1]/freq[-2]) - freq[i] + + # Directions --------------------- + ncd = int(np.floor(nd/7)) + rncd = int(np.round(7*((float(nd)/7)-ncd))) k = 0 - for i in range(0, nl): + for i in range(0, ncd): line = fp.readline() line = line.strip().split() for j in range(0, 7): - auxs[k] = float(line[j]) + dire[k] = float(line[j])*180/pi k = k+1 if rncd > 0: line = fp.readline() line = line.strip().split() - for i in range(0, rnl): - auxs[k] = float(line[i]) + for i in range(0, rncd): + dire[k] = float(line[i])*180/pi k = k+1 - for ic in range(0, nf): - for il in range(0, nd): - dspec[t, ic, il] = auxs[il*nf+ic] - - # Calculate significant wave height - sp1d = np.sum(dspec[t], axis=1) * (np.abs(dire[1] - dire[0])) # Calculate 1D spectrum - hs[t] = 4 * np.sqrt(np.trapz(sp1d, x=freq)) # Calculate significant wave height - - fp.close() - - results['ntime'] = ntime - results['latitude'] = lat - results['longitude'] = lon - results['wind_spd'] = wnds - results['wind_dir'] = wndd - results['freq'] = freq - results['deltafreq'] = dfreq - results['dir'] = dire - results['spec'] = dspec - results['station_name'] = stname - results['hs'] = hs # Add significant wave height to results + nl = int(floor((nf*nd)/7.)) + rnl = int(np.round(7*((float(nf*nd)/7)-nl))) + auxs = np.zeros((nf*nd), 'f') + wnds = np.zeros((nt), 'f') + wndd = np.zeros((nt), 'f') + + for t in range(0, nt): + + cabc = fp.readline() + cabc.strip().split()[0] + mtime[t] = np.double(timegm(strptime( + cabc.strip().split()[0]+cabc.strip().split()[1][0:2], '%Y%m%d%H'))) + cabc = fp.readline() + cabc = cabc.strip().split() + print(cabc) + if len(cabc) >8: + # Format: ["'42085", "'", '17.86', '-66.52', '126.1', '1.12', '146.9'] + namep = cabc[0][1:] + lat = float(cabc[2]) + lon = float(cabc[3]) + depth = float(cabc[4]) + wnds_index = 5 + wndd_index = 6 + elif len(cabc) == 8: + # Format: ["'46021", "'", '57.70-160.00', '50.8', '5.47', '6.9', '0.00', '270.0'] + namep = cabc[0][1:] + lat_lon_str = cabc[2] + lat_lon_str = lat_lon_str.strip("'") + lat_lon_parts = lat_lon_str.split('-') + print(lat_lon_parts) + lat = float(lat_lon_parts[0]) + lon = -float(lat_lon_parts[1]) - else: - sys.exit(f'Station {stname} not included in') - except Exception as e: - sys.exit(f'Error reading spectral file: {str(e)}') + depth = float(cabc[3]) + wnds_index = 4 + wndd_index = 5 + else: + continue # Skip this file as it cannot be processed + # sys.exit('Unrecognized format of cabc') + + wnds[t] = float(cabc[wnds_index]) + wndd[t] = float(cabc[wndd_index]) + + k = 0 + for i in range(0, nl): + line = fp.readline() + line = line.strip().split() + for j in range(0, 7): + auxs[k] = float(line[j]) + k = k+1 + + if rncd > 0: + line = fp.readline() + line = line.strip().split() + for i in range(0, rnl): + auxs[k] = float(line[i]) + k = k+1 + + for ic in range(0, nf): + for il in range(0, nd): + dspec[t, ic, il] = auxs[il*nf+ic] + + fp.close() + del fp + + mdate = pd.to_datetime(mtime, unit='s').strftime( + '%Y-%m-%dT%H:%M:%S.%f') + freq1 = freq*np.nan + freq2 = freq*np.nan + + # ------------------ + # 1D power spectrum + pwst = np.zeros((dspec.shape[0], nf), 'f') + for t in range(0, dspec.shape[0]): + for il in range(0, nf): + pwst[t, il] = sum( + dspec[t, il, :]*(2*np.pi)/nd) + + pwst[t, :] = pwst[t, :]*dfreq[:] + + # organizing directions ----- + adspec = np.copy(dspec) + inddire = int(np.where(dire == min(dire))[0][0]) + for t in range(0, dspec.shape[0]): + adspec[t, :, 0:nd-(inddire+1)] = dspec[t,:, (inddire+1):] + adspec[t, :, nd-(inddire+1):nd] = dspec[t,:, :(inddire+1)] + for i in range(0, nd): + dspec[t, :, i] = adspec[t, :, nd-i-1] + + adspec[t, :, :int(nd/2)] = dspec[t, :, int(nd/2):] + adspec[t, :, int(nd/2):] = dspec[t, :, :int(nd/2)] + dspec[t, :, :] = adspec[t, :, :] + + dire = np.sort(dire) + + # 1D directional spectrum + d1sp = np.zeros((dspec.shape[0], nf), 'f') + + for t in range(0, dspec.shape[0]): + for il in range(0, nf): + a = np.sum(dspec[t, il, :] * np.array( + np.sin((pi*dire)/180.)/np.sum(dspec[t, il, :]))) + b = np.sum(dspec[t, il, :] * np.array( + np.cos((pi*dire)/180.)/np.sum(dspec[t, il, :]))) + aux = math.atan2(a, b)*(180./pi) + if aux < 0: + aux = aux+360. + + d1sp[t, il] = float(aux) + del a, b, aux + + hs = np.sqrt(2 * np.trapz(np.trapz(dspec, x=dire, axis=-1), x=freq, axis=-1)) + tp = freq[np.argmax(np.max(dspec, axis=-1), axis=-1)] + + # build dictionary + result = {'time': mtime, 'date': mdate, 'latitude': lat, 'longitude': lon, 'depth': depth, + 'wind_spd': wnds, 'wind_dir': wndd, 'freq': freq, 'freq1': freq1, 'freq2': freq2, + 'deltafreq': dfreq, 'pspec': pwst, 'theta': dire, 'dmspec': d1sp, 'dirspec': dspec, 'Hs': hs, 'Tp': tp,'station_name': stname} + + results.append(result) + except Exception as e: + print(f"Skipping file {fname} for station {stname}: {str(e)}") + continue return results ->>>>>>> 5c304ea (added .spec and also is able to to the hs calculations in the code) From 8a06fb1886971cfef387eecfb8d691b2db98d2d2 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 22 Apr 2024 07:35:44 +0000 Subject: [PATCH 11/27] Description added to the modelBuoy_collocation.py --- ww3tools/modelBuoy_collocation.py | 1606 +++++++++++++++++------------ 1 file changed, 926 insertions(+), 680 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index 3002b46..008db3e 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -88,747 +88,993 @@ and ww3) to maximize the amount of matchups even when one variable is not available. +How to run .spec file in a format of the .gz file: python3 modelBuoy_collocation.py ww3list.txt 3 +In the ww3list.txt, you have to define the .gz file path. +ww3list.txt is just name for the text file that contains the path for the .gz file.Any name for this file can be defined. + PERSON OF CONTACT: Ricardo M Campos: ricardo.campos@noaa.gov """ -import warnings; warnings.filterwarnings("ignore") +import sys +import os +import gzip +import tarfile +import warnings import numpy as np -from matplotlib.mlab import * -from pylab import * -import xarray as xr import netCDF4 as nc import time from time import strptime from calendar import timegm import wread -# netcdf format -fnetcdf="NETCDF4" - -# Paths -# ndbcp="/data/buoys/NDBC/wparam" -ndbcp="/work/noaa/marine/ricardo.campos/data/buoys/NDBC/ncformat/wparam" -# Copernicus buoys -# copernp="/data/buoys/Copernicus/wtimeseries" -copernp="/work/noaa/marine/ricardo.campos/data/buoys/Copernicus/wtimeseries" -print(' ') - -# Options of including grid and cyclone information -gridinfo=int(0); cyclonemap=int(0); wlist=[]; ftag=''; forecastds=0 -if len(sys.argv) < 2 : - sys.exit(' At least one argument (list of ww3 files) must be informed.') -if len(sys.argv) >= 2 : - # # import os; os.system("ls -d $PWD/*tab.nc > ww3list.txt &") - wlist=np.atleast_1d(np.loadtxt(sys.argv[1],dtype=str)) - ftag=str(sys.argv[1]).split('list')[1].split('.txt')[0] - print(' Reading ww3 list '+str(sys.argv[1])) - print(' Tag '+ftag) -if len(sys.argv) >= 3: - forecastds=int(sys.argv[2]) - if forecastds>0: - print(' Forecast-type data structure') -if len(sys.argv) >= 4: - gridinfo=str(sys.argv[3]) - print(' Using gridInfo '+gridinfo) -if len(sys.argv) >= 5: - cyclonemap=str(sys.argv[4]) - print(' Using cyclone map '+cyclonemap) -if len(sys.argv) > 5: - sys.exit(' Too many inputs') - -# READ DATA -print(" ") -if gridinfo!=0: - # Grid Information - gridmask = wread.mask(gridinfo) - mlat=gridmask['latitude']; mlon=gridmask['longitude'] - mask=gridmask['mask']; distcoast=gridmask['distcoast']; depth=gridmask['depth'] - oni=gridmask['GlobalOceansSeas']; ocnames=gridmask['names_GlobalOceansSeas'] - hsmz=gridmask['HighSeasMarineZones']; hsmznames=gridmask['names_HighSeasMarineZones'] - print(" GridInfo Ok. "+gridinfo) - - # Cyclone Information - if cyclonemap!=0: - cycloneinfo = wread.cyclonemap(cyclonemap) - clat=cycloneinfo['latitude']; clon=cycloneinfo['longitude'] - cmap=cycloneinfo['cmap']; ctime=cycloneinfo['time'] - cinfo=np.array(cycloneinfo['info'].split(':')[1].split(';')) - if np.array_equal(clat,mlat)==True & np.array_equal(clon,mlon)==True: - print(" CycloneMap Ok. "+cyclonemap) - else: - sys.exit(' Error: Cyclone grid and Mask grid are different.') - -# MODEL Point output, search among possible formats ------------------ - -if (str(wlist[0]).split('/')[-1].split('.')[-1]=='bull_tar') or (str(wlist[0]).split('/')[-1].split('.')[-1]=='station_tar'): - for i in range(0,np.size(wlist)): - if str(wlist[i]).split('/')[-1].split('.')[-1]=='bull_tar': - result = wread.bull_tar(wlist[i]) - if str(wlist[i]).split('/')[-1].split('.')[-1]=='station_tar': - result = wread.bull_tar(wlist[i]) - - at=result['time'] - fcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') - if i==0: - stname=result['station_name'] - mtime=np.copy(at) - mfcycle=np.copy(fcycle) - mhs=np.copy(result['hs']) - mtp=np.copy(result['tp']) - if 'dp' in result.keys(): - mdp=np.copy(result['dp']) - else: - mdp=np.copy(mhs)*np.nan - - else: - if (mhs.shape[0]==result['hs'].shape[0]) and (np.size(stname)==np.size(result['station_name'])): - if (stname==result['station_name']).all(): - mtime=np.append(mtime,at) - mfcycle=np.append(mfcycle,fcycle) - mhs=np.append(mhs,result['hs'],axis=1) - mtp=np.append(mtp,result['tp'],axis=1) - if 'dp' in result.keys(): - mdp=np.append(mdp,result['dp'],axis=1) - else: - mdp=np.append(mdp,np.copy(result['hs'])*np.nan,axis=1) - - else: - print(" Stations in "+wlist[i]+" do not match the other tar files. Skipped "+wlist[i]) - - del result,at,fcycle - mdm=np.copy(mhs)*np.nan; mtm=np.copy(mhs)*np.nan # not saved in this file format - print(" ww3 file "+wlist[i]+" OK") - -elif (str(wlist[0]).split('/')[-1].split('.')[-1]=='bull') or (str(wlist[0]).split('/')[-1].split('.')[-1]=='ts'): - if forecastds>0: - forecastds=0 # no 2D time array possible. - print(" Warning: no 2D time array possible. \ - Use multiple tar files (one per forecast cycle), station_tar or bull_tar, containing text files for each station.") - - for i in range(0,np.size(wlist)): - # re-check each file in the list respects the same format - if str(wlist[i]).split('/')[-1].split('.')[-1]=='ts': - result = wread.ts(wlist[i]) - elif str(wlist[i]).split('/')[-1].split('.')[-1]=='bull': - result = wread.bull(wlist[i]) - - at=result['time'] - if i==0: - mfcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') - stname=np.atleast_1d(np.array(result['station_name'])) - mtime=np.copy(at) - mhs=np.copy([result['hs']]) - mtp=np.copy([result['tp']]) - if 'dp' in result.keys(): - mdp=np.copy([result['dp']]) - else: - mdp=np.copy(mhs)*np.nan - - if 'dm' in result.keys(): - mdm=np.copy([result['dm']]) - else: - mdm=np.copy(mhs)*np.nan - - if 'tm' in result.keys(): - mtm=np.copy([result['tm']]) - else: - mtm=np.copy(mhs)*np.nan - - else: - if (mhs.shape[1]==result['hs'].shape[0]): - stname=np.append(stname,np.atleast_1d(np.array(result['station_name']))) - mtime=np.append(mtime,at) - mhs=np.append(mhs,[result['hs']],axis=0) - mtp=np.append(mtp,[result['tp']],axis=0) - if 'dp' in result.keys(): - mdp=np.append(mdp,[result['dp']],axis=0) - else: - mdp=np.append(mdp,[np.copy(result['hs'])*np.nan],axis=0) - - if 'dm' in result.keys(): - mdm=np.append(mdp,[result['dm']],axis=0) - else: - mdm=np.append(mdm,[np.copy(result['hs'])*np.nan],axis=0) - - if 'tm' in result.keys(): - mtm=np.append(mtm,[result['tm']],axis=0) - else: - mtm=np.append(mtm,[np.copy(result['hs'])*np.nan],axis=0) - - else: - print(" Stations in "+wlist[i]+" do not match the other tar files. Skipped "+wlist[i]) - - del result,at - print(" ww3 file "+wlist[i]+" OK") - - -elif str(wlist[0]).split('/')[-1].split('.')[-1]=='nc': - print(" Using ww3 netcdf point output format") - # netcdf point output file - for t in range(0,np.size(wlist)): - try: - f=nc.Dataset(str(wlist[t])) - except: - print(" Cannot open "+wlist[t]) - else: - # list of station/buoy names - if t==0: - auxstationname=f.variables['station_name'][:,:]; stname=[] - for i in range(0,auxstationname.shape[0]): - astname="".join(np.array(auxstationname[i,:]).astype('str')) - if '\t' in astname: - astname=str(astname).replace("\t","") - - stname=np.append(stname,astname); del astname - - funits=f.variables['time'].units - if str(funits).split(' ')[0] == 'seconds': - tincr=1 - elif str(funits).split(' ')[0] == 'hours': - tincr=3600 - elif str(funits).split(' ')[0] == 'days': - tincr=24*3600 - - ahs = np.array(f.variables['hs'][:,:]).T - - if 'th1m' in f.variables.keys(): - adm = np.array(f.variables['th1m'][:,:]).T - else: - adm = np.array(np.copy(ahs*nan)) - - if 'th1p' in f.variables.keys(): - adp = np.array(f.variables['th1p'][:,:]).T - else: - adp = np.array(np.copy(ahs*nan)) - - if 'tr' in f.variables.keys(): - atm = np.array(f.variables['tr'][:,:]).T - else: - atm = np.array(np.copy(ahs*nan)) - - if 'fp' in f.variables.keys(): - auxtp = np.array(f.variables['fp'][:,:]).T - indtp=np.where(auxtp>0.) - if np.size(indtp)>0: - atp=np.copy(auxtp) - atp[indtp]=np.copy(1./atp[indtp]) - del indtp - - del auxtp - - else: - atm = np.array(np.copy(ahs*nan)) - - ftunits=str(f.variables['time'].units).split('since')[1][1::].replace('T',' ').replace('+00:00','') - at = np.array(f.variables['time'][:]*tincr + timegm( strptime(ftunits,'%Y-%m-%d %H:%M:%S') )).astype('double') - - f.close(); del f - fcycle = np.array(np.zeros((at.shape[0]),'d')+at[0]).astype('double') - if t==0: - mhs=np.copy(ahs) - mtm=np.copy(atm) - mtp=np.copy(atp) - mdm=np.copy(adm) - mdp=np.copy(adp) - mtime=np.copy(at) - mfcycle=np.copy(fcycle) - else: - mhs=np.append(mhs,ahs,axis=1) - mtm=np.append(mtm,atm,axis=1) - mtp=np.append(mtp,atp,axis=1) - mdm=np.append(mdm,adm,axis=1) - mdp=np.append(mdp,adp,axis=1) - mtime=np.append(mtime,at) - mfcycle=np.append(mfcycle,fcycle) - - del ahs,atm,atp,adm,adp,at,fcycle - - print(" Read WW3 data OK."); print(' ') - -else: - sys.exit(' Point output file format not recognized: only bull, bull_tar, and .nc implemented.') - # include other text formats: tab50, .ts - -print(" Start building the matchups model/buoy ..."); print(' ') - -# BUOYS ------------------ -bhs=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan -btm=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan -btp=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan -bdm=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan -bdp=np.zeros((np.size(stname),np.size(mtime)),'f')*np.nan -lat=np.zeros(np.size(stname),'f')*np.nan; lon=np.zeros(np.size(stname),'f')*np.nan +# Suppressing warnings +warnings.filterwarnings("ignore") + +# netCDF format specification +fnetcdf = "NETCDF4" + +gridinfo = int(0) +cyclonemap = int(0) +wlist = [] +ftag = '' +forecastds = 0 + + +# Function to unzip and untar files +def unzip_and_untar(gz_file, output_dir): + file_name = os.path.splitext(os.path.basename(gz_file))[0] + base_name = file_name.rsplit('.', 1)[0] + output_file = os.path.join(output_dir, file_name) + + with gzip.open(gz_file, 'rb') as f_in: + with open(output_file, 'wb') as f_out: + f_out.write(f_in.read()) + + with tarfile.open(output_file, 'r') as tar: + tar.extractall(output_dir) + + extracted_folder = os.path.join(output_dir, base_name) + # Creating a list of the extracted files + list_file = os.path.join(output_dir, f'{base_name}_contents.txt') + with open(list_file, 'w') as f: + for root, dirs, files in os.walk(extracted_folder): + for file in files: + f.write(os.path.join(root, file) + '\n') + + # Creating a list of ids between two dots in the filenames + id_file = os.path.join(output_dir, f'{base_name}_id.txt') + with open(id_file, 'w') as f: + for root, dirs, files in os.walk(extracted_folder): + for file in files: + file_parts = file.split('.') + if len(file_parts) >= 3: + id_between_dots = file_parts[1] + f.write(id_between_dots + '\n') + + # Return the path to the extracted folder + return extracted_folder, list_file, id_file + + + +# Main part of the script +if __name__ == "__main__": + mode = sys.argv[1] if len(sys.argv) > 1 else "" + + if mode == "unzip": + if len(sys.argv) != 5: + sys.exit("Usage: python script.py unzip input_gz_file output_directory buoy_path") + + gz_file = sys.argv[2] + output_dir = sys.argv[3] + buoy_path = sys.argv[4] + ndbcp = buoy_path + extracted_folder, list_file, id_file = unzip_and_untar(gz_file, output_dir) + + print("Extraction completed. Extracted files are in:", extracted_folder) + + # Read file names from the extracted folder + file_names = [] + for root, dirs, files in os.walk(extracted_folder): + for file in files: + file_names.append(os.path.join(root, file)) + + # Read id_files (if needed) + id_files = [] + with open(id_file, 'r') as f: + for line in f: + id_files.append(line.strip()) + + # Call the spec_ww3 function with the file names and ID files + results = wread.spec1_ww3(file_names, id_files) + + # Additional processing (if needed) + mfcycle = None + stname = None + mtime = None + mhs = None + mtp = None + mwn = None + mwd = None + mfreq = None + mtm = None + mdm = None + mdp = None + + for i, result in enumerate(results): + at = result['time'] + + if i == 0: + mfcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') + stname = np.atleast_1d(np.array(result['station_name'])) + mtime = np.copy(at) + mhs = np.copy([result['Hs']]) + mtp = np.copy([result['Tp']]) + + if 'wind_spd' in result.keys(): + mwn = np.copy([result['wind_spd']]) + else: + mwn = np.copy(mhs) * np.nan + + if 'wind_dir' in result.keys(): + mwd = np.copy([result['wind_dir']]) + else: + mwd = np.copy(mhs) * np.nan + + if 'freq' in result.keys(): + mfreq = np.copy([result['freq']]) + else: + mfreq = np.copy(mhs) * np.nan + + if 'tm' in result.keys(): + mtm = np.copy([result['tm']]) + else: + mtm = np.copy(mhs) * np.nan + + if 'dp' in result.keys(): + mdp = np.copy([result['dp']]) + else: + mdp = np.copy(mhs) * np.nan + + if 'dm' in result.keys(): + mdm = np.copy([result['dm']]) + else: + mdm = np.copy(mhs) * np.nan + + else: + + if mhs.shape[1] == result['Hs'].shape[0]: + stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) + mtime = np.append(mtime, at) + mhs = np.append(mhs, [result['Hs']], axis=0) + mtp = np.append(mtp, [result['Tp']], axis=0) + + if 'wind_spd' in result.keys(): + mwn = np.append(mwn, [result['wind_spd']], axis=0) + else: + mwn = np.append(mwn, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'wind_dir' in result.keys(): + mwd = np.append(mwd, [result['wind_dir']], axis=0) + else: + mwd = np.append(mwd, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'freq' in result.keys(): + mfreq = np.append(mfreq, [result['freq']], axis=0) + else: + mfreq = np.append(mfreq, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'tm' in result.keys(): + mtm = np.append(mtm, [result['tm']], axis=0) + else: + mtm = np.append(mtm, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'dp' in result.keys(): + mdp = np.append(mdp, [result['dp']], axis=0) + else: + mdp = np.append(mdp, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'dm' in result.keys(): + mdm = np.append(mdp, [result['dm']], axis=0) + else: + mdm = np.append(mdm, [np.copy(result['Hs']) * np.nan], axis=0) + + else: + print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) + + else: + gridinfo = int(0) + cyclonemap = int(0) + wlist = [] + ftag = '' + forecastds = 0 + + if len(sys.argv) < 2: + sys.exit('At least one argument (list of ww3 files) must be informed.') + wlist = np.atleast_1d(np.loadtxt(sys.argv[1], dtype=str)) + ftag = str(sys.argv[1]).split('list')[1].split('.txt')[0] + print('Reading ww3 list ' + str(sys.argv[1])) + print('Tag ' + ftag) + + if len(sys.argv) >= 3: + forecastds = int(sys.argv[2]) + if forecastds > 0: + print('Forecast-type data structure') + + if len(sys.argv) >= 4: + gridinfo = str(sys.argv[3]) + print('Writing gridinfo ' + gridinfo) + + if len(sys.argv) >= 5: + cyclonemap = str(sys.argv[4]) + print('Writing cyclonemap ' + cyclonemap) + + # Paths + ndbcp = "/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" + + # READ DATA + print(" ") + if gridinfo != 0: + # Grid Information + gridmask = wread.mask(gridinfo) + mlat = gridmask['latitude'] + mlon = gridmask['longitude'] + mask = gridmask['mask'] + distcoast = gridmask['distcoast'] + depth = gridmask['depth'] + oni = gridmask['GlobalOceansSeas'] + ocnames = gridmask['names_GlobalOceansSeas'] + hsmz = gridmask['HighSeasMarineZones'] + hsmznames = gridmask['names_HighSeasMarineZones'] + print(" GridInfo Ok. " + gridinfo) + + # Cyclone Information + if cyclonemap != 0: + cycloneinfo = wread.cyclonemap(cyclonemap) + clat = cycloneinfo['latitude'] + clon = cycloneinfo['longitude'] + cmap = cycloneinfo['cmap'] + ctime = cycloneinfo['time'] + cinfo = np.array(cycloneinfo['info'].split(':')[1].split(';')) + if np.array_equal(clat, mlat) == True & np.array_equal(clon, mlon) == True: + print(" CycloneMap Ok. " + cyclonemap) + else: + sys.exit(' Error: Cyclone grid and Mask grid are different.') + + # MODEL Point output, search among possible formats ------------------ + if (str(wlist[0]).split('/')[-1].split('.')[-1] == 'bull_tar') or ( + str(wlist[0]).split('/')[-1].split('.')[-1] == 'station_tar'): + for i in range(0, np.size(wlist)): + if str(wlist[i]).split('/')[-1].split('.')[-1] == 'bull_tar': + result = wread.bull_tar(wlist[i]) + if str(wlist[i]).split('/')[-1].split('.')[-1] == 'station_tar': + result = wread.bull_tar(wlist[i]) + + at = result['time'] + fcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') + if i == 0: + stname = result['station_name'] + mtime = np.copy(at) + mfcycle = np.copy(fcycle) + mhs = np.copy(result['hs']) + mtp = np.copy(result['tp']) + if 'dp' in result.keys(): + mdp = np.copy(result['dp']) + else: + mdp = np.copy(mhs) * np.nan + + if 'wind_spd' in result.keys(): + mwn = np.copy(result['wind_spd']) + else: + mwn = np.copy(mhs) * np.nan + + if 'wind_dir' in result.keys(): + mwd = np.copy(result['wind_dir']) + else: + mwd = np.copy(mhs) * np.nan + + + else: + if (mhs.shape[0] == result['hs'].shape[0]) and ( + np.size(stname) == np.size(result['station_name'])): + if (stname == result['station_name']).all(): + mtime = np.append(mtime, at) + mfcycle = np.append(mfcycle, fcycle) + mhs = np.append(mhs, result['hs'], axis=1) + mtp = np.append(mtp, result['tp'], axis=1) + if 'dp' in result.keys(): + mdp = np.append(mdp, result['dp'], axis=1) + else: + mdp = np.append(mdp, np.copy(result['hs']) * np.nan, axis=1) + + if 'wind_spd' in result.keys(): + mwn = np.append(mwn, result['wind_spd'], axis=1) + else: + mwn = np.append(mwn, np.copy(result['hs']) * np.nan, axis=1) + + if 'wind_dir' in result.keys(): + mwd = np.append(mwd, result['wind_dir'], axis=1) + else: + mwd = np.append(mwd, np.copy(result['hs']) * np.nan, axis=1) + + + else: + print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[ + i]) + + del result, at, fcycle + mdm = np.copy(mhs) * np.nan; + mtm = np.copy(mhs) * np.nan # not saved in this file format + print(" ww3 file " + wlist[i] + " OK") + + elif (str(wlist[0]).split('/')[-1].split('.')[-1] == 'bull') or ( + str(wlist[0]).split('/')[-1].split('.')[-1] == 'ts'): + if forecastds > 0: + forecastds = 0 # no 2D time array possible. + print( + " Warning: no 2D time array possible. Use multiple tar files (one per forecast cycle), station_tar or bull_tar, containing text files for each station.") + + for i in range(0, np.size(wlist)): + # re-check each file in the list respects the same format + if str(wlist[i]).split('/')[-1].split('.')[-1] == 'ts': + result = wread.ts(wlist[i]) + elif str(wlist[i]).split('/')[-1].split('.')[-1] == 'bull': + result = wread.bull(wlist[i]) + + at = result['time'] + if i == 0: + mfcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') + stname = np.atleast_1d(np.array(result['station_name'])) + mtime = np.copy(at) + mhs = np.copy([result['hs']]) + mtp = np.copy([result['tp']]) + if 'dp' in result.keys(): + mdp = np.copy([result['dp']]) + else: + mdp = np.copy(mhs) * np.nan + + if 'dm' in result.keys(): + mdm = np.copy([result['dm']]) + else: + mdm = np.copy(mhs) * np.nan + + if 'tm' in result.keys(): + mtm = np.copy([result['tm']]) + else: + mtm = np.copy(mhs) * np.nan + + else: + if (mhs.shape[1] == result['hs'].shape[0]): + stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) + mtime = np.append(mtime, at) + mhs = np.append(mhs, [result['hs']], axis=0) + mtp = np.append(mtp, [result['tp']], axis=0) + if 'dp' in result.keys(): + mdp = np.append(mdp, [result['dp']], axis=0) + else: + mdp = np.append(mdp, [np.copy(result['hs']) * np.nan], axis=0) + + if 'dm' in result.keys(): + mdm = np.append(mdp, [result['dm']], axis=0) + else: + mdm = np.append(mdm, [np.copy(result['hs']) * np.nan], axis=0) + + if 'tm' in result.keys(): + mtm = np.append(mtm, [result['tm']], axis=0) + else: + mtm = np.append(mtm, [np.copy(result['hs']) * np.nan], axis=0) + + else: + print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) + + del result, at + print(" ww3 file " + wlist[i] + " OK") + + elif str(wlist[0]).split('/')[-1].split('.')[-1] == 'nc': + print(" Using ww3 netcdf point output format") + # netcdf point output file + for t in range(0, np.size(wlist)): + try: + f = nc.Dataset(str(wlist[t])) + except: + print(" Cannot open " + wlist[t]) + else: + # list of station/buoy names + if t == 0: + auxstationname = f.variables['station_name'][:, :]; + stname = [] + for i in range(0, auxstationname.shape[0]): + astname = "".join(np.array(auxstationname[i, :]).astype('str')) + if '\t' in astname: + astname = str(astname).replace("\t", "") + + stname = np.append(stname, astname); + del astname + + funits = f.variables['time'].units + if str(funits).split(' ')[0] == 'seconds': + tincr = 1 + elif str(funits).split(' ')[0] == 'hours': + tincr = 3600 + elif str(funits).split(' ')[0] == 'days': + tincr = 24 * 3600 + + ahs = np.array(f.variables['hs'][:, :]).T + + if 'th1m' in f.variables.keys(): + adm = np.array(f.variables['th1m'][:, :]).T + else: + adm = np.array(np.copy(ahs * nan)) + + if 'th1p' in f.variables.keys(): + adp = np.array(f.variables['th1p'][:, :]).T + else: + adp = np.array(np.copy(ahs * nan)) + + if 'tr' in f.variables.keys(): + atm = np.array(f.variables['tr'][:, :]).T + else: + atm = np.array(np.copy(ahs * nan)) + + if 'fp' in f.variables.keys(): + auxtp = np.array(f.variables['fp'][:, :]).T + indtp = np.where(auxtp > 0.) + if np.size(indtp) > 0: + atp = np.copy(auxtp) + atp[indtp] = np.copy(1. / atp[indtp]) + del indtp + + del auxtp + + else: + atm = np.array(np.copy(ahs * nan)) + + ftunits = str(f.variables['time'].units).split('since')[1][1::].replace('T', ' ').replace('+00:00', + '') + at = np.array( + f.variables['time'][:] * tincr + timegm(strptime(ftunits, '%Y-%m-%d %H:%M:%S'))).astype( + 'double') + + f.close(); + del f + fcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') + if t == 0: + mhs = np.copy(ahs) + mtm = np.copy(atm) + mtp = np.copy(atp) + mdm = np.copy(adm) + mdp = np.copy(adp) + mtime = np.copy(at) + mfcycle = np.copy(fcycle) + else: + mhs = np.append(mhs, ahs, axis=1) + mtm = np.append(mtm, atm, axis=1) + mtp = np.append(mtp, atp, axis=1) + mdm = np.append(mdm, adm, axis=1) + mdp = np.append(mdp, adp, axis=1) + mtime = np.append(mtime, at) + mfcycle = np.append(mfcycle, fcycle) + + del ahs, atm, atp, adm, adp, at, fcycle + + print(" Read WW3 data OK."); + print(' ') + + else: + sys.exit(' Point output file format not recognized: only bull, bull_tar, and .nc implemented.') + # include other text formats: tab50, .ts + + print(" Start building the matchups model/buoy ..."); + print(' ') + + +# BUOYS ------------------ +bwind = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bhs = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +btm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +btp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bdm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bdp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +lat = np.zeros(np.size(stname), 'f') * np.nan +lon = np.zeros(np.size(stname), 'f') * np.nan # help reading NDBC buoys, divided by year -yrange=np.array(np.arange(time.gmtime(mtime.min())[0],time.gmtime(mtime.min())[0]+1,1)).astype('int') +yrange = np.array(np.arange(time.gmtime(mtime.min())[0], time.gmtime(mtime.min())[0] + 1, 1)).astype('int') # loop buoys -for b in range(0,np.size(stname)): - - ahs=[] - try: - - ahs=[];atm=[];atp=[];adm=[];atime=[] - for y in yrange: - - f=nc.Dataset(ndbcp+"/"+stname[b]+"h"+repr(y)+".nc") - if 'wave_height' in f.variables.keys(): - ahs = np.append(ahs,f.variables['wave_height'][:,0,0]) - elif 'hs' in f.variables.keys(): - ahs = np.append(ahs,f.variables['hs'][:,0,0]) - elif 'swh' in f.variables.keys(): - ahs = np.append(ahs,f.variables['swh'][:,0,0]) - - if 'average_wpd' in f.variables.keys(): - atm = np.append(atm,f.variables['average_wpd'][:,0,0]) - else: - atm = np.array(np.copy(ahs*nan)) - - if 'dominant_wpd' in f.variables.keys(): - atp = np.append(atp,f.variables['dominant_wpd'][:,0,0]) - else: - atp = np.array(np.copy(ahs*nan)) - - if 'mean_wave_dir' in f.variables.keys(): - adm = np.append(adm,f.variables['mean_wave_dir'][:,0,0]) - else: - adm = np.array(np.copy(ahs*nan)) - - if 'latitude' in f.variables.keys(): - lat[b] = f.variables['latitude'][:] - elif 'LATITUDE' in f.variables.keys(): - lat[b] = f.variables['LATITUDE'][:] - else: - lat[b] = nan - - if 'longitude' in f.variables.keys(): - lon[b] = f.variables['longitude'][:] - elif 'LONGITUDE' in f.variables.keys(): - lon[b] = f.variables['LONGITUDE'][:] - else: - lon[b] = nan - - atime = np.append(atime,np.array(f.variables['time'][:]).astype('double')) - - f.close(); del f - - adp = adm*np.nan # no peak direction available in this format - - except: - try: - f=nc.Dataset(copernp+"/GL_TS_MO_"+stname[b]+".nc") - if 'VHM0' in f.variables.keys(): - ahs = np.nanmean(f.variables['VHM0'][:,:],axis=1) - elif 'VAVH' in f.variables.keys(): - ahs = np.nanmean(f.variables['VAVH'][:,:],axis=1) - elif 'VGHS' in f.variables.keys(): - ahs = np.nanmean(f.variables['VGHS'][:,:],axis=1) - elif 'significant_swell_wave_height' in f.variables.keys(): - ahs = np.nanmean(f.variables['significant_swell_wave_height'][:,:],axis=1) - elif 'sea_surface_significant_wave_height' in f.variables.keys(): - ahs = np.nanmean(f.variables['sea_surface_significant_wave_height'][:,:],axis=1) - elif 'SWHT' in f.variables.keys(): - ahs = np.nanmean(f.variables['SWHT'][:,:],axis=1) - elif 'wave_height_h1d3' in f.variables.keys(): - ahs = np.nanmean(f.variables['wave_height_h1d3'][:,:],axis=1) - elif 'spectral_significant_wave_height' in f.variables.keys(): - ahs = np.nanmean(f.variables['spectral_significant_wave_height'][:,:],axis=1) - - if 'VTM02' in f.variables.keys(): - atm = np.nanmean(f.variables['VTM02'][:,:],axis=1) - elif 'VGTA' in f.variables.keys(): - atm = np.nanmean(f.variables['VGTA'][:,:],axis=1) - else: - atm = ahs*nan - - if 'VTPK' in f.variables.keys(): - atp = np.nanmean(f.variables['VTPK'][:,:],axis=1) - elif 'dominant_wave_period' in f.variables.keys(): - atp = np.nanmean(f.variables['dominant_wave_period'][:,:],axis=1) - elif 'sea_surface_wave_period_at_spectral_density_maximum' in f.variables.keys(): - atp = np.nanmean(f.variables['sea_surface_wave_period_at_spectral_density_maximum'][:,:],axis=1) - else: - atp = ahs*nan - - if 'VMDR' in f.variables.keys(): - adm = np.nanmean(f.variables['VMDR'][:,:],axis=1) - else: - adm = ahs*nan - - if 'LATITUDE' in f.variables.keys(): - lat[b] = np.nanmean(f.variables['LATITUDE'][:]) - elif 'latitude' in f.variables.keys(): - lat[b] = np.nanmean(f.variables['latitude'][:]) - elif 'LAT' in f.variables.keys(): - lat[b] = np.nanmean(f.variables['LAT'][:]) - elif 'lat' in f.variables.keys(): - lat[b] = np.nanmean(f.variables['lat'][:]) - else: - lat[b] = nan - - if 'LONGITUDE' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['LONGITUDE'][:]) - elif 'LON' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['LON'][:]) - elif 'LONG' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['LONG'][:]) - elif 'longitude' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['longitude'][:]) - elif 'lon' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['lon'][:]) - elif 'long' in f.variables.keys(): - lon[b] = np.nanmean(f.variables['long'][:]) - else: - lon[b] = nan - - adp = ahs*nan # no peak direction available in this format - - if 'TIME' in f.variables.keys(): - atime = np.array(f.variables['TIME'][:]*24*3600 + timegm( strptime('195001010000', '%Y%m%d%H%M') )).astype('double') - elif 'time' in f.variables.keys(): - atime = np.array(f.variables['time'][:]*24*3600 + timegm( strptime('195001010000', '%Y%m%d%H%M') )).astype('double') - - f.close(); del f - except: - ahs=[] - - - if np.size(ahs)>0: - - # First layer of simple quality-control - indq=np.where((ahs>30.)|(ahs<0.0)) - if np.size(indq)>0: - ahs[indq]=np.nan; del indq - - indq=np.where((atm>40.)|(atm<0.0)) - if np.size(indq)>0: - atm[indq]=np.nan; del indq - - indq=np.where((atp>40.)|(atp<0.0)) - if np.size(indq)>0: - atp[indq]=np.nan; del indq - - indq=np.where((adm>360.)|(adm<-180.)) - if np.size(indq)>0: - adm[indq]=np.nan; del indq - - indq=np.where((adp>360.)|(adp<-180.)) - if np.size(indq)>0: - adp[indq]=np.nan; del indq - - c=0 - for t in range(0,np.size(mtime)): - indt=np.where(np.abs(atime-mtime[t])<1800.) - if np.size(indt)>0: - if np.any(ahs[indt[0]].mask==False): - bhs[b,t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask==False]) - c=c+1 - if np.any(atm[indt[0]].mask==False): - btm[b,t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask==False]) - if np.any(atp[indt[0]].mask==False): - btp[b,t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask==False]) - if np.any(adm[indt[0]].mask==False): - bdm[b,t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask==False]) - if np.any(adp[indt[0]].mask==False): - bdp[b,t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask==False]) - - del indt - - # print("counted "+repr(c)+" at "+stname[b]) - - print(" station "+stname[b]+" ok") - del ahs +for b in range(0, np.size(stname)): + + ahs = [] + try: + awm = [] + ahs = [] + atm = [] + atp = [] + adm = [] + atime = [] + for y in yrange: + + f = nc.Dataset(ndbcp + "/" + stname[b] + "h" + repr(y) + ".nc") + if 'wave_height' in f.variables.keys(): + ahs = np.append(ahs, f.variables['wave_height'][:, 0, 0]) + elif 'hs' in f.variables.keys(): + ahs = np.append(ahs, f.variables['hs'][:, 0, 0]) + elif 'swh' in f.variables.keys(): + ahs = np.append(ahs, f.variables['swh'][:, 0, 0]) + + if 'wind_spd' in f.variables.keys(): + awm = np.append(awm, f.variables['wind_spd'][:, 0, 0]) + else: + awm = np.array(np.copy(ahs * nan)) + + if 'average_wpd' in f.variables.keys(): + atm = np.append(atm, f.variables['average_wpd'][:, 0, 0]) + else: + atm = np.array(np.copy(ahs * nan)) + + if 'dominant_wpd' in f.variables.keys(): + atp = np.append(atp, f.variables['dominant_wpd'][:, 0, 0]) + else: + atp = np.array(np.copy(ahs * nan)) + + if 'mean_wave_dir' in f.variables.keys(): + adm = np.append(adm, f.variables['mean_wave_dir'][:, 0, 0]) + else: + adm = np.array(np.copy(ahs * nan)) + + if 'latitude' in f.variables.keys(): + lat[b] = f.variables['latitude'][:] + elif 'LATITUDE' in f.variables.keys(): + lat[b] = f.variables['LATITUDE'][:] + else: + lat[b] = nan + + if 'longitude' in f.variables.keys(): + lon[b] = f.variables['longitude'][:] + elif 'LONGITUDE' in f.variables.keys(): + lon[b] = f.variables['LONGITUDE'][:] + else: + lon[b] = nan + + atime = np.append(atime, np.array(f.variables['time'][:]).astype('double')) + + f.close() + del f + + adp = adm * np.nan # no peak direction available in this format + + if np.size(ahs) > 0: + + # First layer of simple quality-control + indq = np.where((ahs > 30.) | (ahs < 0.0)) + if np.size(indq) > 0: + ahs[indq] = np.nan + del indq + + indq = np.where((atm > 40.) | (atm < 0.0)) + if np.size(indq) > 0: + atm[indq] = np.nan + del indq + + indq = np.where((atp > 40.) | (atp < 0.0)) + if np.size(indq) > 0: + atp[indq] = np.nan + del indq + + indq = np.where((adm > 360.) | (adm < -180.)) + if np.size(indq) > 0: + adm[indq] = np.nan + del indq + + indq = np.where((adp > 360.) | (adp < -180.)) + if np.size(indq) > 0: + adp[indq] = np.nan + del indq + + indq = np.where((awm > 50.) | (awm < 0.0)) + if np.size(indq) > 0: + awm[indq] = np.nan + del indq + + c = 0 + for t in range(0, np.size(mtime)): + indt = np.where(np.abs(atime - mtime[t]) < 1800.) + if np.size(indt) > 0: + if np.any(ahs[indt[0]].mask == False): + bhs[b, t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask == False]) + c = c + 1 + if np.any(atm[indt[0]].mask == False): + btm[b, t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask == False]) + if np.any(atp[indt[0]].mask == False): + btp[b, t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask == False]) + if np.any(adm[indt[0]].mask == False): + bdm[b, t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask == False]) + if np.any(adp[indt[0]].mask == False): + bdp[b, t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask == False]) + if np.any(awm[indt[0]].mask == False): + bwind[b, t] = np.nanmean(awm[indt[0]][awm[indt[0]].mask == False]) + + del indt + + # print("counted "+repr(c)+" at "+stname[b]) + + print(" station " + stname[b] + " ok") +# del ahs + except Exception as e: + print("Error occurred while processing station", stname[b]) + print(e) + +print('bwind:',bwind) +print('bhs:',bhs) + print(' ') # Simple quality-control (range) ind=np.where((bhs>30.)|(bhs<0.0)) if np.size(ind)>0: - bhs[ind]=np.nan; del ind + bhs[ind]=np.nan; del ind ind=np.where((btm>40.)|(btm<0.0)) if np.size(ind)>0: - btm[ind]=np.nan; del ind + btm[ind]=np.nan; del ind ind=np.where((btp>40.)|(btp<0.0)) if np.size(ind)>0: - btp[ind]=np.nan; del ind + btp[ind]=np.nan; del ind ind=np.where((bdm>360.)|(bdm<-180.)) if np.size(ind)>0: - bdm[ind]=np.nan; del ind + bdm[ind]=np.nan; del ind ind=np.where((bdp>360.)|(bdp<-180.)) if np.size(ind)>0: - bdp[ind]=np.nan; del ind + bdp[ind]=np.nan; del ind + +ind=np.where((bwind>50.0)|(bwind<0.0)) +if np.size(ind)>0: + bwind[ind]=np.nan; del ind ind=np.where((mhs>30.)|(mhs<0.0)) if np.size(ind)>0: - mhs[ind]=np.nan; del ind + mhs[ind]=np.nan; del ind ind=np.where((mtm>40.)|(mtm<0.0)) if np.size(ind)>0: - mtm[ind]=np.nan; del ind - + mtm[ind]=np.nan; del ind + ind=np.where((mtp>40.)|(mtp<0.0)) if np.size(ind)>0: - mtp[ind]=np.nan; del ind + mtp[ind]=np.nan; del ind + ind=np.where((mdm>360.)|(mdm<-180.)) if np.size(ind)>0: - mdm[ind]=np.nan; del ind + mdm[ind]=np.nan; del ind ind=np.where((mdp>360.)|(mdp<-180.)) if np.size(ind)>0: - mdp[ind]=np.nan; del ind + mdp[ind]=np.nan; del ind + +ind=np.where((mwn>50.)|(mwn<0.0)) +if np.size(ind)>0: + mwn[ind]=np.nan; del ind # Clean data excluding some stations. Select matchups only when model and buoy are available. ind=np.where( (np.isnan(lat)==False) & (np.isnan(lon)==False) & (np.isnan(np.nanmean(mhs,axis=1))==False) & (np.isnan(np.nanmean(bhs,axis=1))==False) ) if np.size(ind)>0: - stname=np.array(stname[ind[0]]) - lat=np.array(lat[ind[0]]) - lon=np.array(lon[ind[0]]) - mhs=np.array(mhs[ind[0],:]) - mtm=np.array(mtm[ind[0],:]) - mtp=np.array(mtp[ind[0],:]) - mdm=np.array(mdm[ind[0],:]) - mdp=np.array(mdp[ind[0],:]) - bhs=np.array(bhs[ind[0],:]) - btm=np.array(btm[ind[0],:]) - btp=np.array(btp[ind[0],:]) - bdm=np.array(bdm[ind[0],:]) - bdp=np.array(bdp[ind[0],:]) + stname=np.array(stname[ind[0]]) + lat=np.array(lat[ind[0]]) + lon=np.array(lon[ind[0]]) + mhs=np.array(mhs[ind[0],:]) + mtm=np.array(mtm[ind[0],:]) + mtp=np.array(mtp[ind[0],:]) + mdm=np.array(mdm[ind[0],:]) + # mdp=np.array(mdp[ind[0],:]) + mdp = np.array(mdp[ind[0], :]) + mwn=np.array(mwn[ind[0],:]) + bhs=np.array(bhs[ind[0],:]) + btm=np.array(btm[ind[0],:]) + btp=np.array(btp[ind[0],:]) + bdm=np.array(bdm[ind[0],:]) + bdp=np.array(bdp[ind[0],:]) + bwind=np.array(bwind[ind[0],:]) else: - sys.exit(' Error: No matchups Model/Buoy available.') + sys.exit(' Error: No matchups Model/Buoy available.') + print(" Matchups model/buoy complete. Total of "+repr(np.size(ind))+" stations/buoys avaliable."); del ind # Processing grid and/or cyclone information if gridinfo!=0: - print(" Adding extra information ... ") - alon=np.copy(lon); alon[alon<0]=alon[alon<0]+360. - indgplat=[]; indgplon=[] - for i in range(0,lat.shape[0]): - # indexes nearest point. - indgplat = np.append(indgplat,np.where( abs(mlat-lat[i])==abs(mlat-lat[i]).min() )[0][0]) - indgplon = np.append(indgplon,np.where( abs(mlon-alon[i])==abs(mlon-alon[i]).min() )[0][0]) - - indgplat=np.array(indgplat).astype('int'); indgplon=np.array(indgplon).astype('int') - pdistcoast=np.zeros(lat.shape[0],'f')*np.nan - pdepth=np.zeros(lat.shape[0],'f')*np.nan - poni=np.zeros(lat.shape[0],'f')*np.nan - phsmz=np.zeros(lat.shape[0],'f')*np.nan - for i in range(0,lat.shape[0]): - pdistcoast[i]=distcoast[indgplat[i],indgplon[i]] - pdepth[i]=depth[indgplat[i],indgplon[i]] - poni[i]=oni[indgplat[i],indgplon[i]] - phsmz[i]=hsmz[indgplat[i],indgplon[i]] - - print(" Grid Information Included.") - - # Excluding shallow water points too close to the coast (mask information not accurate) - ind=np.where( (np.isnan(pdistcoast)==False) & (np.isnan(pdepth)==False) ) - if np.size(ind)>0: - stname=np.array(stname[ind[0]]) - lat=np.array(lat[ind[0]]) - lon=np.array(lon[ind[0]]) - mhs=np.array(mhs[ind[0],:]) - mtm=np.array(mtm[ind[0],:]) - mtp=np.array(mtp[ind[0],:]) - mdm=np.array(mdm[ind[0],:]) - mdp=np.array(mdp[ind[0],:]) - bhs=np.array(bhs[ind[0],:]) - btm=np.array(btm[ind[0],:]) - btp=np.array(btp[ind[0],:]) - bdm=np.array(bdm[ind[0],:]) - bdp=np.array(bdp[ind[0],:]) - pdistcoast=np.array(pdistcoast[ind[0]]) - pdepth=np.array(pdepth[ind[0]]) - poni=np.array(poni[ind[0]]) - phsmz=np.array(phsmz[ind[0]]) - else: - sys.exit(' Error: No matchups Model/Buoy available after using grid mask.') - - del ind - - if cyclonemap!=0: - fcmap=np.zeros((lat.shape[0],mtime.shape[0]),'f')*np.nan - for t in range(0,np.size(mtime)): - # search for cyclone time index and cyclone map - indt=np.where(np.abs(ctime-mtime[t])<5400.) - if np.size(indt)>0: - for i in range(0,lat.shape[0]): - fcmap[i,t] = np.array(cmap[indt[0][0],indgplat[i],indgplon[i]]) - - del indt - else: - print(' - No cyclone information for this time step: '+repr(t)) - - # print(' Done cyclone analysis at step: '+repr(t)) - - ind=np.where(fcmap<0) - if np.size(ind)>0: - fcmap[ind]=np.nan - - print(" Cyclone Information Included.") + print(" Adding extra information ... ") + alon=np.copy(lon); alon[alon<0]=alon[alon<0]+360. + indgplat=[]; indgplon=[] + for i in range(0,lat.shape[0]): + # indexes nearest point. + indgplat = np.append(indgplat,np.where( abs(mlat-lat[i])==abs(mlat-lat[i]).min() )[0][0]) + indgplon = np.append(indgplon,np.where( abs(mlon-alon[i])==abs(mlon-alon[i]).min() )[0][0]) + + indgplat=np.array(indgplat).astype('int'); indgplon=np.array(indgplon).astype('int') + pdistcoast=np.zeros(lat.shape[0],'f')*np.nan + pdepth=np.zeros(lat.shape[0],'f')*np.nan + poni=np.zeros(lat.shape[0],'f')*np.nan + phsmz=np.zeros(lat.shape[0],'f')*np.nan + for i in range(0,lat.shape[0]): + pdistcoast[i]=distcoast[indgplat[i],indgplon[i]] + pdepth[i]=depth[indgplat[i],indgplon[i]] + poni[i]=oni[indgplat[i],indgplon[i]] + phsmz[i]=hsmz[indgplat[i],indgplon[i]] + + print(" Grid Information Included.") + + # Excluding shallow water points too close to the coast (mask information not accurate) + ind=np.where( (np.isnan(pdistcoast)==False) & (np.isnan(pdepth)==False) ) + if np.size(ind)>0: + stname=np.array(stname[ind[0]]) + lat=np.array(lat[ind[0]]) + lon=np.array(lon[ind[0]]) + mhs=np.array(mhs[ind[0],:]) + mtm=np.array(mtm[ind[0],:]) + mtp=np.array(mtp[ind[0],:]) + mdm=np.array(mdm[ind[0],:]) + mdp=np.array(mdp[ind[0],:]) + mwn=np.array(mwn[ind[0],:]) + bhs=np.array(bhs[ind[0],:]) + btm=np.array(btm[ind[0],:]) + btp=np.array(btp[ind[0],:]) + bdm=np.array(bdm[ind[0],:]) + bdp=np.array(bdp[ind[0],:]) + bwind=np.array(bwind[ind[0],:]) + pdistcoast=np.array(pdistcoast[ind[0]]) + pdepth=np.array(pdepth[ind[0]]) + poni=np.array(poni[ind[0]]) + phsmz=np.array(phsmz[ind[0]]) + else: + sys.exit(' Error: No matchups Model/Buoy available after using grid mask.') + + del ind + + if cyclonemap!=0: + fcmap=np.zeros((lat.shape[0],mtime.shape[0]),'f')*np.nan + for t in range(0,np.size(mtime)): + # search for cyclone time index and cyclone map + indt=np.where(np.abs(ctime-mtime[t])<5400.) + if np.size(indt)>0: + for i in range(0,lat.shape[0]): + fcmap[i,t] = np.array(cmap[indt[0][0],indgplat[i],indgplon[i]]) + + del indt + else: + print(' - No cyclone information for this time step: '+repr(t)) + + # print(' Done cyclone analysis at step: '+repr(t)) + + ind=np.where(fcmap<0) + if np.size(ind)>0: + fcmap[ind]=np.nan + + print(" Cyclone Information Included.") # Edit format if this is forecast model data. Reshape and allocate + + if forecastds>0: - unt=np.unique(mfcycle); mxsz=1 - for i in range(0,unt.shape[0]): - ind=np.where(mfcycle==unt[i])[0] - mxsz=np.max([mxsz,np.size(ind)]) - - for i in range(0,unt.shape[0]): - ind=np.where(mfcycle==unt[i])[0] - if i==0: - nmhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtime=np.zeros((unt.shape[0],mxsz),'double')*np.nan - if cyclonemap!=0: - nfcmap=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - - nmtime[i,0:np.size(ind)]=np.array(mtime[ind]).astype('double') - nmhs[:,i,:][:,0:np.size(ind)]=np.array(mhs[:,ind]) - nmtm[:,i,:][:,0:np.size(ind)]=np.array(mtm[:,ind]) - nmtp[:,i,:][:,0:np.size(ind)]=np.array(mtp[:,ind]) - nmdm[:,i,:][:,0:np.size(ind)]=np.array(mdm[:,ind]) - nmdp[:,i,:][:,0:np.size(ind)]=np.array(mdp[:,ind]) - nbhs[:,i,:][:,0:np.size(ind)]=np.array(bhs[:,ind]) - nbtm[:,i,:][:,0:np.size(ind)]=np.array(btm[:,ind]) - nbtp[:,i,:][:,0:np.size(ind)]=np.array(btp[:,ind]) - nbdm[:,i,:][:,0:np.size(ind)]=np.array(bdm[:,ind]) - nbdp[:,i,:][:,0:np.size(ind)]=np.array(bdp[:,ind]) - if cyclonemap!=0: - nfcmap[:,i,:][:,0:np.size(ind)]=np.array(fcmap[:,ind]) - - ind=np.where( (nmhs>0.0) & (nbhs>0.0) ) + unt=np.unique(mfcycle); mxsz=1 + for i in range(0,unt.shape[0]): + ind=np.where(mfcycle==unt[i])[0] + mxsz=np.max([mxsz,np.size(ind)]) + + for i in range(0,unt.shape[0]): + ind=np.where(mfcycle==unt[i])[0] + if i==0: + nmhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmwn=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nbwind=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + nmtime=np.zeros((unt.shape[0],mxsz),'double')*np.nan + if cyclonemap!=0: + nfcmap=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan + + nmtime[i,0:np.size(ind)]=np.array(mtime[ind]).astype('double') + nmhs[:,i,:][:,0:np.size(ind)]=np.array(mhs[:,ind]) + nmtm[:,i,:][:,0:np.size(ind)]=np.array(mtm[:,ind]) + nmtp[:,i,:][:,0:np.size(ind)]=np.array(mtp[:,ind]) + nmdm[:,i,:][:,0:np.size(ind)]=np.array(mdm[:,ind]) + nmdp[:,i,:][:,0:np.size(ind)]=np.array(mdp[:,ind]) + nmwn[:,i,:][:,0:np.size(ind)]=np.array(mwn[:,ind]) + nbhs[:,i,:][:,0:np.size(ind)]=np.array(bhs[:,ind]) + nbtm[:,i,:][:,0:np.size(ind)]=np.array(btm[:,ind]) + nbtp[:,i,:][:,0:np.size(ind)]=np.array(btp[:,ind]) + nbdm[:,i,:][:,0:np.size(ind)]=np.array(bdm[:,ind]) + nbdp[:,i,:][:,0:np.size(ind)]=np.array(bdp[:,ind]) + nbwind[:,i,:][:,0:np.size(ind)]=np.array(bwind[:,ind]) + + if cyclonemap!=0: + nfcmap[:,i,:][:,0:np.size(ind)]=np.array(fcmap[:,ind]) + + + ind=np.where( (nmhs>0.0) & (nbhs>0.0) ) else: - ind=np.where( (mhs>0.0) & (bhs>0.0) ) + + larger_shape = max(mhs.shape, bhs.shape) + padded_mhs = np.zeros(larger_shape) + padded_bhs = np.zeros(larger_shape) + padded_mhs[:mhs.shape[0], :mhs.shape[1]] = mhs + padded_bhs[:bhs.shape[0], :bhs.shape[1]] = bhs + + + # Create padded arrays for other variables + padded_mtm = np.zeros(larger_shape) # Assuming mtm has the same shape as mhs + padded_mtp = np.zeros(larger_shape) # Assuming mtp has the same shape as mhs + padded_mdm = np.zeros(larger_shape) # Assuming mdm has the same shape as mhs + padded_mdp = np.zeros(larger_shape) # Assuming mdp has the same shape as mhs + padded_mwn = np.zeros(larger_shape) # Assuming mwn has the same shape as mhs + + padded_btm = np.zeros(larger_shape) # Assuming btm has the same shape as bhs + padded_btp = np.zeros(larger_shape) # Assuming btp has the same shape as bhs + padded_bdm = np.zeros(larger_shape) # Assuming bdm has the same shape as bhs + padded_bdp = np.zeros(larger_shape) # Assuming bdp has the same shape as bhs + padded_bwind = np.zeros(larger_shape) # Assuming bwind has the same shape as bhs + + # Copy values from original arrays to padded arrays for other variables + padded_mtm[:mtm.shape[0], :mtm.shape[1]] = mtm + padded_mtp[:mtp.shape[0], :mtp.shape[1]] = mtp + padded_mdm[:mdm.shape[0], :mdm.shape[1]] = mdm + padded_mdp[:mdp.shape[0], :mdp.shape[1]] = mdp + padded_mwn[:mwn.shape[0], :mwn.shape[1]] = mwn + + padded_btm[:btm.shape[0], :btm.shape[1]] = btm + padded_btp[:btp.shape[0], :btp.shape[1]] = btp + padded_bdm[:bdm.shape[0], :bdm.shape[1]] = bdm + padded_bdp[:bdp.shape[0], :bdp.shape[1]] = bdp + padded_bwind[:bwind.shape[0], :bwind.shape[1]] = bwind + +# ind=np.where( (mhs>0.0) & (bhs>0.0) ) + ind = np.where((padded_mhs > 0.0) & (padded_bhs > 0.0)) if np.size(ind)>0: - print(' Total amount of matchups model/buoy: '+repr(np.size(ind))) - - # Save netcdf output file - lon[lon>180.]=lon[lon>180.]-360. - initime=str(time.gmtime(mtime.min())[0])+str(time.gmtime(mtime.min())[1]).zfill(2)+str(time.gmtime(mtime.min())[2]).zfill(2)+str(time.gmtime(mtime.min())[3]).zfill(2) - fintime=str(time.gmtime(mtime.max())[0])+str(time.gmtime(mtime.max())[1]).zfill(2)+str(time.gmtime(mtime.max())[2]).zfill(2)+str(time.gmtime(mtime.max())[3]).zfill(2) - ncfile = nc.Dataset('WW3.Buoy'+ftag+'_'+initime+'to'+fintime+'.nc', "w", format=fnetcdf) - ncfile.history="Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of "+repr(bhs[bhs>0.].shape[0])+" observations or pairs model/observation." - # create dimensions - ncfile.createDimension('buoypoints', bhs.shape[0] ) - if gridinfo!=0: - ncfile.createDimension('GlobalOceansSeas', ocnames.shape[0] ) - ncfile.createDimension('HighSeasMarineZones', hsmznames.shape[0] ) - if cyclonemap!=0: - ncfile.createDimension('cycloneinfo', cinfo.shape[0] ) - vcinfo = ncfile.createVariable('cycloneinfo',dtype('a25'),('cycloneinfo')) - # create variables. - vstname = ncfile.createVariable('buoyID',dtype('a25'),('buoypoints')) - vlat = ncfile.createVariable('latitude',np.dtype('float32').char,('buoypoints')) - vlon = ncfile.createVariable('longitude',np.dtype('float32').char,('buoypoints')) - - if forecastds>0: - ncfile.createDimension('time', nmhs.shape[2] ) - ncfile.createDimension('fcycle', unt.shape[0] ) - vt = ncfile.createVariable('time',np.dtype('float64').char,('fcycle','time')) - vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) - else: - ncfile.createDimension('time', bhs.shape[1] ) - vt = ncfile.createVariable('time',np.dtype('float64').char,('time')) - vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','time')) - vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','time')) - vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','time')) - vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','time')) - vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','time')) - vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','time')) - vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','time')) - vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','time')) - vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','time')) - vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','time')) - - if gridinfo!=0: - vpdistcoast = ncfile.createVariable('distcoast',np.dtype('float32').char,('buoypoints')) - vpdepth = ncfile.createVariable('depth',np.dtype('float32').char,('buoypoints')) - vponi = ncfile.createVariable('GlobalOceansSeas',np.dtype('float32').char,('buoypoints')) - vocnames = ncfile.createVariable('names_GlobalOceansSeas',dtype('a25'),('GlobalOceansSeas')) - vphsmz = ncfile.createVariable('HighSeasMarineZones',np.dtype('float32').char,('buoypoints')) - vhsmznames = ncfile.createVariable('names_HighSeasMarineZones',dtype('a25'),('HighSeasMarineZones')) - if cyclonemap!=0: - if forecastds>0: - vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','fcycle','time')) - else: - vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','time')) - - # Assign units - vlat.units = 'degrees_north' ; vlon.units = 'degrees_east' - vt.units = 'seconds since 1970-01-01T00:00:00+00:00' - vmhs.units='m'; vbhs.units='m' - vmtm.units='s'; vbtm.units='s' - vmtp.units='s'; vbtp.units='s' - vmdm.units='degrees'; vbdm.units='degrees' - vmdp.units='degrees'; vbdp.units='degrees' - if gridinfo!=0: - vpdepth.units='m'; vpdistcoast.units='km' - - # Allocate Data - vstname[:]=stname[:]; vlat[:] = lat[:]; vlon[:] = lon[:] - if forecastds>0: - vt[:,:]=nmtime[:,:] - vmhs[:,:,:]=nmhs[:,:,:] - vmtm[:,:,:]=nmtm[:,:,:] - vmtp[:,:,:]=nmtp[:,:,:] - vmdm[:,:,:]=nmdm[:,:,:] - vmdp[:,:,:]=nmdp[:,:,:] - vbhs[:,:,:]=nbhs[:,:,:] - vbtm[:,:,:]=nbtm[:,:,:] - vbtp[:,:,:]=nbtp[:,:,:] - vbdm[:,:,:]=nbdm[:,:,:] - vbdp[:,:,:]=nbdp[:,:,:] - else: - vt[:]=mtime[:] - vmhs[:,:]=mhs[:,:] - vmtm[:,:]=mtm[:,:] - vmtp[:,:]=mtp[:,:] - vmdm[:,:]=mdm[:,:] - vmdp[:,:]=mdp[:,:] - vbhs[:,:]=bhs[:,:] - vbtm[:,:]=btm[:,:] - vbtp[:,:]=btp[:,:] - vbdm[:,:]=bdm[:,:] - vbdp[:,:]=bdp[:,:] - - if gridinfo!=0: - vpdistcoast[:]=pdistcoast[:] - vpdepth[:]=pdepth[:] - vponi[:]=poni[:]; vocnames[:] = ocnames[:] - vphsmz[:]=phsmz[:]; vhsmznames[:] = hsmznames[:] - if cyclonemap!=0: - vcinfo[:] = cinfo[:] - if forecastds>0: - vcmap[:,:,:]=nfcmap[:,:,:] - else: - vcmap[:,:]=fcmap[:,:] - - ncfile.close() - print(' ') - print('Done. Netcdf ok. New file saved: WW3.Buoy'+ftag+'_'+initime+'to'+fintime+'.nc') + print(' Total amount of matchups model/buoy: '+repr(np.size(ind))) + + # Save netcdf output file + lon[lon>180.]=lon[lon>180.]-360. + initime=str(time.gmtime(mtime.min())[0])+str(time.gmtime(mtime.min())[1]).zfill(2)+str(time.gmtime(mtime.min())[2]).zfill(2)+str(time.gmtime(mtime.min())[3]).zfill(2) + fintime=str(time.gmtime(mtime.max())[0])+str(time.gmtime(mtime.max())[1]).zfill(2)+str(time.gmtime(mtime.max())[2]).zfill(2)+str(time.gmtime(mtime.max())[3]).zfill(2) + ncfile = nc.Dataset('WW3.Buoy'+str(ftag)+'_'+initime+'to'+fintime+'.nc', "w", format=fnetcdf) + ncfile.history="Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of "+repr(bhs[bhs>0.].shape[0])+" observations or pairs model/observation." + # create dimensions + ncfile.createDimension('buoypoints', bhs.shape[0] ) + if gridinfo!=0: + ncfile.createDimension('GlobalOceansSeas', ocnames.shape[0] ) + ncfile.createDimension('HighSeasMarineZones', hsmznames.shape[0] ) + if cyclonemap!=0: + ncfile.createDimension('cycloneinfo', cinfo.shape[0] ) + vcinfo = ncfile.createVariable('cycloneinfo',dtype('a25'),('cycloneinfo')) + # create variables. + vstname = ncfile.createVariable('buoyID',type('a25'),('buoypoints')) + vlat = ncfile.createVariable('latitude',np.dtype('float32').char,('buoypoints')) + vlon = ncfile.createVariable('longitude',np.dtype('float32').char,('buoypoints')) + + if forecastds>0: + ncfile.createDimension('time', nmhs.shape[2] ) + ncfile.createDimension('fcycle', unt.shape[0] ) + vt = ncfile.createVariable('time',np.dtype('float64').char,('fcycle','time')) + vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vmwn = ncfile.createVariable('model_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) + + vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) + vbwind = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) + + else: + ncfile.createDimension('time', bhs.shape[1] ) + vt = ncfile.createVariable('time',np.dtype('float64').char,('time')) + vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','time')) + vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','time')) + vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','time')) + vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','time')) + vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','time')) + vmwn = ncfile.createVariable('model_wind',np.dtype('float32').char,('buoypoints','time')) + + vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','time')) + vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','time')) + vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','time')) + vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','time')) + vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','time')) + vbwind = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','time')) + + + + + if gridinfo!=0: + vpdistcoast = ncfile.createVariable('distcoast',np.dtype('float32').char,('buoypoints')) + vpdepth = ncfile.createVariable('depth',np.dtype('float32').char,('buoypoints')) + vponi = ncfile.createVariable('GlobalOceansSeas',np.dtype('float32').char,('buoypoints')) + vocnames = ncfile.createVariable('names_GlobalOceansSeas',dtype('a25'),('GlobalOceansSeas')) + vphsmz = ncfile.createVariable('HighSeasMarineZones',np.dtype('float32').char,('buoypoints')) + vhsmznames = ncfile.createVariable('names_HighSeasMarineZones',dtype('a25'),('HighSeasMarineZones')) + if cyclonemap!=0: + if forecastds>0: + vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','fcycle','time')) + else: + vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','time')) + + # Assign units + vlat.units = 'degrees_north' ; vlon.units = 'degrees_east' + vt.units = 'seconds since 1970-01-01T00:00:00+00:00' + vmhs.units='m'; vbhs.units='m' + vmtm.units='s'; vbtm.units='s' + vmtp.units='s'; vbtp.units='s' + vmdm.units='degrees'; vbdm.units='degrees' + vmdp.units='degrees'; vbdp.units='degrees' + vmwn.unit='m/s';vbwind.unit='m/s' + + if gridinfo!=0: + vpdepth.units='m'; vpdistcoast.units='km' + + # Allocate Data + vstname[:]=stname[:]; vlat[:] = lat[:]; vlon[:] = lon[:] + if forecastds>0: + vt[:,:]=nmtime[:,:] + vmhs[:,:,:]=nmhs[:,:,:] + vmtm[:,:,:]=nmtm[:,:,:] + vmtp[:,:,:]=nmtp[:,:,:] + vmdm[:,:,:]=nmdm[:,:,:] + vmdp[:,:,:]=nmdp[:,:,:] + vmwn[:,:,:]=nmwn[:,:,:] + vbhs[:,:,:]=nbhs[:,:,:] + vbtm[:,:,:]=nbtm[:,:,:] + vbtp[:,:,:]=nbtp[:,:,:] + vbdm[:,:,:]=nbdm[:,:,:] + vbdp[:,:,:]=nbdp[:,:,:] + vbwind[:,:,:]=nbwind[:,:,:] + + else: + vt[:]=mtime[:] + vmhs[:,:]=padded_mhs[:,:] + vmtm[:,:]=padded_mtm[:,:] + vmtp[:,:]=padded_mtp[:,:] + vmdm[:,:]=padded_mdm[:,:] + vmdp[:,:]=padded_mdp[:,:] + vmwn[:,:]=padded_mwn[:,:] + vbhs[:,:]=padded_bhs[:,:] + vbtm[:,:]=btm[:,:] + vbtp[:,:]=btp[:,:] + vbdm[:,:]=bdm[:,:] + vbdp[:,:]=bdp[:,:] + vbwind[:,:]=bwind[:,:] + + + + + if gridinfo!=0: + vpdistcoast[:]=pdistcoast[:] + vpdepth[:]=pdepth[:] + vponi[:]=poni[:]; vocnames[:] = ocnames[:] + vphsmz[:]=phsmz[:]; vhsmznames[:] = hsmznames[:] + if cyclonemap!=0: + vcinfo[:] = cinfo[:] + if forecastds>0: + vcmap[:,:,:]=nfcmap[:,:,:] + else: + vcmap[:,:]=fcmap[:,:] + + ncfile.close() + print(' ') + print('Done. Netcdf ok. New file saved: WW3.Buoy'+str(ftag)+'_'+initime+'to'+fintime+'.nc') From 11b026d49a054b643cf6673061979ccaf2b64c8d Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 22 Apr 2024 15:14:41 +0000 Subject: [PATCH 12/27] removed extra comments --- ww3tools/wread.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/ww3tools/wread.py b/ww3tools/wread.py index 527866c..df71f33 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -60,7 +60,7 @@ """ ======= ->>>>>>> 12a8a9d (wind variable added to the .spec in a list of buoy nams and boy IDs) + import matplotlib import time import timeit @@ -82,7 +82,6 @@ import tarfile import math -<<<<<<< HEAD def readconfig(fname): """ @@ -134,7 +133,7 @@ def readconfig(fname): def mask(*args): ======= def spec_ww3(*args): ->>>>>>> 12a8a9d (wind variable added to the .spec in a list of buoy nams and boy IDs) + ''' WAVEWATCH III, wave spectrum, netcdf (.nc) or text (.spec) format Input: file names (list of file names), and station names (list of station names) @@ -142,7 +141,6 @@ def spec_ww3(*args): time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pwst,d1sp,dire,dspec,wnds,wndd ''' -<<<<<<< HEAD print(" reading ww3_tools mask ...") try: f=nc.Dataset(fname) @@ -1576,7 +1574,7 @@ def spec_ww3(*args): fp.close(); del fp - # mdate = [date2num(datetime.datetime.utcfromtimestamp(time_stamp)) for time_stamp in mtime] + mdate = pd.to_datetime(mtime, unit='s').strftime('%Y-%m-%dT%H:%M:%S.%f') freq1=freq*np.nan; freq2=freq*np.nan @@ -1626,8 +1624,6 @@ def spec_ww3(*args): del mtime,mdate,lat,lon,wnds,wndd,freq,freq1,freq2,dfreq,pwst,dire,d1sp,dspec -<<<<<<< HEAD -======= #added a function to read the txt files def read_text_file(fname_txtfile): @@ -1653,7 +1649,7 @@ def read_text_file(fname_txtfile): # Station names stname.append(str(tar.getmembers()[t].name).split('/')[-1].split('.')[-2]) -======= + if len(args) < 2: sys.exit(' Two inputs are required: list of file names and list of station names') @@ -1668,7 +1664,7 @@ def read_text_file(fname_txtfile): file_names = args[0] station_names = args[1] results = [] ->>>>>>> 12a8a9d (wind variable added to the .spec in a list of buoy nams and boy IDs) + for fname in fnames: for stname in stnames: From 99e2ed1eeea5194cadb138da76ba633439402763 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 22 Apr 2024 15:27:46 +0000 Subject: [PATCH 13/27] updated instruction on how to use the .spec (gz) --- ww3tools/modelBuoy_collocation.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index 008db3e..c832356 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -88,9 +88,19 @@ and ww3) to maximize the amount of matchups even when one variable is not available. -How to run .spec file in a format of the .gz file: python3 modelBuoy_collocation.py ww3list.txt 3 -In the ww3list.txt, you have to define the .gz file path. -ww3list.txt is just name for the text file that contains the path for the .gz file.Any name for this file can be defined. +.spec update: + +How to run .spec file in a format of the .gz file: +Here is an example of how the job script should look like: +# Define variables +input_gz_file="multi_1.t11z.spec_tar.gz" +output_directory="./" (any directory) +buoy_path="/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" (This path is on Hera. The path for Orion is different) + +# Process data command +python3 modelBuoy_collocation.py unzip "$input_gz_file" "$output_directory" "$buoy_path" + + PERSON OF CONTACT: Ricardo M Campos: ricardo.campos@noaa.gov From c0aa513eb8c93246c70b23bb886271064bb32969 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 22 Apr 2024 21:10:34 +0000 Subject: [PATCH 14/27] description updated --- ww3tools/wread.py | 79 +++++++++++++++++------------------------------ 1 file changed, 28 insertions(+), 51 deletions(-) diff --git a/ww3tools/wread.py b/ww3tools/wread.py index df71f33..936a81c 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1,4 +1,3 @@ -<<<<<<< HEAD #!/usr/bin/env python3 # -*- coding: utf-8 -*- @@ -59,11 +58,8 @@ """ -======= - import matplotlib import time -import timeit from time import strptime from calendar import timegm import pandas as pd @@ -71,10 +67,9 @@ import netCDF4 as nc import numpy as np from pylab import * -import yaml -import re -import os +import matplotlib.pyplot as plt import sys +import pandas as pd from matplotlib import ticker # import pickle import sys @@ -131,15 +126,15 @@ def readconfig(fname): def mask(*args): -======= -def spec_ww3(*args): - ''' - WAVEWATCH III, wave spectrum, netcdf (.nc) or text (.spec) format - Input: file names (list of file names), and station names (list of station names) - Output: list of dictionaries containing: - time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pwst,d1sp,dire,dspec,wnds,wndd + Read gridmask netcdf file generated with prepGridMask.py + Input: file name (example: gridInfo_GEFSv12.nc) + Output: dictionary containing the arrays and string names ''' + if len(args) == 1: + fname=str(args[0]) + else: + sys.exit(' Too many inputs') print(" reading ww3_tools mask ...") try: @@ -1574,7 +1569,7 @@ def spec_ww3(*args): fp.close(); del fp - + # mdate = [date2num(datetime.datetime.utcfromtimestamp(time_stamp)) for time_stamp in mtime] mdate = pd.to_datetime(mtime, unit='s').strftime('%Y-%m-%dT%H:%M:%S.%f') freq1=freq*np.nan; freq2=freq*np.nan @@ -1624,31 +1619,13 @@ def spec_ww3(*args): del mtime,mdate,lat,lon,wnds,wndd,freq,freq1,freq2,dfreq,pwst,dire,d1sp,dspec -#added a function to read the txt files - -def read_text_file(fname_txtfile): - try: - # Attempt to open and read the file name from the txt file - with open(fname_txtfile, 'r') as f: - lines = f.readlines() - if len(lines) != 1: - raise ValueError("The txt file should contain only one line with the file name.") - fname = lines[0].strip() - except FileNotFoundError: - sys.exit('Text file not found.') - except Exception as e: - sys.exit(f'Error reading txt file: {str(e)}') - - results = {} - stname = [] - - try: - tar = tarfile.open(fname, "r:gz") # Open the tar file - - for t in range(0, len(tar.getmembers())): - # Station names - - stname.append(str(tar.getmembers()[t].name).split('/')[-1].split('.')[-2]) +def spec1_ww3(*args): + ''' + WAVEWATCH III, wave spectrum, netcdf (.nc) or text (.spec) format + Input: file names (list of file names), and station names (list of station names) + Output: list of dictionaries containing: + time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pwst,d1sp,dire,dspec,wnds,wndd + ''' if len(args) < 2: sys.exit(' Two inputs are required: list of file names and list of station names') @@ -1665,11 +1642,10 @@ def read_text_file(fname_txtfile): station_names = args[1] results = [] - for fname in fnames: for stname in stnames: try: - # Text format (only one point allowed here, same as WW3/NOAA operational) + fp = open(fname) nt = fp.read().count(stname) fp.close() @@ -1750,9 +1726,9 @@ def read_text_file(fname_txtfile): cabc.strip().split()[0]+cabc.strip().split()[1][0:2], '%Y%m%d%H'))) cabc = fp.readline() cabc = cabc.strip().split() - print(cabc) + if len(cabc) >8: - # Format: ["'42085", "'", '17.86', '-66.52', '126.1', '1.12', '146.9'] + namep = cabc[0][1:] lat = float(cabc[2]) lon = float(cabc[3]) @@ -1760,12 +1736,11 @@ def read_text_file(fname_txtfile): wnds_index = 5 wndd_index = 6 elif len(cabc) == 8: - # Format: ["'46021", "'", '57.70-160.00', '50.8', '5.47', '6.9', '0.00', '270.0'] + namep = cabc[0][1:] lat_lon_str = cabc[2] lat_lon_str = lat_lon_str.strip("'") lat_lon_parts = lat_lon_str.split('-') - print(lat_lon_parts) lat = float(lat_lon_parts[0]) lon = -float(lat_lon_parts[1]) @@ -1773,8 +1748,8 @@ def read_text_file(fname_txtfile): wnds_index = 4 wndd_index = 5 else: - continue # Skip this file as it cannot be processed - # sys.exit('Unrecognized format of cabc') + continue + wnds[t] = float(cabc[wnds_index]) wndd[t] = float(cabc[wndd_index]) @@ -1847,8 +1822,11 @@ def read_text_file(fname_txtfile): d1sp[t, il] = float(aux) del a, b, aux - hs = np.sqrt(2 * np.trapz(np.trapz(dspec, x=dire, axis=-1), x=freq, axis=-1)) - tp = freq[np.argmax(np.max(dspec, axis=-1), axis=-1)] + m0 = np.sum(pwst , axis=1) + hs = 4 * np.sqrt(m0) + max_index = np.argmax(pwst, axis=1) + f_max = freq[max_index] + tp = 1 / f_max # build dictionary result = {'time': mtime, 'date': mdate, 'latitude': lat, 'longitude': lon, 'depth': depth, @@ -1862,4 +1840,3 @@ def read_text_file(fname_txtfile): return results - From a1e3ed5fe18d3d004de855b3312659f59f0f8765 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Sun, 5 May 2024 22:56:09 +0000 Subject: [PATCH 15/27] Fixed the parameter in the unzip_and_untar function to use extracted_folder instead of output_dir to ensure correct file extraction. --- ww3tools/modelBuoy_collocation.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index c832356..365d49d 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -142,10 +142,15 @@ def unzip_and_untar(gz_file, output_dir): with open(output_file, 'wb') as f_out: f_out.write(f_in.read()) - with tarfile.open(output_file, 'r') as tar: - tar.extractall(output_dir) +# with tarfile.open(output_file, 'r') as tar: +# tar.extractall(output_dir) extracted_folder = os.path.join(output_dir, base_name) + + with tarfile.open(output_file, 'r') as tar: + tar.extractall(extracted_folder) + + # Creating a list of the extracted files list_file = os.path.join(output_dir, f'{base_name}_contents.txt') with open(list_file, 'w') as f: From 0adf9f50caaa167ce8c7a52678f516a3cf7515b7 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Sat, 25 May 2024 00:49:23 +0000 Subject: [PATCH 16/27] The code is updated to be able to process the spec.gz buoy file formats --- ww3tools/modelBuoy_collocation.py | 976 ++++++++++++++---------------- ww3tools/wread.py | 287 ++++----- 2 files changed, 569 insertions(+), 694 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index 365d49d..7c69334 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -11,6 +11,7 @@ v1.3 11/17/2022 v1.4 12/08/2022 v1.5 01/31/2023 + v1.6 05/21/2024 PURPOSE: Collocation/pairing ww3 point output results with wave buoys. @@ -33,7 +34,7 @@ USAGE: Input WW3 point outputs are utilized (not grids), with option of - reading different formats: netcdf, text (bull and ts), +modelBuoy_collocation.py reading different formats: netcdf, text (bull and ts), or tar files with multiple bull or ts files. For the observations, it uses two public buoy databases, NDBC and Copernicus, which (at least one) must have been previously @@ -55,6 +56,26 @@ multiple forecast data files: nohup python3 modelBuoy_collocation.py ww3list.gfs-d36.GSE1.5.txt 2 gridInfo.nc CycloneMap_2021.nc >> nohup_modelBuoy_collocation.out 2>&1 & +- From 05/21/24 if you want to run this you have to define the (buoy_path) variable in the command line or in the job script. This recent does not accept the +hard coded ndbcp path anymore. + +USAGE (spec.gz): + In wread.py the ww3_spec1 is responsible for reading and accepting the spec.gz files like,ex:gfswave.t00z.spec_tar.gz + sort of files. + to run the code one should have to define these values in the job script or the command line: + +# Define the arguments + input_gz_file="./gfswave.t00z.spec_tar.gz" + output_directory="./" (Where the users wants to have the output saved) + buoy_path="/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" (The path should be changed based on the users buoy path directory) + model_name="HR3a" # Define the model name + forecast_ds="1" # Indicator for forecast data structure, set to 1 or 0 based on requirements + (If it is 1 or greater than 1, it does the forecast structure) + +# Process data for each date + python3 modelBuoy_collocation.py spec.gz "$input_gz_file" "$output_directory" "$buoy_path" "$model_name " "$forecast_ds" + + OUTPUT: netcdf file WW3.Buoy*.nc containing matchups of buoy and ww3 data, for the stations (lat/lon) where both data sources are available. @@ -87,19 +108,8 @@ dimensions), and check if variable names exist in the netcdf file (buoy and ww3) to maximize the amount of matchups even when one variable is not available. - -.spec update: - -How to run .spec file in a format of the .gz file: -Here is an example of how the job script should look like: -# Define variables -input_gz_file="multi_1.t11z.spec_tar.gz" -output_directory="./" (any directory) -buoy_path="/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" (This path is on Hera. The path for Orion is different) - -# Process data command -python3 modelBuoy_collocation.py unzip "$input_gz_file" "$output_directory" "$buoy_path" - + 05/21/2024 : Ghazal Mohammadpour , developed the code to be able to process + the spec.gz files. PERSON OF CONTACT: @@ -118,6 +128,8 @@ from time import strptime from calendar import timegm import wread +import glob +import shutil # Suppressing warnings warnings.filterwarnings("ignore") @@ -129,8 +141,6 @@ cyclonemap = int(0) wlist = [] ftag = '' -forecastds = 0 - # Function to unzip and untar files def unzip_and_untar(gz_file, output_dir): @@ -142,15 +152,11 @@ def unzip_and_untar(gz_file, output_dir): with open(output_file, 'wb') as f_out: f_out.write(f_in.read()) -# with tarfile.open(output_file, 'r') as tar: -# tar.extractall(output_dir) - extracted_folder = os.path.join(output_dir, base_name) with tarfile.open(output_file, 'r') as tar: tar.extractall(extracted_folder) - # Creating a list of the extracted files list_file = os.path.join(output_dir, f'{base_name}_contents.txt') with open(list_file, 'w') as f: @@ -167,23 +173,24 @@ def unzip_and_untar(gz_file, output_dir): if len(file_parts) >= 3: id_between_dots = file_parts[1] f.write(id_between_dots + '\n') - + # Return the path to the extracted folder return extracted_folder, list_file, id_file - # Main part of the script if __name__ == "__main__": mode = sys.argv[1] if len(sys.argv) > 1 else "" - if mode == "unzip": - if len(sys.argv) != 5: + if mode == "spec.gz": + if len(sys.argv) != 7: sys.exit("Usage: python script.py unzip input_gz_file output_directory buoy_path") gz_file = sys.argv[2] output_dir = sys.argv[3] buoy_path = sys.argv[4] + model_name = sys.argv[5] + forecastds = int(sys.argv[6]) # Convert forecast indicator to integer ndbcp = buoy_path extracted_folder, list_file, id_file = unzip_and_untar(gz_file, output_dir) @@ -298,6 +305,7 @@ def unzip_and_untar(gz_file, output_dir): else: print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) + else: gridinfo = int(0) cyclonemap = int(0) @@ -305,28 +313,28 @@ def unzip_and_untar(gz_file, output_dir): ftag = '' forecastds = 0 - if len(sys.argv) < 2: + if len(sys.argv) < 3: sys.exit('At least one argument (list of ww3 files) must be informed.') wlist = np.atleast_1d(np.loadtxt(sys.argv[1], dtype=str)) ftag = str(sys.argv[1]).split('list')[1].split('.txt')[0] print('Reading ww3 list ' + str(sys.argv[1])) print('Tag ' + ftag) - if len(sys.argv) >= 3: - forecastds = int(sys.argv[2]) - if forecastds > 0: - print('Forecast-type data structure') + forecastds = int(sys.argv[2]) if len(sys.argv) >= 4 else 0 + if forecastds > 0: + print('Forecast-type data structure') - if len(sys.argv) >= 4: - gridinfo = str(sys.argv[3]) + gridinfo = str(sys.argv[3]) if len(sys.argv) >= 5 else int(0) + if gridinfo != 0: print('Writing gridinfo ' + gridinfo) - if len(sys.argv) >= 5: - cyclonemap = str(sys.argv[4]) + cyclonemap = str(sys.argv[4]) if len(sys.argv) >= 6 else int(0) + if cyclonemap != 0: print('Writing cyclonemap ' + cyclonemap) # Paths - ndbcp = "/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" + ndbcp = str(sys.argv[-1]) # Always get the last argument for ndbcp path + print('Using ndbcp path: ' + ndbcp) # READ DATA print(" ") @@ -389,7 +397,7 @@ def unzip_and_untar(gz_file, output_dir): else: mwd = np.copy(mhs) * np.nan - + else: if (mhs.shape[0] == result['hs'].shape[0]) and ( np.size(stname) == np.size(result['station_name'])): @@ -415,8 +423,7 @@ def unzip_and_untar(gz_file, output_dir): else: - print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[ - i]) + print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) del result, at, fcycle mdm = np.copy(mhs) * np.nan; @@ -520,17 +527,17 @@ def unzip_and_untar(gz_file, output_dir): if 'th1m' in f.variables.keys(): adm = np.array(f.variables['th1m'][:, :]).T else: - adm = np.array(np.copy(ahs * nan)) + adm = np.array(np.copy(ahs * np.nan)) if 'th1p' in f.variables.keys(): adp = np.array(f.variables['th1p'][:, :]).T else: - adp = np.array(np.copy(ahs * nan)) + adp = np.array(np.copy(ahs * np.nan)) if 'tr' in f.variables.keys(): atm = np.array(f.variables['tr'][:, :]).T else: - atm = np.array(np.copy(ahs * nan)) + atm = np.array(np.copy(ahs * np.nan)) if 'fp' in f.variables.keys(): auxtp = np.array(f.variables['fp'][:, :]).T @@ -543,7 +550,7 @@ def unzip_and_untar(gz_file, output_dir): del auxtp else: - atm = np.array(np.copy(ahs * nan)) + atm = np.array(np.copy(ahs * np.nan)) ftunits = str(f.variables['time'].units).split('since')[1][1::].replace('T', ' ').replace('+00:00', '') @@ -560,6 +567,7 @@ def unzip_and_untar(gz_file, output_dir): mtp = np.copy(atp) mdm = np.copy(adm) mdp = np.copy(adp) + mwn = np.copy(awm) mtime = np.copy(at) mfcycle = np.copy(fcycle) else: @@ -568,6 +576,7 @@ def unzip_and_untar(gz_file, output_dir): mtp = np.append(mtp, atp, axis=1) mdm = np.append(mdm, adm, axis=1) mdp = np.append(mdp, adp, axis=1) + mwn = np.append(mwn, awm, axis=1) mtime = np.append(mtime, at) mfcycle = np.append(mfcycle, fcycle) @@ -584,512 +593,439 @@ def unzip_and_untar(gz_file, output_dir): print(' ') -# BUOYS ------------------ -bwind = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -bhs = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -btm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -btp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -bdm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -bdp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan -lat = np.zeros(np.size(stname), 'f') * np.nan -lon = np.zeros(np.size(stname), 'f') * np.nan -# help reading NDBC buoys, divided by year -yrange = np.array(np.arange(time.gmtime(mtime.min())[0], time.gmtime(mtime.min())[0] + 1, 1)).astype('int') -# loop buoys -for b in range(0, np.size(stname)): - - ahs = [] - try: - awm = [] + # BUOYS ------------------ + bwind = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan + bhs = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan + btm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan + btp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan + bdm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan + bdp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan + lat = np.zeros(np.size(stname), 'f') * np.nan + lon = np.zeros(np.size(stname), 'f') * np.nan + # help reading NDBC buoys, divided by year + yrange = np.array(np.arange(time.gmtime(mtime.min())[0], time.gmtime(mtime.min())[0] + 1, 1)).astype('int') + # loop buoys + for b in range(0, np.size(stname)): + ahs = [] - atm = [] - atp = [] - adm = [] - atime = [] - for y in yrange: - - f = nc.Dataset(ndbcp + "/" + stname[b] + "h" + repr(y) + ".nc") - if 'wave_height' in f.variables.keys(): - ahs = np.append(ahs, f.variables['wave_height'][:, 0, 0]) - elif 'hs' in f.variables.keys(): - ahs = np.append(ahs, f.variables['hs'][:, 0, 0]) - elif 'swh' in f.variables.keys(): - ahs = np.append(ahs, f.variables['swh'][:, 0, 0]) - - if 'wind_spd' in f.variables.keys(): - awm = np.append(awm, f.variables['wind_spd'][:, 0, 0]) - else: - awm = np.array(np.copy(ahs * nan)) + try: + awm = [] + ahs = [] + atm = [] + atp = [] + adm = [] + atime = [] + for y in yrange: + + f = nc.Dataset(ndbcp + "/" + stname[b] + "h" + repr(y) + ".nc") + if 'wave_height' in f.variables.keys(): + ahs = np.append(ahs, f.variables['wave_height'][:, 0, 0]) + elif 'hs' in f.variables.keys(): + ahs = np.append(ahs, f.variables['hs'][:, 0, 0]) + elif 'swh' in f.variables.keys(): + ahs = np.append(ahs, f.variables['swh'][:, 0, 0]) + + if 'wind_spd' in f.variables.keys(): + awm = np.append(awm, f.variables['wind_spd'][:, 0, 0]) + else: + awm = np.array(np.copy(ahs * np.nan)) - if 'average_wpd' in f.variables.keys(): - atm = np.append(atm, f.variables['average_wpd'][:, 0, 0]) - else: - atm = np.array(np.copy(ahs * nan)) + if 'average_wpd' in f.variables.keys(): + atm = np.append(atm, f.variables['average_wpd'][:, 0, 0]) + else: + atm = np.array(np.copy(ahs * np.nan)) - if 'dominant_wpd' in f.variables.keys(): - atp = np.append(atp, f.variables['dominant_wpd'][:, 0, 0]) - else: - atp = np.array(np.copy(ahs * nan)) + if 'dominant_wpd' in f.variables.keys(): + atp = np.append(atp, f.variables['dominant_wpd'][:, 0, 0]) + else: + atp = np.array(np.copy(ahs * np.nan)) - if 'mean_wave_dir' in f.variables.keys(): - adm = np.append(adm, f.variables['mean_wave_dir'][:, 0, 0]) - else: - adm = np.array(np.copy(ahs * nan)) + if 'mean_wave_dir' in f.variables.keys(): + adm = np.append(adm, f.variables['mean_wave_dir'][:, 0, 0]) + else: + adm = np.array(np.copy(ahs * np.nan)) - if 'latitude' in f.variables.keys(): - lat[b] = f.variables['latitude'][:] - elif 'LATITUDE' in f.variables.keys(): - lat[b] = f.variables['LATITUDE'][:] - else: - lat[b] = nan + if 'latitude' in f.variables.keys(): + lat[b] = f.variables['latitude'][:] + elif 'LATITUDE' in f.variables.keys(): + lat[b] = f.variables['LATITUDE'][:] + else: + lat[b] = np.nan - if 'longitude' in f.variables.keys(): - lon[b] = f.variables['longitude'][:] - elif 'LONGITUDE' in f.variables.keys(): - lon[b] = f.variables['LONGITUDE'][:] - else: - lon[b] = nan - - atime = np.append(atime, np.array(f.variables['time'][:]).astype('double')) - - f.close() - del f - - adp = adm * np.nan # no peak direction available in this format - - if np.size(ahs) > 0: - - # First layer of simple quality-control - indq = np.where((ahs > 30.) | (ahs < 0.0)) - if np.size(indq) > 0: - ahs[indq] = np.nan - del indq - - indq = np.where((atm > 40.) | (atm < 0.0)) - if np.size(indq) > 0: - atm[indq] = np.nan - del indq - - indq = np.where((atp > 40.) | (atp < 0.0)) - if np.size(indq) > 0: - atp[indq] = np.nan - del indq - - indq = np.where((adm > 360.) | (adm < -180.)) - if np.size(indq) > 0: - adm[indq] = np.nan - del indq - - indq = np.where((adp > 360.) | (adp < -180.)) - if np.size(indq) > 0: - adp[indq] = np.nan - del indq - - indq = np.where((awm > 50.) | (awm < 0.0)) - if np.size(indq) > 0: - awm[indq] = np.nan - del indq - - c = 0 - for t in range(0, np.size(mtime)): - indt = np.where(np.abs(atime - mtime[t]) < 1800.) - if np.size(indt) > 0: - if np.any(ahs[indt[0]].mask == False): - bhs[b, t] = np.nanmean(ahs[indt[0]][ahs[indt[0]].mask == False]) - c = c + 1 - if np.any(atm[indt[0]].mask == False): - btm[b, t] = np.nanmean(atm[indt[0]][atm[indt[0]].mask == False]) - if np.any(atp[indt[0]].mask == False): - btp[b, t] = np.nanmean(atp[indt[0]][atp[indt[0]].mask == False]) - if np.any(adm[indt[0]].mask == False): - bdm[b, t] = np.nanmean(adm[indt[0]][adm[indt[0]].mask == False]) - if np.any(adp[indt[0]].mask == False): - bdp[b, t] = np.nanmean(adp[indt[0]][adp[indt[0]].mask == False]) - if np.any(awm[indt[0]].mask == False): - bwind[b, t] = np.nanmean(awm[indt[0]][awm[indt[0]].mask == False]) - - del indt - - # print("counted "+repr(c)+" at "+stname[b]) - - print(" station " + stname[b] + " ok") -# del ahs - except Exception as e: - print("Error occurred while processing station", stname[b]) - print(e) - -print('bwind:',bwind) -print('bhs:',bhs) - - -print(' ') -# Simple quality-control (range) -ind=np.where((bhs>30.)|(bhs<0.0)) -if np.size(ind)>0: - bhs[ind]=np.nan; del ind - -ind=np.where((btm>40.)|(btm<0.0)) -if np.size(ind)>0: - btm[ind]=np.nan; del ind - -ind=np.where((btp>40.)|(btp<0.0)) -if np.size(ind)>0: - btp[ind]=np.nan; del ind - -ind=np.where((bdm>360.)|(bdm<-180.)) -if np.size(ind)>0: - bdm[ind]=np.nan; del ind - -ind=np.where((bdp>360.)|(bdp<-180.)) -if np.size(ind)>0: - bdp[ind]=np.nan; del ind - -ind=np.where((bwind>50.0)|(bwind<0.0)) -if np.size(ind)>0: - bwind[ind]=np.nan; del ind - -ind=np.where((mhs>30.)|(mhs<0.0)) -if np.size(ind)>0: - mhs[ind]=np.nan; del ind - -ind=np.where((mtm>40.)|(mtm<0.0)) -if np.size(ind)>0: - mtm[ind]=np.nan; del ind - -ind=np.where((mtp>40.)|(mtp<0.0)) -if np.size(ind)>0: - mtp[ind]=np.nan; del ind - - -ind=np.where((mdm>360.)|(mdm<-180.)) -if np.size(ind)>0: - mdm[ind]=np.nan; del ind - -ind=np.where((mdp>360.)|(mdp<-180.)) -if np.size(ind)>0: - mdp[ind]=np.nan; del ind - -ind=np.where((mwn>50.)|(mwn<0.0)) -if np.size(ind)>0: - mwn[ind]=np.nan; del ind - -# Clean data excluding some stations. Select matchups only when model and buoy are available. -ind=np.where( (np.isnan(lat)==False) & (np.isnan(lon)==False) & (np.isnan(np.nanmean(mhs,axis=1))==False) & (np.isnan(np.nanmean(bhs,axis=1))==False) ) -if np.size(ind)>0: - stname=np.array(stname[ind[0]]) - lat=np.array(lat[ind[0]]) - lon=np.array(lon[ind[0]]) - mhs=np.array(mhs[ind[0],:]) - mtm=np.array(mtm[ind[0],:]) - mtp=np.array(mtp[ind[0],:]) - mdm=np.array(mdm[ind[0],:]) - # mdp=np.array(mdp[ind[0],:]) + if 'longitude' in f.variables.keys(): + lon[b] = f.variables['longitude'][:] + elif 'LONGITUDE' in f.variables.keys(): + lon[b] = f.variables['LONGITUDE'][:] + else: + lon[b] = np.nan + + atime = np.append(atime, np.array(f.variables['time'][:]).astype('double')) + + f.close() + del f + + adp = adm * np.nan # no peak direction available in this format + + if np.size(ahs) > 0: + + # First layer of simple quality-control + indq = np.where((ahs > 30.) | (ahs < 0.0)) + if np.size(indq) > 0: + ahs[indq] = np.nan + del indq + + indq = np.where((atm > 40.) | (atm < 0.0)) + if np.size(indq) > 0: + atm[indq] = np.nan + del indq + + indq = np.where((atp > 40.) | (atp < 0.0)) + if np.size(indq) > 0: + atp[indq] = np.nan + del indq + + indq = np.where((adm > 360.) | (adm < -180.)) + if np.size(indq) > 0: + adm[indq] = np.nan + del indq + + indq = np.where((adp > 360.) | (adp < -180.)) + if np.size(indq) > 0: + adp[indq] = np.nan + del indq + + indq = np.where((awm > 50.) | (awm < 0.0)) + if np.size(indq) > 0: + awm[indq] = np.nan + del indq + + c = 0 + for t in mtime: + indt = np.where(np.abs(atime - t) < 1800.)[0] + if np.size(indt) > 0: + if np.any(ahs[indt].mask == False): + bhs[b, c] = np.nanmean(ahs[indt][ahs[indt].mask == False]) + if np.any(atm[indt].mask == False): + btm[b, c] = np.nanmean(atm[indt][atm[indt].mask == False]) + if np.any(atp[indt].mask == False): + btp[b, c] = np.nanmean(atp[indt][atp[indt].mask == False]) + if np.any(adm[indt].mask == False): + bdm[b, c] = np.nanmean(adm[indt][adm[indt].mask == False]) + if np.any(adp[indt].mask == False): + bdp[b, c] = np.nanmean(adp[indt][adp[indt].mask == False]) + if np.any(awm[indt].mask == False): + bwind[b, c] = np.nanmean(awm[indt][awm[indt].mask == False]) + c += 1 + + # print("counted "+repr(c)+" at "+stname[b]) + + print(" station " + stname[b] + " ok") + + except Exception as e: + print("Error occurred while processing station", stname[b]) + print(e) + + print('bwind:', bwind) + print('bhs:', bhs) + + + print(' ') + + + # Simple quality-control (range) + ind = np.where((bhs > 30.) | (bhs < 0.0)) + if np.size(ind) > 0: + bhs[ind] = np.nan; + del ind + + ind = np.where((btm > 40.) | (btm < 0.0)) + if np.size(ind) > 0: + btm[ind] = np.nan; + del ind + + ind = np.where((btp > 40.) | (btp < 0.0)) + if np.size(ind) > 0: + btp[ind] = np.nan; + del ind + + ind = np.where((bdm > 360.) | (bdm < -180.)) + if np.size(ind) > 0: + bdm[ind] = np.nan; + del ind + + ind = np.where((bdp > 360.) | (bdp < -180.)) + if np.size(ind) > 0: + bdp[ind] = np.nan; + del ind + + ind = np.where((bwind > 50.0) | (bwind < 0.0)) + if np.size(ind) > 0: + bwind[ind] = np.nan; + del ind + + ind = np.where((mhs > 30.) | (mhs < 0.0)) + if np.size(ind) > 0: + mhs[ind] = np.nan; + del ind + + ind = np.where((mtm > 40.) | (mtm < 0.0)) + if np.size(ind) > 0: + mtm[ind] = np.nan; + del ind + + ind = np.where((mtp > 40.) | (mtp < 0.0)) + if np.size(ind) > 0: + mtp[ind] = np.nan; + del ind + + + ind = np.where((mdm > 360.) | (mdm < -180.)) + if np.size(ind) > 0: + mdm[ind] = np.nan; + del ind + + ind = np.where((mdp > 360.) | (mdp < -180.)) + if np.size(ind) > 0: + mdp[ind] = np.nan; + del ind + + ind = np.where((mwn > 50.) | (mwn < 0.0)) + if np.size(ind) > 0: + mwn[ind] = np.nan; + del ind + + # Clean data excluding some stations. Select matchups only when model and buoy are available. + + ind = np.where((np.isnan(lat) == False) & (np.isnan(lon) == False) & (np.isnan(np.nanmean(mhs, axis=1)) == False) & (np.isnan(np.nanmean(bhs, axis=1)) == False)) + if np.size(ind) > 0: + stname = np.array(stname[ind[0]]) + lat = np.array(lat[ind[0]]) + lon = np.array(lon[ind[0]]) + mhs = np.array(mhs[ind[0], :]) + mtm = np.array(mtm[ind[0], :]) + mtp = np.array(mtp[ind[0], :]) + mdm = np.array(mdm[ind[0], :]) mdp = np.array(mdp[ind[0], :]) - mwn=np.array(mwn[ind[0],:]) - bhs=np.array(bhs[ind[0],:]) - btm=np.array(btm[ind[0],:]) - btp=np.array(btp[ind[0],:]) - bdm=np.array(bdm[ind[0],:]) - bdp=np.array(bdp[ind[0],:]) - bwind=np.array(bwind[ind[0],:]) -else: + mwn = np.array(mwn[ind[0], :]) + bhs = np.array(bhs[ind[0], :]) + btm = np.array(btm[ind[0], :]) + btp = np.array(btp[ind[0], :]) + bdm = np.array(bdm[ind[0], :]) + bdp = np.array(bdp[ind[0], :]) + bwind = np.array(bwind[ind[0], :]) + else: sys.exit(' Error: No matchups Model/Buoy available.') -print(" Matchups model/buoy complete. Total of "+repr(np.size(ind))+" stations/buoys avaliable."); del ind - -# Processing grid and/or cyclone information -if gridinfo!=0: - print(" Adding extra information ... ") - alon=np.copy(lon); alon[alon<0]=alon[alon<0]+360. - indgplat=[]; indgplon=[] - for i in range(0,lat.shape[0]): - # indexes nearest point. - indgplat = np.append(indgplat,np.where( abs(mlat-lat[i])==abs(mlat-lat[i]).min() )[0][0]) - indgplon = np.append(indgplon,np.where( abs(mlon-alon[i])==abs(mlon-alon[i]).min() )[0][0]) - - indgplat=np.array(indgplat).astype('int'); indgplon=np.array(indgplon).astype('int') - pdistcoast=np.zeros(lat.shape[0],'f')*np.nan - pdepth=np.zeros(lat.shape[0],'f')*np.nan - poni=np.zeros(lat.shape[0],'f')*np.nan - phsmz=np.zeros(lat.shape[0],'f')*np.nan - for i in range(0,lat.shape[0]): - pdistcoast[i]=distcoast[indgplat[i],indgplon[i]] - pdepth[i]=depth[indgplat[i],indgplon[i]] - poni[i]=oni[indgplat[i],indgplon[i]] - phsmz[i]=hsmz[indgplat[i],indgplon[i]] - - print(" Grid Information Included.") - - # Excluding shallow water points too close to the coast (mask information not accurate) - ind=np.where( (np.isnan(pdistcoast)==False) & (np.isnan(pdepth)==False) ) - if np.size(ind)>0: - stname=np.array(stname[ind[0]]) - lat=np.array(lat[ind[0]]) - lon=np.array(lon[ind[0]]) - mhs=np.array(mhs[ind[0],:]) - mtm=np.array(mtm[ind[0],:]) - mtp=np.array(mtp[ind[0],:]) - mdm=np.array(mdm[ind[0],:]) - mdp=np.array(mdp[ind[0],:]) - mwn=np.array(mwn[ind[0],:]) - bhs=np.array(bhs[ind[0],:]) - btm=np.array(btm[ind[0],:]) - btp=np.array(btp[ind[0],:]) - bdm=np.array(bdm[ind[0],:]) - bdp=np.array(bdp[ind[0],:]) - bwind=np.array(bwind[ind[0],:]) - pdistcoast=np.array(pdistcoast[ind[0]]) - pdepth=np.array(pdepth[ind[0]]) - poni=np.array(poni[ind[0]]) - phsmz=np.array(phsmz[ind[0]]) - else: - sys.exit(' Error: No matchups Model/Buoy available after using grid mask.') + print(" Matchups model/buoy complete. Total of " + repr(np.size(ind)) + " stations/buoys available."); + del ind - del ind + # Edit format if this is forecast model data. Reshape and allocate + if forecastds > 0: + unt = np.unique(mfcycle) + mxsz = 1 - if cyclonemap!=0: - fcmap=np.zeros((lat.shape[0],mtime.shape[0]),'f')*np.nan - for t in range(0,np.size(mtime)): - # search for cyclone time index and cyclone map - indt=np.where(np.abs(ctime-mtime[t])<5400.) - if np.size(indt)>0: - for i in range(0,lat.shape[0]): - fcmap[i,t] = np.array(cmap[indt[0][0],indgplat[i],indgplon[i]]) + for i in range(0, unt.shape[0]): + ind = np.where(mfcycle == unt[i])[0] + mxsz = np.max([mxsz, np.size(ind)]) - del indt - else: - print(' - No cyclone information for this time step: '+repr(t)) - - # print(' Done cyclone analysis at step: '+repr(t)) - - ind=np.where(fcmap<0) - if np.size(ind)>0: - fcmap[ind]=np.nan - - print(" Cyclone Information Included.") - -# Edit format if this is forecast model data. Reshape and allocate - - -if forecastds>0: - unt=np.unique(mfcycle); mxsz=1 - for i in range(0,unt.shape[0]): - ind=np.where(mfcycle==unt[i])[0] - mxsz=np.max([mxsz,np.size(ind)]) - - for i in range(0,unt.shape[0]): - ind=np.where(mfcycle==unt[i])[0] - if i==0: - nmhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmwn=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbhs=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbtm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbtp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbdm=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbdp=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nbwind=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - nmtime=np.zeros((unt.shape[0],mxsz),'double')*np.nan - if cyclonemap!=0: - nfcmap=np.zeros((mhs.shape[0],unt.shape[0],mxsz),'f')*np.nan - - nmtime[i,0:np.size(ind)]=np.array(mtime[ind]).astype('double') - nmhs[:,i,:][:,0:np.size(ind)]=np.array(mhs[:,ind]) - nmtm[:,i,:][:,0:np.size(ind)]=np.array(mtm[:,ind]) - nmtp[:,i,:][:,0:np.size(ind)]=np.array(mtp[:,ind]) - nmdm[:,i,:][:,0:np.size(ind)]=np.array(mdm[:,ind]) - nmdp[:,i,:][:,0:np.size(ind)]=np.array(mdp[:,ind]) - nmwn[:,i,:][:,0:np.size(ind)]=np.array(mwn[:,ind]) - nbhs[:,i,:][:,0:np.size(ind)]=np.array(bhs[:,ind]) - nbtm[:,i,:][:,0:np.size(ind)]=np.array(btm[:,ind]) - nbtp[:,i,:][:,0:np.size(ind)]=np.array(btp[:,ind]) - nbdm[:,i,:][:,0:np.size(ind)]=np.array(bdm[:,ind]) - nbdp[:,i,:][:,0:np.size(ind)]=np.array(bdp[:,ind]) - nbwind[:,i,:][:,0:np.size(ind)]=np.array(bwind[:,ind]) - - if cyclonemap!=0: - nfcmap[:,i,:][:,0:np.size(ind)]=np.array(fcmap[:,ind]) - - - ind=np.where( (nmhs>0.0) & (nbhs>0.0) ) - -else: - - larger_shape = max(mhs.shape, bhs.shape) - padded_mhs = np.zeros(larger_shape) - padded_bhs = np.zeros(larger_shape) - padded_mhs[:mhs.shape[0], :mhs.shape[1]] = mhs - padded_bhs[:bhs.shape[0], :bhs.shape[1]] = bhs - - - # Create padded arrays for other variables - padded_mtm = np.zeros(larger_shape) # Assuming mtm has the same shape as mhs - padded_mtp = np.zeros(larger_shape) # Assuming mtp has the same shape as mhs - padded_mdm = np.zeros(larger_shape) # Assuming mdm has the same shape as mhs - padded_mdp = np.zeros(larger_shape) # Assuming mdp has the same shape as mhs - padded_mwn = np.zeros(larger_shape) # Assuming mwn has the same shape as mhs - - padded_btm = np.zeros(larger_shape) # Assuming btm has the same shape as bhs - padded_btp = np.zeros(larger_shape) # Assuming btp has the same shape as bhs - padded_bdm = np.zeros(larger_shape) # Assuming bdm has the same shape as bhs - padded_bdp = np.zeros(larger_shape) # Assuming bdp has the same shape as bhs - padded_bwind = np.zeros(larger_shape) # Assuming bwind has the same shape as bhs - - # Copy values from original arrays to padded arrays for other variables - padded_mtm[:mtm.shape[0], :mtm.shape[1]] = mtm - padded_mtp[:mtp.shape[0], :mtp.shape[1]] = mtp - padded_mdm[:mdm.shape[0], :mdm.shape[1]] = mdm - padded_mdp[:mdp.shape[0], :mdp.shape[1]] = mdp - padded_mwn[:mwn.shape[0], :mwn.shape[1]] = mwn - - padded_btm[:btm.shape[0], :btm.shape[1]] = btm - padded_btp[:btp.shape[0], :btp.shape[1]] = btp - padded_bdm[:bdm.shape[0], :bdm.shape[1]] = bdm - padded_bdp[:bdp.shape[0], :bdp.shape[1]] = bdp - padded_bwind[:bwind.shape[0], :bwind.shape[1]] = bwind - - - -# ind=np.where( (mhs>0.0) & (bhs>0.0) ) - ind = np.where((padded_mhs > 0.0) & (padded_bhs > 0.0)) -if np.size(ind)>0: - print(' Total amount of matchups model/buoy: '+repr(np.size(ind))) + for i in range(0, unt.shape[0]): + ind = np.where(mfcycle == unt[i])[0] + if i == 0: + nmhs = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmtm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmtp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmdm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmdp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmwn = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbhs = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbtm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbtp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbdm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbdp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbwind = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmtime = np.zeros((unt.shape[0], mxsz), 'double') * np.nan + if cyclonemap != 0: + nfcmap = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + + nmtime[i, 0:np.size(ind)] = np.array(mtime[ind]).astype('double') + nmhs[:, i, :][:, 0:np.size(ind)] = np.array(mhs[:, ind]) + nmtm[:, i, :][:, 0:np.size(ind)] = np.array(mtm[:, ind]) + nmtp[:, i, :][:, 0:np.size(ind)] = np.array(mtp[:, ind]) + nmdm[:, i, :][:, 0:np.size(ind)] = np.array(mdm[:, ind]) + nmdp[:, i, :][:, 0:np.size(ind)] = np.array(mdp[:, ind]) + nmwn[:, i, :][:, 0:np.size(ind)] = np.array(mwn[:, ind]) + nbhs[:, i, :][:, 0:np.size(ind)] = np.array(bhs[:, ind]) + nbtm[:, i, :][:, 0:np.size(ind)] = np.array(btm[:, ind]) + nbtp[:, i, :][:, 0:np.size(ind)] = np.array(btp[:, ind]) + nbdm[:, i, :][:, 0:np.size(ind)] = np.array(bdm[:, ind]) + nbdp[:, i, :][:, 0:np.size(ind)] = np.array(bdp[:, ind]) + nbwind[:, i, :][:, 0:np.size(ind)] = np.array(bwind[:, ind]) + if cyclonemap != 0: + nfcmap[:, i, :][:, 0:np.size(ind)] = np.array(fcmap[:, ind]) + + ind = np.where((nmhs > 0.0) & (nbhs > 0.0)) + + else: + ind = np.where((mhs > 0.0) & (bhs > 0.0)) + + if np.size(ind) > 0: + print(' Total amount of matchups model/buoy: ' + repr(np.size(ind))) # Save netcdf output file - lon[lon>180.]=lon[lon>180.]-360. - initime=str(time.gmtime(mtime.min())[0])+str(time.gmtime(mtime.min())[1]).zfill(2)+str(time.gmtime(mtime.min())[2]).zfill(2)+str(time.gmtime(mtime.min())[3]).zfill(2) - fintime=str(time.gmtime(mtime.max())[0])+str(time.gmtime(mtime.max())[1]).zfill(2)+str(time.gmtime(mtime.max())[2]).zfill(2)+str(time.gmtime(mtime.max())[3]).zfill(2) - ncfile = nc.Dataset('WW3.Buoy'+str(ftag)+'_'+initime+'to'+fintime+'.nc', "w", format=fnetcdf) - ncfile.history="Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of "+repr(bhs[bhs>0.].shape[0])+" observations or pairs model/observation." + lon[lon > 180.] = lon[lon > 180.] - 360. + initime = str(time.gmtime(mtime.min())[0]) + str(time.gmtime(mtime.min())[1]).zfill(2) + str(time.gmtime(mtime.min())[2]).zfill(2) + str(time.gmtime(mtime.min())[3]).zfill(2) + fintime = str(time.gmtime(mtime.max())[0]) + str(time.gmtime(mtime.max())[1]).zfill(2) + str(time.gmtime(mtime.max())[2]).zfill(2) + str(time.gmtime(mtime.max())[3]).zfill(2) + + # Ensure model_name is defined + if 'model_name' not in locals(): + model_name = '' + + ncfile = nc.Dataset(f'WW3.{model_name}Buoy{ftag}_{initime}to{fintime}.nc', "w", format=fnetcdf) + print(f"Model Name: {model_name}, Tag: {ftag}, Start Time: {initime}, End Time: {fintime}") + ncfile.history = "Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of " + repr(bhs[bhs > 0.].shape[0]) + " observations or pairs model/observation." + # create dimensions - ncfile.createDimension('buoypoints', bhs.shape[0] ) - if gridinfo!=0: - ncfile.createDimension('GlobalOceansSeas', ocnames.shape[0] ) - ncfile.createDimension('HighSeasMarineZones', hsmznames.shape[0] ) - if cyclonemap!=0: - ncfile.createDimension('cycloneinfo', cinfo.shape[0] ) - vcinfo = ncfile.createVariable('cycloneinfo',dtype('a25'),('cycloneinfo')) + ncfile.createDimension('buoypoints', bhs.shape[0]) + if gridinfo != 0: + ncfile.createDimension('GlobalOceansSeas', ocnames.shape[0]) + ncfile.createDimension('HighSeasMarineZones', hsmznames.shape[0]) + if cyclonemap != 0: + ncfile.createDimension('cycloneinfo', cinfo.shape[0]) + vcinfo = ncfile.createVariable('cycloneinfo', dtype('a25'), ('cycloneinfo')) + # create variables. - vstname = ncfile.createVariable('buoyID',type('a25'),('buoypoints')) - vlat = ncfile.createVariable('latitude',np.dtype('float32').char,('buoypoints')) - vlon = ncfile.createVariable('longitude',np.dtype('float32').char,('buoypoints')) - - if forecastds>0: - ncfile.createDimension('time', nmhs.shape[2] ) - ncfile.createDimension('fcycle', unt.shape[0] ) - vt = ncfile.createVariable('time',np.dtype('float64').char,('fcycle','time')) - vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vmwn = ncfile.createVariable('model_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) - - vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','fcycle','time')) - vbwind = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','fcycle','time')) + vstname = ncfile.createVariable('buoyID', type('a25'), ('buoypoints')) + vlat = ncfile.createVariable('latitude', np.dtype('float32').char, ('buoypoints')) + vlon = ncfile.createVariable('longitude', np.dtype('float32').char, ('buoypoints')) + + if forecastds > 0: + ncfile.createDimension('time', nmhs.shape[2]) + ncfile.createDimension('fcycle', unt.shape[0]) + vt = ncfile.createVariable('time', np.dtype('float64').char, ('fcycle', 'time')) + vmhs = ncfile.createVariable('model_hs', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + vmtm = ncfile.createVariable('model_tm', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + vmtp = ncfile.createVariable('model_tp', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + vmdm = ncfile.createVariable('model_dm', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + vmdp = ncfile.createVariable('model_dp', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + vmwn = ncfile.createVariable('model_wind', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + + vbhs = ncfile.createVariable('obs_hs', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + vbtm = ncfile.createVariable('obs_tm', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + vbtp = ncfile.createVariable('obs_tp', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + vbdm = ncfile.createVariable('obs_dm', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + vbdp = ncfile.createVariable('obs_dp', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + vbwind = ncfile.createVariable('obs_wind', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) else: - ncfile.createDimension('time', bhs.shape[1] ) - vt = ncfile.createVariable('time',np.dtype('float64').char,('time')) - vmhs = ncfile.createVariable('model_hs',np.dtype('float32').char,('buoypoints','time')) - vmtm = ncfile.createVariable('model_tm',np.dtype('float32').char,('buoypoints','time')) - vmtp = ncfile.createVariable('model_tp',np.dtype('float32').char,('buoypoints','time')) - vmdm = ncfile.createVariable('model_dm',np.dtype('float32').char,('buoypoints','time')) - vmdp = ncfile.createVariable('model_dp',np.dtype('float32').char,('buoypoints','time')) - vmwn = ncfile.createVariable('model_wind',np.dtype('float32').char,('buoypoints','time')) - - vbhs = ncfile.createVariable('obs_hs',np.dtype('float32').char,('buoypoints','time')) - vbtm = ncfile.createVariable('obs_tm',np.dtype('float32').char,('buoypoints','time')) - vbtp = ncfile.createVariable('obs_tp',np.dtype('float32').char,('buoypoints','time')) - vbdm = ncfile.createVariable('obs_dm',np.dtype('float32').char,('buoypoints','time')) - vbdp = ncfile.createVariable('obs_dp',np.dtype('float32').char,('buoypoints','time')) - vbwind = ncfile.createVariable('obs_wind',np.dtype('float32').char,('buoypoints','time')) - - - - - if gridinfo!=0: - vpdistcoast = ncfile.createVariable('distcoast',np.dtype('float32').char,('buoypoints')) - vpdepth = ncfile.createVariable('depth',np.dtype('float32').char,('buoypoints')) - vponi = ncfile.createVariable('GlobalOceansSeas',np.dtype('float32').char,('buoypoints')) - vocnames = ncfile.createVariable('names_GlobalOceansSeas',dtype('a25'),('GlobalOceansSeas')) - vphsmz = ncfile.createVariable('HighSeasMarineZones',np.dtype('float32').char,('buoypoints')) - vhsmznames = ncfile.createVariable('names_HighSeasMarineZones',dtype('a25'),('HighSeasMarineZones')) - if cyclonemap!=0: - if forecastds>0: - vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','fcycle','time')) - else: - vcmap = ncfile.createVariable('cyclone',np.dtype('float32').char,('buoypoints','time')) + ncfile.createDimension('time', bhs.shape[1]) + vt = ncfile.createVariable('time', np.dtype('float64').char, ('time')) + vmhs = ncfile.createVariable('model_hs', np.dtype('float32').char, ('buoypoints', 'time')) + vmtm = ncfile.createVariable('model_tm', np.dtype('float32').char, ('buoypoints', 'time')) + vmtp = ncfile.createVariable('model_tp', np.dtype('float32').char, ('buoypoints', 'time')) + vmdm = ncfile.createVariable('model_dm', np.dtype('float32').char, ('buoypoints', 'time')) + vmdp = ncfile.createVariable('model_dp', np.dtype('float32').char, ('buoypoints', 'time')) + vmwn = ncfile.createVariable('model_wind', np.dtype('float32').char, ('buoypoints', 'time')) + + vbhs = ncfile.createVariable('obs_hs', np.dtype('float32').char, ('buoypoints', 'time')) + vbtm = ncfile.createVariable('obs_tm', np.dtype('float32').char, ('buoypoints', 'time')) + vbtp = ncfile.createVariable('obs_tp', np.dtype('float32').char, ('buoypoints', 'time')) + vbdm = ncfile.createVariable('obs_dm', np.dtype('float32').char, ('buoypoints', 'time')) + vbdp = ncfile.createVariable('obs_dp', np.dtype('float32').char, ('buoypoints', 'time')) + vbwind = ncfile.createVariable('obs_wind', np.dtype('float32').char, ('buoypoints', 'time')) + + if gridinfo != 0: + vpdistcoast = ncfile.createVariable('distcoast', np.dtype('float32').char, ('buoypoints')) + vpdepth = ncfile.createVariable('depth', np.dtype('float32').char, ('buoypoints')) + vponi = ncfile.createVariable('GlobalOceansSeas', np.dtype('float32').char, ('buoypoints')) + vocnames = ncfile.createVariable('names_GlobalOceansSeas', dtype('a25'), ('GlobalOceansSeas')) + vphsmz = ncfile.createVariable('HighSeasMarineZones', np.dtype('float32').char, ('buoypoints')) + vhsmznames = ncfile.createVariable('names_HighSeasMarineZones', dtype('a25'), ('HighSeasMarineZones')) + if cyclonemap != 0: + if forecastds > 0: + vcmap = ncfile.createVariable('cyclone', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) + else: + vcmap = ncfile.createVariable('cyclone', np.dtype('float32').char, ('buoypoints', 'time')) # Assign units - vlat.units = 'degrees_north' ; vlon.units = 'degrees_east' + vlat.units = 'degrees_north' + vlon.units = 'degrees_east' vt.units = 'seconds since 1970-01-01T00:00:00+00:00' - vmhs.units='m'; vbhs.units='m' - vmtm.units='s'; vbtm.units='s' - vmtp.units='s'; vbtp.units='s' - vmdm.units='degrees'; vbdm.units='degrees' - vmdp.units='degrees'; vbdp.units='degrees' - vmwn.unit='m/s';vbwind.unit='m/s' + vmhs.units = 'm' + vbhs.units = 'm' + vmtm.units = 's' + vbtm.units = 's' + vmtp.units = 's' + vbtp.units = 's' + vmdm.units = 'degrees' + vbdm.units = 'degrees' + vmdp.units = 'degrees' + vbdp.units = 'degrees' + vmwn.unit = 'm/s' + vbwind.unit = 'm/s' - if gridinfo!=0: - vpdepth.units='m'; vpdistcoast.units='km' + if gridinfo != 0: + vpdepth.units = 'm' + vpdistcoast.units = 'km' # Allocate Data - vstname[:]=stname[:]; vlat[:] = lat[:]; vlon[:] = lon[:] - if forecastds>0: - vt[:,:]=nmtime[:,:] - vmhs[:,:,:]=nmhs[:,:,:] - vmtm[:,:,:]=nmtm[:,:,:] - vmtp[:,:,:]=nmtp[:,:,:] - vmdm[:,:,:]=nmdm[:,:,:] - vmdp[:,:,:]=nmdp[:,:,:] - vmwn[:,:,:]=nmwn[:,:,:] - vbhs[:,:,:]=nbhs[:,:,:] - vbtm[:,:,:]=nbtm[:,:,:] - vbtp[:,:,:]=nbtp[:,:,:] - vbdm[:,:,:]=nbdm[:,:,:] - vbdp[:,:,:]=nbdp[:,:,:] - vbwind[:,:,:]=nbwind[:,:,:] + vstname[:] = stname[:] + vlat[:] = lat[:] + vlon[:] = lon[:] + if forecastds > 0: + vt[:, :] = nmtime[:, :] + vmhs[:, :, :] = nmhs[:, :, :] + vmtm[:, :, :] = nmtm[:, :, :] + vmtp[:, :, :] = nmtp[:, :, :] + vmdm[:, :, :] = nmdm[:, :, :] + vmdp[:, :, :] = nmdp[:, :, :] + vmwn[:, :, :] = nmwn[:, :, :] + vbhs[:, :, :] = nbhs[:, :, :] + vbtm[:, :, :] = nbtm[:, :, :] + vbtp[:, :, :] = nbtp[:, :, :] + vbdm[:, :, :] = nbdm[:, :, :] + vbdp[:, :, :] = nbdp[:, :, :] + vbwind[:, :, :] = nbwind[:, :, :] - else: - vt[:]=mtime[:] - vmhs[:,:]=padded_mhs[:,:] - vmtm[:,:]=padded_mtm[:,:] - vmtp[:,:]=padded_mtp[:,:] - vmdm[:,:]=padded_mdm[:,:] - vmdp[:,:]=padded_mdp[:,:] - vmwn[:,:]=padded_mwn[:,:] - vbhs[:,:]=padded_bhs[:,:] - vbtm[:,:]=btm[:,:] - vbtp[:,:]=btp[:,:] - vbdm[:,:]=bdm[:,:] - vbdp[:,:]=bdp[:,:] - vbwind[:,:]=bwind[:,:] - - - - - if gridinfo!=0: - vpdistcoast[:]=pdistcoast[:] - vpdepth[:]=pdepth[:] - vponi[:]=poni[:]; vocnames[:] = ocnames[:] - vphsmz[:]=phsmz[:]; vhsmznames[:] = hsmznames[:] - if cyclonemap!=0: - vcinfo[:] = cinfo[:] - if forecastds>0: - vcmap[:,:,:]=nfcmap[:,:,:] - else: - vcmap[:,:]=fcmap[:,:] + if gridinfo != 0: + vpdistcoast[:] = pdistcoast[:] + vpdepth[:] = pdepth[:] + vponi[:] = poni[:] + vocnames[:] = ocnames[:] + vphsmz[:] = phsmz[:] + vhsmznames[:] = hsmznames[:] + if cyclonemap != 0: + vcinfo[:] = cinfo[:] + if forecastds > 0: + vcmap[:, :, :] = nfcmap[:, :, :] + else: + vcmap[:, :] = fcmap[:, :] ncfile.close() print(' ') - print('Done. Netcdf ok. New file saved: WW3.Buoy'+str(ftag)+'_'+initime+'to'+fintime+'.nc') + print(f'Done. Netcdf ok. New file saved: WW3.{model_name}_Buoy{ftag}_{initime}to{fintime}.nc') + + # File deletion section + files_to_delete = [ + os.path.join(output_dir, '*_contents.txt'), + os.path.join(output_dir, '*_id.txt'), + os.path.join(output_dir, '*.spec_tar') + ] + + for pattern in files_to_delete: + for file in glob.glob(pattern): + try: + os.remove(file) + print(f'Deleted file: {file}') + except OSError as e: + print(f'Error deleting file {file}: {e}') + + # Folder deletion section + try: + shutil.rmtree(extracted_folder) + print(f'Deleted folder: {extracted_folder}') + except OSError as e: + print(f'Error deleting folder {extracted_folder}: {e}') + + print('Temporary files and folder deleted.') diff --git a/ww3tools/wread.py b/ww3tools/wread.py index 936a81c..5b9d8f5 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1626,9 +1626,9 @@ def spec1_ww3(*args): Output: list of dictionaries containing: time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pwst,d1sp,dire,dspec,wnds,wndd ''' - + if len(args) < 2: - sys.exit(' Two inputs are required: list of file names and list of station names') + sys.exit('Two inputs are required: list of file names and list of station names') fnames = args[0] stnames = args[1] @@ -1636,202 +1636,141 @@ def spec1_ww3(*args): if len(args) > 2: sk = int(args[2]) if len(args) > 3: - sys.exit(' Too many inputs') + sys.exit('Too many inputs') - file_names = args[0] - station_names = args[1] results = [] for fname in fnames: for stname in stnames: try: + with open(fname) as fp: + nt = fp.read().count(stname) # No change - fp = open(fname) - nt = fp.read().count(stname) - fp.close() - del fp if nt >= 1: - # Open file and read the first parameters - fp = open(fname) - cabc = fp.readline() - cabc = cabc.strip().split() - nf = int(cabc[3]) # number of frequencies - nd = int(cabc[4]) # number of directions - npo = int(cabc[5]) # number of point outputs - - freq = zeros(nf, 'f') - dire = zeros(nd, 'f') - dspec = zeros((nt, nf, nd), 'f') - adire = zeros(dire.shape) - adspec = zeros(dspec.shape) - mtime = np.zeros((nt), 'd') - - # Frequencies -------------------- - ncf = int(np.floor(nf/8)) - rncf = int(np.round(8*((float(nf)/8)-ncf))) - k = 0 - for i in range(0, ncf): - line = fp.readline() - line = line.strip().split() - for j in range(0, 8): - freq[k] = float(line[j]) - k = k + 1 - - if rncf > 0: - line = fp.readline() - line = line.strip().split() - for i in range(0, rncf): - freq[k] = float(line[i]) - k = k + 1 - - # DF in frequency (dfreq) - dfreq = np.zeros(freq.shape[0], 'f') - for i in range(0, freq.shape[0]): - if i == 0 or i == (freq.shape[0]-1): - dfreq[i] = freq[i] * \ - (1 + (((freq[-1]/freq[-2])-1)/2)) - freq[i] - else: - dfreq[i] = freq[i] * \ - (freq[-1]/freq[-2]) - freq[i] - - # Directions --------------------- - ncd = int(np.floor(nd/7)) - rncd = int(np.round(7*((float(nd)/7)-ncd))) - k = 0 - for i in range(0, ncd): - line = fp.readline() - line = line.strip().split() - for j in range(0, 7): - dire[k] = float(line[j])*180/pi - k = k+1 - - if rncd > 0: - line = fp.readline() - line = line.strip().split() - for i in range(0, rncd): - dire[k] = float(line[i])*180/pi - k = k+1 - - nl = int(floor((nf*nd)/7.)) - rnl = int(np.round(7*((float(nf*nd)/7)-nl))) - auxs = np.zeros((nf*nd), 'f') - wnds = np.zeros((nt), 'f') - wndd = np.zeros((nt), 'f') - - for t in range(0, nt): - - cabc = fp.readline() - cabc.strip().split()[0] - mtime[t] = np.double(timegm(strptime( - cabc.strip().split()[0]+cabc.strip().split()[1][0:2], '%Y%m%d%H'))) - cabc = fp.readline() - cabc = cabc.strip().split() - - if len(cabc) >8: - - namep = cabc[0][1:] - lat = float(cabc[2]) - lon = float(cabc[3]) - depth = float(cabc[4]) - wnds_index = 5 - wndd_index = 6 - elif len(cabc) == 8: - - namep = cabc[0][1:] - lat_lon_str = cabc[2] - lat_lon_str = lat_lon_str.strip("'") - lat_lon_parts = lat_lon_str.split('-') - lat = float(lat_lon_parts[0]) - lon = -float(lat_lon_parts[1]) - - depth = float(cabc[3]) - wnds_index = 4 - wndd_index = 5 - else: - continue - + with open(fname) as fp: + cabc = fp.readline().strip().split() + nf = int(cabc[3]) + nd = int(cabc[4]) + npo = int(cabc[5]) + + freq = np.zeros(nf, 'f') + dire = np.zeros(nd, 'f') + dspec = np.zeros((nt, nf, nd), 'f') + adire = np.zeros(dire.shape) + adspec = np.zeros(dspec.shape) + mtime = np.zeros((nt), 'd') + + k = 0 + # Reading frequencies + for i in range(0, int(np.floor(nf/8))): # Changed loop condition to int(np.floor(nf/8)) + line = fp.readline().strip().split() + for j in range(8): + freq[k] = float(line[j]) + k += 1 + + if (nf % 8) > 0: + line = fp.readline().strip().split() + for i in range(nf % 8): + freq[k] = float(line[i]) + k += 1 - wnds[t] = float(cabc[wnds_index]) - wndd[t] = float(cabc[wndd_index]) + dfreq = np.diff(freq, prepend=freq[0]) # Simplified dfreq calculation using np.diff k = 0 - for i in range(0, nl): - line = fp.readline() - line = line.strip().split() - for j in range(0, 7): - auxs[k] = float(line[j]) - k = k+1 - - if rncd > 0: - line = fp.readline() - line = line.strip().split() - for i in range(0, rnl): - auxs[k] = float(line[i]) - k = k+1 - - for ic in range(0, nf): - for il in range(0, nd): - dspec[t, ic, il] = auxs[il*nf+ic] - - fp.close() - del fp - - mdate = pd.to_datetime(mtime, unit='s').strftime( - '%Y-%m-%dT%H:%M:%S.%f') - freq1 = freq*np.nan - freq2 = freq*np.nan - - # ------------------ - # 1D power spectrum - pwst = np.zeros((dspec.shape[0], nf), 'f') - for t in range(0, dspec.shape[0]): - for il in range(0, nf): - pwst[t, il] = sum( - dspec[t, il, :]*(2*np.pi)/nd) + # Reading directions + for i in range(0, int(np.floor(nd/7))): # Changed loop condition to int(np.floor(nd/7)) + line = fp.readline().strip().split() + for j in range(7): + dire[k] = float(line[j]) * 180 / np.pi # Added conversion to degrees + k += 1 + + if (nd % 7) > 0: + line = fp.readline().strip().split() + for i in range(nd % 7): + dire[k] = float(line[i]) * 180 / np.pi # Added conversion to degrees + k += 1 + + auxs = np.zeros((nf * nd), 'f') + wnds = np.zeros((nt), 'f') + wndd = np.zeros((nt), 'f') + + for t in range(nt): + cabc = fp.readline().strip().split() + mtime[t] = np.double(timegm(strptime(cabc[0] + cabc[1][0:2], '%Y%m%d%H'))) + cabc = fp.readline().strip().split() + + if len(cabc) > 8: + lat, lon, depth = float(cabc[2]), float(cabc[3]), float(cabc[4]) + wnds[t], wndd[t] = float(cabc[5]), float(cabc[6]) + elif len(cabc) == 8: + lat_lon_parts = cabc[2].strip("'").split('-') + lat, lon, depth = float(lat_lon_parts[0]), -float(lat_lon_parts[1]), float(cabc[3]) + wnds[t], wndd[t] = float(cabc[4]), float(cabc[5]) + else: + continue + + k = 0 + for i in range(0, int(np.floor((nf*nd)/7.))): # Changed loop condition to int(np.floor((nf*nd)/7.)) + line = fp.readline().strip().split() + for j in range(7): + auxs[k] = float(line[j]) + k += 1 + + if (nf * nd % 7) > 0: + line = fp.readline().strip().split() + for i in range(nf * nd % 7): + auxs[k] = float(line[i]) + k += 1 + + for ic in range(nf): + for il in range(nd): + dspec[t, ic, il] = auxs[il * nf + ic] - pwst[t, :] = pwst[t, :]*dfreq[:] + mdate = pd.to_datetime(mtime, unit='s').strftime('%Y-%m-%dT%H:%M:%S.%f') + freq1 = np.copy(freq) + freq2 = np.copy(freq) + + pwst = np.zeros((dspec.shape[0], nf), 'f') + for t in range(dspec.shape[0]): + for il in range(nf): + pwst[t, il] = sum(dspec[t, il, :] * (2 * np.pi) / nd) + pwst[t, :] *= dfreq - # organizing directions ----- adspec = np.copy(dspec) - inddire = int(np.where(dire == min(dire))[0][0]) - for t in range(0, dspec.shape[0]): - adspec[t, :, 0:nd-(inddire+1)] = dspec[t,:, (inddire+1):] - adspec[t, :, nd-(inddire+1):nd] = dspec[t,:, :(inddire+1)] - for i in range(0, nd): - dspec[t, :, i] = adspec[t, :, nd-i-1] - - adspec[t, :, :int(nd/2)] = dspec[t, :, int(nd/2):] - adspec[t, :, int(nd/2):] = dspec[t, :, :int(nd/2)] + inddire = np.argmin(dire) + for t in range(dspec.shape[0]): + adspec[t, :, 0:nd - (inddire + 1)] = dspec[t, :, (inddire + 1):] + adspec[t, :, nd - (inddire + 1):nd] = dspec[t, :, :(inddire + 1)] + for i in range(nd): + dspec[t, :, i] = adspec[t, :, nd - i - 1] + adspec[t, :, :int(nd / 2)] = dspec[t, :, int(nd / 2):] + adspec[t, :, int(nd / 2):] = dspec[t, :, :int(nd / 2)] dspec[t, :, :] = adspec[t, :, :] dire = np.sort(dire) - # 1D directional spectrum d1sp = np.zeros((dspec.shape[0], nf), 'f') - - for t in range(0, dspec.shape[0]): - for il in range(0, nf): - a = np.sum(dspec[t, il, :] * np.array( - np.sin((pi*dire)/180.)/np.sum(dspec[t, il, :]))) - b = np.sum(dspec[t, il, :] * np.array( - np.cos((pi*dire)/180.)/np.sum(dspec[t, il, :]))) - aux = math.atan2(a, b)*(180./pi) + for t in range(dspec.shape[0]): + for il in range(nf): + a = np.sum(dspec[t, il, :] * np.sin((np.pi * dire) / 180.) / np.sum(dspec[t, il, :])) + b = np.sum(dspec[t, il, :] * np.cos((np.pi * dire) / 180.) / np.sum(dspec[t, il, :])) + aux = np.degrees(np.arctan2(a, b)) # Changed to np.degrees if aux < 0: - aux = aux+360. + aux += 360. + d1sp[t, il] = aux - d1sp[t, il] = float(aux) - del a, b, aux - - m0 = np.sum(pwst , axis=1) + m0 = np.sum(pwst, axis=1) hs = 4 * np.sqrt(m0) max_index = np.argmax(pwst, axis=1) f_max = freq[max_index] - tp = 1 / f_max - - # build dictionary - result = {'time': mtime, 'date': mdate, 'latitude': lat, 'longitude': lon, 'depth': depth, - 'wind_spd': wnds, 'wind_dir': wndd, 'freq': freq, 'freq1': freq1, 'freq2': freq2, - 'deltafreq': dfreq, 'pspec': pwst, 'theta': dire, 'dmspec': d1sp, 'dirspec': dspec, 'Hs': hs, 'Tp': tp,'station_name': stname} + tp = 1 / f_max + + result = { + 'time': mtime, 'date': mdate, 'latitude': lat, 'longitude': lon, 'depth': depth, + 'wind_spd': wnds, 'wind_dir': wndd, 'freq': freq, 'freq1': freq1, 'freq2': freq2, + 'deltafreq': dfreq, 'pspec': pwst, 'theta': dire, 'dmspec': d1sp, 'dirspec': dspec, + 'Hs': hs, 'Tp': tp, 'station_name': stname + } results.append(result) except Exception as e: From a2d5b33780e53b6f7ffcdac9020fd14384f52323 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Sat, 25 May 2024 01:18:33 +0000 Subject: [PATCH 17/27] hs and tp added to the .spec file format --- ww3tools/modelBuoy_collocation.py | 3 +-- ww3tools/wread.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index 7c69334..bc4e53d 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -421,7 +421,6 @@ def unzip_and_untar(gz_file, output_dir): else: mwd = np.append(mwd, np.copy(result['hs']) * np.nan, axis=1) - else: print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) @@ -560,6 +559,7 @@ def unzip_and_untar(gz_file, output_dir): f.close(); del f + fcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') if t == 0: mhs = np.copy(ahs) @@ -1028,4 +1028,3 @@ def unzip_and_untar(gz_file, output_dir): print(f'Error deleting folder {extracted_folder}: {e}') print('Temporary files and folder deleted.') - diff --git a/ww3tools/wread.py b/ww3tools/wread.py index 5b9d8f5..79d2e4e 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1619,6 +1619,7 @@ def spec_ww3(*args): del mtime,mdate,lat,lon,wnds,wndd,freq,freq1,freq2,dfreq,pwst,dire,d1sp,dspec + def spec1_ww3(*args): ''' WAVEWATCH III, wave spectrum, netcdf (.nc) or text (.spec) format @@ -1778,4 +1779,3 @@ def spec1_ww3(*args): continue return results - From 9ff9e0e60764351be2a3676dfafcacbaf79d2748 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Sat, 25 May 2024 01:24:05 +0000 Subject: [PATCH 18/27] The code is updated to be able to process the spec.gz buoy file formats --- ww3tools/modelBuoy_collocation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index bc4e53d..a35f946 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -421,6 +421,7 @@ def unzip_and_untar(gz_file, output_dir): else: mwd = np.append(mwd, np.copy(result['hs']) * np.nan, axis=1) + else: print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) @@ -559,7 +560,6 @@ def unzip_and_untar(gz_file, output_dir): f.close(); del f - fcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') if t == 0: mhs = np.copy(ahs) From e00676e831867c041cefb4525d14fe4cd6e3c898 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Tue, 28 May 2024 23:12:20 +0000 Subject: [PATCH 19/27] updated the script with the proper description and the parsing issue for concatenated lattitude and longitude values --- ww3tools/wread.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/ww3tools/wread.py b/ww3tools/wread.py index 79d2e4e..36be25c 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1622,12 +1622,10 @@ def spec_ww3(*args): def spec1_ww3(*args): ''' - WAVEWATCH III, wave spectrum, netcdf (.nc) or text (.spec) format - Input: file names (list of file names), and station names (list of station names) - Output: list of dictionaries containing: - time(seconds since 1970),time(datetime64),lat,lon; Arrays: freq,dfreq,pwst,d1sp,dire,dspec,wnds,wndd + WAVEWATCH III, .spec.gz reader. This new function is responsible for reading the model station data + with the .spec.gz format. ''' - + if len(args) < 2: sys.exit('Two inputs are required: list of file names and list of station names') @@ -1645,8 +1643,8 @@ def spec1_ww3(*args): for stname in stnames: try: with open(fname) as fp: - nt = fp.read().count(stname) # No change - + nt = fp.read().count(stname) + if nt >= 1: with open(fname) as fp: cabc = fp.readline().strip().split() @@ -1663,7 +1661,7 @@ def spec1_ww3(*args): k = 0 # Reading frequencies - for i in range(0, int(np.floor(nf/8))): # Changed loop condition to int(np.floor(nf/8)) + for i in range(0, int(np.floor(nf/8))): line = fp.readline().strip().split() for j in range(8): freq[k] = float(line[j]) @@ -1675,20 +1673,20 @@ def spec1_ww3(*args): freq[k] = float(line[i]) k += 1 - dfreq = np.diff(freq, prepend=freq[0]) # Simplified dfreq calculation using np.diff + dfreq = np.diff(freq, prepend=freq[0]) k = 0 # Reading directions - for i in range(0, int(np.floor(nd/7))): # Changed loop condition to int(np.floor(nd/7)) + for i in range(0, int(np.floor(nd/7))): line = fp.readline().strip().split() for j in range(7): - dire[k] = float(line[j]) * 180 / np.pi # Added conversion to degrees + dire[k] = float(line[j]) * 180 / np.pi k += 1 if (nd % 7) > 0: line = fp.readline().strip().split() for i in range(nd % 7): - dire[k] = float(line[i]) * 180 / np.pi # Added conversion to degrees + dire[k] = float(line[i]) * 180 / np.pi k += 1 auxs = np.zeros((nf * nd), 'f') @@ -1699,19 +1697,24 @@ def spec1_ww3(*args): cabc = fp.readline().strip().split() mtime[t] = np.double(timegm(strptime(cabc[0] + cabc[1][0:2], '%Y%m%d%H'))) cabc = fp.readline().strip().split() - + if len(cabc) > 8: lat, lon, depth = float(cabc[2]), float(cabc[3]), float(cabc[4]) wnds[t], wndd[t] = float(cabc[5]), float(cabc[6]) elif len(cabc) == 8: lat_lon_parts = cabc[2].strip("'").split('-') - lat, lon, depth = float(lat_lon_parts[0]), -float(lat_lon_parts[1]), float(cabc[3]) + if len(lat_lon_parts) == 2: + lat, lon, depth = float(lat_lon_parts[0]), -float(lat_lon_parts[1]), float(cabc[3]) + else: + lat = float(cabc[2][:6]) + lon = float(cabc[2][6:]) + depth = float(cabc[3]) wnds[t], wndd[t] = float(cabc[4]), float(cabc[5]) else: continue k = 0 - for i in range(0, int(np.floor((nf*nd)/7.))): # Changed loop condition to int(np.floor((nf*nd)/7.)) + for i in range(0, int(np.floor((nf*nd)/7.))): line = fp.readline().strip().split() for j in range(7): auxs[k] = float(line[j]) @@ -1755,7 +1758,7 @@ def spec1_ww3(*args): for il in range(nf): a = np.sum(dspec[t, il, :] * np.sin((np.pi * dire) / 180.) / np.sum(dspec[t, il, :])) b = np.sum(dspec[t, il, :] * np.cos((np.pi * dire) / 180.) / np.sum(dspec[t, il, :])) - aux = np.degrees(np.arctan2(a, b)) # Changed to np.degrees + aux = np.degrees(np.arctan2(a, b)) if aux < 0: aux += 360. d1sp[t, il] = aux From e0cb3e7e9fdfaef0cd479f2fb747da7314aa2b21 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Thu, 30 May 2024 18:12:37 +0000 Subject: [PATCH 20/27] This commit updates the calculation of total observations in the ncfile.history to multiply the number of time points by the number of buoy points. This change ensures that the history metadata accurately reflects the total potential data points across all buoys and time intervals, providing a clear overview of the dataset's comprehensive scope. --- ww3tools/modelBuoy_collocation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index a35f946..aadffff 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -883,7 +883,10 @@ def unzip_and_untar(gz_file, output_dir): ncfile = nc.Dataset(f'WW3.{model_name}Buoy{ftag}_{initime}to{fintime}.nc', "w", format=fnetcdf) print(f"Model Name: {model_name}, Tag: {ftag}, Start Time: {initime}, End Time: {fintime}") - ncfile.history = "Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of " + repr(bhs[bhs > 0.].shape[0]) + " observations or pairs model/observation." + num_time_points = nmhs.shape[2] # Total number of time points + num_buoy_points = bhs.shape[0] # Total number of buoy points + total_observations = num_time_points * num_buoy_points + ncfile.history = "Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of " + str(total_observations) + " observations or pairs model/observation." # create dimensions ncfile.createDimension('buoypoints', bhs.shape[0]) From e7ba943df5f7737a09b4ab922ce0782800cd5335 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Thu, 30 May 2024 18:15:26 +0000 Subject: [PATCH 21/27] Improve dfreq calculation to reflect logarithmic frequency spacing. --- ww3tools/wread.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/ww3tools/wread.py b/ww3tools/wread.py index 36be25c..0ba8fb4 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1620,10 +1620,13 @@ def spec_ww3(*args): + def spec1_ww3(*args): ''' - WAVEWATCH III, .spec.gz reader. This new function is responsible for reading the model station data - with the .spec.gz format. + WAVEWATCH III, wave spectrum, netcdf (.nc) or text (.spec) format + Input: file names (list of file names), and station names (list of station names) + Output: list of dictionaries containing: + time(seconds since 1970), time(datetime64), lat, lon; Arrays: freq, dfreq, pwst, d1sp, dire, dspec, wnds, wndd ''' if len(args) < 2: @@ -1661,7 +1664,7 @@ def spec1_ww3(*args): k = 0 # Reading frequencies - for i in range(0, int(np.floor(nf/8))): + for i in range(0, int(np.floor(nf / 8))): line = fp.readline().strip().split() for j in range(8): freq[k] = float(line[j]) @@ -1673,11 +1676,20 @@ def spec1_ww3(*args): freq[k] = float(line[i]) k += 1 - dfreq = np.diff(freq, prepend=freq[0]) + # Calculate dfreq using geometric progression + dfreq = np.zeros(freq.shape[0], 'f') + alpha = (freq[-1] / freq[-2]) + for i in range(freq.shape[0]): + if i == 0: + dfreq[i] = freq[i] * (np.sqrt(alpha) - 1) + elif i == (freq.shape[0] - 1): + dfreq[i] = freq[i] * (1 - 1 / np.sqrt(alpha)) + else: + dfreq[i] = freq[i] * (np.sqrt(alpha) - 1 / np.sqrt(alpha)) k = 0 # Reading directions - for i in range(0, int(np.floor(nd/7))): + for i in range(0, int(np.floor(nd / 7))): line = fp.readline().strip().split() for j in range(7): dire[k] = float(line[j]) * 180 / np.pi @@ -1714,7 +1726,7 @@ def spec1_ww3(*args): continue k = 0 - for i in range(0, int(np.floor((nf*nd)/7.))): + for i in range(0, int(np.floor((nf * nd) / 7.))): line = fp.readline().strip().split() for j in range(7): auxs[k] = float(line[j]) @@ -1782,3 +1794,4 @@ def spec1_ww3(*args): continue return results + From 26dc8128f70922b9db3419eb08ec94b6228ed1f9 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Fri, 31 May 2024 00:32:16 +0000 Subject: [PATCH 22/27] commented out the mtime --- ww3tools/modelBuoy_collocation.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index aadffff..3a8c99d 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -268,7 +268,6 @@ def unzip_and_untar(gz_file, output_dir): if mhs.shape[1] == result['Hs'].shape[0]: stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) - mtime = np.append(mtime, at) mhs = np.append(mhs, [result['Hs']], axis=0) mtp = np.append(mtp, [result['Tp']], axis=0) @@ -305,6 +304,18 @@ def unzip_and_untar(gz_file, output_dir): else: print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) + # Print values after loop for inspection + print('mfcycle:', mfcycle) + print('stname:', stname) + print('mtime:', mtime) + print('mhs:', mhs) + print('mtp:', mtp) + print('mwn:', mwn) + print('mwd:', mwd) + print('mfreq:', mfreq) + print('mtm:', mtm) + print('mdp:', mdp) + print('mdm:', mdm) else: gridinfo = int(0) @@ -813,6 +824,9 @@ def unzip_and_untar(gz_file, output_dir): bdm = np.array(bdm[ind[0], :]) bdp = np.array(bdp[ind[0], :]) bwind = np.array(bwind[ind[0], :]) + + print("bhs:",bhs.shape) + else: sys.exit(' Error: No matchups Model/Buoy available.') @@ -883,10 +897,7 @@ def unzip_and_untar(gz_file, output_dir): ncfile = nc.Dataset(f'WW3.{model_name}Buoy{ftag}_{initime}to{fintime}.nc', "w", format=fnetcdf) print(f"Model Name: {model_name}, Tag: {ftag}, Start Time: {initime}, End Time: {fintime}") - num_time_points = nmhs.shape[2] # Total number of time points - num_buoy_points = bhs.shape[0] # Total number of buoy points - total_observations = num_time_points * num_buoy_points - ncfile.history = "Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of " + str(total_observations) + " observations or pairs model/observation." + ncfile.history = "Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of " +repr(bhs[bhs>0.].shape[0])+" observations or pairs model/observation." # create dimensions ncfile.createDimension('buoypoints', bhs.shape[0]) @@ -1031,3 +1042,4 @@ def unzip_and_untar(gz_file, output_dir): print(f'Error deleting folder {extracted_folder}: {e}') print('Temporary files and folder deleted.') + From 1c50b8feb494c87f6f35615a5e66cc0fbbf3a687 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Fri, 31 May 2024 00:35:48 +0000 Subject: [PATCH 23/27] removed the checking comments --- ww3tools/modelBuoy_collocation.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index 3a8c99d..33a73e2 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -304,18 +304,6 @@ def unzip_and_untar(gz_file, output_dir): else: print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) - # Print values after loop for inspection - print('mfcycle:', mfcycle) - print('stname:', stname) - print('mtime:', mtime) - print('mhs:', mhs) - print('mtp:', mtp) - print('mwn:', mwn) - print('mwd:', mwd) - print('mfreq:', mfreq) - print('mtm:', mtm) - print('mdp:', mdp) - print('mdm:', mdm) else: gridinfo = int(0) @@ -825,8 +813,6 @@ def unzip_and_untar(gz_file, output_dir): bdp = np.array(bdp[ind[0], :]) bwind = np.array(bwind[ind[0], :]) - print("bhs:",bhs.shape) - else: sys.exit(' Error: No matchups Model/Buoy available.') From 2ea55cba8d60e5d3e3018844efda67b69b993808 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 10 Jun 2024 04:54:21 +0000 Subject: [PATCH 24/27] added the initial_condition as a global value and also added the fcst_hr value in the output --- ww3tools/modelBuoy_collocation.py | 35 +++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index 33a73e2..98f05b2 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -426,7 +426,7 @@ def unzip_and_untar(gz_file, output_dir): del result, at, fcycle mdm = np.copy(mhs) * np.nan; - mtm = np.copy(mhs) * np.nan # not saved in this file format + mtm = np.copy(mhs) * np.nan print(" ww3 file " + wlist[i] + " OK") elif (str(wlist[0]).split('/')[-1].split('.')[-1] == 'bull') or ( @@ -714,9 +714,8 @@ def unzip_and_untar(gz_file, output_dir): bdp[b, c] = np.nanmean(adp[indt][adp[indt].mask == False]) if np.any(awm[indt].mask == False): bwind[b, c] = np.nanmean(awm[indt][awm[indt].mask == False]) - c += 1 + c += 1 - # print("counted "+repr(c)+" at "+stname[b]) print(" station " + stname[b] + " ok") @@ -724,12 +723,6 @@ def unzip_and_untar(gz_file, output_dir): print("Error occurred while processing station", stname[b]) print(e) - print('bwind:', bwind) - print('bhs:', bhs) - - - print(' ') - # Simple quality-control (range) ind = np.where((bhs > 30.) | (bhs < 0.0)) @@ -884,6 +877,8 @@ def unzip_and_untar(gz_file, output_dir): ncfile = nc.Dataset(f'WW3.{model_name}Buoy{ftag}_{initime}to{fintime}.nc', "w", format=fnetcdf) print(f"Model Name: {model_name}, Tag: {ftag}, Start Time: {initime}, End Time: {fintime}") ncfile.history = "Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of " +repr(bhs[bhs>0.].shape[0])+" observations or pairs model/observation." + ncfile.initial_condition = initime + ncfile.time_units = "seconds since 1970-01-01T00:00:00+00:00" # create dimensions ncfile.createDimension('buoypoints', bhs.shape[0]) @@ -899,6 +894,8 @@ def unzip_and_untar(gz_file, output_dir): vlat = ncfile.createVariable('latitude', np.dtype('float32').char, ('buoypoints')) vlon = ncfile.createVariable('longitude', np.dtype('float32').char, ('buoypoints')) + + if forecastds > 0: ncfile.createDimension('time', nmhs.shape[2]) ncfile.createDimension('fcycle', unt.shape[0]) @@ -964,6 +961,7 @@ def unzip_and_untar(gz_file, output_dir): vmwn.unit = 'm/s' vbwind.unit = 'm/s' + if gridinfo != 0: vpdepth.units = 'm' vpdistcoast.units = 'km' @@ -972,6 +970,10 @@ def unzip_and_untar(gz_file, output_dir): vstname[:] = stname[:] vlat[:] = lat[:] vlon[:] = lon[:] + initime_unix = timegm(strptime(initime, '%Y%m%d%H')) + + + if forecastds > 0: vt[:, :] = nmtime[:, :] vmhs[:, :, :] = nmhs[:, :, :] @@ -1001,6 +1003,21 @@ def unzip_and_untar(gz_file, output_dir): else: vcmap[:, :] = fcmap[:, :] + fcst_hr = np.full_like(mtime, np.nan, dtype='float64') + for i in range(mtime.shape[0]): + if mtime.ndim > 1: + for j in range(mtime.shape[1]): + if not np.isnan(mtime[i, j]): + fcst_hr[i, j] = (mtime[i, j] - initime_unix) / 3600 + else: + if not np.isnan(mtime[i]): + fcst_hr[i] = (mtime[i] - initime_unix) / 3600 + + # Add fcst_hr + vfcst_hr = ncfile.createVariable('fcst_hr', np.dtype('float64').char, ('buoypoints', 'time')) + vfcst_hr.units = 'hours' + vfcst_hr[:] = fcst_hr + ncfile.close() print(' ') print(f'Done. Netcdf ok. New file saved: WW3.{model_name}_Buoy{ftag}_{initime}to{fintime}.nc') From 6a60e5e825bc26098e546311573741fae1eb4e6e Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 10 Jun 2024 04:56:00 +0000 Subject: [PATCH 25/27] added the ability to read another spec file variable format --- ww3tools/wread.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ww3tools/wread.py b/ww3tools/wread.py index 0ba8fb4..08c296a 100644 --- a/ww3tools/wread.py +++ b/ww3tools/wread.py @@ -1722,6 +1722,12 @@ def spec1_ww3(*args): lon = float(cabc[2][6:]) depth = float(cabc[3]) wnds[t], wndd[t] = float(cabc[4]), float(cabc[5]) + elif len(cabc) == 7: + + lat_lon_parts = cabc[1].split() + lat, lon = float(lat_lon_parts[0]), float(lat_lon_parts[1]) + depth = float(cabc[2]) + wnds[t], wndd[t] = float(cabc[3]), float(cabc[4]) else: continue @@ -1794,4 +1800,3 @@ def spec1_ww3(*args): continue return results - From da9cbe2750fb5e2fabd0d4d5d41c73a641e8cdf0 Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Mon, 24 Jun 2024 14:55:56 +0000 Subject: [PATCH 26/27] Updated instruction on how to use format other than spec.gz --- ww3tools/modelBuoy_collocation.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index 98f05b2..fcdc7b5 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -76,6 +76,17 @@ python3 modelBuoy_collocation.py spec.gz "$input_gz_file" "$output_directory" "$buoy_path" "$model_name " "$forecast_ds" +Note: Updated way for submitting formats(bulltar) other than spec.gz: + + +buoy_path="/scratch2/NCEPDEV/marine/Matthew.Masarik/dat/buoys/NDBC/ncformat/wparam" +The buoy path should be defined in the job script. + +# Process data for each date +python3 modelBuoy_collocation.py ww3list.txt(any text file name that user defined) 2 $buoy_path +(The buoy path should be the last item) + + OUTPUT: netcdf file WW3.Buoy*.nc containing matchups of buoy and ww3 data, for the stations (lat/lon) where both data sources are available. From 03364054aac0735803d9f95df957ffb5cf386b0e Mon Sep 17 00:00:00 2001 From: Ghazal-Mohammadpour Date: Tue, 25 Jun 2024 18:19:42 +0000 Subject: [PATCH 27/27] added Cyclonemap and gridinfo to this version --- ww3tools/modelBuoy_collocation.py | 1179 ++++++++++++++++------------- 1 file changed, 648 insertions(+), 531 deletions(-) diff --git a/ww3tools/modelBuoy_collocation.py b/ww3tools/modelBuoy_collocation.py index fcdc7b5..ff9fb8a 100644 --- a/ww3tools/modelBuoy_collocation.py +++ b/ww3tools/modelBuoy_collocation.py @@ -148,46 +148,49 @@ # netCDF format specification fnetcdf = "NETCDF4" -gridinfo = int(0) -cyclonemap = int(0) +gridinfo = None +cyclonemap = None wlist = [] ftag = '' # Function to unzip and untar files def unzip_and_untar(gz_file, output_dir): - file_name = os.path.splitext(os.path.basename(gz_file))[0] - base_name = file_name.rsplit('.', 1)[0] - output_file = os.path.join(output_dir, file_name) - - with gzip.open(gz_file, 'rb') as f_in: - with open(output_file, 'wb') as f_out: - f_out.write(f_in.read()) - - extracted_folder = os.path.join(output_dir, base_name) - - with tarfile.open(output_file, 'r') as tar: - tar.extractall(extracted_folder) - - # Creating a list of the extracted files - list_file = os.path.join(output_dir, f'{base_name}_contents.txt') - with open(list_file, 'w') as f: - for root, dirs, files in os.walk(extracted_folder): - for file in files: - f.write(os.path.join(root, file) + '\n') - - # Creating a list of ids between two dots in the filenames - id_file = os.path.join(output_dir, f'{base_name}_id.txt') - with open(id_file, 'w') as f: - for root, dirs, files in os.walk(extracted_folder): - for file in files: - file_parts = file.split('.') - if len(file_parts) >= 3: - id_between_dots = file_parts[1] - f.write(id_between_dots + '\n') - - # Return the path to the extracted folder - return extracted_folder, list_file, id_file - + try: + file_name = os.path.splitext(os.path.basename(gz_file))[0] + base_name = file_name.rsplit('.', 1)[0] + output_file = os.path.join(output_dir, file_name) + + with gzip.open(gz_file, 'rb') as f_in: + with open(output_file, 'wb') as f_out: + f_out.write(f_in.read()) + + extracted_folder = os.path.join(output_dir, base_name) + + with tarfile.open(output_file, 'r') as tar: + tar.extractall(extracted_folder) + + # Creating a list of the extracted files + list_file = os.path.join(output_dir, f'{base_name}_contents.txt') + with open(list_file, 'w') as f: + for root, dirs, files in os.walk(extracted_folder): + for file in files: + f.write(os.path.join(root, file) + '\n') + + # Creating a list of ids between two dots in the filenames + id_file = os.path.join(output_dir, f'{base_name}_id.txt') + with open(id_file, 'w') as f: + for root, dirs, files in os.walk(extracted_folder): + for file in files: + file_parts = file.split('.') + if len(file_parts) >= 3: + id_between_dots = file_parts[1] + f.write(id_between_dots + '\n') + + # Return the path to the extracted folder + return extracted_folder, list_file, id_file + except Exception as e: + print(f"Error in unzip_and_untar: {e}") + sys.exit(1) # Main part of the script if __name__ == "__main__": @@ -195,17 +198,28 @@ def unzip_and_untar(gz_file, output_dir): if mode == "spec.gz": if len(sys.argv) != 7: - sys.exit("Usage: python script.py unzip input_gz_file output_directory buoy_path") + sys.exit("Usage: python script.py spec.gz input_gz_file output_directory buoy_path model_name forecastds") gz_file = sys.argv[2] output_dir = sys.argv[3] buoy_path = sys.argv[4] model_name = sys.argv[5] forecastds = int(sys.argv[6]) # Convert forecast indicator to integer + + # Debugging lines + print(f"gz_file: {gz_file}") + print(f"output_dir: {output_dir}") + print(f"buoy_path: {buoy_path}") + print(f"model_name: {model_name}") + print(f"forecastds: {forecastds}") + + print(f"Unzipping and untarring {gz_file} to {output_dir}") ndbcp = buoy_path extracted_folder, list_file, id_file = unzip_and_untar(gz_file, output_dir) print("Extraction completed. Extracted files are in:", extracted_folder) + print("Contents list file:", list_file) + print("ID list file:", id_file) # Read file names from the extracted folder file_names = [] @@ -213,14 +227,32 @@ def unzip_and_untar(gz_file, output_dir): for file in files: file_names.append(os.path.join(root, file)) + if not file_names: + print("No files found in the extracted folder.") + sys.exit(1) + + print("List of files to process:", file_names) + # Read id_files (if needed) id_files = [] with open(id_file, 'r') as f: for line in f: id_files.append(line.strip()) + if not id_files: + print("No IDs found in the ID list file.") + sys.exit(1) + + print("List of IDs:", id_files) + # Call the spec_ww3 function with the file names and ID files - results = wread.spec1_ww3(file_names, id_files) + try: + print("Calling wread.spec1_ww3...") + results =wread. spec1_ww3(file_names, id_files) + print("wread.spec1_ww3 call completed.") + except Exception as e: + print(f"Error in wread.spec1_ww3: {e}") + sys.exit(1) # Additional processing (if needed) mfcycle = None @@ -235,90 +267,109 @@ def unzip_and_untar(gz_file, output_dir): mdm = None mdp = None - for i, result in enumerate(results): - at = result['time'] - - if i == 0: - mfcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') - stname = np.atleast_1d(np.array(result['station_name'])) - mtime = np.copy(at) - mhs = np.copy([result['Hs']]) - mtp = np.copy([result['Tp']]) - - if 'wind_spd' in result.keys(): - mwn = np.copy([result['wind_spd']]) - else: - mwn = np.copy(mhs) * np.nan - - if 'wind_dir' in result.keys(): - mwd = np.copy([result['wind_dir']]) - else: - mwd = np.copy(mhs) * np.nan - - if 'freq' in result.keys(): - mfreq = np.copy([result['freq']]) - else: - mfreq = np.copy(mhs) * np.nan - - if 'tm' in result.keys(): - mtm = np.copy([result['tm']]) - else: - mtm = np.copy(mhs) * np.nan - - if 'dp' in result.keys(): - mdp = np.copy([result['dp']]) - else: - mdp = np.copy(mhs) * np.nan - - if 'dm' in result.keys(): - mdm = np.copy([result['dm']]) - else: - mdm = np.copy(mhs) * np.nan - - else: + try: + for i, result in enumerate(results): + print(f"Processing result {i+1}/{len(results)}...") + at = result['time'] - if mhs.shape[1] == result['Hs'].shape[0]: - stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) - mhs = np.append(mhs, [result['Hs']], axis=0) - mtp = np.append(mtp, [result['Tp']], axis=0) + if i == 0: + mfcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') + stname = np.atleast_1d(np.array(result['station_name'])) + mtime = np.copy(at) + mhs = np.copy([result['Hs']]) + mtp = np.copy([result['Tp']]) if 'wind_spd' in result.keys(): - mwn = np.append(mwn, [result['wind_spd']], axis=0) + mwn = np.copy([result['wind_spd']]) else: - mwn = np.append(mwn, [np.copy(result['Hs']) * np.nan], axis=0) + mwn = np.copy(mhs) * np.nan if 'wind_dir' in result.keys(): - mwd = np.append(mwd, [result['wind_dir']], axis=0) + mwd = np.copy([result['wind_dir']]) else: - mwd = np.append(mwd, [np.copy(result['Hs']) * np.nan], axis=0) + mwd = np.copy(mhs) * np.nan if 'freq' in result.keys(): - mfreq = np.append(mfreq, [result['freq']], axis=0) + mfreq = np.copy([result['freq']]) else: - mfreq = np.append(mfreq, [np.copy(result['Hs']) * np.nan], axis=0) + mfreq = np.copy(mhs) * np.nan if 'tm' in result.keys(): - mtm = np.append(mtm, [result['tm']], axis=0) + mtm = np.copy([result['tm']]) else: - mtm = np.append(mtm, [np.copy(result['Hs']) * np.nan], axis=0) + mtm = np.copy(mhs) * np.nan if 'dp' in result.keys(): - mdp = np.append(mdp, [result['dp']], axis=0) + mdp = np.copy([result['dp']]) else: - mdp = np.append(mdp, [np.copy(result['Hs']) * np.nan], axis=0) + mdp = np.copy(mhs) * np.nan if 'dm' in result.keys(): - mdm = np.append(mdp, [result['dm']], axis=0) + mdm = np.copy([result['dm']]) else: - mdm = np.append(mdm, [np.copy(result['Hs']) * np.nan], axis=0) + mdm = np.copy(mhs) * np.nan else: - print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) + if mhs.shape[1] == result['Hs'].shape[0]: + stname = np.append(stname, np.atleast_1d(np.array(result['station_name']))) + mhs = np.append(mhs, [result['Hs']], axis=0) + mtp = np.append(mtp, [result['Tp']], axis=0) + + if 'wind_spd' in result.keys(): + mwn = np.append(mwn, [result['wind_spd']], axis=0) + else: + mwn = np.append(mwn, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'wind_dir' in result.keys(): + mwd = np.append(mwd, [result['wind_dir']], axis=0) + else: + mwd = np.append(mwd, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'freq' in result.keys(): + mfreq = np.append(mfreq, [result['freq']], axis=0) + else: + mfreq = np.append(mfreq, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'tm' in result.keys(): + mtm = np.append(mtm, [result['tm']], axis=0) + else: + mtm = np.append(mtm, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'dp' in result.keys(): + mdp = np.append(mdp, [result['dp']], axis=0) + else: + mdp = np.append(mdp, [np.copy(result['Hs']) * np.nan], axis=0) + + if 'dm' in result.keys(): + mdm = np.append(mdp, [result['dm']], axis=0) + else: + mdm = np.append(mdm, [np.copy(result['Hs']) * np.nan], axis=0) + + else: + print(" Stations in file do not match the other tar files. Skipped") + + print("Completed processing results.") + + # Add additional debugging statements here + print("Checking final data structures...") + print(f"mfcycle: {mfcycle}") + print(f"stname: {stname}") + print(f"mtime: {mtime}") + print(f"mhs: {mhs}") + print(f"mtp: {mtp}") + print(f"mwn: {mwn}") + print(f"mwd: {mwd}") + print(f"mfreq: {mfreq}") + print(f"mtm: {mtm}") + print(f"mdm: {mdm}") + print(f"mdp: {mdp}") + + except Exception as e: + print(f"Error during processing: {e}") + sys.exit(1) else: - gridinfo = int(0) - cyclonemap = int(0) wlist = [] ftag = '' forecastds = 0 @@ -334,21 +385,32 @@ def unzip_and_untar(gz_file, output_dir): if forecastds > 0: print('Forecast-type data structure') - gridinfo = str(sys.argv[3]) if len(sys.argv) >= 5 else int(0) - if gridinfo != 0: - print('Writing gridinfo ' + gridinfo) + # Determine if gridinfo and cyclonemap are provided + if len(sys.argv) >= 6: + ndbcp = sys.argv[3] + gridinfo = sys.argv[4] + cyclonemap = sys.argv[5] + print('Using ndbcp path:', ndbcp) + print('Using gridinfo:', gridinfo) + print('Using cyclonemap:', cyclonemap) + else: + # If gridinfo and cyclonemap are not both provided, use the last argument as buoy_path + buoy_path = sys.argv[-1] + ndbcp = buoy_path + # Check if gridinfo is provided without cyclonemap or vice versa + if 'gridinfo' in sys.argv: + gridinfo = sys.argv[3] + print('Using gridinfo:', gridinfo) - cyclonemap = str(sys.argv[4]) if len(sys.argv) >= 6 else int(0) - if cyclonemap != 0: - print('Writing cyclonemap ' + cyclonemap) + if 'cyclonemap' in sys.argv: + cyclonemap = sys.argv[3] + print('Using cyclonemap:', cyclonemap) - # Paths - ndbcp = str(sys.argv[-1]) # Always get the last argument for ndbcp path - print('Using ndbcp path: ' + ndbcp) + print('Using ndbcp path:', buoy_path) # READ DATA print(" ") - if gridinfo != 0: + if gridinfo: # Grid Information gridmask = wread.mask(gridinfo) mlat = gridmask['latitude'] @@ -363,19 +425,19 @@ def unzip_and_untar(gz_file, output_dir): print(" GridInfo Ok. " + gridinfo) # Cyclone Information - if cyclonemap != 0: + if cyclonemap: cycloneinfo = wread.cyclonemap(cyclonemap) clat = cycloneinfo['latitude'] clon = cycloneinfo['longitude'] cmap = cycloneinfo['cmap'] ctime = cycloneinfo['time'] cinfo = np.array(cycloneinfo['info'].split(':')[1].split(';')) - if np.array_equal(clat, mlat) == True & np.array_equal(clon, mlon) == True: + if np.array_equal(clat, mlat) & np.array_equal(clon, mlon): print(" CycloneMap Ok. " + cyclonemap) else: sys.exit(' Error: Cyclone grid and Mask grid are different.') - # MODEL Point output, search among possible formats ------------------ + # MODEL Point output, search among possible formats if (str(wlist[0]).split('/')[-1].split('.')[-1] == 'bull_tar') or ( str(wlist[0]).split('/')[-1].split('.')[-1] == 'station_tar'): for i in range(0, np.size(wlist)): @@ -407,7 +469,6 @@ def unzip_and_untar(gz_file, output_dir): else: mwd = np.copy(mhs) * np.nan - else: if (mhs.shape[0] == result['hs'].shape[0]) and ( np.size(stname) == np.size(result['station_name'])): @@ -431,13 +492,12 @@ def unzip_and_untar(gz_file, output_dir): else: mwd = np.append(mwd, np.copy(result['hs']) * np.nan, axis=1) - else: print(" Stations in " + wlist[i] + " do not match the other tar files. Skipped " + wlist[i]) del result, at, fcycle - mdm = np.copy(mhs) * np.nan; - mtm = np.copy(mhs) * np.nan + mdm = np.copy(mhs) * np.nan + mtm = np.copy(mhs) * np.nan print(" ww3 file " + wlist[i] + " OK") elif (str(wlist[0]).split('/')[-1].split('.')[-1] == 'bull') or ( @@ -514,14 +574,14 @@ def unzip_and_untar(gz_file, output_dir): else: # list of station/buoy names if t == 0: - auxstationname = f.variables['station_name'][:, :]; + auxstationname = f.variables['station_name'][:, :] stname = [] for i in range(0, auxstationname.shape[0]): astname = "".join(np.array(auxstationname[i, :]).astype('str')) if '\t' in astname: astname = str(astname).replace("\t", "") - stname = np.append(stname, astname); + stname = np.append(stname, astname) del astname funits = f.variables['time'].units @@ -562,13 +622,11 @@ def unzip_and_untar(gz_file, output_dir): else: atm = np.array(np.copy(ahs * np.nan)) - ftunits = str(f.variables['time'].units).split('since')[1][1::].replace('T', ' ').replace('+00:00', - '') + ftunits = str(f.variables['time'].units).split('since')[1][1::].replace('T', ' ').replace('+00:00', '') at = np.array( - f.variables['time'][:] * tincr + timegm(strptime(ftunits, '%Y-%m-%d %H:%M:%S'))).astype( - 'double') + f.variables['time'][:] * tincr + timegm(strptime(ftunits, '%Y-%m-%d %H:%M:%S'))).astype('double') - f.close(); + f.close() del f fcycle = np.array(np.zeros((at.shape[0]), 'd') + at[0]).astype('double') if t == 0: @@ -592,214 +650,260 @@ def unzip_and_untar(gz_file, output_dir): del ahs, atm, atp, adm, adp, at, fcycle - print(" Read WW3 data OK."); + print(" Read WW3 data OK.") print(' ') else: sys.exit(' Point output file format not recognized: only bull, bull_tar, and .nc implemented.') # include other text formats: tab50, .ts - print(" Start building the matchups model/buoy ..."); + print(" Start building the matchups model/buoy ...") print(' ') - # BUOYS ------------------ - bwind = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan - bhs = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan - btm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan - btp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan - bdm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan - bdp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan - lat = np.zeros(np.size(stname), 'f') * np.nan - lon = np.zeros(np.size(stname), 'f') * np.nan - # help reading NDBC buoys, divided by year - yrange = np.array(np.arange(time.gmtime(mtime.min())[0], time.gmtime(mtime.min())[0] + 1, 1)).astype('int') - # loop buoys - for b in range(0, np.size(stname)): - ahs = [] - try: - awm = [] - ahs = [] - atm = [] - atp = [] - adm = [] - atime = [] - for y in yrange: - - f = nc.Dataset(ndbcp + "/" + stname[b] + "h" + repr(y) + ".nc") - if 'wave_height' in f.variables.keys(): - ahs = np.append(ahs, f.variables['wave_height'][:, 0, 0]) - elif 'hs' in f.variables.keys(): - ahs = np.append(ahs, f.variables['hs'][:, 0, 0]) - elif 'swh' in f.variables.keys(): - ahs = np.append(ahs, f.variables['swh'][:, 0, 0]) - - if 'wind_spd' in f.variables.keys(): - awm = np.append(awm, f.variables['wind_spd'][:, 0, 0]) - else: - awm = np.array(np.copy(ahs * np.nan)) - - if 'average_wpd' in f.variables.keys(): - atm = np.append(atm, f.variables['average_wpd'][:, 0, 0]) - else: - atm = np.array(np.copy(ahs * np.nan)) +# BUOYS +bwind = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bhs = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +btm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +btp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bdm = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +bdp = np.zeros((np.size(stname), np.size(mtime)), 'f') * np.nan +lat = np.zeros(np.size(stname), 'f') * np.nan +lon = np.zeros(np.size(stname), 'f') * np.nan - if 'dominant_wpd' in f.variables.keys(): - atp = np.append(atp, f.variables['dominant_wpd'][:, 0, 0]) - else: - atp = np.array(np.copy(ahs * np.nan)) - - if 'mean_wave_dir' in f.variables.keys(): - adm = np.append(adm, f.variables['mean_wave_dir'][:, 0, 0]) - else: - adm = np.array(np.copy(ahs * np.nan)) - - if 'latitude' in f.variables.keys(): - lat[b] = f.variables['latitude'][:] - elif 'LATITUDE' in f.variables.keys(): - lat[b] = f.variables['LATITUDE'][:] - else: - lat[b] = np.nan +# Help reading NDBC buoys, divided by year +yrange = np.array(np.arange(time.gmtime(mtime.min())[0], time.gmtime(mtime.min())[0] + 1, 1)).astype('int') - if 'longitude' in f.variables.keys(): - lon[b] = f.variables['longitude'][:] - elif 'LONGITUDE' in f.variables.keys(): - lon[b] = f.variables['LONGITUDE'][:] - else: - lon[b] = np.nan - - atime = np.append(atime, np.array(f.variables['time'][:]).astype('double')) - - f.close() - del f - - adp = adm * np.nan # no peak direction available in this format - - if np.size(ahs) > 0: - - # First layer of simple quality-control - indq = np.where((ahs > 30.) | (ahs < 0.0)) - if np.size(indq) > 0: - ahs[indq] = np.nan - del indq - - indq = np.where((atm > 40.) | (atm < 0.0)) - if np.size(indq) > 0: - atm[indq] = np.nan - del indq - - indq = np.where((atp > 40.) | (atp < 0.0)) - if np.size(indq) > 0: - atp[indq] = np.nan - del indq - - indq = np.where((adm > 360.) | (adm < -180.)) - if np.size(indq) > 0: - adm[indq] = np.nan - del indq - - indq = np.where((adp > 360.) | (adp < -180.)) - if np.size(indq) > 0: - adp[indq] = np.nan - del indq - - indq = np.where((awm > 50.) | (awm < 0.0)) - if np.size(indq) > 0: - awm[indq] = np.nan - del indq - - c = 0 - for t in mtime: - indt = np.where(np.abs(atime - t) < 1800.)[0] - if np.size(indt) > 0: - if np.any(ahs[indt].mask == False): - bhs[b, c] = np.nanmean(ahs[indt][ahs[indt].mask == False]) - if np.any(atm[indt].mask == False): - btm[b, c] = np.nanmean(atm[indt][atm[indt].mask == False]) - if np.any(atp[indt].mask == False): - btp[b, c] = np.nanmean(atp[indt][atp[indt].mask == False]) - if np.any(adm[indt].mask == False): - bdm[b, c] = np.nanmean(adm[indt][adm[indt].mask == False]) - if np.any(adp[indt].mask == False): - bdp[b, c] = np.nanmean(adp[indt][adp[indt].mask == False]) - if np.any(awm[indt].mask == False): - bwind[b, c] = np.nanmean(awm[indt][awm[indt].mask == False]) - c += 1 - - - print(" station " + stname[b] + " ok") +# Loop buoys +for b in range(0, np.size(stname)): + ahs = [] + try: + awm = [] + ahs = [] + atm = [] + atp = [] + adm = [] + atime = [] + for y in yrange: + f = nc.Dataset(ndbcp + "/" + stname[b] + "h" + repr(y) + ".nc") + if 'wave_height' in f.variables.keys(): + ahs = np.append(ahs, f.variables['wave_height'][:, 0, 0]) + elif 'hs' in f.variables.keys(): + ahs = np.append(ahs, f.variables['hs'][:, 0, 0]) + elif 'swh' in f.variables.keys(): + ahs = np.append(ahs, f.variables['swh'][:, 0, 0]) + + if 'wind_spd' in f.variables.keys(): + awm = np.append(awm, f.variables['wind_spd'][:, 0, 0]) + else: + awm = np.array(np.copy(ahs * np.nan)) - except Exception as e: - print("Error occurred while processing station", stname[b]) - print(e) + if 'average_wpd' in f.variables.keys(): + atm = np.append(atm, f.variables['average_wpd'][:, 0, 0]) + else: + atm = np.array(np.copy(ahs * np.nan)) + if 'dominant_wpd' in f.variables.keys(): + atp = np.append(atp, f.variables['dominant_wpd'][:, 0, 0]) + else: + atp = np.array(np.copy(ahs * np.nan)) - # Simple quality-control (range) - ind = np.where((bhs > 30.) | (bhs < 0.0)) - if np.size(ind) > 0: - bhs[ind] = np.nan; - del ind + if 'mean_wave_dir' in f.variables.keys(): + adm = np.append(adm, f.variables['mean_wave_dir'][:, 0, 0]) + else: + adm = np.array(np.copy(ahs * np.nan)) - ind = np.where((btm > 40.) | (btm < 0.0)) - if np.size(ind) > 0: - btm[ind] = np.nan; - del ind + if 'latitude' in f.variables.keys(): + lat[b] = f.variables['latitude'][:] + elif 'LATITUDE' in f.variables.keys(): + lat[b] = f.variables['LATITUDE'][:] + else: + lat[b] = np.nan - ind = np.where((btp > 40.) | (btp < 0.0)) - if np.size(ind) > 0: - btp[ind] = np.nan; - del ind + if 'longitude' in f.variables.keys(): + lon[b] = f.variables['longitude'][:] + elif 'LONGITUDE' in f.variables.keys(): + lon[b] = f.variables['LONGITUDE'][:] + else: + lon[b] = np.nan + + atime = np.append(atime, np.array(f.variables['time'][:]).astype('double')) + + f.close() + del f + + adp = adm * np.nan # no peak direction available in this format + + if np.size(ahs) > 0: + # First layer of simple quality-control + indq = np.where((ahs > 30.) | (ahs < 0.0)) + if np.size(indq) > 0: + ahs[indq] = np.nan + del indq + + indq = np.where((atm > 40.) | (atm < 0.0)) + if np.size(indq) > 0: + atm[indq] = np.nan + del indq + + indq = np.where((atp > 40.) | (atp < 0.0)) + if np.size(indq) > 0: + atp[indq] = np.nan + del indq + + indq = np.where((adm > 360.) | (adm < -180.)) + if np.size(indq) > 0: + adm[indq] = np.nan + del indq + + indq = np.where((adp > 360.) | (adp < -180.)) + if np.size(indq) > 0: + adp[indq] = np.nan + del indq + + indq = np.where((awm > 50.) | (awm < 0.0)) + if np.size(indq) > 0: + awm[indq] = np.nan + del indq + + c = 0 + for t in mtime: + indt = np.where(np.abs(atime - t) < 1800.)[0] + if np.size(indt) > 0: + if np.any(ahs[indt].mask == False): + bhs[b, c] = np.nanmean(ahs[indt][ahs[indt].mask == False]) + if np.any(atm[indt].mask == False): + btm[b, c] = np.nanmean(atm[indt][atm[indt].mask == False]) + if np.any(atp[indt].mask == False): + btp[b, c] = np.nanmean(atp[indt][atp[indt].mask == False]) + if np.any(adm[indt].mask == False): + bdm[b, c] = np.nanmean(adm[indt][adm[indt].mask == False]) + if np.any(adp[indt].mask == False): + bdp[b, c] = np.nanmean(adp[indt][adp[indt].mask == False]) + if np.any(awm[indt].mask == False): + bwind[b, c] = np.nanmean(awm[indt][awm[indt].mask == False]) + c += 1 + + print(" station " + stname[b] + " ok") + + except Exception as e: + print("Error occurred while processing station", stname[b]) + print(e) + +# Simple quality-control (range) +ind = np.where((bhs > 30.) | (bhs < 0.0)) +if np.size(ind) > 0: + bhs[ind] = np.nan + del ind - ind = np.where((bdm > 360.) | (bdm < -180.)) - if np.size(ind) > 0: - bdm[ind] = np.nan; - del ind +ind = np.where((btm > 40.) | (btm < 0.0)) +if np.size(ind) > 0: + btm[ind] = np.nan + del ind - ind = np.where((bdp > 360.) | (bdp < -180.)) - if np.size(ind) > 0: - bdp[ind] = np.nan; - del ind +ind = np.where((btp > 40.) | (btp < 0.0)) +if np.size(ind) > 0: + btp[ind] = np.nan + del ind - ind = np.where((bwind > 50.0) | (bwind < 0.0)) - if np.size(ind) > 0: - bwind[ind] = np.nan; - del ind +ind = np.where((bdm > 360.) | (bdm < -180.)) +if np.size(ind) > 0: + bdm[ind] = np.nan + del ind - ind = np.where((mhs > 30.) | (mhs < 0.0)) - if np.size(ind) > 0: - mhs[ind] = np.nan; - del ind +ind = np.where((bdp > 360.) | (bdp < -180.)) +if np.size(ind) > 0: + bdp[ind] = np.nan + del ind - ind = np.where((mtm > 40.) | (mtm < 0.0)) - if np.size(ind) > 0: - mtm[ind] = np.nan; - del ind +ind = np.where((bwind > 50.0) | (bwind < 0.0)) +if np.size(ind) > 0: + bwind[ind] = np.nan + del ind - ind = np.where((mtp > 40.) | (mtp < 0.0)) - if np.size(ind) > 0: - mtp[ind] = np.nan; - del ind +ind = np.where((mhs > 30.) | (mhs < 0.0)) +if np.size(ind) > 0: + mhs[ind] = np.nan + del ind +ind = np.where((mtm > 40.) | (mtm < 0.0)) +if np.size(ind) > 0: + mtm[ind] = np.nan + del ind - ind = np.where((mdm > 360.) | (mdm < -180.)) - if np.size(ind) > 0: - mdm[ind] = np.nan; - del ind +ind = np.where((mtp > 40.) | (mtp < 0.0)) +if np.size(ind) > 0: + mtp[ind] = np.nan + del ind - ind = np.where((mdp > 360.) | (mdp < -180.)) - if np.size(ind) > 0: - mdp[ind] = np.nan; - del ind +ind = np.where((mdm > 360.) | (mdm < -180.)) +if np.size(ind) > 0: + mdm[ind] = np.nan + del ind - ind = np.where((mwn > 50.) | (mwn < 0.0)) - if np.size(ind) > 0: - mwn[ind] = np.nan; - del ind +ind = np.where((mdp > 360.) | (mdp < -180.)) +if np.size(ind) > 0: + mdp[ind] = np.nan + del ind - # Clean data excluding some stations. Select matchups only when model and buoy are available. +ind = np.where((mwn > 50.) | (mwn < 0.0)) +if np.size(ind) > 0: + mwn[ind] = np.nan + del ind - ind = np.where((np.isnan(lat) == False) & (np.isnan(lon) == False) & (np.isnan(np.nanmean(mhs, axis=1)) == False) & (np.isnan(np.nanmean(bhs, axis=1)) == False)) +# Clean data excluding some stations. Select matchups only when model and buoy are available. +ind = np.where((np.isnan(lat) == False) & (np.isnan(lon) == False) & (np.isnan(np.nanmean(mhs, axis=1)) == False) & (np.isnan(np.nanmean(bhs, axis=1)) == False)) +if np.size(ind) > 0: + stname = np.array(stname[ind[0]]) + lat = np.array(lat[ind[0]]) + lon = np.array(lon[ind[0]]) + mhs = np.array(mhs[ind[0], :]) + mtm = np.array(mtm[ind[0], :]) + mtp = np.array(mtp[ind[0], :]) + mdm = np.array(mdm[ind[0], :]) + mdp = np.array(mdp[ind[0], :]) + mwn = np.array(mwn[ind[0], :]) + bhs = np.array(bhs[ind[0], :]) + btm = np.array(btm[ind[0], :]) + btp = np.array(btp[ind[0], :]) + bdm = np.array(bdm[ind[0], :]) + bdp = np.array(bdp[ind[0], :]) + bwind = np.array(bwind[ind[0], :]) +else: + sys.exit(' Error: No matchups Model/Buoy available.') + +print(" Matchups model/buoy complete. Total of " + repr(np.size(ind)) + " stations/buoys available.") +del ind + +# Processing grid and/or cyclone information +if gridinfo: + print(" Adding extra information ... ") + alon = np.copy(lon) + alon[alon < 0] = alon[alon < 0] + 360. + indgplat = [] + indgplon = [] + for i in range(0, lat.shape[0]): + # indexes nearest point. + indgplat = np.append(indgplat, np.where(abs(mlat - lat[i]) == abs(mlat - lat[i]).min())[0][0]) + indgplon = np.append(indgplon, np.where(abs(mlon - alon[i]) == abs(mlon - alon[i]).min())[0][0]) + + indgplat = np.array(indgplat).astype('int') + indgplon = np.array(indgplon).astype('int') + pdistcoast = np.zeros(lat.shape[0], 'f') * np.nan + pdepth = np.zeros(lat.shape[0], 'f') * np.nan + poni = np.zeros(lat.shape[0], 'f') * np.nan + phsmz = np.zeros(lat.shape[0], 'f') * np.nan + for i in range(0, lat.shape[0]): + pdistcoast[i] = distcoast[indgplat[i], indgplon[i]] + pdepth[i] = depth[indgplat[i], indgplon[i]] + poni[i] = oni[indgplat[i], indgplon[i]] + phsmz[i] = hsmz[indgplat[i], indgplon[i]] + + print(" Grid Information Included.") + + # Excluding shallow water points too close to the coast (mask information not accurate) + ind = np.where((np.isnan(pdistcoast) == False) & (np.isnan(pdepth) == False)) if np.size(ind) > 0: stname = np.array(stname[ind[0]]) lat = np.array(lat[ind[0]]) @@ -809,251 +913,264 @@ def unzip_and_untar(gz_file, output_dir): mtp = np.array(mtp[ind[0], :]) mdm = np.array(mdm[ind[0], :]) mdp = np.array(mdp[ind[0], :]) - mwn = np.array(mwn[ind[0], :]) bhs = np.array(bhs[ind[0], :]) btm = np.array(btm[ind[0], :]) btp = np.array(btp[ind[0], :]) bdm = np.array(bdm[ind[0], :]) bdp = np.array(bdp[ind[0], :]) - bwind = np.array(bwind[ind[0], :]) - + pdistcoast = np.array(pdistcoast[ind[0]]) + pdepth = np.array(pdepth[ind[0]]) + poni = np.array(poni[ind[0]]) + phsmz = np.array(phsmz[ind[0]]) else: - sys.exit(' Error: No matchups Model/Buoy available.') + sys.exit(' Error: No matchups Model/Buoy available after using grid mask.') - - print(" Matchups model/buoy complete. Total of " + repr(np.size(ind)) + " stations/buoys available."); del ind - # Edit format if this is forecast model data. Reshape and allocate - if forecastds > 0: - unt = np.unique(mfcycle) - mxsz = 1 - - for i in range(0, unt.shape[0]): - ind = np.where(mfcycle == unt[i])[0] - mxsz = np.max([mxsz, np.size(ind)]) - - for i in range(0, unt.shape[0]): - ind = np.where(mfcycle == unt[i])[0] - if i == 0: - nmhs = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nmtm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nmtp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nmdm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nmdp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nmwn = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nbhs = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nbtm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nbtp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nbdm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nbdp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nbwind = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - nmtime = np.zeros((unt.shape[0], mxsz), 'double') * np.nan - if cyclonemap != 0: - nfcmap = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan - - nmtime[i, 0:np.size(ind)] = np.array(mtime[ind]).astype('double') - nmhs[:, i, :][:, 0:np.size(ind)] = np.array(mhs[:, ind]) - nmtm[:, i, :][:, 0:np.size(ind)] = np.array(mtm[:, ind]) - nmtp[:, i, :][:, 0:np.size(ind)] = np.array(mtp[:, ind]) - nmdm[:, i, :][:, 0:np.size(ind)] = np.array(mdm[:, ind]) - nmdp[:, i, :][:, 0:np.size(ind)] = np.array(mdp[:, ind]) - nmwn[:, i, :][:, 0:np.size(ind)] = np.array(mwn[:, ind]) - nbhs[:, i, :][:, 0:np.size(ind)] = np.array(bhs[:, ind]) - nbtm[:, i, :][:, 0:np.size(ind)] = np.array(btm[:, ind]) - nbtp[:, i, :][:, 0:np.size(ind)] = np.array(btp[:, ind]) - nbdm[:, i, :][:, 0:np.size(ind)] = np.array(bdm[:, ind]) - nbdp[:, i, :][:, 0:np.size(ind)] = np.array(bdp[:, ind]) - nbwind[:, i, :][:, 0:np.size(ind)] = np.array(bwind[:, ind]) - if cyclonemap != 0: - nfcmap[:, i, :][:, 0:np.size(ind)] = np.array(fcmap[:, ind]) - - ind = np.where((nmhs > 0.0) & (nbhs > 0.0)) - - else: - ind = np.where((mhs > 0.0) & (bhs > 0.0)) - - if np.size(ind) > 0: - print(' Total amount of matchups model/buoy: ' + repr(np.size(ind))) - - # Save netcdf output file - lon[lon > 180.] = lon[lon > 180.] - 360. - initime = str(time.gmtime(mtime.min())[0]) + str(time.gmtime(mtime.min())[1]).zfill(2) + str(time.gmtime(mtime.min())[2]).zfill(2) + str(time.gmtime(mtime.min())[3]).zfill(2) - fintime = str(time.gmtime(mtime.max())[0]) + str(time.gmtime(mtime.max())[1]).zfill(2) + str(time.gmtime(mtime.max())[2]).zfill(2) + str(time.gmtime(mtime.max())[3]).zfill(2) - - # Ensure model_name is defined - if 'model_name' not in locals(): - model_name = '' - - ncfile = nc.Dataset(f'WW3.{model_name}Buoy{ftag}_{initime}to{fintime}.nc', "w", format=fnetcdf) - print(f"Model Name: {model_name}, Tag: {ftag}, Start Time: {initime}, End Time: {fintime}") - ncfile.history = "Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of " +repr(bhs[bhs>0.].shape[0])+" observations or pairs model/observation." - ncfile.initial_condition = initime - ncfile.time_units = "seconds since 1970-01-01T00:00:00+00:00" - - # create dimensions - ncfile.createDimension('buoypoints', bhs.shape[0]) - if gridinfo != 0: - ncfile.createDimension('GlobalOceansSeas', ocnames.shape[0]) - ncfile.createDimension('HighSeasMarineZones', hsmznames.shape[0]) - if cyclonemap != 0: - ncfile.createDimension('cycloneinfo', cinfo.shape[0]) - vcinfo = ncfile.createVariable('cycloneinfo', dtype('a25'), ('cycloneinfo')) - - # create variables. - vstname = ncfile.createVariable('buoyID', type('a25'), ('buoypoints')) - vlat = ncfile.createVariable('latitude', np.dtype('float32').char, ('buoypoints')) - vlon = ncfile.createVariable('longitude', np.dtype('float32').char, ('buoypoints')) + if cyclonemap: + fcmap = np.zeros((lat.shape[0], mtime.shape[0]), 'f') * np.nan + for t in range(0, np.size(mtime)): + # search for cyclone time index and cyclone map + indt = np.where(np.abs(ctime - mtime[t]) < 5400.) + if np.size(indt) > 0: + for i in range(0, lat.shape[0]): + fcmap[i, t] = np.array(cmap[indt[0][0], indgplat[i], indgplon[i]]) + del indt + else: + print(' - No cyclone information for this time step: ' + repr(t)) + + ind = np.where(fcmap < 0) + if np.size(ind) > 0: + fcmap[ind] = np.nan + + print(" Cyclone Information Included.") + +# Edit format if this is forecast model data. Reshape and allocate +if forecastds > 0: + unt = np.unique(mfcycle) + mxsz = 1 + + for i in range(0, unt.shape[0]): + ind = np.where(mfcycle == unt[i])[0] + mxsz = np.max([mxsz, np.size(ind)]) + + for i in range(0, unt.shape[0]): + ind = np.where(mfcycle == unt[i])[0] + if i == 0: + nmhs = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmtm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmtp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmdm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmdp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmwn = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbhs = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbtm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbtp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbdm = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbdp = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nbwind = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + nmtime = np.zeros((unt.shape[0], mxsz), 'double') * np.nan + if cyclonemap: + nfcmap = np.zeros((mhs.shape[0], unt.shape[0], mxsz), 'f') * np.nan + + nmtime[i, 0:np.size(ind)] = np.array(mtime[ind]).astype('double') + nmhs[:, i, :][:, 0:np.size(ind)] = np.array(mhs[:, ind]) + nmtm[:, i, :][:, 0:np.size(ind)] = np.array(mtm[:, ind]) + nmtp[:, i, :][:, 0:np.size(ind)] = np.array(mtp[:, ind]) + nmdm[:, i, :][:, 0:np.size(ind)] = np.array(mdm[:, ind]) + nmdp[:, i, :][:, 0:np.size(ind)] = np.array(mdp[:, ind]) + nmwn[:, i, :][:, 0:np.size(ind)] = np.array(mwn[:, ind]) + nbhs[:, i, :][:, 0:np.size(ind)] = np.array(bhs[:, ind]) + nbtm[:, i, :][:, 0:np.size(ind)] = np.array(btm[:, ind]) + nbtp[:, i, :][:, 0:np.size(ind)] = np.array(btp[:, ind]) + nbdm[:, i, :][:, 0:np.size(ind)] = np.array(bdm[:, ind]) + nbdp[:, i, :][:, 0:np.size(ind)] = np.array(bdp[:, ind]) + nbwind[:, i, :][:, 0:np.size(ind)] = np.array(bwind[:, ind]) + if cyclonemap: + nfcmap[:, i, :][:, 0:np.size(ind)] = np.array(fcmap[:, ind]) + + ind = np.where((nmhs > 0.0) & (nbhs > 0.0)) + +else: + ind = np.where((mhs > 0.0) & (bhs > 0.0)) + +if np.size(ind) > 0: + print(' Total amount of matchups model/buoy: ' + repr(np.size(ind))) + + # Save netcdf output file + lon[lon > 180.] = lon[lon > 180.] - 360. + initime = str(time.gmtime(mtime.min())[0]) + str(time.gmtime(mtime.min())[1]).zfill(2) + str(time.gmtime(mtime.min())[2]).zfill(2) + str(time.gmtime(mtime.min())[3]).zfill(2) + fintime = str(time.gmtime(mtime.max())[0]) + str(time.gmtime(mtime.max())[1]).zfill(2) + str(time.gmtime(mtime.max())[2]).zfill(2) + str(time.gmtime(mtime.max())[3]).zfill(2) + + # Ensure model_name is defined + if 'model_name' not in locals(): + model_name = '' + + ncfile = nc.Dataset(f'WW3.{model_name}Buoy{ftag}_{initime}to{fintime}.nc', "w", format=fnetcdf) + print(f"Model Name: {model_name}, Tag: {ftag}, Start Time: {initime}, End Time: {fintime}") + ncfile.history = "Matchups of WAVEWATCHIII point output (table) and NDBC and Copernicus Buoys. Total of " + repr(bhs[bhs > 0.].shape[0]) + " observations or pairs model/observation." + ncfile.initial_condition = initime + ncfile.time_units = "seconds since 1970-01-01T00:00:00+00:00" + + # Create dimensions + ncfile.createDimension('buoypoints', bhs.shape[0]) + if gridinfo: + ncfile.createDimension('GlobalOceansSeas', ocnames.shape[0]) + ncfile.createDimension('HighSeasMarineZones', hsmznames.shape[0]) + if cyclonemap: + ncfile.createDimension('cycloneinfo', cinfo.shape[0]) + vcinfo = ncfile.createVariable('cycloneinfo', 'S1', ('cycloneinfo')) + + # Create variables + vstname = ncfile.createVariable('buoyID', 'S1', ('buoypoints')) + vlat = ncfile.createVariable('latitude', 'f4', ('buoypoints')) + vlon = ncfile.createVariable('longitude', 'f4', ('buoypoints')) + if forecastds > 0: + ncfile.createDimension('time', nmhs.shape[2]) + ncfile.createDimension('fcycle', unt.shape[0]) + vt = ncfile.createVariable('time', 'f8', ('fcycle', 'time')) + vmhs = ncfile.createVariable('model_hs', 'f4', ('buoypoints', 'fcycle', 'time')) + vmtm = ncfile.createVariable('model_tm', 'f4', ('buoypoints', 'fcycle', 'time')) + vmtp = ncfile.createVariable('model_tp', 'f4', ('buoypoints', 'fcycle', 'time')) + vmdm = ncfile.createVariable('model_dm', 'f4', ('buoypoints', 'fcycle', 'time')) + vmdp = ncfile.createVariable('model_dp', 'f4', ('buoypoints', 'fcycle', 'time')) + vmwn = ncfile.createVariable('model_wind', 'f4', ('buoypoints', 'fcycle', 'time')) + + vbhs = ncfile.createVariable('obs_hs', 'f4', ('buoypoints', 'fcycle', 'time')) + vbtm = ncfile.createVariable('obs_tm', 'f4', ('buoypoints', 'fcycle', 'time')) + vbtp = ncfile.createVariable('obs_tp', 'f4', ('buoypoints', 'fcycle', 'time')) + vbdm = ncfile.createVariable('obs_dm', 'f4', ('buoypoints', 'fcycle', 'time')) + vbdp = ncfile.createVariable('obs_dp', 'f4', ('buoypoints', 'fcycle', 'time')) + vbwind = ncfile.createVariable('obs_wind', 'f4', ('buoypoints', 'fcycle', 'time')) + else: + ncfile.createDimension('time', bhs.shape[1]) + vt = ncfile.createVariable('time', 'f8', ('time')) + vmhs = ncfile.createVariable('model_hs', 'f4', ('buoypoints', 'time')) + vmtm = ncfile.createVariable('model_tm', 'f4', ('buoypoints', 'time')) + vmtp = ncfile.createVariable('model_tp', 'f4', ('buoypoints', 'time')) + vmdm = ncfile.createVariable('model_dm', 'f4', ('buoypoints', 'time')) + vmdp = ncfile.createVariable('model_dp', 'f4', ('buoypoints', 'time')) + vmwn = ncfile.createVariable('model_wind', 'f4', ('buoypoints', 'time')) + + vbhs = ncfile.createVariable('obs_hs', 'f4', ('buoypoints', 'time')) + vbtm = ncfile.createVariable('obs_tm', 'f4', ('buoypoints', 'time')) + vbtp = ncfile.createVariable('obs_tp', 'f4', ('buoypoints', 'time')) + vbdm = ncfile.createVariable('obs_dm', 'f4', ('buoypoints', 'time')) + vbdp = ncfile.createVariable('obs_dp', 'f4', ('buoypoints', 'time')) + vbwind = ncfile.createVariable('obs_wind', 'f4', ('buoypoints', 'time')) + + if gridinfo: + vpdistcoast = ncfile.createVariable('distcoast', 'f4', ('buoypoints')) + vpdepth = ncfile.createVariable('depth', 'f4', ('buoypoints')) + vponi = ncfile.createVariable('GlobalOceansSeas', 'f4', ('buoypoints')) + vocnames = ncfile.createVariable('names_GlobalOceansSeas', 'S1', ('GlobalOceansSeas')) + vphsmz = ncfile.createVariable('HighSeasMarineZones', 'f4', ('buoypoints')) + vhsmznames = ncfile.createVariable('names_HighSeasMarineZones', 'S1', ('HighSeasMarineZones')) + if cyclonemap: if forecastds > 0: - ncfile.createDimension('time', nmhs.shape[2]) - ncfile.createDimension('fcycle', unt.shape[0]) - vt = ncfile.createVariable('time', np.dtype('float64').char, ('fcycle', 'time')) - vmhs = ncfile.createVariable('model_hs', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - vmtm = ncfile.createVariable('model_tm', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - vmtp = ncfile.createVariable('model_tp', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - vmdm = ncfile.createVariable('model_dm', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - vmdp = ncfile.createVariable('model_dp', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - vmwn = ncfile.createVariable('model_wind', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - - vbhs = ncfile.createVariable('obs_hs', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - vbtm = ncfile.createVariable('obs_tm', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - vbtp = ncfile.createVariable('obs_tp', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - vbdm = ncfile.createVariable('obs_dm', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - vbdp = ncfile.createVariable('obs_dp', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - vbwind = ncfile.createVariable('obs_wind', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - + vcmap = ncfile.createVariable('cyclone', 'f4', ('buoypoints', 'fcycle', 'time')) else: - ncfile.createDimension('time', bhs.shape[1]) - vt = ncfile.createVariable('time', np.dtype('float64').char, ('time')) - vmhs = ncfile.createVariable('model_hs', np.dtype('float32').char, ('buoypoints', 'time')) - vmtm = ncfile.createVariable('model_tm', np.dtype('float32').char, ('buoypoints', 'time')) - vmtp = ncfile.createVariable('model_tp', np.dtype('float32').char, ('buoypoints', 'time')) - vmdm = ncfile.createVariable('model_dm', np.dtype('float32').char, ('buoypoints', 'time')) - vmdp = ncfile.createVariable('model_dp', np.dtype('float32').char, ('buoypoints', 'time')) - vmwn = ncfile.createVariable('model_wind', np.dtype('float32').char, ('buoypoints', 'time')) - - vbhs = ncfile.createVariable('obs_hs', np.dtype('float32').char, ('buoypoints', 'time')) - vbtm = ncfile.createVariable('obs_tm', np.dtype('float32').char, ('buoypoints', 'time')) - vbtp = ncfile.createVariable('obs_tp', np.dtype('float32').char, ('buoypoints', 'time')) - vbdm = ncfile.createVariable('obs_dm', np.dtype('float32').char, ('buoypoints', 'time')) - vbdp = ncfile.createVariable('obs_dp', np.dtype('float32').char, ('buoypoints', 'time')) - vbwind = ncfile.createVariable('obs_wind', np.dtype('float32').char, ('buoypoints', 'time')) - - if gridinfo != 0: - vpdistcoast = ncfile.createVariable('distcoast', np.dtype('float32').char, ('buoypoints')) - vpdepth = ncfile.createVariable('depth', np.dtype('float32').char, ('buoypoints')) - vponi = ncfile.createVariable('GlobalOceansSeas', np.dtype('float32').char, ('buoypoints')) - vocnames = ncfile.createVariable('names_GlobalOceansSeas', dtype('a25'), ('GlobalOceansSeas')) - vphsmz = ncfile.createVariable('HighSeasMarineZones', np.dtype('float32').char, ('buoypoints')) - vhsmznames = ncfile.createVariable('names_HighSeasMarineZones', dtype('a25'), ('HighSeasMarineZones')) - if cyclonemap != 0: - if forecastds > 0: - vcmap = ncfile.createVariable('cyclone', np.dtype('float32').char, ('buoypoints', 'fcycle', 'time')) - else: - vcmap = ncfile.createVariable('cyclone', np.dtype('float32').char, ('buoypoints', 'time')) - - # Assign units - vlat.units = 'degrees_north' - vlon.units = 'degrees_east' - vt.units = 'seconds since 1970-01-01T00:00:00+00:00' - vmhs.units = 'm' - vbhs.units = 'm' - vmtm.units = 's' - vbtm.units = 's' - vmtp.units = 's' - vbtp.units = 's' - vmdm.units = 'degrees' - vbdm.units = 'degrees' - vmdp.units = 'degrees' - vbdp.units = 'degrees' - vmwn.unit = 'm/s' - vbwind.unit = 'm/s' - - - if gridinfo != 0: - vpdepth.units = 'm' - vpdistcoast.units = 'km' - - # Allocate Data - vstname[:] = stname[:] - vlat[:] = lat[:] - vlon[:] = lon[:] - initime_unix = timegm(strptime(initime, '%Y%m%d%H')) - - + vcmap = ncfile.createVariable('cyclone', 'f4', ('buoypoints', 'time')) + + # Assign units + vlat.units = 'degrees_north' + vlon.units = 'degrees_east' + vt.units = 'seconds since 1970-01-01T00:00:00+00:00' + vmhs.units = 'm' + vbhs.units = 'm' + vmtm.units = 's' + vbtm.units = 's' + vmtp.units = 's' + vbtp.units = 's' + vmdm.units = 'degrees' + vbdm.units = 'degrees' + vmdp.units = 'degrees' + vbdp.units = 'degrees' + vmwn.unit = 'm/s' + vbwind.unit = 'm/s' + + if gridinfo: + vpdepth.units = 'm' + vpdistcoast.units = 'km' + + # Allocate Data + vstname[:] = stname[:] + vlat[:] = lat[:] + vlon[:] = lon[:] + initime_unix = timegm(strptime(initime, '%Y%m%d%H')) + if forecastds > 0: + vt[:, :] = nmtime[:, :] + vmhs[:, :, :] = nmhs[:, :, :] + vmtm[:, :, :] = nmtm[:, :, :] + vmtp[:, :, :] = nmtp[:, :, :] + vmdm[:, :, :] = nmdm[:, :, :] + vmdp[:, :, :] = nmdp[:, :, :] + vmwn[:, :, :] = nmwn[:, :, :] + vbhs[:, :, :] = nbhs[:, :, :] + vbtm[:, :, :] = nbtm[:, :, :] + vbtp[:, :, :] = nbtp[:, :, :] + vbdm[:, :, :] = nbdm[:, :, :] + vbdp[:, :, :] = nbdp[:, :, :] + vbwind[:, :, :] = nbwind[:, :, :] + + if gridinfo: + vpdistcoast[:] = pdistcoast[:] + vpdepth[:] = pdepth[:] + vponi[:] = poni[:] + vocnames[:] = ocnames[:] + vphsmz[:] = phsmz[:] + vhsmznames[:] = hsmznames[:] + if cyclonemap: + vcinfo[:] = cinfo[:] if forecastds > 0: - vt[:, :] = nmtime[:, :] - vmhs[:, :, :] = nmhs[:, :, :] - vmtm[:, :, :] = nmtm[:, :, :] - vmtp[:, :, :] = nmtp[:, :, :] - vmdm[:, :, :] = nmdm[:, :, :] - vmdp[:, :, :] = nmdp[:, :, :] - vmwn[:, :, :] = nmwn[:, :, :] - vbhs[:, :, :] = nbhs[:, :, :] - vbtm[:, :, :] = nbtm[:, :, :] - vbtp[:, :, :] = nbtp[:, :, :] - vbdm[:, :, :] = nbdm[:, :, :] - vbdp[:, :, :] = nbdp[:, :, :] - vbwind[:, :, :] = nbwind[:, :, :] - - if gridinfo != 0: - vpdistcoast[:] = pdistcoast[:] - vpdepth[:] = pdepth[:] - vponi[:] = poni[:] - vocnames[:] = ocnames[:] - vphsmz[:] = phsmz[:] - vhsmznames[:] = hsmznames[:] - if cyclonemap != 0: - vcinfo[:] = cinfo[:] - if forecastds > 0: - vcmap[:, :, :] = nfcmap[:, :, :] - else: - vcmap[:, :] = fcmap[:, :] - - fcst_hr = np.full_like(mtime, np.nan, dtype='float64') - for i in range(mtime.shape[0]): - if mtime.ndim > 1: - for j in range(mtime.shape[1]): - if not np.isnan(mtime[i, j]): - fcst_hr[i, j] = (mtime[i, j] - initime_unix) / 3600 - else: - if not np.isnan(mtime[i]): - fcst_hr[i] = (mtime[i] - initime_unix) / 3600 - - # Add fcst_hr - vfcst_hr = ncfile.createVariable('fcst_hr', np.dtype('float64').char, ('buoypoints', 'time')) - vfcst_hr.units = 'hours' - vfcst_hr[:] = fcst_hr - - ncfile.close() - print(' ') - print(f'Done. Netcdf ok. New file saved: WW3.{model_name}_Buoy{ftag}_{initime}to{fintime}.nc') - - # File deletion section - files_to_delete = [ - os.path.join(output_dir, '*_contents.txt'), - os.path.join(output_dir, '*_id.txt'), - os.path.join(output_dir, '*.spec_tar') - ] - - for pattern in files_to_delete: - for file in glob.glob(pattern): - try: - os.remove(file) - print(f'Deleted file: {file}') - except OSError as e: - print(f'Error deleting file {file}: {e}') - - # Folder deletion section - try: - shutil.rmtree(extracted_folder) - print(f'Deleted folder: {extracted_folder}') - except OSError as e: - print(f'Error deleting folder {extracted_folder}: {e}') - - print('Temporary files and folder deleted.') + vcmap[:, :, :] = nfcmap[:, :, :] + else: + vcmap[:, :] = fcmap[:, :] + + fcst_hr = np.full_like(mtime, np.nan, dtype='float64') + for i in range(mtime.shape[0]): + if mtime.ndim > 1: + for j in range(mtime.shape[1]): + if not np.isnan(mtime[i, j]): + fcst_hr[i, j] = (mtime[i, j] - initime_unix) / 3600 + else: + if not np.isnan(mtime[i]): + fcst_hr[i] = (mtime[i] - initime_unix) / 3600 + + # Add fcst_hr + vfcst_hr = ncfile.createVariable('fcst_hr', 'f8', ('buoypoints', 'time')) + vfcst_hr.units = 'hours' + vfcst_hr[:] = fcst_hr + + ncfile.close() + print(' ') + print(f'Done. Netcdf ok. New file saved: WW3.{model_name}_Buoy{ftag}_{initime}to{fintime}.nc') + + # File deletion section + files_to_delete = [ + os.path.join(output_dir, '*_contents.txt'), + os.path.join(output_dir, '*_id.txt'), + os.path.join(output_dir, '*.spec_tar') + ] + + for pattern in files_to_delete: + for file in glob.glob(pattern): + try: + os.remove(file) + print(f'Deleted file: {file}') + except OSError as e: + print(f'Error deleting file {file}: {e}') + + # Folder deletion section + try: + shutil.rmtree(extracted_folder) + print(f'Deleted folder: {extracted_folder}') + except OSError as e: + print(f'Error deleting folder {extracted_folder}: {e}') + + print('Temporary files and folder deleted.')