From 56cb1b7337b18e2e6e770c685ed7802f37124517 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Fri, 25 Aug 2023 18:40:33 +0000 Subject: [PATCH] need to update file_conventions.py --- pyaerocom/config.py | 33 ++++++++++++++++++++++------- pyaerocom/data/file_conventions.ini | 4 ++++ pyaerocom/io/fileconventions.py | 1 - pyaerocom/io/readgridded.py | 2 +- pyaerocom/plugins/tropomi/reader.py | 12 +++++------ 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/pyaerocom/config.py b/pyaerocom/config.py index e5de58112..99d53c8ce 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -1,4 +1,4 @@ -reimport getpass +import getpass import logging import os from configparser import ConfigParser @@ -136,7 +136,8 @@ class Config: CLIM_RESAMPLE_HOW = "mean" # median, ... # as a function of climatological frequency CLIM_MIN_COUNT = dict( - daily=30, monthly=5 # at least 30 daily measurements in each month over whole period + daily=30, + monthly=5, # at least 30 daily measurements in each month over whole period ) # analogue to daily ... # names for the satellite data sets @@ -176,7 +177,11 @@ class Config: # this dictionary links environment ID's with corresponding subdirectory # names that are required to exist in order to load this environment - _check_subdirs_cfg = {"metno": "aerocom", "users-db": "AMAP", "local-db": "modeldata"} + _check_subdirs_cfg = { + "metno": "aerocom", + "users-db": "AMAP", + "local-db": "modeldata", + } with resources.path("pyaerocom.data", "variables.ini") as path: _var_info_file = str(path) @@ -229,7 +234,9 @@ def __init__(self, config_file=None, try_infer_environment=True): if config_file is not None: if not os.path.exists(config_file): - raise FileNotFoundError(f"input config file does not exist {config_file}") + raise FileNotFoundError( + f"input config file does not exist {config_file}" + ) elif not config_file.endswith("ini"): raise ValueError("Need path to an ini file for input config_file") @@ -406,7 +413,9 @@ def DOWNLOAD_DATADIR(self, val): try: os.mkdir(val) except Exception: - raise OSError(f"Input directory {val} does not exist and can also not be created") + raise OSError( + f"Input directory {val} does not exist and can also not be created" + ) self._downloaddatadir = val @property @@ -439,7 +448,9 @@ def CACHEDIR(self): try: return chk_make_subdir(self.cache_basedir, self.user) except Exception as e: - logger.warning(f"Failed to access CACHEDIR: {repr(e)}\nDeactivating caching") + logger.warning( + f"Failed to access CACHEDIR: {repr(e)}\nDeactivating caching" + ) self._caching_active = False @CACHEDIR.setter @@ -467,7 +478,9 @@ def CACHING(self, val): @property def VAR_PARAM(self): """Deprecated name, please use :attr:`VARS` instead""" - logger.warning("Deprecated (but still functional) name VAR_PARAM. Please use VARS") + logger.warning( + "Deprecated (but still functional) name VAR_PARAM. Please use VARS" + ) return self.VARS @property @@ -717,7 +730,11 @@ def reload(self, keep_basedirs=True): self.read_config(self.last_config_file, keep_basedirs) def read_config( - self, config_file, basedir=None, init_obslocs_ungridded=False, init_data_search_dirs=False + self, + config_file, + basedir=None, + init_obslocs_ungridded=False, + init_data_search_dirs=False, ): """ Import paths from one of the config ini files diff --git a/pyaerocom/data/file_conventions.ini b/pyaerocom/data/file_conventions.ini index 034cb8cc5..4a18fcd20 100644 --- a/pyaerocom/data/file_conventions.ini +++ b/pyaerocom/data/file_conventions.ini @@ -18,3 +18,7 @@ year_pos=-2 var_pos=-3 ts_pos=-4 data_id_pos=1 + +[cso] +file_sep=_ +year_pos=-3 \ No newline at end of file diff --git a/pyaerocom/io/fileconventions.py b/pyaerocom/io/fileconventions.py index 06c8c1aad..10a338b42 100644 --- a/pyaerocom/io/fileconventions.py +++ b/pyaerocom/io/fileconventions.py @@ -48,7 +48,6 @@ def __init__( data_id_pos=None, from_file=None, ): - self.name = name self.file_sep = file_sep diff --git a/pyaerocom/io/readgridded.py b/pyaerocom/io/readgridded.py index 40c2bd1b6..9d12f69e3 100755 --- a/pyaerocom/io/readgridded.py +++ b/pyaerocom/io/readgridded.py @@ -694,12 +694,12 @@ def _update_file_convention(self, files): FileNotFoundError if none of the input files matches a registered convention. """ - breakpoint() for file in files: try: self.file_convention.from_file(os.path.basename(file)) return except Exception: + breakpoint() pass raise FileNotFoundError( diff --git a/pyaerocom/plugins/tropomi/reader.py b/pyaerocom/plugins/tropomi/reader.py index 973685ac2..3c5fce97c 100644 --- a/pyaerocom/plugins/tropomi/reader.py +++ b/pyaerocom/plugins/tropomi/reader.py @@ -30,9 +30,7 @@ "value", ) -TMP_DATA_DIR = ( - "/lustre/storeB/project/fou/kl/sesam/archive/CSO-gridded/xEMEP__r01x01__qa08/NO2/2023/01" -) +TMP_DATA_DIR = "/lustre/storeB/project/fou/kl/sesam/archive/CSO-gridded/xEMEP__r01x01__qa08/NO2/2022/10/" class ReadTropomiL3(ReadGridded): @@ -41,7 +39,7 @@ class ReadTropomiL3(ReadGridded): e.g., Tropospheric vertical column of nitrogen dioxide """ - _FILEMASK = "S5p_*.nc" + _FILEMASK = "CSO_output_*_gridded.nc" __version__ = "0.01" DATA_ID = const.TROPOMI_NAME @@ -49,5 +47,7 @@ class ReadTropomiL3(ReadGridded): def __init__(self, data_id=None, data_dir=None): if data_dir is None: data_dir = TMP_DATA_DIR - # raise Exception("Need a data_dir") - super().__init__(data_dir=data_dir) + breakpoint() + # raise Exception("Need a data_dir") + super().__init__(data_dir=data_dir, file_convention="cso") + breakpoint()