SESAM Tropomi readers and data

pyaerocom/config.py

@@ -91,6 +91,9 @@ class Config:
     #: MEP name
     MEP_NAME = "MEP"
 
+    # TROPOMI name
+    TROPOMI_NAME = "TROPOMI"
+
     #: boolean specifying wheter EBAS DB is copied to local cache for faster
     #: access, defaults to True
     EBAS_DB_LOCAL_CACHE = True
@@ -133,7 +136,8 @@ class Config:
     CLIM_RESAMPLE_HOW = "mean"  # median, ...
     # as a function of climatological frequency
     CLIM_MIN_COUNT = dict(
-        daily=30, monthly=5  # at least 30 daily measurements in each month over whole period
+        daily=30,
+        monthly=5,  # at least 30 daily measurements in each month over whole period
     )  # analogue to daily ...
 
     # names for the satellite data sets
@@ -173,7 +177,11 @@ class Config:
 
     # this dictionary links environment ID's with corresponding subdirectory
     # names that are required to exist in order to load this environment
-    _check_subdirs_cfg = {"metno": "aerocom", "users-db": "AMAP", "local-db": "modeldata"}
+    _check_subdirs_cfg = {
+        "metno": "aerocom",
+        "users-db": "AMAP",
+        "local-db": "modeldata",
+    }
 
     with resources.path("pyaerocom.data", "variables.ini") as path:
         _var_info_file = str(path)
@@ -193,7 +201,6 @@ class Config:
     _LUSTRE_CHECK_PATH = "/project/aerocom/aerocom1/"
 
     def __init__(self, config_file=None, try_infer_environment=True):
-
         # Directories
         self._outputdir = None
         self._cache_basedir = None
@@ -227,7 +234,9 @@ def __init__(self, config_file=None, try_infer_environment=True):
 
         if config_file is not None:
             if not os.path.exists(config_file):
-                raise FileNotFoundError(f"input config file does not exist {config_file}")
+                raise FileNotFoundError(
+                    f"input config file does not exist {config_file}"
+                )
             elif not config_file.endswith("ini"):
                 raise ValueError("Need path to an ini file for input config_file")
 
@@ -278,7 +287,6 @@ def _basedirs_search_db(self):
         return [self.ROOTDIR, self.HOMEDIR]
 
     def _infer_config_from_basedir(self, basedir):
-
         basedir = os.path.normpath(basedir)
         for env_id, chk_sub in self._check_subdirs_cfg.items():
             chkdir = os.path.join(basedir, chk_sub)
@@ -405,7 +413,9 @@ def DOWNLOAD_DATADIR(self, val):
             try:
                 os.mkdir(val)
             except Exception:
-                raise OSError(f"Input directory {val} does not exist and can also not be created")
+                raise OSError(
+                    f"Input directory {val} does not exist and can also not be created"
+                )
         self._downloaddatadir = val
 
     @property
@@ -438,7 +448,9 @@ def CACHEDIR(self):
         try:
             return chk_make_subdir(self.cache_basedir, self.user)
         except Exception as e:
-            logger.warning(f"Failed to access CACHEDIR: {repr(e)}\nDeactivating caching")
+            logger.warning(
+                f"Failed to access CACHEDIR: {repr(e)}\nDeactivating caching"
+            )
             self._caching_active = False
 
     @CACHEDIR.setter
@@ -466,7 +478,9 @@ def CACHING(self, val):
     @property
     def VAR_PARAM(self):
         """Deprecated name, please use :attr:`VARS` instead"""
-        logger.warning("Deprecated (but still functional) name VAR_PARAM. Please use VARS")
+        logger.warning(
+            "Deprecated (but still functional) name VAR_PARAM. Please use VARS"
+        )
         return self.VARS
 
     @property
@@ -716,7 +730,11 @@ def reload(self, keep_basedirs=True):
         self.read_config(self.last_config_file, keep_basedirs)
 
     def read_config(
-        self, config_file, basedir=None, init_obslocs_ungridded=False, init_data_search_dirs=False
+        self,
+        config_file,
+        basedir=None,
+        init_obslocs_ungridded=False,
+        init_data_search_dirs=False,
     ):
         """
         Import paths from one of the config ini files

pyaerocom/data/file_conventions.ini

@@ -18,3 +18,10 @@ year_pos=-2
 var_pos=-3
 ts_pos=-4
 data_id_pos=1
+
+[cso]
+file_sep=_ 
+year_pos=-3
+var_pos=None
+ts_pos=None
+data_id_pos=None

pyaerocom/data/paths.ini

@@ -99,9 +99,24 @@ EEA_NRT = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/EEA_AQeRep.NRT/renamed/
 EEA_V2 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/EEA_AQeRep.v2/renamed/
 
 AIR_NOW = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/MACC_INSITU_AirNow
-MARCO_POLO = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/CHINA_MP_NRT
 MEP = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/MEP/aggregated/
 
+# SESAM TROPOMI
+# LB: These are Level 2 data
+#TROPOMI_GLOBAL_NO2_OFFL = ${BASEDIR}/fou/kl/archive/CSO-data/global/S5p/NO2/OFFL
+#TROPOMI_GLOBAL_NO2_RPRO = ${BASEDIR}/fou/kl/archive/CSO-data/global/S5p/NO2/RPRO
+#TROPOMI_XEMEP_NO2_OFFL = ${BASEDIR}/fou/kl/archive/CSO-data/xEMEP/S5p/NO2/OFFL
+#TROPOMI_XEMEP_NO2_RPRO = ${BASEDIR}/fou/kl/archive/CSO-data/xEMEP/S5p/NO2/RPRO
+#TROPOMI_XEMEP_SO2_OFFL = ${BASEDIR}/fou/kl/archive/CSO-data/xEMEP/S5p/SO2/OFFL
+#TROPOMI_XEMEP_SO2_RPRO = ${BASEDIR}/fou/kl/archive/CSO-data/xEMEP/S5p/SO2/RPRO
+#TROPOMI_XEMEP_HCHO_OFFL = ${BASEDIR}/fou/kl/archive/CSO-data/xEMEP/S5p/HCHO/OFFL
+#TROPOMI_XEMEP_HCHO_RPRO = CSO-data/xEMEP/S5p/HCHO/RPRO
+
+TROPOMI_XEMEP_r01x01=${BASEDIR}/fou/kl/archive/archive/CSO-gridded/xEMEP__r01x01__qa08/NO2
+
+
+
+
 [obsnames]
 #names of the different obs networks
 #Aeronet V2
@@ -145,9 +160,12 @@ GHOST_EBAS_MONTHLY = GHOST.EBAS.monthly
 EEA_NRT = EEAAQeRep.NRT
 EEA_V2 = EEAAQeRep.v2
 AIR_NOW = AirNow
-MARCO_POLO = MarcoPolo
 MEP = MEP
 
+
+
+
+
 [parameters]
 #parameters definition
 ObsOnlyModelname = OBSERVATIONS-ONLY
@@ -183,4 +201,14 @@ EEA = 2013
 EARLINET = 2000
 EEA_NRT = 2020
 EEA_V2 = 2016
-MEP = 2013
+MEP = 2013
+
+# TROPOMI
+TROPOMI_GLOBAL_NO2_OFFL = 2018
+TROPOMI_GLOBAL_NO2_RPRO = 2018
+TROPOMI_XEMEP_NO2_OFFL = 2018
+TROPOMI_XEMEP_NO2_RPRO = 2018
+TROPOMI_XEMEP_SO2_OFFL = 2018
+TROPOMI_XEMEP_SO2_RPRO = 2018
+TROPOMI_XEMEP_HCHO_OFFL = 2018
+TROPOMI_XEMEP_HCHO_RPRO = 2018

pyaerocom/io/fileconventions.py

@@ -6,6 +6,14 @@
 from pyaerocom.exceptions import FileConventionError
 from pyaerocom.tstype import TsType
 
+# CSO names to aerocom names
+CSO_VAR_MAP = {
+    "NO2": "vmrno2",
+    "HCHO": "vmrhcho",
+    "SO2": "vmrso2",
+    "CO": "vmrco",
+}
+
 
 class FileConventionRead:
     """Class that represents a file naming convention for reading Aerocom files
@@ -48,7 +56,6 @@ def __init__(
         data_id_pos=None,
         from_file=None,
     ):
-
         self.name = name
         self.file_sep = file_sep
 
@@ -110,7 +117,11 @@ def from_file(self, file):
         """
 
         if basename(file).count("_") >= 4:
-            self.import_default("aerocom3")
+            if "CSO" in basename(file):  # LB: This needs testing
+                self.import_default("cso")
+            else:
+                self.import_default("aerocom3")
+
         elif basename(file).count(".") >= 4:
             self.import_default("aerocom2")
         else:
@@ -120,6 +131,15 @@ def from_file(self, file):
         self.check_validity(file)
         return self
 
+    def from_filepath(
+        self, filepath
+    ):  # LB: for the CSO file convention, some info is in the filepath, so deal with this case separately
+        if "CSO" in basename(filepath):
+            self.import_default("cso")
+
+        self.check_validity_filepath(filepath)
+        return
+
     def check_validity(self, file):
         """Check if filename is valid"""
         info = self.get_info_from_file(file)
@@ -131,6 +151,18 @@ def check_validity(self, file):
         elif not (const.MIN_YEAR <= year <= const.MAX_YEAR):
             raise FileConventionError(f"Invalid year {info['year']} in filename {basename(file)}")
 
+    def check_validity_filepath(self, filepath):
+        info = self.get_info_from_filepath(filepath)
+        year = info["year"]
+        if not TsType.valid(info["ts_type"]):
+            raise FileConventionError(
+                f"Invalid ts_type {info['ts_type']} in filename {basename(filepath)}"
+            )
+        elif not (const.MIN_YEAR <= year <= const.MAX_YEAR):
+            raise FileConventionError(
+                f"Invalid year {info['year']} in filename {basename(filepath)}"
+            )
+
     def _info_from_aerocom3(self, file: str) -> dict:
         """Extract info from filename Aerocom 3 convention
 
@@ -258,10 +290,60 @@ def _info_from_aerocom2(self, file: str) -> dict:
             )
         return info
 
+    def _info_from_cso(self, filepath: str) -> dict:
+        """Extract info from filename CSO convention
+
+        Parameters
+        -----------
+        file : str
+            netcdf file name
+
+        Returns
+        -------
+        dict
+            dictionary containing infos that were extracted from filename
+        """
+        info = self.info_init
+        spl_filepath = splitext(filepath)[0].split("/")  # split filepath on forward slash
+        try:
+            info["year"] = int(spl_filepath[-3])
+        except Exception:
+            raise FileConventionError(
+                f"Failed to extract year information from file {basename(filepath)} "
+                f"using file convention {self.name}"
+            )
+        try:
+            info["var_name"] = CSO_VAR_MAP[spl_filepath[-4]]
+
+        except Exception:
+            raise FileConventionError(
+                f"Failed to extract variable information from file {basename(filepath)} "
+                f"using file convention {self.name}"
+            )
+
+        try:
+            info["ts_type"] = "daily"  # this is a hack for now until we consider other ts_types
+        except Exception:
+            raise FileConventionError(
+                f"Failed to extract ts_type from file {basename(filepath)} "
+                f"using file convention {self.name}"
+            )
+
+        try:
+            info["data_id"] = ".".join(spl_filepath[-6:-4])
+        except Exception:
+            raise FileConventionError(
+                f"Failed to extract name from file {basename(filepath)} "
+                f"using file convention {self.name}"
+            )
+
+        # info["vert_code"] = "space"
+        return info
+
     def get_info_from_file(self, file: str) -> dict:
         """Identify convention from a file
 
-        Currently only two conventions (aerocom2 and aerocom3) exist that are
+        Currently only three conventions (aerocom2, aerocom3, cso) exist that are
         identified by the delimiter used.
 
         Parameters
@@ -295,6 +377,12 @@ def get_info_from_file(self, file: str) -> dict:
             return self._info_from_aerocom3(file)
         if self.name == "aerocom2":
             return self._info_from_aerocom2(file)
+
+        raise FileConventionError(f"Unknown {self.name}")
+
+    def get_info_from_filepath(self, filepath: str) -> dict:
+        if self.name == "cso":  # LB: new file convention for gridded CSO files we will have
+            return self._info_from_cso(filepath)
         raise FileConventionError(f"Unknown {self.name}")
 
     def string_mask(self, data_id, var, year, ts_type, vert_which=None):