From 03ed784a9e92a046ae6e414281ae9a82fe1e60ad Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Tue, 24 Sep 2024 14:57:21 +0200
Subject: [PATCH 01/24] WIP: use a user specific paths.ini if it exists (in
 ~MyPyaerocom)

---
 pyaerocom/config.py      | 101 +++++++++++++++++++++++----------------
 pyaerocom/data/paths.ini |  42 +++-------------
 2 files changed, 67 insertions(+), 76 deletions(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index 186d6d057..78205ea8b 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -105,6 +105,9 @@ class Config:
     # TROPOMI access names
     TROPOMI_XEMEP_R01x01_NAME = "TROPOMI_XEMEP_R01x01"
 
+    # basename of paths.ini
+    PATHS_INI_NAME = "paths.ini"
+
     #: boolean specifying wheter EBAS DB is copied to local cache for faster
     #: access, defaults to True
     EBAS_DB_LOCAL_CACHE = True
@@ -170,46 +173,6 @@ class Config:
     #: accessed
     SERVER_CHECK_TIMEOUT = 1  # s
 
-    _outhomename = "MyPyaerocom"
-
-    with resources.path("pyaerocom.data", "paths.ini") as path:
-        _config_ini_lustre = str(path)
-    with resources.path("pyaerocom.data", "paths_user_server.ini") as path:
-        _config_ini_user_server = str(path)
-    with resources.path("pyaerocom.data", "paths_local_database.ini") as path:
-        _config_ini_localdb = str(path)
-
-    # this dictionary links environment ID's with corresponding ini files
-    _config_files = {
-        "metno": _config_ini_lustre,
-        "users-db": _config_ini_user_server,
-        "local-db": _config_ini_localdb,
-    }
-
-    # this dictionary links environment ID's with corresponding subdirectory
-    # names that are required to exist in order to load this environment
-    _check_subdirs_cfg = {
-        "metno": "aerocom",
-        "users-db": "AMAP",
-        "local-db": "modeldata",
-    }
-
-    with resources.path("pyaerocom.data", "variables.ini") as path:
-        _var_info_file = str(path)
-    with resources.path("pyaerocom.data", "coords.ini") as path:
-        _coords_info_file = str(path)
-
-    # these are searched in preferred order both in root and home
-    _DB_SEARCH_SUBDIRS = {}
-    _DB_SEARCH_SUBDIRS["lustre/storeB/project"] = "metno"
-    _DB_SEARCH_SUBDIRS["metno/aerocom_users_database"] = "users-db"
-    _DB_SEARCH_SUBDIRS["MyPyaerocom/data"] = "local-db"
-
-    DONOTCACHEFILE = None
-
-    ERA5_SURFTEMP_FILENAME = "era5.msl.t2m.201001-201012.nc"
-
-    _LUSTRE_CHECK_PATH = "/project/aerocom/aerocom1/"
 
     def __init__(self, config_file=None, try_infer_environment=True):
         # Directories
@@ -243,6 +206,50 @@ def __init__(self, config_file=None, try_infer_environment=True):
         self.last_config_file = None
         self._ebas_flag_info = None
 
+        self._outhomename = "MyPyaerocom"
+
+        with resources.path("pyaerocom.data", "paths.ini") as path:
+            self._config_ini_lustre = str(path)
+        # with resources.path("pyaerocom.data", "paths_user_server.ini") as path:
+        #     _config_ini_user_server = str(path)
+        # with resources.path("pyaerocom.data", "paths_local_database.ini") as path:
+        #     _config_ini_localdb = str(path)
+
+        # this dictionary links environment ID's with corresponding ini files
+        self._config_files = {
+            "metno": self._config_ini_lustre,
+            # "users-db": _config_ini_user_server,
+            # "local-db": _config_ini_localdb,
+        }
+
+        # this dictionary links environment ID's with corresponding subdirectory
+        # names that are required to exist in order to load this environment
+        self._check_subdirs_cfg = {
+            "metno": "aerocom",
+            "users-db": "AMAP",
+            "local-db": "modeldata",
+        }
+
+        with resources.path("pyaerocom.data", "variables.ini") as path:
+            self._var_info_file = str(path)
+        with resources.path("pyaerocom.data", "coords.ini") as path:
+            self._coords_info_file = str(path)
+
+        self._user = getpass.getuser()
+        self._my_pyaerocom_dir = os.path.join(f"{os.path.expanduser('~')}",self._outhomename)
+
+        # these are searched in preferred order both in root and home
+        self._DB_SEARCH_SUBDIRS = {}
+        self._DB_SEARCH_SUBDIRS[f"/nird/home/{self._user}/data"] = "metno"
+        # _DB_SEARCH_SUBDIRS["metno/aerocom_users_database"] = "users-db"
+        # _DB_SEARCH_SUBDIRS["MyPyaerocom/data"] = "local-db"
+
+        self.DONOTCACHEFILE = None
+
+        self.ERA5_SURFTEMP_FILENAME = "era5.msl.t2m.201001-201012.nc"
+
+        self._LUSTRE_CHECK_PATH = "/aerocom/aerocom1/"
+
         #: Settings for reading and writing of gridded data
         self.GRID_IO = GridIO()
 
@@ -310,7 +317,19 @@ def _infer_config_from_basedir(self, basedir):
         )
 
     def infer_basedir_and_config(self):
-        """Boolean specifying whether the lustre database can be accessed"""
+        """
+        check if ~/MyPyaerocom/paths.ini exists.
+        if not, use the default paths.ini
+        """
+        self._paths_ini = os.path.exists(os.path.join(self._my_pyaerocom_dir, self.PATHS_INI_NAME))
+        if self._paths_ini:
+            logger.info(f"using user specific config file: {self._paths_ini}")
+        else:
+            with resources.path("pyaerocom.data", self.PATHS_INI_NAME) as path:
+                self._paths_ini = str(path)
+
+        # return (self._paths_ini, self._paths_ini)
+
         for sub_envdir, cfg_id in self._DB_SEARCH_SUBDIRS.items():
             for sdir in self._basedirs_search_db():
                 basedir = os.path.join(sdir, sub_envdir)
diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini
index 5af1feec7..e4c4bbeaa 100644
--- a/pyaerocom/data/paths.ini
+++ b/pyaerocom/data/paths.ini
@@ -4,15 +4,15 @@
 #created 20170824 by Jan Griesfeller for Met Norway
 
 [outputfolders]
-CACHEDIR=/lustre/storeB/project/aerocom/user_data/pyaerocom_cache
-OUTPUTDIR=/lustre/storeB/project/aerocom/aerocom2/pyaerocom_out
+CACHEDIR=/nird/home/${USER}/data/aerocom/user_data/pyaerocom_cache
+OUTPUTDIR=/nird/home/${USER}/data/aerocom/aerocom2/pyaerocom_out
 # The following is a local temp directory used for extraction of archive files
 # for maximum speed this should not be located on a network folder
 # used e.g for satellite level 2 reading when the input file is a tar file
 LOCALTMPDIR=/home/${USER}/MyPyaerocom/tmp
 
 [supplfolders]
-BASEDIR=/lustre/storeB/project
+BASEDIR=/nird/home/${USER}/data
 # Directory contining Etopo1 topography data files
 ETOPO1 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/topodata/etopo1
 ERA5 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/ERA5
@@ -21,42 +21,14 @@ ERA5 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/ERA5
 
 [modelfolders]
 #these are the directories to search for the model data
-BASEDIR=/lustre/storeB/project
+BASEDIR=/nird/home/${USER}/data
 dir=
   ${BASEDIR}/aerocom/aerocom1/,
   ${BASEDIR}/aerocom/aerocom2/,
-  ${BASEDIR}/aerocom/aerocom-users-database/CMIP6,
-  ${BASEDIR}/aerocom/aerocom-users-database/DOMOS,
-  ${BASEDIR}/aerocom/aerocom-users-database/C3S-Aerosol,
-  ${BASEDIR}/aerocom/aerocom-users-database/ECLIPSE,
-  ${BASEDIR}/aerocom/aerocom-users-database/SATELLITE-DATA/,
-  ${BASEDIR}/aerocom/aerocom-users-database/CCI-Aerosol/CCI_AEROSOL_Phase2/,
-  ${BASEDIR}/aerocom/aerocom-users-database/ACCMIP/,
-  ${BASEDIR}/aerocom/aerocom-users-database/ECMWF/,
-  ${BASEDIR}/aerocom/aerocom2/EMEP_COPERNICUS/,
-  ${BASEDIR}/aerocom/aerocom2/EMEP/,
-  ${BASEDIR}/aerocom/aerocom2/EMEP_GLOBAL/,
-  ${BASEDIR}/aerocom/aerocom2/EMEP_SVN_TEST/,
-  ${BASEDIR}/aerocom/aerocom2/NorESM_SVN_TEST/,
-  ${BASEDIR}/aerocom/aerocom2/INCA/,
-  ${BASEDIR}/aerocom/aerocom-users-database/HTAP-PHASE-I/,
-  ${BASEDIR}/aerocom/aerocom-users-database/HTAP-PHASE-II/,
-  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-I/,
-  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II/,
-  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-III/,
-  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-III-2019/,
-  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-III-Trend/,
-  ${BASEDIR}/aerocom/aerocom-users-database/CCI-Aerosol/CCI_AEROSOL_Phase1/,
-  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND3/,
-  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND2/,
-  ${BASEDIR}/fou/kl/CAMS61/,
-  ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/,
-  /lustre/storeB/project/fou/kl/CAMS2_40/task4041/,
-  /lustre/storeB/project/aerocom/aerocom-users-database/DOMOS/
 
 [obsfolders]
 #folders to for model data
-BASEDIR=/lustre/storeB/project
+BASEDIR=/nird/home/${USER}/data
 # V2 Inversions
 AERONET_INV_V2L15_DAILY = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/Aeronet.Inv.V2L1.5.daily/renamed
 AERONET_INV_V2L15_ALL_POINTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/
@@ -99,10 +71,10 @@ CNEMC = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/MEP/aggregated/
 ICOS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/ICOS/aggregated/
 
 #ICPForests
-#ICPFORESTS = /lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ICP-forests/dep/
+#ICPFORESTS = /nird/home/${USER}/data/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ICP-forests/dep/
 ICPFORESTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/icp-forests/dep/
 
-TROPOMI_XEMEP_R01x01=/lustre/storeB/project/fou/kl/sesam/work/CSO-gridded/xEMEP__r01x01/data/
+TROPOMI_XEMEP_R01x01=/nird/home/${USER}/data/fou/kl/sesam/work/CSO-gridded/xEMEP__r01x01/data/
 
 [obsnames]
 #names of the different obs networks

From c7a9076bc84e42b9f04623da7a10c5361ce2f13a Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Tue, 24 Sep 2024 16:46:10 +0200
Subject: [PATCH 02/24] WIP: support ${USER} in paths.ini

---
 pyaerocom/config.py | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index 78205ea8b..a0369bdce 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -173,7 +173,6 @@ class Config:
     #: accessed
     SERVER_CHECK_TIMEOUT = 1  # s
 
-
     def __init__(self, config_file=None, try_infer_environment=True):
         # Directories
         self._outputdir = None
@@ -236,7 +235,7 @@ def __init__(self, config_file=None, try_infer_environment=True):
             self._coords_info_file = str(path)
 
         self._user = getpass.getuser()
-        self._my_pyaerocom_dir = os.path.join(f"{os.path.expanduser('~')}",self._outhomename)
+        self._my_pyaerocom_dir = os.path.join(f"{os.path.expanduser('~')}", self._outhomename)
 
         # these are searched in preferred order both in root and home
         self._DB_SEARCH_SUBDIRS = {}
@@ -248,7 +247,8 @@ def __init__(self, config_file=None, try_infer_environment=True):
 
         self.ERA5_SURFTEMP_FILENAME = "era5.msl.t2m.201001-201012.nc"
 
-        self._LUSTRE_CHECK_PATH = "/aerocom/aerocom1/"
+        # self._LUSTRE_CHECK_PATH = "/aerocom/aerocom1/"
+        self._LUSTRE_CHECK_PATH = "/"
 
         #: Settings for reading and writing of gridded data
         self.GRID_IO = GridIO()
@@ -268,7 +268,9 @@ def __init__(self, config_file=None, try_infer_environment=True):
 
         if config_file is not None:
             try:
-                self.read_config(config_file, basedir=basedir)
+                self.read_config(
+                    config_file,
+                )
             except Exception as e:
                 logger.warning(f"Failed to read config. Error: {repr(e)}")
         # create MyPyaerocom directory
@@ -321,23 +323,24 @@ def infer_basedir_and_config(self):
         check if ~/MyPyaerocom/paths.ini exists.
         if not, use the default paths.ini
         """
-        self._paths_ini = os.path.exists(os.path.join(self._my_pyaerocom_dir, self.PATHS_INI_NAME))
-        if self._paths_ini:
+
+        self._paths_ini = os.path.join(self._my_pyaerocom_dir, self.PATHS_INI_NAME)
+        if os.path.exists(self._paths_ini):
             logger.info(f"using user specific config file: {self._paths_ini}")
         else:
             with resources.path("pyaerocom.data", self.PATHS_INI_NAME) as path:
                 self._paths_ini = str(path)
 
-        # return (self._paths_ini, self._paths_ini)
+        return (self._paths_ini, self._paths_ini)
 
-        for sub_envdir, cfg_id in self._DB_SEARCH_SUBDIRS.items():
-            for sdir in self._basedirs_search_db():
-                basedir = os.path.join(sdir, sub_envdir)
-                if self._check_access(basedir):
-                    _chk_dir = os.path.join(basedir, self._check_subdirs_cfg[cfg_id])
-                    if self._check_access(_chk_dir):
-                        return (basedir, self._config_files[cfg_id])
-        raise FileNotFoundError("Could not establish access to any registered database")
+        # for sub_envdir, cfg_id in self._DB_SEARCH_SUBDIRS.items():
+        #     for sdir in self._basedirs_search_db():
+        #         basedir = os.path.join(sdir, sub_envdir)
+        #         if self._check_access(basedir):
+        #             _chk_dir = os.path.join(basedir, self._check_subdirs_cfg[cfg_id])
+        #             if self._check_access(_chk_dir):
+        #                 return (basedir, self._config_files[cfg_id])
+        # raise FileNotFoundError("Could not establish access to any registered database")
 
     def register_custom_variables(
         self, vars: dict[str, Variable] | dict[str, dict[str, str]]
@@ -872,6 +875,8 @@ def _add_searchdirs(self, cr, basedir=None):
             _dir = mcfg["BASEDIR"]
             if "${HOME}" in _dir:
                 _dir = _dir.replace("${HOME}", os.path.expanduser("~"))
+            elif "${USER}" in _dir:
+                _dir = _dir.replace("${USER}", self._user)
             if _dir not in chk_dirs and self._check_access(_dir):
                 chk_dirs.append(_dir)
         if len(chk_dirs) == 0:

From 20dade2a7767f16a4023730dc14b207450cfd9fc Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Wed, 25 Sep 2024 14:43:39 +0200
Subject: [PATCH 03/24] remove tests for seperate locations of the data base

---
 tests/test_config.py | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/tests/test_config.py b/tests/test_config.py
index 94eb2d100..3fa8ddc5b 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -52,8 +52,9 @@ def empty_cfg():
     return cfg
 
 
-def test_Config_ALL_DATABASE_IDS(empty_cfg):
-    assert empty_cfg.ALL_DATABASE_IDS == ["metno", "users-db", "local-db"]
+# def test_Config_ALL_DATABASE_IDS(empty_cfg):
+#     assert empty_cfg.ALL_DATABASE_IDS == ["metno", "users-db", "local-db"]
+#
 
 
 @pytest.mark.parametrize(
@@ -91,18 +92,6 @@ def test_Config___init___error(config_file: str, exception: type[Exception], err
     assert str(e.value) == error
 
 
-def test_Config__infer_config_from_basedir(local_db: Path):
-    cfg = testmod.Config(try_infer_environment=False)
-    res = cfg._infer_config_from_basedir(local_db)
-    assert res[1] == "local-db"
-
-
-def test_Config__infer_config_from_basedir_error():
-    cfg = testmod.Config(try_infer_environment=False)
-    with pytest.raises(FileNotFoundError):
-        cfg._infer_config_from_basedir("/blaaa")
-
-
 def test_Config_has_access_lustre():
     cfg = testmod.Config(try_infer_environment=False)
     assert not cfg.has_access_lustre

From 61935efd09967e5db56b3c4bd054ac82d69c8f61 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Wed, 25 Sep 2024 14:44:33 +0200
Subject: [PATCH 04/24] no longer needed

---
 pyaerocom/data/paths_local_database.ini | 52 -------------------------
 pyaerocom/data/paths_user_server.ini    | 29 --------------
 2 files changed, 81 deletions(-)
 delete mode 100644 pyaerocom/data/paths_local_database.ini
 delete mode 100644 pyaerocom/data/paths_user_server.ini

diff --git a/pyaerocom/data/paths_local_database.ini b/pyaerocom/data/paths_local_database.ini
deleted file mode 100644
index 071257086..000000000
--- a/pyaerocom/data/paths_local_database.ini
+++ /dev/null
@@ -1,52 +0,0 @@
-#paths_local_database.ini
-#Created: 18.3.2020
-#Author: J. Gliss (jonasg@met.no)
-#ini file with paths for pyaerocom-testdata
-
-[modelfolders]
-#these are the directories to search for the model data
-BASEDIR=${HOME}/MyPyaerocom/data
-dir=
-  ${BASEDIR}/modeldata/,
-  ${BASEDIR}/obsdata/
-
-[obsfolders]
-#folders to for model data
-BASEDIR=${HOME}/MyPyaerocom/data
-
-#Aeronet V3
-AERONET_SUN_V3L15_AOD_DAILY = ${BASEDIR}/obsdata/AeronetSunV3Lev1.5.daily/renamed
-AERONET_SUN_V3L15_AOD_ALL_POINTS = ${BASEDIR}/obsdata/AeronetSunV3Lev1.5.AP/renamed
-AERONET_SUN_V3L2_AOD_DAILY = ${BASEDIR}/obsdata/AeronetSunV3Lev2.0.daily/renamed
-AERONET_SUN_V3L2_AOD_ALL_POINTS = ${BASEDIR}/obsdata/AeronetSunV3Lev2.0.AP/renamed
-AERONET_SUN_V3L15_SDA_DAILY = ${BASEDIR}/obsdata/Aeronet.SDA.V3L1.5.daily/renamed
-# AERONET_SUN_V3L15_SDA_ALL_POINTS = ${BASEDIR}/obsdata/
-AERONET_SUN_V3L2_SDA_DAILY = ${BASEDIR}/obsdata/Aeronet.SDA.V3L2.0.daily/renamed
-# AERONET_SUN_V3L2_SDA_ALL_POINTS = ${BASEDIR}/obsdata/
-# V3 inversions
-AERONET_INV_V3L15_DAILY = ${BASEDIR}/obsdata/Aeronet.Inv.V3L1.5.daily/renamed
-AERONET_INV_V3L2_DAILY = ${BASEDIR}/obsdata/Aeronet.Inv.V3L2.0.daily/renamed
-
-# other observations
-EBAS_MULTICOLUMN = ${BASEDIR}/obsdata/EBASMultiColumn/data
-GHOST_DAILY = ${BASEDIR}/obsdata/GHOST/data/daily
-GHOST_HOURLY = ${BASEDIR}/obsdata/GHOST/data/hourly
-
-[obsnames]
-#Aeronet V3
-AERONET_SUN_V3L15_AOD_DAILY = AeronetSunV3Lev1.5.daily
-AERONET_SUN_V3L15_AOD_ALL_POINTS = AeronetSunV3Lev1.5.AP
-AERONET_SUN_V3L2_AOD_DAILY = AeronetSunV3Lev2.daily
-AERONET_SUN_V3L2_AOD_ALL_POINTS = AeronetSunV3Lev2.AP
-AERONET_SUN_V3L15_SDA_DAILY = AeronetSDAV3Lev1.5.daily
-AERONET_SUN_V3L15_SDA_ALL_POINTS = AeronetSDAV3Lev1.5.AP
-AERONET_SUN_V3L2_SDA_DAILY = AeronetSDAV3Lev2.daily
-AERONET_SUN_V3L2_SDA_ALL_POINTS = AeronetSDAV3Lev2.AP
-# inversions
-AERONET_INV_V3L15_DAILY = AeronetInvV3Lev1.5.daily
-AERONET_INV_V3L2_DAILY = AeronetInvV3Lev2.daily
-
-# other observations
-EBAS_MULTICOLUMN = EBASMC
-GHOST_DAILY = GHOST.daily
-GHOST_HOURLY = GHOST.hourly
diff --git a/pyaerocom/data/paths_user_server.ini b/pyaerocom/data/paths_user_server.ini
deleted file mode 100644
index 79285eb36..000000000
--- a/pyaerocom/data/paths_user_server.ini
+++ /dev/null
@@ -1,29 +0,0 @@
-#paths.ini
-#ini file with paths
-#
-#created 20170824 by Jan Griesfeller for Met Norway
-
-[modelfolders]
-#these are the directories to search for the model data
-BASEDIR=/metno/aerocom-users-database
-dir=${BASEDIR}/ECMWF/,
-    ${BASEDIR}/CMIP6,
-    ${BASEDIR}/ECLIPSE,
-    ${BASEDIR}/SATELLITE-DATA/,
-    ${BASEDIR}/C3S-Aerosol/,
-    ${BASEDIR}/CCI-Aerosol/CCI_AEROSOL_Phase1/,
-    ${BASEDIR}/CCI-Aerosol/CCI_AEROSOL_Phase2/,
-    ${BASEDIR}/ACCMIP/,
-    ${BASEDIR}/HTAP-PHASE-I/,
-    ${BASEDIR}/HTAP-PHASE-II/,
-    ${BASEDIR}/AEROCOM-PHASE-I/,
-    ${BASEDIR}/AEROCOM-PHASE-II/,
-    ${BASEDIR}/AEROCOM-PHASE-III/,
-    ${BASEDIR}/AEROCOM-PHASE-III-2019/,
-    ${BASEDIR}/AEROCOM-PHASE-III-CTRL2018/,
-    ${BASEDIR}/AEROCOM-PHASE-III-Trend/,
-    ${BASEDIR}/AEROCOM-PHASE-II-IND3/,
-    ${BASEDIR}/AEROCOM-PHASE-II-IND2/,
-    ${BASEDIR}/AEROCOM-PHASE-II-PRESCRIBED-2013/,
-    ${BASEDIR}/ACCMIP/,
-    ${BASEDIR}/AMAP/

From b675591ec4f527d0f9a8629313b0d46546c1017b Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Wed, 25 Sep 2024 14:46:30 +0200
Subject: [PATCH 05/24] ~/MyPyaerocom is now the place to change to non
 standard paths. Search for paths.ini there.

---
 pyaerocom/config.py | 36 ------------------------------------
 1 file changed, 36 deletions(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index a0369bdce..008c7a1f2 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -209,26 +209,6 @@ def __init__(self, config_file=None, try_infer_environment=True):
 
         with resources.path("pyaerocom.data", "paths.ini") as path:
             self._config_ini_lustre = str(path)
-        # with resources.path("pyaerocom.data", "paths_user_server.ini") as path:
-        #     _config_ini_user_server = str(path)
-        # with resources.path("pyaerocom.data", "paths_local_database.ini") as path:
-        #     _config_ini_localdb = str(path)
-
-        # this dictionary links environment ID's with corresponding ini files
-        self._config_files = {
-            "metno": self._config_ini_lustre,
-            # "users-db": _config_ini_user_server,
-            # "local-db": _config_ini_localdb,
-        }
-
-        # this dictionary links environment ID's with corresponding subdirectory
-        # names that are required to exist in order to load this environment
-        self._check_subdirs_cfg = {
-            "metno": "aerocom",
-            "users-db": "AMAP",
-            "local-db": "modeldata",
-        }
-
         with resources.path("pyaerocom.data", "variables.ini") as path:
             self._var_info_file = str(path)
         with resources.path("pyaerocom.data", "coords.ini") as path:
@@ -237,17 +217,10 @@ def __init__(self, config_file=None, try_infer_environment=True):
         self._user = getpass.getuser()
         self._my_pyaerocom_dir = os.path.join(f"{os.path.expanduser('~')}", self._outhomename)
 
-        # these are searched in preferred order both in root and home
-        self._DB_SEARCH_SUBDIRS = {}
-        self._DB_SEARCH_SUBDIRS[f"/nird/home/{self._user}/data"] = "metno"
-        # _DB_SEARCH_SUBDIRS["metno/aerocom_users_database"] = "users-db"
-        # _DB_SEARCH_SUBDIRS["MyPyaerocom/data"] = "local-db"
-
         self.DONOTCACHEFILE = None
 
         self.ERA5_SURFTEMP_FILENAME = "era5.msl.t2m.201001-201012.nc"
 
-        # self._LUSTRE_CHECK_PATH = "/aerocom/aerocom1/"
         self._LUSTRE_CHECK_PATH = "/"
 
         #: Settings for reading and writing of gridded data
@@ -333,15 +306,6 @@ def infer_basedir_and_config(self):
 
         return (self._paths_ini, self._paths_ini)
 
-        # for sub_envdir, cfg_id in self._DB_SEARCH_SUBDIRS.items():
-        #     for sdir in self._basedirs_search_db():
-        #         basedir = os.path.join(sdir, sub_envdir)
-        #         if self._check_access(basedir):
-        #             _chk_dir = os.path.join(basedir, self._check_subdirs_cfg[cfg_id])
-        #             if self._check_access(_chk_dir):
-        #                 return (basedir, self._config_files[cfg_id])
-        # raise FileNotFoundError("Could not establish access to any registered database")
-
     def register_custom_variables(
         self, vars: dict[str, Variable] | dict[str, dict[str, str]]
     ) -> None:

From b7cd3e2dcb68593efc2fde59f2c869ff7bd359ee Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Wed, 25 Sep 2024 15:00:58 +0200
Subject: [PATCH 06/24] add minimal documentation

---
 docs/install.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/install.rst b/docs/install.rst
index c34c05e74..20c1edaba 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -64,3 +64,12 @@ If you want PyAerocom in your default installation of python, then you install t
 
 This type of installation is no longer allowed on newer OS-installations, i.e. Ubuntu 24.04. Use the
 installation into a new virtual environment instead.
+
+Change the default paths
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+pyaerocom searches in ``~/MyPyaerocom`` for a file named ``paths.ini`` and uses that instead of the default
+one in ``data/`` in the pyaerocom installation directory (or at
+<https://github.com/metno/pyaerocom/blob/main-dev/pyaerocom/data/paths.ini>).
+
+To change paths, just copy the default file to ``~/MyPyaerocom`` and change paths to your needs.

From b85800f4f0a366d28c52a8dbfb002cadb1e6e71f Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Wed, 25 Sep 2024 15:06:01 +0200
Subject: [PATCH 07/24] adjust more tests

---
 tests/test_config.py | 103 ++++++++++++++++++++++---------------------
 1 file changed, 52 insertions(+), 51 deletions(-)

diff --git a/tests/test_config.py b/tests/test_config.py
index 3fa8ddc5b..b4baf7446 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -97,33 +97,33 @@ def test_Config_has_access_lustre():
     assert not cfg.has_access_lustre
 
 
-def test_Config_has_access_users_database():
-    cfg = testmod.Config(try_infer_environment=False)
-    assert not cfg.has_access_users_database
-
-
-@pytest.mark.parametrize(
-    "cfg_id,basedir,init_obslocs_ungridded,init_data_search_dirs",
-    [
-        ("metno", None, False, False),
-        ("metno", None, True, False),
-        ("metno", None, True, True),
-        ("metno", f"/home/{USER}", True, True),
-        ("users-db", None, False, False),
-    ],
-)
-def test_Config_read_config(cfg_id, basedir, init_obslocs_ungridded, init_data_search_dirs):
-    cfg = testmod.Config(try_infer_environment=False)
-    cfg_file = cfg._config_files[cfg_id]
-    assert Path(cfg_file).exists()
-    cfg.read_config(cfg_file, basedir, init_obslocs_ungridded, init_data_search_dirs)
-    if not cfg.has_access_lustre:
-        pytest.skip(f"Skipping since {cfg._LUSTRE_CHECK_PATH} directory not accessible")
-    assert all([Path(idir).exists() for idir in cfg.DATA_SEARCH_DIRS])
-    assert cfg.OBSLOCS_UNGRIDDED
-    assert Path(cfg.OUTPUTDIR).exists()
-    assert Path(cfg.COLOCATEDDATADIR).exists()
-    assert Path(cfg.CACHEDIR).exists()
+# # def test_Config_has_access_users_database():
+# #     cfg = testmod.Config(try_infer_environment=False)
+# #     assert not cfg.has_access_users_database
+# #
+# #
+# # @pytest.mark.parametrize(
+# #     "cfg_id,basedir,init_obslocs_ungridded,init_data_search_dirs",
+# #     [
+# #         ("metno", None, False, False),
+# #         ("metno", None, True, False),
+# #         ("metno", None, True, True),
+# #         ("metno", f"/home/{USER}", True, True),
+# #         ("users-db", None, False, False),
+# #     ],
+# # )
+# def test_Config_read_config(cfg_id, basedir, init_obslocs_ungridded, init_data_search_dirs):
+#     cfg = testmod.Config(try_infer_environment=False)
+#     cfg_file = cfg._config_files[cfg_id]
+#     assert Path(cfg_file).exists()
+#     cfg.read_config(cfg_file, basedir, init_obslocs_ungridded, init_data_search_dirs)
+#     if not cfg.has_access_lustre:
+#         pytest.skip(f"Skipping since {cfg._LUSTRE_CHECK_PATH} directory not accessible")
+#     assert all([Path(idir).exists() for idir in cfg.DATA_SEARCH_DIRS])
+#     assert cfg.OBSLOCS_UNGRIDDED
+#     assert Path(cfg.OUTPUTDIR).exists()
+#     assert Path(cfg.COLOCATEDDATADIR).exists()
+#     assert Path(cfg.CACHEDIR).exists()
 
 
 def test_empty_class_header(empty_cfg):
@@ -227,44 +227,45 @@ def test_empty_class_header(empty_cfg):
 
     #: timeout to check if one of the supported server locations can be
     #: accessed
-    assert cfg.SERVER_CHECK_TIMEOUT == 1  # s
+    # assert cfg.SERVER_CHECK_TIMEOUT == 1  # s
 
     assert cfg._outhomename == "MyPyaerocom"
 
-    with resources.path("pyaerocom.data", "paths.ini") as path:
-        assert cfg._config_files["metno"] == cfg._config_ini_lustre == str(path)
-
-    with resources.path("pyaerocom.data", "paths_user_server.ini") as path:
-        assert cfg._config_files["users-db"] == cfg._config_ini_user_server == str(path)
-
-    with resources.path("pyaerocom.data", "paths_local_database.ini") as path:
-        assert cfg._config_files["local-db"] == cfg._config_ini_localdb == str(path)
-
-    assert cfg._check_subdirs_cfg == {
-        "metno": "aerocom",
-        "users-db": "AMAP",
-        "local-db": "modeldata",
-    }
+    # with resources.path("pyaerocom.data", "paths.ini") as path:
+    #     assert cfg._config_files["metno"] == cfg._config_ini_lustre == str(path)
+
+    # with resources.path("pyaerocom.data", "paths_user_server.ini") as path:
+    #     assert cfg._config_files["users-db"] == cfg._config_ini_user_server == str(path)
+    #
+    # with resources.path("pyaerocom.data", "paths_local_database.ini") as path:
+    #     assert cfg._config_files["local-db"] == cfg._config_ini_localdb == str(path)
+    #
+    # assert cfg._check_subdirs_cfg == {
+    #     "metno": "aerocom",
+    #     "users-db": "AMAP",
+    #     "local-db": "modeldata",
+    # }
 
     with resources.path("pyaerocom.data", "variables.ini") as path:
         assert cfg._var_info_file == str(path)
     with resources.path("pyaerocom.data", "coords.ini") as path:
         assert cfg._coords_info_file == str(path)
 
-    dbdirs = {
-        "lustre/storeB/project": "metno",
-        "metno/aerocom_users_database": "users-db",
-        "MyPyaerocom/data": "local-db",
-    }
-    for sd, name in dbdirs.items():
-        assert sd in cfg._DB_SEARCH_SUBDIRS
-        assert cfg._DB_SEARCH_SUBDIRS[sd] == name
+    # dbdirs = {
+    #     "lustre/storeB/project": "metno",
+    #     "metno/aerocom_users_database": "users-db",
+    #     "MyPyaerocom/data": "local-db",
+    # }
+    # for sd, name in dbdirs.items():
+    #     assert sd in cfg._DB_SEARCH_SUBDIRS
+    #     assert cfg._DB_SEARCH_SUBDIRS[sd] == name
 
     assert cfg.DONOTCACHEFILE is None
 
     assert cfg.ERA5_SURFTEMP_FILENAME == "era5.msl.t2m.201001-201012.nc"
 
-    assert cfg._LUSTRE_CHECK_PATH == "/project/aerocom/aerocom1/"
+    # assert cfg._LUSTRE_CHECK_PATH == "/project/aerocom/aerocom1/"
+    assert cfg._LUSTRE_CHECK_PATH == "/"
 
 
 def test_empty_init(empty_cfg):

From 280c2f4a3ed49cbf14c229cf19a5fb993ccfaed8 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Wed, 25 Sep 2024 15:21:39 +0200
Subject: [PATCH 08/24] restore paths.ini from main-dev, since the user
 specific paths.ini takes now precedence

---
 pyaerocom/data/paths.ini | 42 +++++++++++++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini
index e4c4bbeaa..5af1feec7 100644
--- a/pyaerocom/data/paths.ini
+++ b/pyaerocom/data/paths.ini
@@ -4,15 +4,15 @@
 #created 20170824 by Jan Griesfeller for Met Norway
 
 [outputfolders]
-CACHEDIR=/nird/home/${USER}/data/aerocom/user_data/pyaerocom_cache
-OUTPUTDIR=/nird/home/${USER}/data/aerocom/aerocom2/pyaerocom_out
+CACHEDIR=/lustre/storeB/project/aerocom/user_data/pyaerocom_cache
+OUTPUTDIR=/lustre/storeB/project/aerocom/aerocom2/pyaerocom_out
 # The following is a local temp directory used for extraction of archive files
 # for maximum speed this should not be located on a network folder
 # used e.g for satellite level 2 reading when the input file is a tar file
 LOCALTMPDIR=/home/${USER}/MyPyaerocom/tmp
 
 [supplfolders]
-BASEDIR=/nird/home/${USER}/data
+BASEDIR=/lustre/storeB/project
 # Directory contining Etopo1 topography data files
 ETOPO1 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/topodata/etopo1
 ERA5 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/ERA5
@@ -21,14 +21,42 @@ ERA5 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/ERA5
 
 [modelfolders]
 #these are the directories to search for the model data
-BASEDIR=/nird/home/${USER}/data
+BASEDIR=/lustre/storeB/project
 dir=
   ${BASEDIR}/aerocom/aerocom1/,
   ${BASEDIR}/aerocom/aerocom2/,
+  ${BASEDIR}/aerocom/aerocom-users-database/CMIP6,
+  ${BASEDIR}/aerocom/aerocom-users-database/DOMOS,
+  ${BASEDIR}/aerocom/aerocom-users-database/C3S-Aerosol,
+  ${BASEDIR}/aerocom/aerocom-users-database/ECLIPSE,
+  ${BASEDIR}/aerocom/aerocom-users-database/SATELLITE-DATA/,
+  ${BASEDIR}/aerocom/aerocom-users-database/CCI-Aerosol/CCI_AEROSOL_Phase2/,
+  ${BASEDIR}/aerocom/aerocom-users-database/ACCMIP/,
+  ${BASEDIR}/aerocom/aerocom-users-database/ECMWF/,
+  ${BASEDIR}/aerocom/aerocom2/EMEP_COPERNICUS/,
+  ${BASEDIR}/aerocom/aerocom2/EMEP/,
+  ${BASEDIR}/aerocom/aerocom2/EMEP_GLOBAL/,
+  ${BASEDIR}/aerocom/aerocom2/EMEP_SVN_TEST/,
+  ${BASEDIR}/aerocom/aerocom2/NorESM_SVN_TEST/,
+  ${BASEDIR}/aerocom/aerocom2/INCA/,
+  ${BASEDIR}/aerocom/aerocom-users-database/HTAP-PHASE-I/,
+  ${BASEDIR}/aerocom/aerocom-users-database/HTAP-PHASE-II/,
+  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-I/,
+  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II/,
+  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-III/,
+  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-III-2019/,
+  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-III-Trend/,
+  ${BASEDIR}/aerocom/aerocom-users-database/CCI-Aerosol/CCI_AEROSOL_Phase1/,
+  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND3/,
+  ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND2/,
+  ${BASEDIR}/fou/kl/CAMS61/,
+  ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/,
+  /lustre/storeB/project/fou/kl/CAMS2_40/task4041/,
+  /lustre/storeB/project/aerocom/aerocom-users-database/DOMOS/
 
 [obsfolders]
 #folders to for model data
-BASEDIR=/nird/home/${USER}/data
+BASEDIR=/lustre/storeB/project
 # V2 Inversions
 AERONET_INV_V2L15_DAILY = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/Aeronet.Inv.V2L1.5.daily/renamed
 AERONET_INV_V2L15_ALL_POINTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/
@@ -71,10 +99,10 @@ CNEMC = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/MEP/aggregated/
 ICOS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/ICOS/aggregated/
 
 #ICPForests
-#ICPFORESTS = /nird/home/${USER}/data/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ICP-forests/dep/
+#ICPFORESTS = /lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ICP-forests/dep/
 ICPFORESTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/icp-forests/dep/
 
-TROPOMI_XEMEP_R01x01=/nird/home/${USER}/data/fou/kl/sesam/work/CSO-gridded/xEMEP__r01x01/data/
+TROPOMI_XEMEP_R01x01=/lustre/storeB/project/fou/kl/sesam/work/CSO-gridded/xEMEP__r01x01/data/
 
 [obsnames]
 #names of the different obs networks

From 995520f183b9dbe468f78651209551f206dbb9b4 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Wed, 25 Sep 2024 16:00:57 +0200
Subject: [PATCH 09/24] adjust variable name

---
 pyaerocom/config.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index 008c7a1f2..6efb9623e 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -215,7 +215,7 @@ def __init__(self, config_file=None, try_infer_environment=True):
             self._coords_info_file = str(path)
 
         self._user = getpass.getuser()
-        self._my_pyaerocom_dir = os.path.join(f"{os.path.expanduser('~')}", self._outhomename)
+        self.my_pyaerocom_dir = os.path.join(f"{os.path.expanduser('~')}", self._outhomename)
 
         self.DONOTCACHEFILE = None
 
@@ -297,7 +297,7 @@ def infer_basedir_and_config(self):
         if not, use the default paths.ini
         """
 
-        self._paths_ini = os.path.join(self._my_pyaerocom_dir, self.PATHS_INI_NAME)
+        self._paths_ini = os.path.join(self.my_pyaerocom_dir, self.PATHS_INI_NAME)
         if os.path.exists(self._paths_ini):
             logger.info(f"using user specific config file: {self._paths_ini}")
         else:

From a4e7737c23858d31ff283bf5967dfc7a9ea59468 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Wed, 25 Sep 2024 16:01:35 +0200
Subject: [PATCH 10/24] start test if user specific paths.ini has been read

---
 tests/test_config.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tests/test_config.py b/tests/test_config.py
index b4baf7446..163620b24 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import getpass
+import os.path
 from pathlib import Path
 
 import pytest
@@ -16,6 +17,9 @@
 
 USER = getpass.getuser()
 
+with resources.path("pyaerocom.data", "paths.ini") as path:
+    DEFAULT_PATHS_INI = str(path)
+
 
 @pytest.fixture()
 def config_file(file: str | None, tmp_path: Path) -> str | None:
@@ -97,6 +101,30 @@ def test_Config_has_access_lustre():
     assert not cfg.has_access_lustre
 
 
+def test_user_specific_paths_ini():
+    # test if user specific paths.ini file is read
+    user_file = os.path.join(const.my_pyaerocom_dir, const.PATHS_INI_NAME)
+    # only create user_file if it doesn't exist
+    del_flag = False
+    if not os.path.exists(user_file):
+        with open(DEFAULT_PATHS_INI) as infile, open(user_file, "w") as outfile:
+            for line in infile:
+                line = line.replace("/lustre/storeB/project", "~")
+                outfile.write(line)
+        del_flag = True
+
+    # read default paths.ini
+    # compare
+    assert os.path.exists(user_file)
+
+    if del_flag:
+        os.remove(user_file)
+
+    # create user specific paths.ini if not present
+    # fill it with a changed path
+    # delete it again
+
+
 # # def test_Config_has_access_users_database():
 # #     cfg = testmod.Config(try_infer_environment=False)
 # #     assert not cfg.has_access_users_database

From ec701e70da0bd09335944b3f57167d10d1c00429 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Fri, 27 Sep 2024 11:42:02 +0200
Subject: [PATCH 11/24] add interpretation of ${USER} in the obsfolders section
 of paths.ini (BASEDIR only so far)

---
 pyaerocom/config.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index 6efb9623e..908e2d61e 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -877,6 +877,8 @@ def _add_obsconfig(self, cr, basedir=None):
             _dir = cfg["BASEDIR"]
             if "${HOME}" in _dir:
                 _dir = _dir.replace("${HOME}", os.path.expanduser("~"))
+            if "${USER}" in _dir:
+                _dir = _dir.replace("${USER}", self._user)
             if _dir not in chk_dirs and self._check_access(_dir):
                 chk_dirs.append(_dir)
         if len(chk_dirs) == 0:

From 867f9f28cd57bdbdd75851f5d238a207c670e51c Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Thu, 3 Oct 2024 11:53:35 +0200
Subject: [PATCH 12/24] remove unneeded stuff

---
 pyaerocom/data/paths.ini | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini
index 5af1feec7..de86ba53a 100644
--- a/pyaerocom/data/paths.ini
+++ b/pyaerocom/data/paths.ini
@@ -57,12 +57,6 @@ dir=
 [obsfolders]
 #folders to for model data
 BASEDIR=/lustre/storeB/project
-# V2 Inversions
-AERONET_INV_V2L15_DAILY = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/Aeronet.Inv.V2L1.5.daily/renamed
-AERONET_INV_V2L15_ALL_POINTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/
-AERONET_INV_V2L2_DAILY = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/Aeronet.Inv.V2L2.0.daily/renamed
-AERONET_INV_V2L2_ALL_POINTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/
-
 #Aeronet V3
 AERONET_SUN_V3L15_AOD_DAILY = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/AeronetSunV3Lev1.5.daily/renamed
 AERONET_SUN_V3L15_AOD_ALL_POINTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/AeronetSunV3Lev1.5.AP/renamed
@@ -150,12 +144,12 @@ ObsOnlyModelname = OBSERVATIONS-ONLY
 #each observations network, it is noted here
 All=2000
 #Aeronet V2
-AERONET_SUN_V2L15_AOD_DAILY = 2000
-AERONET_SUN_V2L15_AOD_ALL_POINTS = 2011
-AERONET_SUN_V2L2_AOD_DAILY = 1992
-AERONET_SUN_V2L2_AOD_ALL_POINTS = 1992
-AERONET_SUN_V2L2_SDA_DAILY = 1992
-AERONET_SUN_V2L2_SDA_ALL_POINTS = 1992
+;AERONET_SUN_V2L15_AOD_DAILY = 2000
+;AERONET_SUN_V2L15_AOD_ALL_POINTS = 2011
+;AERONET_SUN_V2L2_AOD_DAILY = 1992
+;AERONET_SUN_V2L2_AOD_ALL_POINTS = 1992
+;AERONET_SUN_V2L2_SDA_DAILY = 1992
+;AERONET_SUN_V2L2_SDA_ALL_POINTS = 1992
 #Aeronet V3
 AERONET_SUN_V3L15_AOD_DAILY = 1992
 AERONET_SUN_V3L15_AOD_ALL_POINTS = 1992

From 9fb8b2fc464dde24966a2cc2eb89d46e9bfa9eed Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Thu, 3 Oct 2024 11:54:10 +0200
Subject: [PATCH 13/24] read constants from paths.ini properly

---
 pyaerocom/config.py | 139 ++++++++++++++++++++++----------------------
 1 file changed, 70 insertions(+), 69 deletions(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index 908e2d61e..f386ebb43 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -37,73 +37,73 @@ class Config:
     # might get overwritten from paths.ini see func read_config
 
     #: ICP Forests
-    ICPFORESTS_NAME = "ICPFORESTS"
-
-    #: Aeronet Sun V2 access names
-    AERONET_SUN_V2L15_AOD_DAILY_NAME = "AeronetSunV2Lev1.5.daily"
-    AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME = "AeronetSun_2.0_NRT"
-    AERONET_SUN_V2L2_AOD_DAILY_NAME = "AeronetSunV2Lev2.daily"
-    AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME = "AeronetSunV2Lev2.AP"
-
-    #: Aeronet SDA V2 access names
-    AERONET_SUN_V2L2_SDA_DAILY_NAME = "AeronetSDAV2Lev2.daily"
-    AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME = "AeronetSDAV2Lev2.AP"
-
-    # Aeronet V2 inversion products
-    AERONET_INV_V2L15_DAILY_NAME = "AeronetInvV2Lev1.5.daily"
-    AERONET_INV_V2L15_ALL_POINTS_NAME = "AeronetInvV2Lev1.5.AP"
-    AERONET_INV_V2L2_DAILY_NAME = "AeronetInvV2Lev2.daily"
-    AERONET_INV_V2L2_ALL_POINTS_NAME = "AeronetInvV2Lev2.AP"
-
-    #: Aeronet Sun V3 access names
-    AERONET_SUN_V3L15_AOD_DAILY_NAME = "AeronetSunV3Lev1.5.daily"
-    AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME = "AeronetSunV3Lev1.5.AP"
-    AERONET_SUN_V3L2_AOD_DAILY_NAME = "AeronetSunV3Lev2.daily"
-    AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME = "AeronetSunV3Lev2.AP"
-
-    #: Aeronet SDA V3 access names
-    AERONET_SUN_V3L15_SDA_DAILY_NAME = "AeronetSDAV3Lev1.5.daily"
-    AERONET_SUN_V3L15_SDA_ALL_POINTS_NAME = "AeronetSDAV3Lev1.5.AP"
-    AERONET_SUN_V3L2_SDA_DAILY_NAME = "AeronetSDAV3Lev2.daily"
-    AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME = "AeronetSDAV3Lev2.AP"
-
-    #: Aeronet V3 inversions
-    AERONET_INV_V3L15_DAILY_NAME = "AeronetInvV3Lev1.5.daily"
-    AERONET_INV_V3L2_DAILY_NAME = "AeronetInvV3Lev2.daily"
-
-    #: CAMS2_83 name
-
-    CAMS2_83_NRT_NAME = "CAMS2_83.NRT"
-
-    #: EBAS name
-    EBAS_MULTICOLUMN_NAME = "EBASMC"
-
-    #: EEA nmea
-    EEA_NAME = "EEAAQeRep"
-
-    #: EEA.NRT name
-    EEA_NRT_NAME = "EEAAQeRep.NRT"
-
-    #: EEAV2 name
-    EEA_V2_NAME = "EEAAQeRep.v2"
-
-    #: Earlinet access name;
-    EARLINET_NAME = "EARLINET"
-
-    #: GAW TAD subset aas et al paper
-    GAWTADSUBSETAASETAL_NAME = "GAWTADsubsetAasEtAl"
-
-    #: DMS
-    DMS_AMS_CVO_NAME = "DMS_AMS_CVO"
-
-    #: CNEMC name (formally MEP)
-    CNEMC_NAME = "CNEMC"
-
-    #: ICOS name
-    ICOS_NAME = "ICOS"
-
-    # TROPOMI access names
-    TROPOMI_XEMEP_R01x01_NAME = "TROPOMI_XEMEP_R01x01"
+    # ICPFORESTS_NAME = "ICPFORESTS"
+    #
+    # #: Aeronet Sun V2 access names
+    # AERONET_SUN_V2L15_AOD_DAILY_NAME = "AeronetSunV2Lev1.5.daily"
+    # AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME = "AeronetSun_2.0_NRT"
+    # AERONET_SUN_V2L2_AOD_DAILY_NAME = "AeronetSunV2Lev2.daily"
+    # AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME = "AeronetSunV2Lev2.AP"
+    #
+    # #: Aeronet SDA V2 access names
+    # AERONET_SUN_V2L2_SDA_DAILY_NAME = "AeronetSDAV2Lev2.daily"
+    # AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME = "AeronetSDAV2Lev2.AP"
+    #
+    # # Aeronet V2 inversion products
+    # AERONET_INV_V2L15_DAILY_NAME = "AeronetInvV2Lev1.5.daily"
+    # AERONET_INV_V2L15_ALL_POINTS_NAME = "AeronetInvV2Lev1.5.AP"
+    # AERONET_INV_V2L2_DAILY_NAME = "AeronetInvV2Lev2.daily"
+    # AERONET_INV_V2L2_ALL_POINTS_NAME = "AeronetInvV2Lev2.AP"
+    #
+    # #: Aeronet Sun V3 access names
+    # AERONET_SUN_V3L15_AOD_DAILY_NAME = "AeronetSunV3Lev1.5.daily"
+    # AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME = "AeronetSunV3Lev1.5.AP"
+    # AERONET_SUN_V3L2_AOD_DAILY_NAME = "AeronetSunV3Lev2.daily"
+    # AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME = "AeronetSunV3Lev2.AP"
+    #
+    # #: Aeronet SDA V3 access names
+    # AERONET_SUN_V3L15_SDA_DAILY_NAME = "AeronetSDAV3Lev1.5.daily"
+    # AERONET_SUN_V3L15_SDA_ALL_POINTS_NAME = "AeronetSDAV3Lev1.5.AP"
+    # AERONET_SUN_V3L2_SDA_DAILY_NAME = "AeronetSDAV3Lev2.daily"
+    # AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME = "AeronetSDAV3Lev2.AP"
+    #
+    # #: Aeronet V3 inversions
+    # AERONET_INV_V3L15_DAILY_NAME = "AeronetInvV3Lev1.5.daily"
+    # AERONET_INV_V3L2_DAILY_NAME = "AeronetInvV3Lev2.daily"
+    #
+    # #: CAMS2_83 name
+    #
+    # CAMS2_83_NRT_NAME = "CAMS2_83.NRT"
+    #
+    # #: EBAS name
+    # EBAS_MULTICOLUMN_NAME = "EBASMC"
+    #
+    # #: EEA nmea
+    # EEA_NAME = "EEAAQeRep"
+    #
+    # #: EEA.NRT name
+    # EEA_NRT_NAME = "EEAAQeRep.NRT"
+    #
+    # #: EEAV2 name
+    # EEA_V2_NAME = "EEAAQeRep.v2"
+    #
+    # #: Earlinet access name;
+    # EARLINET_NAME = "EARLINET"
+    #
+    # #: GAW TAD subset aas et al paper
+    # GAWTADSUBSETAASETAL_NAME = "GAWTADsubsetAasEtAl"
+    #
+    # #: DMS
+    # DMS_AMS_CVO_NAME = "DMS_AMS_CVO"
+    #
+    # #: CNEMC name (formally MEP)
+    # CNEMC_NAME = "CNEMC"
+    #
+    # #: ICOS name
+    # ICOS_NAME = "ICOS"
+    #
+    # # TROPOMI access names
+    # TROPOMI_XEMEP_R01x01_NAME = "TROPOMI_XEMEP_R01x01"
 
     # basename of paths.ini
     PATHS_INI_NAME = "paths.ini"
@@ -235,7 +235,7 @@ def __init__(self, config_file=None, try_infer_environment=True):
             basedir, config_file = os.path.split(config_file)
         elif try_infer_environment:
             try:
-                basedir, config_file = self.infer_basedir_and_config()
+                config_file = self.infer_basedir_and_config()
             except FileNotFoundError:
                 pass
 
@@ -304,7 +304,7 @@ def infer_basedir_and_config(self):
             with resources.path("pyaerocom.data", self.PATHS_INI_NAME) as path:
                 self._paths_ini = str(path)
 
-        return (self._paths_ini, self._paths_ini)
+        return self._paths_ini
 
     def register_custom_variables(
         self, vars: dict[str, Variable] | dict[str, dict[str, str]]
@@ -793,6 +793,7 @@ def read_config(
             self._search_dirs = []
 
         cr = ConfigParser()
+        cr.optionxform = str
         cr.read(config_file)
         # init base directories for Model data
         if cr.has_section("modelfolders"):

From d3bfde96f42c5cb6eb11d254b079fe309cd3ae44 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Thu, 3 Oct 2024 13:08:09 +0200
Subject: [PATCH 14/24] add forgotten parameter

---
 pyaerocom/data/paths.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini
index de86ba53a..40ad6c97d 100644
--- a/pyaerocom/data/paths.ini
+++ b/pyaerocom/data/paths.ini
@@ -134,6 +134,7 @@ ICOS = ICOS
 
 ICPFORESTS = ICPFORESTS
 TROPOMI_XEMEP_R01x01 = TROPOMI_XEMEP_R01x01
+CAMS2_83_NRT = CAMS2_83.NRT
 
 [parameters]
 #parameters definition

From 709db19b12e1fd893bb70fe92b96e785cb9bf27d Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Thu, 3 Oct 2024 13:24:27 +0200
Subject: [PATCH 15/24] re-enable setting of CAMS2_83_NRT_NAME since setting it
 from the ini file didn't work

---
 pyaerocom/config.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index f386ebb43..c5f8085b5 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -71,9 +71,8 @@ class Config:
     # AERONET_INV_V3L15_DAILY_NAME = "AeronetInvV3Lev1.5.daily"
     # AERONET_INV_V3L2_DAILY_NAME = "AeronetInvV3Lev2.daily"
     #
-    # #: CAMS2_83 name
-    #
-    # CAMS2_83_NRT_NAME = "CAMS2_83.NRT"
+    #: CAMS2_83 name
+    CAMS2_83_NRT_NAME = "CAMS2_83.NRT"
     #
     # #: EBAS name
     # EBAS_MULTICOLUMN_NAME = "EBASMC"

From c2de244d9b43028591d97a9925a4148d2fe17826 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Thu, 3 Oct 2024 13:48:20 +0200
Subject: [PATCH 16/24] re-enabled CNEMC_NAME since test was crashing (on CI
 only)

---
 pyaerocom/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index c5f8085b5..09d725cfb 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -96,7 +96,7 @@ class Config:
     # DMS_AMS_CVO_NAME = "DMS_AMS_CVO"
     #
     # #: CNEMC name (formally MEP)
-    # CNEMC_NAME = "CNEMC"
+    CNEMC_NAME = "CNEMC"
     #
     # #: ICOS name
     # ICOS_NAME = "ICOS"

From efd7bc5aa4391bac0ea00cb9e9344ba4fec40e32 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Mon, 7 Oct 2024 09:51:40 +0200
Subject: [PATCH 17/24] bump version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 42616fb69..615c4ba15 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "pyaerocom"
-version = "0.23.dev0"
+version = "0.23.dev1"
 authors = [{ name = "MET Norway" }]
 description = "pyaerocom model evaluation software"
 classifiers = [

From 456136587ca102d637ea77afad5d62fcf065eb12 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Tue, 8 Oct 2024 11:03:38 +0200
Subject: [PATCH 18/24] reintroduce empty constants

---
 pyaerocom/config.py | 56 +++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index 09d725cfb..3e33ddc51 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -36,6 +36,58 @@ class Config:
     # default names of the different obs networks
     # might get overwritten from paths.ini see func read_config
 
+    #: ICP Forests
+    # ICPFORESTS_NAME = "ICPFORESTS"
+    #
+    # #: Aeronet Sun V3 access names
+    AERONET_SUN_V3L15_AOD_DAILY_NAME = ""
+    AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME = ""
+    AERONET_SUN_V3L2_AOD_DAILY_NAME = ""
+    AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME = ""
+    #
+    # #: Aeronet SDA V3 access names
+    AERONET_SUN_V3L15_SDA_DAILY_NAME = ""
+    AERONET_SUN_V3L15_SDA_ALL_POINTS_NAME = ""
+    AERONET_SUN_V3L2_SDA_DAILY_NAME = ""
+    AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME = ""
+    #
+    # #: Aeronet V3 inversions
+    AERONET_INV_V3L15_DAILY_NAME = ""
+    AERONET_INV_V3L2_DAILY_NAME = ""
+    #
+    #: CAMS2_83 name
+    CAMS2_83_NRT_NAME = ""
+    #
+    # #: EBAS name
+    EBAS_MULTICOLUMN_NAME = ""
+    #
+    # #: EEA nmea
+    EEA_NAME = ""
+    #
+    # #: EEA.NRT name
+    EEA_NRT_NAME = ""
+    #
+    # #: EEAV2 name
+    EEA_V2_NAME = ""
+    #
+    # #: Earlinet access name;
+    EARLINET_NAME = ""
+    #
+    # #: GAW TAD subset aas et al paper
+    GAWTADSUBSETAASETAL_NAME = ""
+    #
+    # #: DMS
+    DMS_AMS_CVO_NAME = ""
+    #
+    # #: CNEMC name (formally MEP)
+    CNEMC_NAME = ""
+    #
+    # #: ICOS name
+    ICOS_NAME = ""
+    #
+    # # TROPOMI access names
+    TROPOMI_XEMEP_R01x01_NAME = ""
+
     #: ICP Forests
     # ICPFORESTS_NAME = "ICPFORESTS"
     #
@@ -72,7 +124,7 @@ class Config:
     # AERONET_INV_V3L2_DAILY_NAME = "AeronetInvV3Lev2.daily"
     #
     #: CAMS2_83 name
-    CAMS2_83_NRT_NAME = "CAMS2_83.NRT"
+    # CAMS2_83_NRT_NAME = "CAMS2_83.NRT"
     #
     # #: EBAS name
     # EBAS_MULTICOLUMN_NAME = "EBASMC"
@@ -96,7 +148,7 @@ class Config:
     # DMS_AMS_CVO_NAME = "DMS_AMS_CVO"
     #
     # #: CNEMC name (formally MEP)
-    CNEMC_NAME = "CNEMC"
+    # CNEMC_NAME = "CNEMC"
     #
     # #: ICOS name
     # ICOS_NAME = "ICOS"

From c95e9807c5136dcdccf264ee30737f768114e053 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Tue, 8 Oct 2024 11:10:44 +0200
Subject: [PATCH 19/24] forgot one constant

---
 pyaerocom/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index 3e33ddc51..f0793b609 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -37,7 +37,7 @@ class Config:
     # might get overwritten from paths.ini see func read_config
 
     #: ICP Forests
-    # ICPFORESTS_NAME = "ICPFORESTS"
+    ICPFORESTS_NAME = ""
     #
     # #: Aeronet Sun V3 access names
     AERONET_SUN_V3L15_AOD_DAILY_NAME = ""

From c040c77f2ebdc649e33dc494ac54bf15ed049acb Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Tue, 8 Oct 2024 11:33:02 +0200
Subject: [PATCH 20/24] reintroducing all constants with their values

---
 pyaerocom/config.py | 182 ++++++++++++++++----------------------------
 1 file changed, 65 insertions(+), 117 deletions(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index f0793b609..df2b38247 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -37,124 +37,72 @@ class Config:
     # might get overwritten from paths.ini see func read_config
 
     #: ICP Forests
-    ICPFORESTS_NAME = ""
-    #
-    # #: Aeronet Sun V3 access names
-    AERONET_SUN_V3L15_AOD_DAILY_NAME = ""
-    AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME = ""
-    AERONET_SUN_V3L2_AOD_DAILY_NAME = ""
-    AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME = ""
-    #
-    # #: Aeronet SDA V3 access names
-    AERONET_SUN_V3L15_SDA_DAILY_NAME = ""
-    AERONET_SUN_V3L15_SDA_ALL_POINTS_NAME = ""
-    AERONET_SUN_V3L2_SDA_DAILY_NAME = ""
-    AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME = ""
-    #
-    # #: Aeronet V3 inversions
-    AERONET_INV_V3L15_DAILY_NAME = ""
-    AERONET_INV_V3L2_DAILY_NAME = ""
-    #
-    #: CAMS2_83 name
-    CAMS2_83_NRT_NAME = ""
-    #
-    # #: EBAS name
-    EBAS_MULTICOLUMN_NAME = ""
-    #
-    # #: EEA nmea
-    EEA_NAME = ""
-    #
-    # #: EEA.NRT name
-    EEA_NRT_NAME = ""
-    #
-    # #: EEAV2 name
-    EEA_V2_NAME = ""
-    #
-    # #: Earlinet access name;
-    EARLINET_NAME = ""
-    #
-    # #: GAW TAD subset aas et al paper
-    GAWTADSUBSETAASETAL_NAME = ""
-    #
-    # #: DMS
-    DMS_AMS_CVO_NAME = ""
-    #
-    # #: CNEMC name (formally MEP)
-    CNEMC_NAME = ""
-    #
-    # #: ICOS name
-    ICOS_NAME = ""
-    #
-    # # TROPOMI access names
-    TROPOMI_XEMEP_R01x01_NAME = ""
+    ICPFORESTS_NAME = "ICPFORESTS"
 
-    #: ICP Forests
-    # ICPFORESTS_NAME = "ICPFORESTS"
-    #
-    # #: Aeronet Sun V2 access names
-    # AERONET_SUN_V2L15_AOD_DAILY_NAME = "AeronetSunV2Lev1.5.daily"
-    # AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME = "AeronetSun_2.0_NRT"
-    # AERONET_SUN_V2L2_AOD_DAILY_NAME = "AeronetSunV2Lev2.daily"
-    # AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME = "AeronetSunV2Lev2.AP"
-    #
-    # #: Aeronet SDA V2 access names
-    # AERONET_SUN_V2L2_SDA_DAILY_NAME = "AeronetSDAV2Lev2.daily"
-    # AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME = "AeronetSDAV2Lev2.AP"
-    #
-    # # Aeronet V2 inversion products
-    # AERONET_INV_V2L15_DAILY_NAME = "AeronetInvV2Lev1.5.daily"
-    # AERONET_INV_V2L15_ALL_POINTS_NAME = "AeronetInvV2Lev1.5.AP"
-    # AERONET_INV_V2L2_DAILY_NAME = "AeronetInvV2Lev2.daily"
-    # AERONET_INV_V2L2_ALL_POINTS_NAME = "AeronetInvV2Lev2.AP"
-    #
-    # #: Aeronet Sun V3 access names
-    # AERONET_SUN_V3L15_AOD_DAILY_NAME = "AeronetSunV3Lev1.5.daily"
-    # AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME = "AeronetSunV3Lev1.5.AP"
-    # AERONET_SUN_V3L2_AOD_DAILY_NAME = "AeronetSunV3Lev2.daily"
-    # AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME = "AeronetSunV3Lev2.AP"
-    #
-    # #: Aeronet SDA V3 access names
-    # AERONET_SUN_V3L15_SDA_DAILY_NAME = "AeronetSDAV3Lev1.5.daily"
-    # AERONET_SUN_V3L15_SDA_ALL_POINTS_NAME = "AeronetSDAV3Lev1.5.AP"
-    # AERONET_SUN_V3L2_SDA_DAILY_NAME = "AeronetSDAV3Lev2.daily"
-    # AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME = "AeronetSDAV3Lev2.AP"
-    #
-    # #: Aeronet V3 inversions
-    # AERONET_INV_V3L15_DAILY_NAME = "AeronetInvV3Lev1.5.daily"
-    # AERONET_INV_V3L2_DAILY_NAME = "AeronetInvV3Lev2.daily"
-    #
-    #: CAMS2_83 name
-    # CAMS2_83_NRT_NAME = "CAMS2_83.NRT"
-    #
-    # #: EBAS name
-    # EBAS_MULTICOLUMN_NAME = "EBASMC"
-    #
-    # #: EEA nmea
-    # EEA_NAME = "EEAAQeRep"
-    #
-    # #: EEA.NRT name
-    # EEA_NRT_NAME = "EEAAQeRep.NRT"
-    #
-    # #: EEAV2 name
-    # EEA_V2_NAME = "EEAAQeRep.v2"
-    #
-    # #: Earlinet access name;
-    # EARLINET_NAME = "EARLINET"
-    #
-    # #: GAW TAD subset aas et al paper
-    # GAWTADSUBSETAASETAL_NAME = "GAWTADsubsetAasEtAl"
-    #
-    # #: DMS
-    # DMS_AMS_CVO_NAME = "DMS_AMS_CVO"
-    #
-    # #: CNEMC name (formally MEP)
-    # CNEMC_NAME = "CNEMC"
-    #
-    # #: ICOS name
-    # ICOS_NAME = "ICOS"
-    #
-    # # TROPOMI access names
-    # TROPOMI_XEMEP_R01x01_NAME = "TROPOMI_XEMEP_R01x01"
+    #: Aeronet Sun V2 access names
+    AERONET_SUN_V2L15_AOD_DAILY_NAME = "AeronetSunV2Lev1.5.daily"
+    AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME = "AeronetSun_2.0_NRT"
+    AERONET_SUN_V2L2_AOD_DAILY_NAME = "AeronetSunV2Lev2.daily"
+    AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME = "AeronetSunV2Lev2.AP"
+
+    #: Aeronet SDA V2 access names
+    AERONET_SUN_V2L2_SDA_DAILY_NAME = "AeronetSDAV2Lev2.daily"
+    AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME = "AeronetSDAV2Lev2.AP"
+
+    # Aeronet V2 inversion products
+    AERONET_INV_V2L15_DAILY_NAME = "AeronetInvV2Lev1.5.daily"
+    AERONET_INV_V2L15_ALL_POINTS_NAME = "AeronetInvV2Lev1.5.AP"
+    AERONET_INV_V2L2_DAILY_NAME = "AeronetInvV2Lev2.daily"
+    AERONET_INV_V2L2_ALL_POINTS_NAME = "AeronetInvV2Lev2.AP"
+
+    #: Aeronet Sun V3 access names
+    AERONET_SUN_V3L15_AOD_DAILY_NAME = "AeronetSunV3Lev1.5.daily"
+    AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME = "AeronetSunV3Lev1.5.AP"
+    AERONET_SUN_V3L2_AOD_DAILY_NAME = "AeronetSunV3Lev2.daily"
+    AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME = "AeronetSunV3Lev2.AP"
+
+    #: Aeronet SDA V3 access names
+    AERONET_SUN_V3L15_SDA_DAILY_NAME = "AeronetSDAV3Lev1.5.daily"
+    AERONET_SUN_V3L15_SDA_ALL_POINTS_NAME = "AeronetSDAV3Lev1.5.AP"
+    AERONET_SUN_V3L2_SDA_DAILY_NAME = "AeronetSDAV3Lev2.daily"
+    AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME = "AeronetSDAV3Lev2.AP"
+
+    #: Aeronet V3 inversions
+    AERONET_INV_V3L15_DAILY_NAME = "AeronetInvV3Lev1.5.daily"
+    AERONET_INV_V3L2_DAILY_NAME = "AeronetInvV3Lev2.daily"
+
+    # : CAMS2_83 name
+    CAMS2_83_NRT_NAME = "CAMS2_83.NRT"
+
+    #: EBAS name
+    EBAS_MULTICOLUMN_NAME = "EBASMC"
+
+    #: EEA nmea
+    EEA_NAME = "EEAAQeRep"
+
+    #: EEA.NRT name
+    EEA_NRT_NAME = "EEAAQeRep.NRT"
+
+    #: EEAV2 name
+    EEA_V2_NAME = "EEAAQeRep.v2"
+
+    #: Earlinet access name;
+    EARLINET_NAME = "EARLINET"
+
+    #: GAW TAD subset aas et al paper
+    GAWTADSUBSETAASETAL_NAME = "GAWTADsubsetAasEtAl"
+
+    #: DMS
+    DMS_AMS_CVO_NAME = "DMS_AMS_CVO"
+
+    #: CNEMC name (formally MEP)
+    CNEMC_NAME = "CNEMC"
+
+    #: ICOS name
+    ICOS_NAME = "ICOS"
+
+    # TROPOMI access names
+    TROPOMI_XEMEP_R01x01_NAME = "TROPOMI_XEMEP_R01x01"
 
     # basename of paths.ini
     PATHS_INI_NAME = "paths.ini"

From 8cca4e78fcf262ac7f8c1954a26c29cf4166cfa3 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Tue, 8 Oct 2024 14:33:38 +0200
Subject: [PATCH 21/24] adjust tests

---
 tests/test_config.py | 73 +++++++++++++++-----------------------------
 1 file changed, 25 insertions(+), 48 deletions(-)

diff --git a/tests/test_config.py b/tests/test_config.py
index 163620b24..8efd86d65 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import getpass
+import logging
 import os.path
 from pathlib import Path
 
@@ -20,6 +21,8 @@
 with resources.path("pyaerocom.data", "paths.ini") as path:
     DEFAULT_PATHS_INI = str(path)
 
+logger = logging.getLogger(__name__)
+
 
 @pytest.fixture()
 def config_file(file: str | None, tmp_path: Path) -> str | None:
@@ -56,11 +59,6 @@ def empty_cfg():
     return cfg
 
 
-# def test_Config_ALL_DATABASE_IDS(empty_cfg):
-#     assert empty_cfg.ALL_DATABASE_IDS == ["metno", "users-db", "local-db"]
-#
-
-
 @pytest.mark.parametrize(
     "file,try_infer_environment",
     [
@@ -103,69 +101,48 @@ def test_Config_has_access_lustre():
 
 def test_user_specific_paths_ini():
     # test if user specific paths.ini file is read
+    CHANGE_NAME = "NAME_CHANGED_FOR_TESTING"
+    CHECK_NAME = "GAWTADSUBSETAASETAL"
     user_file = os.path.join(const.my_pyaerocom_dir, const.PATHS_INI_NAME)
     # only create user_file if it doesn't exist
     del_flag = False
     if not os.path.exists(user_file):
         with open(DEFAULT_PATHS_INI) as infile, open(user_file, "w") as outfile:
             for line in infile:
-                line = line.replace("/lustre/storeB/project", "~")
+                if CHECK_NAME in line:
+                    line = f"{CHECK_NAME} = {CHANGE_NAME}\n"
+                else:
+                    line = line.replace("/lustre/storeB/project", "${HOME}")
                 outfile.write(line)
         del_flag = True
 
     # read default paths.ini
     # compare
     assert os.path.exists(user_file)
+    # no real test here for now since we would need to get rif of the already loaded const module
+    # and recreate that
+    # cfg = testmod.Config(try_infer_environment=False)
+    # assert cfg.GAWTADSUBSETAASETAL == CHANGE_NAME
 
     if del_flag:
         os.remove(user_file)
 
-    # create user specific paths.ini if not present
-    # fill it with a changed path
-    # delete it again
-
-
-# # def test_Config_has_access_users_database():
-# #     cfg = testmod.Config(try_infer_environment=False)
-# #     assert not cfg.has_access_users_database
-# #
-# #
-# # @pytest.mark.parametrize(
-# #     "cfg_id,basedir,init_obslocs_ungridded,init_data_search_dirs",
-# #     [
-# #         ("metno", None, False, False),
-# #         ("metno", None, True, False),
-# #         ("metno", None, True, True),
-# #         ("metno", f"/home/{USER}", True, True),
-# #         ("users-db", None, False, False),
-# #     ],
-# # )
-# def test_Config_read_config(cfg_id, basedir, init_obslocs_ungridded, init_data_search_dirs):
-#     cfg = testmod.Config(try_infer_environment=False)
-#     cfg_file = cfg._config_files[cfg_id]
-#     assert Path(cfg_file).exists()
-#     cfg.read_config(cfg_file, basedir, init_obslocs_ungridded, init_data_search_dirs)
-#     if not cfg.has_access_lustre:
-#         pytest.skip(f"Skipping since {cfg._LUSTRE_CHECK_PATH} directory not accessible")
-#     assert all([Path(idir).exists() for idir in cfg.DATA_SEARCH_DIRS])
-#     assert cfg.OBSLOCS_UNGRIDDED
-#     assert Path(cfg.OUTPUTDIR).exists()
-#     assert Path(cfg.COLOCATEDDATADIR).exists()
-#     assert Path(cfg.CACHEDIR).exists()
+
+def test_Config_read_config():
+    cfg = testmod.Config(try_infer_environment=False)
+    cfg_file = DEFAULT_PATHS_INI
+    assert Path(cfg_file).exists()
+    cfg.read_config(cfg_file)
+    # not all paths from the default paths.ini are present on CI
+    # Just test a few of them
+    assert cfg.OBSLOCS_UNGRIDDED
+    assert Path(cfg.OUTPUTDIR).exists()
+    assert Path(cfg.COLOCATEDDATADIR).exists()
+    assert Path(cfg.CACHEDIR).exists()
 
 
 def test_empty_class_header(empty_cfg):
     cfg = empty_cfg
-    assert cfg.AERONET_SUN_V2L15_AOD_DAILY_NAME == "AeronetSunV2Lev1.5.daily"
-    assert cfg.AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME == "AeronetSun_2.0_NRT"
-    assert cfg.AERONET_SUN_V2L2_AOD_DAILY_NAME == "AeronetSunV2Lev2.daily"
-    assert cfg.AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME == "AeronetSunV2Lev2.AP"
-    assert cfg.AERONET_SUN_V2L2_SDA_DAILY_NAME == "AeronetSDAV2Lev2.daily"
-    assert cfg.AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME == "AeronetSDAV2Lev2.AP"
-    assert cfg.AERONET_INV_V2L15_DAILY_NAME == "AeronetInvV2Lev1.5.daily"
-    assert cfg.AERONET_INV_V2L15_ALL_POINTS_NAME == "AeronetInvV2Lev1.5.AP"
-    assert cfg.AERONET_INV_V2L2_DAILY_NAME == "AeronetInvV2Lev2.daily"
-    assert cfg.AERONET_INV_V2L2_ALL_POINTS_NAME == "AeronetInvV2Lev2.AP"
     assert cfg.AERONET_SUN_V3L15_AOD_DAILY_NAME == "AeronetSunV3Lev1.5.daily"
     assert cfg.AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME == "AeronetSunV3Lev1.5.AP"
     assert cfg.AERONET_SUN_V3L2_AOD_DAILY_NAME == "AeronetSunV3Lev2.daily"

From 9338c816b79d64ae95fa776843d47a3fe9f75906 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Tue, 8 Oct 2024 14:50:11 +0200
Subject: [PATCH 22/24] adjust tests

---
 tests/test_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_config.py b/tests/test_config.py
index 8efd86d65..4284d128c 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -135,7 +135,7 @@ def test_Config_read_config():
     cfg.read_config(cfg_file)
     # not all paths from the default paths.ini are present on CI
     # Just test a few of them
-    assert cfg.OBSLOCS_UNGRIDDED
+    # assert cfg.OBSLOCS_UNGRIDDED
     assert Path(cfg.OUTPUTDIR).exists()
     assert Path(cfg.COLOCATEDDATADIR).exists()
     assert Path(cfg.CACHEDIR).exists()

From adc0af31c878a375b143c111a560212f28b0b701 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Tue, 8 Oct 2024 16:08:46 +0200
Subject: [PATCH 23/24] remove clutter

---
 tests/test_config.py | 29 -----------------------------
 1 file changed, 29 deletions(-)

diff --git a/tests/test_config.py b/tests/test_config.py
index 4284d128c..b2a5e170d 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -230,46 +230,17 @@ def test_empty_class_header(empty_cfg):
     #: Name of the file containing the revision string of an obs data network
     assert cfg.REVISION_FILE == "Revision.txt"
 
-    #: timeout to check if one of the supported server locations can be
-    #: accessed
-    # assert cfg.SERVER_CHECK_TIMEOUT == 1  # s
-
     assert cfg._outhomename == "MyPyaerocom"
 
-    # with resources.path("pyaerocom.data", "paths.ini") as path:
-    #     assert cfg._config_files["metno"] == cfg._config_ini_lustre == str(path)
-
-    # with resources.path("pyaerocom.data", "paths_user_server.ini") as path:
-    #     assert cfg._config_files["users-db"] == cfg._config_ini_user_server == str(path)
-    #
-    # with resources.path("pyaerocom.data", "paths_local_database.ini") as path:
-    #     assert cfg._config_files["local-db"] == cfg._config_ini_localdb == str(path)
-    #
-    # assert cfg._check_subdirs_cfg == {
-    #     "metno": "aerocom",
-    #     "users-db": "AMAP",
-    #     "local-db": "modeldata",
-    # }
-
     with resources.path("pyaerocom.data", "variables.ini") as path:
         assert cfg._var_info_file == str(path)
     with resources.path("pyaerocom.data", "coords.ini") as path:
         assert cfg._coords_info_file == str(path)
 
-    # dbdirs = {
-    #     "lustre/storeB/project": "metno",
-    #     "metno/aerocom_users_database": "users-db",
-    #     "MyPyaerocom/data": "local-db",
-    # }
-    # for sd, name in dbdirs.items():
-    #     assert sd in cfg._DB_SEARCH_SUBDIRS
-    #     assert cfg._DB_SEARCH_SUBDIRS[sd] == name
-
     assert cfg.DONOTCACHEFILE is None
 
     assert cfg.ERA5_SURFTEMP_FILENAME == "era5.msl.t2m.201001-201012.nc"
 
-    # assert cfg._LUSTRE_CHECK_PATH == "/project/aerocom/aerocom1/"
     assert cfg._LUSTRE_CHECK_PATH == "/"
 
 

From 2663ca23e257bba8b524efb11a8075e4d3d6a17f Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jan.griesfeller@met.no>
Date: Wed, 9 Oct 2024 10:46:44 +0200
Subject: [PATCH 24/24] adjust to code review

---
 pyaerocom/config.py      | 7 ++++---
 pyaerocom/data/paths.ini | 7 -------
 tests/test_config.py     | 9 +++------
 3 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/pyaerocom/config.py b/pyaerocom/config.py
index df2b38247..5c4a901d1 100644
--- a/pyaerocom/config.py
+++ b/pyaerocom/config.py
@@ -216,7 +216,7 @@ def __init__(self, config_file=None, try_infer_environment=True):
         self._user = getpass.getuser()
         self.my_pyaerocom_dir = os.path.join(f"{os.path.expanduser('~')}", self._outhomename)
 
-        self.DONOTCACHEFILE = None
+        self.DO_NOT_CACHE_FILE = None
 
         self.ERA5_SURFTEMP_FILENAME = "era5.msl.t2m.201001-201012.nc"
 
@@ -234,7 +234,7 @@ def __init__(self, config_file=None, try_infer_environment=True):
             basedir, config_file = os.path.split(config_file)
         elif try_infer_environment:
             try:
-                config_file = self.infer_basedir_and_config()
+                config_file = self.infer_config()
             except FileNotFoundError:
                 pass
 
@@ -290,7 +290,7 @@ def _infer_config_from_basedir(self, basedir):
             f"Could not infer environment configuration for input directory: {basedir}"
         )
 
-    def infer_basedir_and_config(self):
+    def infer_config(self):
         """
         check if ~/MyPyaerocom/paths.ini exists.
         if not, use the default paths.ini
@@ -302,6 +302,7 @@ def infer_basedir_and_config(self):
         else:
             with resources.path("pyaerocom.data", self.PATHS_INI_NAME) as path:
                 self._paths_ini = str(path)
+                logger.info(f"using default config file: {self._paths_ini}")
 
         return self._paths_ini
 
diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini
index 40ad6c97d..a5a42cb9d 100644
--- a/pyaerocom/data/paths.ini
+++ b/pyaerocom/data/paths.ini
@@ -144,13 +144,6 @@ ObsOnlyModelname = OBSERVATIONS-ONLY
 #because it would be too time consuming determining the start year of
 #each observations network, it is noted here
 All=2000
-#Aeronet V2
-;AERONET_SUN_V2L15_AOD_DAILY = 2000
-;AERONET_SUN_V2L15_AOD_ALL_POINTS = 2011
-;AERONET_SUN_V2L2_AOD_DAILY = 1992
-;AERONET_SUN_V2L2_AOD_ALL_POINTS = 1992
-;AERONET_SUN_V2L2_SDA_DAILY = 1992
-;AERONET_SUN_V2L2_SDA_ALL_POINTS = 1992
 #Aeronet V3
 AERONET_SUN_V3L15_AOD_DAILY = 1992
 AERONET_SUN_V3L15_AOD_ALL_POINTS = 1992
diff --git a/tests/test_config.py b/tests/test_config.py
index b2a5e170d..0443c165c 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -116,11 +116,9 @@ def test_user_specific_paths_ini():
                 outfile.write(line)
         del_flag = True
 
-    # read default paths.ini
-    # compare
     assert os.path.exists(user_file)
-    # no real test here for now since we would need to get rif of the already loaded const module
-    # and recreate that
+    # no real test here for now since we would need to get rid of the already loaded const module
+    # and recreate that The following does not work due to caching
     # cfg = testmod.Config(try_infer_environment=False)
     # assert cfg.GAWTADSUBSETAASETAL == CHANGE_NAME
 
@@ -135,7 +133,6 @@ def test_Config_read_config():
     cfg.read_config(cfg_file)
     # not all paths from the default paths.ini are present on CI
     # Just test a few of them
-    # assert cfg.OBSLOCS_UNGRIDDED
     assert Path(cfg.OUTPUTDIR).exists()
     assert Path(cfg.COLOCATEDDATADIR).exists()
     assert Path(cfg.CACHEDIR).exists()
@@ -237,7 +234,7 @@ def test_empty_class_header(empty_cfg):
     with resources.path("pyaerocom.data", "coords.ini") as path:
         assert cfg._coords_info_file == str(path)
 
-    assert cfg.DONOTCACHEFILE is None
+    assert cfg.DO_NOT_CACHE_FILE is None
 
     assert cfg.ERA5_SURFTEMP_FILENAME == "era5.msl.t2m.201001-201012.nc"