diff --git a/docs/install.rst b/docs/install.rst index c34c05e74..20c1edaba 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -64,3 +64,12 @@ If you want PyAerocom in your default installation of python, then you install t This type of installation is no longer allowed on newer OS-installations, i.e. Ubuntu 24.04. Use the installation into a new virtual environment instead. + +Change the default paths +^^^^^^^^^^^^^^^^^^^^^^^^ + +pyaerocom searches in ``~/MyPyaerocom`` for a file named ``paths.ini`` and uses that instead of the default +one in ``data/`` in the pyaerocom installation directory (or at +). + +To change paths, just copy the default file to ``~/MyPyaerocom`` and change paths to your needs. diff --git a/pyaerocom/config.py b/pyaerocom/config.py index 186d6d057..5c4a901d1 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -71,8 +71,7 @@ class Config: AERONET_INV_V3L15_DAILY_NAME = "AeronetInvV3Lev1.5.daily" AERONET_INV_V3L2_DAILY_NAME = "AeronetInvV3Lev2.daily" - #: CAMS2_83 name - + # : CAMS2_83 name CAMS2_83_NRT_NAME = "CAMS2_83.NRT" #: EBAS name @@ -105,6 +104,9 @@ class Config: # TROPOMI access names TROPOMI_XEMEP_R01x01_NAME = "TROPOMI_XEMEP_R01x01" + # basename of paths.ini + PATHS_INI_NAME = "paths.ini" + #: boolean specifying wheter EBAS DB is copied to local cache for faster #: access, defaults to True EBAS_DB_LOCAL_CACHE = True @@ -170,47 +172,6 @@ class Config: #: accessed SERVER_CHECK_TIMEOUT = 1 # s - _outhomename = "MyPyaerocom" - - with resources.path("pyaerocom.data", "paths.ini") as path: - _config_ini_lustre = str(path) - with resources.path("pyaerocom.data", "paths_user_server.ini") as path: - _config_ini_user_server = str(path) - with resources.path("pyaerocom.data", "paths_local_database.ini") as path: - _config_ini_localdb = str(path) - - # this dictionary links environment ID's with corresponding ini files - _config_files = { - "metno": _config_ini_lustre, - "users-db": _config_ini_user_server, - "local-db": _config_ini_localdb, - } - - # this dictionary links environment ID's with corresponding subdirectory - # names that are required to exist in order to load this environment - _check_subdirs_cfg = { - "metno": "aerocom", - "users-db": "AMAP", - "local-db": "modeldata", - } - - with resources.path("pyaerocom.data", "variables.ini") as path: - _var_info_file = str(path) - with resources.path("pyaerocom.data", "coords.ini") as path: - _coords_info_file = str(path) - - # these are searched in preferred order both in root and home - _DB_SEARCH_SUBDIRS = {} - _DB_SEARCH_SUBDIRS["lustre/storeB/project"] = "metno" - _DB_SEARCH_SUBDIRS["metno/aerocom_users_database"] = "users-db" - _DB_SEARCH_SUBDIRS["MyPyaerocom/data"] = "local-db" - - DONOTCACHEFILE = None - - ERA5_SURFTEMP_FILENAME = "era5.msl.t2m.201001-201012.nc" - - _LUSTRE_CHECK_PATH = "/project/aerocom/aerocom1/" - def __init__(self, config_file=None, try_infer_environment=True): # Directories self._outputdir = None @@ -243,6 +204,24 @@ def __init__(self, config_file=None, try_infer_environment=True): self.last_config_file = None self._ebas_flag_info = None + self._outhomename = "MyPyaerocom" + + with resources.path("pyaerocom.data", "paths.ini") as path: + self._config_ini_lustre = str(path) + with resources.path("pyaerocom.data", "variables.ini") as path: + self._var_info_file = str(path) + with resources.path("pyaerocom.data", "coords.ini") as path: + self._coords_info_file = str(path) + + self._user = getpass.getuser() + self.my_pyaerocom_dir = os.path.join(f"{os.path.expanduser('~')}", self._outhomename) + + self.DO_NOT_CACHE_FILE = None + + self.ERA5_SURFTEMP_FILENAME = "era5.msl.t2m.201001-201012.nc" + + self._LUSTRE_CHECK_PATH = "/" + #: Settings for reading and writing of gridded data self.GRID_IO = GridIO() @@ -255,13 +234,15 @@ def __init__(self, config_file=None, try_infer_environment=True): basedir, config_file = os.path.split(config_file) elif try_infer_environment: try: - basedir, config_file = self.infer_basedir_and_config() + config_file = self.infer_config() except FileNotFoundError: pass if config_file is not None: try: - self.read_config(config_file, basedir=basedir) + self.read_config( + config_file, + ) except Exception as e: logger.warning(f"Failed to read config. Error: {repr(e)}") # create MyPyaerocom directory @@ -309,16 +290,21 @@ def _infer_config_from_basedir(self, basedir): f"Could not infer environment configuration for input directory: {basedir}" ) - def infer_basedir_and_config(self): - """Boolean specifying whether the lustre database can be accessed""" - for sub_envdir, cfg_id in self._DB_SEARCH_SUBDIRS.items(): - for sdir in self._basedirs_search_db(): - basedir = os.path.join(sdir, sub_envdir) - if self._check_access(basedir): - _chk_dir = os.path.join(basedir, self._check_subdirs_cfg[cfg_id]) - if self._check_access(_chk_dir): - return (basedir, self._config_files[cfg_id]) - raise FileNotFoundError("Could not establish access to any registered database") + def infer_config(self): + """ + check if ~/MyPyaerocom/paths.ini exists. + if not, use the default paths.ini + """ + + self._paths_ini = os.path.join(self.my_pyaerocom_dir, self.PATHS_INI_NAME) + if os.path.exists(self._paths_ini): + logger.info(f"using user specific config file: {self._paths_ini}") + else: + with resources.path("pyaerocom.data", self.PATHS_INI_NAME) as path: + self._paths_ini = str(path) + logger.info(f"using default config file: {self._paths_ini}") + + return self._paths_ini def register_custom_variables( self, vars: dict[str, Variable] | dict[str, dict[str, str]] @@ -807,6 +793,7 @@ def read_config( self._search_dirs = [] cr = ConfigParser() + cr.optionxform = str cr.read(config_file) # init base directories for Model data if cr.has_section("modelfolders"): @@ -853,6 +840,8 @@ def _add_searchdirs(self, cr, basedir=None): _dir = mcfg["BASEDIR"] if "${HOME}" in _dir: _dir = _dir.replace("${HOME}", os.path.expanduser("~")) + elif "${USER}" in _dir: + _dir = _dir.replace("${USER}", self._user) if _dir not in chk_dirs and self._check_access(_dir): chk_dirs.append(_dir) if len(chk_dirs) == 0: @@ -889,6 +878,8 @@ def _add_obsconfig(self, cr, basedir=None): _dir = cfg["BASEDIR"] if "${HOME}" in _dir: _dir = _dir.replace("${HOME}", os.path.expanduser("~")) + if "${USER}" in _dir: + _dir = _dir.replace("${USER}", self._user) if _dir not in chk_dirs and self._check_access(_dir): chk_dirs.append(_dir) if len(chk_dirs) == 0: diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini index 5af1feec7..a5a42cb9d 100644 --- a/pyaerocom/data/paths.ini +++ b/pyaerocom/data/paths.ini @@ -57,12 +57,6 @@ dir= [obsfolders] #folders to for model data BASEDIR=/lustre/storeB/project -# V2 Inversions -AERONET_INV_V2L15_DAILY = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/Aeronet.Inv.V2L1.5.daily/renamed -AERONET_INV_V2L15_ALL_POINTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/ -AERONET_INV_V2L2_DAILY = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/Aeronet.Inv.V2L2.0.daily/renamed -AERONET_INV_V2L2_ALL_POINTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/ - #Aeronet V3 AERONET_SUN_V3L15_AOD_DAILY = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/AeronetSunV3Lev1.5.daily/renamed AERONET_SUN_V3L15_AOD_ALL_POINTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/AeronetSunV3Lev1.5.AP/renamed @@ -140,6 +134,7 @@ ICOS = ICOS ICPFORESTS = ICPFORESTS TROPOMI_XEMEP_R01x01 = TROPOMI_XEMEP_R01x01 +CAMS2_83_NRT = CAMS2_83.NRT [parameters] #parameters definition @@ -149,13 +144,6 @@ ObsOnlyModelname = OBSERVATIONS-ONLY #because it would be too time consuming determining the start year of #each observations network, it is noted here All=2000 -#Aeronet V2 -AERONET_SUN_V2L15_AOD_DAILY = 2000 -AERONET_SUN_V2L15_AOD_ALL_POINTS = 2011 -AERONET_SUN_V2L2_AOD_DAILY = 1992 -AERONET_SUN_V2L2_AOD_ALL_POINTS = 1992 -AERONET_SUN_V2L2_SDA_DAILY = 1992 -AERONET_SUN_V2L2_SDA_ALL_POINTS = 1992 #Aeronet V3 AERONET_SUN_V3L15_AOD_DAILY = 1992 AERONET_SUN_V3L15_AOD_ALL_POINTS = 1992 diff --git a/pyaerocom/data/paths_local_database.ini b/pyaerocom/data/paths_local_database.ini deleted file mode 100644 index 071257086..000000000 --- a/pyaerocom/data/paths_local_database.ini +++ /dev/null @@ -1,52 +0,0 @@ -#paths_local_database.ini -#Created: 18.3.2020 -#Author: J. Gliss (jonasg@met.no) -#ini file with paths for pyaerocom-testdata - -[modelfolders] -#these are the directories to search for the model data -BASEDIR=${HOME}/MyPyaerocom/data -dir= - ${BASEDIR}/modeldata/, - ${BASEDIR}/obsdata/ - -[obsfolders] -#folders to for model data -BASEDIR=${HOME}/MyPyaerocom/data - -#Aeronet V3 -AERONET_SUN_V3L15_AOD_DAILY = ${BASEDIR}/obsdata/AeronetSunV3Lev1.5.daily/renamed -AERONET_SUN_V3L15_AOD_ALL_POINTS = ${BASEDIR}/obsdata/AeronetSunV3Lev1.5.AP/renamed -AERONET_SUN_V3L2_AOD_DAILY = ${BASEDIR}/obsdata/AeronetSunV3Lev2.0.daily/renamed -AERONET_SUN_V3L2_AOD_ALL_POINTS = ${BASEDIR}/obsdata/AeronetSunV3Lev2.0.AP/renamed -AERONET_SUN_V3L15_SDA_DAILY = ${BASEDIR}/obsdata/Aeronet.SDA.V3L1.5.daily/renamed -# AERONET_SUN_V3L15_SDA_ALL_POINTS = ${BASEDIR}/obsdata/ -AERONET_SUN_V3L2_SDA_DAILY = ${BASEDIR}/obsdata/Aeronet.SDA.V3L2.0.daily/renamed -# AERONET_SUN_V3L2_SDA_ALL_POINTS = ${BASEDIR}/obsdata/ -# V3 inversions -AERONET_INV_V3L15_DAILY = ${BASEDIR}/obsdata/Aeronet.Inv.V3L1.5.daily/renamed -AERONET_INV_V3L2_DAILY = ${BASEDIR}/obsdata/Aeronet.Inv.V3L2.0.daily/renamed - -# other observations -EBAS_MULTICOLUMN = ${BASEDIR}/obsdata/EBASMultiColumn/data -GHOST_DAILY = ${BASEDIR}/obsdata/GHOST/data/daily -GHOST_HOURLY = ${BASEDIR}/obsdata/GHOST/data/hourly - -[obsnames] -#Aeronet V3 -AERONET_SUN_V3L15_AOD_DAILY = AeronetSunV3Lev1.5.daily -AERONET_SUN_V3L15_AOD_ALL_POINTS = AeronetSunV3Lev1.5.AP -AERONET_SUN_V3L2_AOD_DAILY = AeronetSunV3Lev2.daily -AERONET_SUN_V3L2_AOD_ALL_POINTS = AeronetSunV3Lev2.AP -AERONET_SUN_V3L15_SDA_DAILY = AeronetSDAV3Lev1.5.daily -AERONET_SUN_V3L15_SDA_ALL_POINTS = AeronetSDAV3Lev1.5.AP -AERONET_SUN_V3L2_SDA_DAILY = AeronetSDAV3Lev2.daily -AERONET_SUN_V3L2_SDA_ALL_POINTS = AeronetSDAV3Lev2.AP -# inversions -AERONET_INV_V3L15_DAILY = AeronetInvV3Lev1.5.daily -AERONET_INV_V3L2_DAILY = AeronetInvV3Lev2.daily - -# other observations -EBAS_MULTICOLUMN = EBASMC -GHOST_DAILY = GHOST.daily -GHOST_HOURLY = GHOST.hourly diff --git a/pyaerocom/data/paths_user_server.ini b/pyaerocom/data/paths_user_server.ini deleted file mode 100644 index 79285eb36..000000000 --- a/pyaerocom/data/paths_user_server.ini +++ /dev/null @@ -1,29 +0,0 @@ -#paths.ini -#ini file with paths -# -#created 20170824 by Jan Griesfeller for Met Norway - -[modelfolders] -#these are the directories to search for the model data -BASEDIR=/metno/aerocom-users-database -dir=${BASEDIR}/ECMWF/, - ${BASEDIR}/CMIP6, - ${BASEDIR}/ECLIPSE, - ${BASEDIR}/SATELLITE-DATA/, - ${BASEDIR}/C3S-Aerosol/, - ${BASEDIR}/CCI-Aerosol/CCI_AEROSOL_Phase1/, - ${BASEDIR}/CCI-Aerosol/CCI_AEROSOL_Phase2/, - ${BASEDIR}/ACCMIP/, - ${BASEDIR}/HTAP-PHASE-I/, - ${BASEDIR}/HTAP-PHASE-II/, - ${BASEDIR}/AEROCOM-PHASE-I/, - ${BASEDIR}/AEROCOM-PHASE-II/, - ${BASEDIR}/AEROCOM-PHASE-III/, - ${BASEDIR}/AEROCOM-PHASE-III-2019/, - ${BASEDIR}/AEROCOM-PHASE-III-CTRL2018/, - ${BASEDIR}/AEROCOM-PHASE-III-Trend/, - ${BASEDIR}/AEROCOM-PHASE-II-IND3/, - ${BASEDIR}/AEROCOM-PHASE-II-IND2/, - ${BASEDIR}/AEROCOM-PHASE-II-PRESCRIBED-2013/, - ${BASEDIR}/ACCMIP/, - ${BASEDIR}/AMAP/ diff --git a/pyproject.toml b/pyproject.toml index 392d619c3..0606a7909 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi" [project] name = "pyaerocom" -version = "0.23.dev0" +version = "0.23.dev1" authors = [{ name = "MET Norway" }] description = "pyaerocom model evaluation software" classifiers = [ diff --git a/tests/test_config.py b/tests/test_config.py index 94eb2d100..0443c165c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,8 @@ from __future__ import annotations import getpass +import logging +import os.path from pathlib import Path import pytest @@ -16,6 +18,11 @@ USER = getpass.getuser() +with resources.path("pyaerocom.data", "paths.ini") as path: + DEFAULT_PATHS_INI = str(path) + +logger = logging.getLogger(__name__) + @pytest.fixture() def config_file(file: str | None, tmp_path: Path) -> str | None: @@ -52,10 +59,6 @@ def empty_cfg(): return cfg -def test_Config_ALL_DATABASE_IDS(empty_cfg): - assert empty_cfg.ALL_DATABASE_IDS == ["metno", "users-db", "local-db"] - - @pytest.mark.parametrize( "file,try_infer_environment", [ @@ -91,47 +94,45 @@ def test_Config___init___error(config_file: str, exception: type[Exception], err assert str(e.value) == error -def test_Config__infer_config_from_basedir(local_db: Path): - cfg = testmod.Config(try_infer_environment=False) - res = cfg._infer_config_from_basedir(local_db) - assert res[1] == "local-db" - - -def test_Config__infer_config_from_basedir_error(): - cfg = testmod.Config(try_infer_environment=False) - with pytest.raises(FileNotFoundError): - cfg._infer_config_from_basedir("/blaaa") - - def test_Config_has_access_lustre(): cfg = testmod.Config(try_infer_environment=False) assert not cfg.has_access_lustre -def test_Config_has_access_users_database(): +def test_user_specific_paths_ini(): + # test if user specific paths.ini file is read + CHANGE_NAME = "NAME_CHANGED_FOR_TESTING" + CHECK_NAME = "GAWTADSUBSETAASETAL" + user_file = os.path.join(const.my_pyaerocom_dir, const.PATHS_INI_NAME) + # only create user_file if it doesn't exist + del_flag = False + if not os.path.exists(user_file): + with open(DEFAULT_PATHS_INI) as infile, open(user_file, "w") as outfile: + for line in infile: + if CHECK_NAME in line: + line = f"{CHECK_NAME} = {CHANGE_NAME}\n" + else: + line = line.replace("/lustre/storeB/project", "${HOME}") + outfile.write(line) + del_flag = True + + assert os.path.exists(user_file) + # no real test here for now since we would need to get rid of the already loaded const module + # and recreate that The following does not work due to caching + # cfg = testmod.Config(try_infer_environment=False) + # assert cfg.GAWTADSUBSETAASETAL == CHANGE_NAME + + if del_flag: + os.remove(user_file) + + +def test_Config_read_config(): cfg = testmod.Config(try_infer_environment=False) - assert not cfg.has_access_users_database - - -@pytest.mark.parametrize( - "cfg_id,basedir,init_obslocs_ungridded,init_data_search_dirs", - [ - ("metno", None, False, False), - ("metno", None, True, False), - ("metno", None, True, True), - ("metno", f"/home/{USER}", True, True), - ("users-db", None, False, False), - ], -) -def test_Config_read_config(cfg_id, basedir, init_obslocs_ungridded, init_data_search_dirs): - cfg = testmod.Config(try_infer_environment=False) - cfg_file = cfg._config_files[cfg_id] + cfg_file = DEFAULT_PATHS_INI assert Path(cfg_file).exists() - cfg.read_config(cfg_file, basedir, init_obslocs_ungridded, init_data_search_dirs) - if not cfg.has_access_lustre: - pytest.skip(f"Skipping since {cfg._LUSTRE_CHECK_PATH} directory not accessible") - assert all([Path(idir).exists() for idir in cfg.DATA_SEARCH_DIRS]) - assert cfg.OBSLOCS_UNGRIDDED + cfg.read_config(cfg_file) + # not all paths from the default paths.ini are present on CI + # Just test a few of them assert Path(cfg.OUTPUTDIR).exists() assert Path(cfg.COLOCATEDDATADIR).exists() assert Path(cfg.CACHEDIR).exists() @@ -139,16 +140,6 @@ def test_Config_read_config(cfg_id, basedir, init_obslocs_ungridded, init_data_s def test_empty_class_header(empty_cfg): cfg = empty_cfg - assert cfg.AERONET_SUN_V2L15_AOD_DAILY_NAME == "AeronetSunV2Lev1.5.daily" - assert cfg.AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME == "AeronetSun_2.0_NRT" - assert cfg.AERONET_SUN_V2L2_AOD_DAILY_NAME == "AeronetSunV2Lev2.daily" - assert cfg.AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME == "AeronetSunV2Lev2.AP" - assert cfg.AERONET_SUN_V2L2_SDA_DAILY_NAME == "AeronetSDAV2Lev2.daily" - assert cfg.AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME == "AeronetSDAV2Lev2.AP" - assert cfg.AERONET_INV_V2L15_DAILY_NAME == "AeronetInvV2Lev1.5.daily" - assert cfg.AERONET_INV_V2L15_ALL_POINTS_NAME == "AeronetInvV2Lev1.5.AP" - assert cfg.AERONET_INV_V2L2_DAILY_NAME == "AeronetInvV2Lev2.daily" - assert cfg.AERONET_INV_V2L2_ALL_POINTS_NAME == "AeronetInvV2Lev2.AP" assert cfg.AERONET_SUN_V3L15_AOD_DAILY_NAME == "AeronetSunV3Lev1.5.daily" assert cfg.AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME == "AeronetSunV3Lev1.5.AP" assert cfg.AERONET_SUN_V3L2_AOD_DAILY_NAME == "AeronetSunV3Lev2.daily" @@ -236,46 +227,18 @@ def test_empty_class_header(empty_cfg): #: Name of the file containing the revision string of an obs data network assert cfg.REVISION_FILE == "Revision.txt" - #: timeout to check if one of the supported server locations can be - #: accessed - assert cfg.SERVER_CHECK_TIMEOUT == 1 # s - assert cfg._outhomename == "MyPyaerocom" - with resources.path("pyaerocom.data", "paths.ini") as path: - assert cfg._config_files["metno"] == cfg._config_ini_lustre == str(path) - - with resources.path("pyaerocom.data", "paths_user_server.ini") as path: - assert cfg._config_files["users-db"] == cfg._config_ini_user_server == str(path) - - with resources.path("pyaerocom.data", "paths_local_database.ini") as path: - assert cfg._config_files["local-db"] == cfg._config_ini_localdb == str(path) - - assert cfg._check_subdirs_cfg == { - "metno": "aerocom", - "users-db": "AMAP", - "local-db": "modeldata", - } - with resources.path("pyaerocom.data", "variables.ini") as path: assert cfg._var_info_file == str(path) with resources.path("pyaerocom.data", "coords.ini") as path: assert cfg._coords_info_file == str(path) - dbdirs = { - "lustre/storeB/project": "metno", - "metno/aerocom_users_database": "users-db", - "MyPyaerocom/data": "local-db", - } - for sd, name in dbdirs.items(): - assert sd in cfg._DB_SEARCH_SUBDIRS - assert cfg._DB_SEARCH_SUBDIRS[sd] == name - - assert cfg.DONOTCACHEFILE is None + assert cfg.DO_NOT_CACHE_FILE is None assert cfg.ERA5_SURFTEMP_FILENAME == "era5.msl.t2m.201001-201012.nc" - assert cfg._LUSTRE_CHECK_PATH == "/project/aerocom/aerocom1/" + assert cfg._LUSTRE_CHECK_PATH == "/" def test_empty_init(empty_cfg):