ICOS CO2 Reader

pyaerocom/aeroval/fairmode_stats.py

@@ -30,7 +30,7 @@ def _RMSU(mean: float, std: float, spec: str) -> float:
     RV = SPECIES[spec]["RV"]
     alpha = SPECIES[spec]["alpha"]
 
-    in_sqrt = (1 - alpha**2) * (mean**2 + std**2) + alpha**2 * RV**2
+    in_sqrt = (1 - alpha ** 2) * (mean ** 2 + std ** 2) + alpha ** 2 * RV ** 2
 
     return UrRV * np.sqrt(in_sqrt)
 
@@ -44,7 +44,7 @@ def _fairmode_sign(mod_std: float, obs_std: float, R: float) -> float:
 
 def _crms(mod_std: float, obs_std: float, R: float) -> float:
     """Returns the Centered Root Mean Squared Error"""
-    return np.sqrt(mod_std**2 + obs_std**2 - 2 * mod_std * obs_std * R)
+    return np.sqrt(mod_std ** 2 + obs_std ** 2 - 2 * mod_std * obs_std * R)
 
 
 def _mqi(rms: float, rmsu: float, *, beta: float) -> float:

pyaerocom/config.py

@@ -91,6 +91,9 @@ class Config:
     #: MEP name
     MEP_NAME = "MEP"
 
+    #: ICOS name
+    ICOS_NAME = "ICOS"
+
     #: boolean specifying wheter EBAS DB is copied to local cache for faster
     #: access, defaults to True
     EBAS_DB_LOCAL_CACHE = True
@@ -193,7 +196,6 @@ class Config:
     _LUSTRE_CHECK_PATH = "/project/aerocom/aerocom1/"
 
     def __init__(self, config_file=None, try_infer_environment=True):
-
         # Directories
         self._outputdir = None
         self._cache_basedir = None
@@ -278,7 +280,6 @@ def _basedirs_search_db(self):
         return [self.ROOTDIR, self.HOMEDIR]
 
     def _infer_config_from_basedir(self, basedir):
-
         basedir = os.path.normpath(basedir)
         for env_id, chk_sub in self._check_subdirs_cfg.items():
             chkdir = os.path.join(basedir, chk_sub)

pyaerocom/data/paths.ini

@@ -101,6 +101,7 @@ EEA_V2 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/EEA_AQeRep.v2/renamed/
 AIR_NOW = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/MACC_INSITU_AirNow
 MARCO_POLO = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/CHINA_MP_NRT
 MEP = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/MEP/aggregated/
+ICOS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/ICOS/aggregated/
 
 [obsnames]
 #names of the different obs networks
@@ -147,6 +148,7 @@ EEA_V2 = EEAAQeRep.v2
 AIR_NOW = AirNow
 MARCO_POLO = MarcoPolo
 MEP = MEP
+ICOS = ICOS
 
 [parameters]
 #parameters definition
@@ -183,4 +185,5 @@ EEA = 2013
 EARLINET = 2000
 EEA_NRT = 2020
 EEA_V2 = 2016
-MEP = 2013
+MEP = 2013
+ICOS = 2008

pyaerocom/data/variables.ini

@@ -3027,6 +3027,16 @@ maximum = 100000000
 dimensions = time,lat,lon
 comments_and_purpose = Major HTAP output diagnostic: Air Quality and Impact analysis
 
+[vmrco2]
+var_name = vmrco2
+description = CO2 Volume Mixing Ratio
+standard_name = mole_fraction_of_carbon_dioxide_in_air
+var_type = volume mixing ratios
+unit = nmol mol-1
+minimum = 0
+maximum = 100000000
+dimensions = time,lat,lon
+
 [vmrhno3]
 var_name = vmrhno3
 description = HNO3 Volume Mixing Ratio

pyaerocom/plugins/icos/reader.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import logging
+import re
+from collections import defaultdict
+from collections.abc import Iterable
+from functools import cached_property, lru_cache
+from pathlib import Path
+
+import xarray as xr
+
+from pyaerocom import const
+from pyaerocom.plugins.mep.reader import ReadMEP
+
+logger = logging.getLogger(__name__)
+
+
+class ReadICOS(ReadMEP):
+    """Class for reading ICOS (CO2) observations. HARP format so based on MEP reader
+
+    Args:
+        ReadMEP (class): Base class for this reader, based on ReadUngriddedBase
+
+    """
+
+    #: Mask for identifying datafiles
+    _FILEMASK = "icos-co2-*.nc"
+
+    #: Version log of this class (for caching)
+    __version__ = "0.01"
+
+    #: Name of the dataset (OBS_ID)
+    DATA_ID = const.ICOS_NAME
+
+    #: List of all datasets supported by this interface
+    SUPPORTED_DATASETS = [DATA_ID]
+
+    #: There is no global ts_type but it is specified in the data files...
+    TS_TYPE = "variable"
+
+    #: sampling frequencies found in data files
+    TS_TYPES_FILE = {"hour": "hourly", "day": "daily"}
+
+    #: field name of the start time of the measurement (in lower case)
+    START_TIME_NAME = "datetime_start"
+
+    #: filed name of the end time of the measurement (in lower case)
+    END_TIME_NAME = "datetime_stop"
+
+    #: there's no general instrument name in the data
+    INSTRUMENT_NAME = "unknown"
+
+    DATA_PRODUCT = ""
+
+    #: functions used to convert variables that are computed
+    AUX_FUNS = {}
+
+    VAR_MAPPING = {"vmrco2": "CO2_volume_mixing_ratio"}
+
+    # STATION_REGEX = re.compile("icos-co2-nrt-(.*)-.*-.*-.*.nc")
+    STATION_REGEX = re.compile("icos-co2-(.*)-.*-.*.nc")
+
+    DEFAULT_VARS = list(VAR_MAPPING)
+
+    DATASET_NAME = DATA_ID
+
+    PROVIDES_VARIABLES = list(VAR_MAPPING) + list(AUX_FUNS)
+
+    def __init__(self, data_id=None, data_dir=None):
+        if data_dir is None:
+            data_dir = const.OBSLOCS_UNGRIDDED[const.ICOS_NAME]
+
+        super().__init__(data_id=data_id, data_dir=data_dir)
+        self.files = sorted(map(str, self.FOUND_FILES))

tests/plugins/icos/test_reader.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from pyaerocom import const
+from pyaerocom.plugins.mep.reader import ReadMEP
+from pyaerocom.plugins.icos.reader import ReadICOS
+from tests.conftest import lustre_unavail
+
+try:
+    ICOS_PATH = Path(const.OBSLOCS_UNGRIDDED[const.ICOS_NAME])
+except KeyError:
+    pytest.mark.skip(reason=f"ICOS path not initialised due to non existence in CI")
+
+VARS_DEFAULT = {"vmrco2"}
+VARS_PROVIDED = VARS_DEFAULT  # | {} add more if ever needed
+
+STATION_NAMES = ("bir", "gat", "hpb")
+
+
+@lustre_unavail
+@pytest.fixture(scope="module")
+def reader() -> ReadICOS:
+    return ReadICOS(data_dir=str(ICOS_PATH))
+
+
+@lustre_unavail
+@pytest.fixture()
+def station_files(station: str) -> list[Path]:
+    files = sorted(ICOS_PATH.rglob(f"icos-co2-{station}*.nc"))
+    assert files, f"no files for {station}"
+    return files
+
+
+@lustre_unavail
+def test_DATASET_NAME(reader: ReadICOS):
+    assert reader.DATASET_NAME == "ICOS"
+
+
+@lustre_unavail
+def test_DEFAULT_VARS(reader: ReadICOS):
+    assert set(reader.DEFAULT_VARS) >= VARS_DEFAULT
+
+
+@lustre_unavail
+def test_files(reader: ReadICOS):
+    assert reader.files, "no stations files found"
+    assert len(reader.files) >= 1, "found less files than expected"
+
+
+@lustre_unavail
+def test_FOUND_FILES(reader: ReadICOS):
+    assert reader.FOUND_FILES, "no stations files found"
+    assert len(reader.FOUND_FILES) >= 1, "found less files than expected"
+
+
+@lustre_unavail
+@pytest.mark.parametrize("station", STATION_NAMES)
+def test_stations(reader: ReadICOS, station: str):
+    assert reader.stations()[station], f"no {station} station files"
+
+
+@lustre_unavail
+def test_PROVIDES_VARIABLES(reader: ReadICOS):
+    return set(reader.PROVIDES_VARIABLES) >= VARS_PROVIDED
+
+
+@lustre_unavail
+@pytest.mark.parametrize("station", STATION_NAMES)
+def test_read_file(reader: ReadICOS, station_files: list[str]):
+    data = reader.read_file(station_files[-1])
+    assert set(data.contains_vars) == VARS_DEFAULT
+
+
+@lustre_unavail
+def test_read_file_error(reader: ReadICOS):
+    bad_station_file = "not-a-file.nc"
+    with pytest.raises(ValueError) as e:
+        reader.read_file(bad_station_file)
+    assert str(e.value) == f"missing {bad_station_file}"
+
+
+@lustre_unavail
+@pytest.mark.parametrize("station", STATION_NAMES)
+def test_read(reader: ReadICOS, station_files: list[str]):
+    data = reader.read(VARS_PROVIDED, station_files, first_file=0, last_file=1)
+    assert set(data.contains_vars) == VARS_PROVIDED
+
+
+@lustre_unavail
+def test_read_error(reader: ReadICOS):
+    bad_variable_name = "not-a-variable"
+    with pytest.raises(ValueError) as e:
+        reader.read((bad_variable_name,))
+    assert str(e.value) == f"Unsupported variables: {bad_variable_name}"
+
+
+@lustre_unavail
+def test_reader_gives_correct_mep_path(reader: ReadICOS):
+    assert str(ICOS_PATH) == reader.data_dir