From 0e07a83cc2b2cca9ee94cbde17d6ffd93594d6cb Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Wed, 10 Aug 2022 11:58:57 +0000 Subject: [PATCH 01/82] starts ipc forests reader --- pyaerocom/io/ipcforests/metadata.py | 10 ++++++++++ pyaerocom/io/ipcforests/reader.py | 0 2 files changed, 10 insertions(+) create mode 100644 pyaerocom/io/ipcforests/metadata.py create mode 100644 pyaerocom/io/ipcforests/reader.py diff --git a/pyaerocom/io/ipcforests/metadata.py b/pyaerocom/io/ipcforests/metadata.py new file mode 100644 index 000000000..f6b894c13 --- /dev/null +++ b/pyaerocom/io/ipcforests/metadata.py @@ -0,0 +1,10 @@ +class Variables: + def __init__(self) -> None: + pass + + +class MetadataReader: + def __init__(self, dir: str) -> None: + self.dir = dir + + self.add_dir = dir + "" diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py new file mode 100644 index 000000000..e69de29bb From b165d8ba84a7a6e80517f2744451435d9c8ac994 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Thu, 8 Sep 2022 13:00:10 +0000 Subject: [PATCH 02/82] Halfway through making reader --- pyaerocom/config.py | 3 + pyaerocom/data/paths.ini | 3 + pyaerocom/io/ipcforests/__init__.py | 3 + pyaerocom/io/ipcforests/metadata.py | 312 +++++++++++++++++++++++++++- pyaerocom/io/ipcforests/reader.py | 249 ++++++++++++++++++++++ 5 files changed, 569 insertions(+), 1 deletion(-) create mode 100644 pyaerocom/io/ipcforests/__init__.py diff --git a/pyaerocom/config.py b/pyaerocom/config.py index 2e685dfe1..e323d5568 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -88,6 +88,9 @@ class Config: #: DMS DMS_AMS_CVO_NAME = "DMS_AMS_CVO" + #: IPC Forests + IPCFORESTS_NAME = "IPCFOREST" + #: boolean specifying wheter EBAS DB is copied to local cache for faster #: access, defaults to True EBAS_DB_LOCAL_CACHE = True diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini index ef7cb6b6d..9d85e81ea 100644 --- a/pyaerocom/data/paths.ini +++ b/pyaerocom/data/paths.ini @@ -101,6 +101,9 @@ EEA_V2 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/EEA_AQeRep.v2/renamed/ AIR_NOW = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/MACC_INSITU_AirNow MARCO_POLO = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/CHINA_MP_NRT +#IPCForests +IPCFORESTS = /lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ipc-forests/dep/ + [obsnames] #names of the different obs networks #Aeronet V2 diff --git a/pyaerocom/io/ipcforests/__init__.py b/pyaerocom/io/ipcforests/__init__.py new file mode 100644 index 000000000..eea436a37 --- /dev/null +++ b/pyaerocom/io/ipcforests/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/pyaerocom/io/ipcforests/metadata.py b/pyaerocom/io/ipcforests/metadata.py index f6b894c13..7d112d9b5 100644 --- a/pyaerocom/io/ipcforests/metadata.py +++ b/pyaerocom/io/ipcforests/metadata.py @@ -1,10 +1,320 @@ +from datetime import datetime, timedelta +from typing import Tuple + +import pandas as pd +from pandas import date_range + +COUNTRY_CODES = { + 1: "FR", + 2: "BE", + 3: "NL", + 4: "DE", + 5: "IT", + 6: "UK", + 7: "IE", + 8: "DK", + 9: "GR", + 10: "PT", + 11: "ES", + 12: "LU", + 13: "SE", + 14: "AT", + 15: "FI", + 50: "CH", + 51: "HU", + 52: "RO", + 53: "PL", + 54: "SK", + 55: "NO", + 56: "LT", + 57: "HR", + 58: "CZ", + 59: "EE", + 60: "SI", + 61: "MD", + 62: "RU", + 63: "BG", + 64: "LV", + 65: "BY", + 66: "CY", + 67: "CS", + 68: "AD", + 95: "cn", + 80: "ME", + 96: "AZ", + 72: "TR", +} + +COUNTRIES = { + 1: "France", + 2: "Belgium", + 3: "Netherlands", + 4: "Germany", + 5: "Italy", + 6: "United Kingdom", + 7: "Ireland", + 8: "Denmark", + 9: "Greece", + 10: "Portugal", + 11: "Spain", + 12: "Luxembourg", + 13: "Sweden", + 14: "Austria", + 15: "Finland", + 50: "Switzerland", + 51: "Hungary", + 52: "Romania", + 53: "Poland", + 54: "Slovak Republic", + 55: "Norway", + 56: "Lithuania", + 57: "Croatia", + 58: "Czech Republic", + 59: "Estonia", + 60: "Slovenia", + 61: "Republic of Moldova", + 62: "Russia", + 63: "Bulgaria", + 64: "Latvia", + 65: "Belarus", + 66: "Cyprus", + 67: "Serbia", + 68: "Andorra", + 95: "Canaries (Spain)", + 80: "Montenegro", + 96: "Azores (Portugal)", + 72: "Türkiye", +} + + class Variables: def __init__(self) -> None: pass +class Station: + def __init__( + self, + country_code: int, + plot_code: int, + sampler_code: int, + lat: str, + lon: str, + alt: int, + partner_code: int, + ts_type: str, + ) -> None: + self.country_code = country_code + self.plot_code = plot_code + self.sampler_code = sampler_code + self.lat = lat + self.lon = lon + self.alt = alt + self.partner_code = partner_code + self.ts_type = ts_type + + self.data: dict[str, list[float]] = {} + self.dtime: dict[str, list[datetime]] = {} + + self.country = COUNTRIES[country_code] + + self.station_name = self.get_station_name(country_code, plot_code, sampler_code) + + self.var_info: dict[str, dict[str, str | list[float]]] = {} + + @staticmethod + def get_station_name( + country_code: int, + plot_code: int, + sampler_code: int, + ) -> str: + return f"{COUNTRY_CODES[country_code]}-{plot_code}-{sampler_code}" + + def _add_species_to_var_info(self, species: str, unit: str) -> None: + self.var_info[species] = dict( + ts_type=self.ts_type, + ts_type_src=self.ts_type, + units=unit, + ) + + def add_measurement(self, species: str, time: datetime, measurement: float, unit: str) -> None: + if species not in self.var_info: + self._add_species_to_var_info(species, unit) + if species not in self.dtime: + self.dtime[species] = [] + self.data[species] = [] + + self.dtime[species].append(time) + self.data[species].append(measurement) + + def get_timeseries(self, species: str) -> pd.Series: + return pd.Series(self.data[species], index=self.dtime[species]) + + +class SurveyYear: + def __init__(self, year: int, start: datetime, stop: datetime, periods: int) -> None: + self.year = year + self.start = start + self.stop = stop + self.periods = periods + + self.daterange = date_range(start, stop, periods) + self.days = (self.stop - self.start).days / self.periods + + self.ts_type = self._get_tstype() + + def get_date(self, period: int) -> datetime: + if period > self.periods or period <= 0: + + raise ValueError(f"The period {period} needs to be in the range 1-{self.periods}") + return self.start + timedelta(days=self.days * (period - 1)) + # return self.daterange[period - 1] + + def _get_tstype(self) -> str: + + days = (self.stop - self.start).days / self.periods + + if self.days >= 26: + return "monthly" + elif self.days >= 6: + return "weekly" + else: + return "daily" + + +class Plot: + def __init__( + self, + country_code: int, + plot_code: int, + sampler_code: int, + lat: str, + lon: str, + alt: int, + partner_code: int, + ) -> None: + self.country_code = country_code + self.plot_code = plot_code + self.sampler_code = sampler_code + self.lat = lat + self.lon = lon + self.alt = alt + self.partner_code = partner_code + + self.periods = {} + self.survey_years: dict[int, SurveyYear] = {} + + def add_survey_year(self, year: int, start: str, stop: str, periods: int) -> None: + start_dt = datetime.strptime(start, "%Y-%m-%d") + stop_dt = datetime.strptime(stop, "%Y-%m-%d") + self.survey_years[year] = SurveyYear(year, start_dt, stop_dt, periods) + + def get_date(self, year: int, period: int) -> datetime: + return self.survey_years[year].get_date(period) + + +class Plots: + def __init__(self, plot_file: str) -> None: + self.plot_file: str = plot_file + self.plots: dict[int, dict[int, dict[int, Plot]]] = {} + + def read_file(self, altitudes: dict[str, int]) -> dict[int, dict[int, dict[int, Plot]]]: + plots: dict[int, dict[int, dict[int, Plot]]] = {} + print(f"Starting to read plot metadata") + with open(self.plot_file, "r") as f: + f.readline() + for line in f: + words = line.split(";") + if words[0] == "": + continue + + survey_year = int(words[0]) + country_code = int(words[1]) + partner_code = int(words[2]) + plot_code = int(words[3]) + sampler_code = int(words[4]) + + lat = words[6] + lon = words[7] + alt_code = words[8] + start = words[9] + stop = words[10] + periods = int(words[11]) + + alt = altitudes[alt_code] + + if start == "" or stop == "": + continue + + if country_code not in plots: + plots[country_code] = {} + if plot_code not in plots[country_code]: + plots[country_code][plot_code] = {} + if sampler_code not in plots[country_code][plot_code]: + plots[country_code][plot_code][sampler_code] = Plot( + country_code, plot_code, sampler_code, lat, lon, alt, partner_code + ) + + plots[country_code][plot_code][sampler_code].add_survey_year( + survey_year, start, stop, periods + ) + print(f"Done read plot metadata") + self.plots = plots + return plots + + def get_ts_type(self, year: int, country_code: int, plot_code: int, sampler_code: int) -> str: + return self.plots[country_code][plot_code][sampler_code].survey_years[year].ts_type + + def get_date( + self, year: int, country_code: int, plot_code: int, sampler_code: int, period: int + ) -> datetime: + return self.plots[country_code][plot_code][sampler_code].get_date(year, period) + + def get_days(self, year: int, country_code: int, plot_code: int, sampler_code: int) -> float: + start = self.plots[country_code][plot_code][sampler_code].survey_years[year].start + stop = self.plots[country_code][plot_code][sampler_code].survey_years[year].stop + days = self.plots[country_code][plot_code][sampler_code].survey_years[year].days + + if start == stop: + raise ValueError(f"start {start} is the same as stop {stop}") + + return days + + def get_position( + self, year: int, country_code: int, plot_code: int, sampler_code: int + ) -> Tuple[str, str, int]: + lat = self.plots[country_code][plot_code][sampler_code].lat + lon = self.plots[country_code][plot_code][sampler_code].lon + alt = self.plots[country_code][plot_code][sampler_code].alt + + return lat, lon, alt + + class MetadataReader: def __init__(self, dir: str) -> None: self.dir = dir - self.add_dir = dir + "" + self.add_dir = dir + "/adds" + + self.altitudes = self._get_altitude_dir() + + self.plots = Plots(self.dir + "/dp_pld.csv") + self.plots.read_file(self.altitudes) + + def _get_altitude_dir(self) -> dict[str, int]: + altitudes = {} + with open(self.add_dir + "/dictionaries/d_altitude.csv") as f: + f.readline() + for line in f: + words = line.split(";") + altitudes[words[0]] = int(words[4]) + (int(words[5]) - int(words[4])) // 2 + + return altitudes + + +if __name__ == "__main__": + metadata = MetadataReader( + "/lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ipc-forests/dep" + ) + breakpoint() + # print(metadata.altitudes) diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index e69de29bb..aecf38d40 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -0,0 +1,249 @@ +import fnmatch +import logging +import os +import re +from statistics import quantiles + +import numpy as np +from geonum.atmosphere import T0_STD, p0 +from metadata import MetadataReader, Station +from tqdm import tqdm + +from pyaerocom import const +from pyaerocom._lowlevel_helpers import BrowseDict +from pyaerocom.io.readungriddedbase import ReadUngriddedBase +from pyaerocom.molmasses import get_molmass +from pyaerocom.stationdata import StationData +from pyaerocom.tstype import TsType +from pyaerocom.ungriddeddata import UngriddedData + +logger = logging.getLogger(__name__) + + +class ReadIPCForest(ReadUngriddedBase): + + #: version log of this class (for caching) + __version__ = "0.1_" + ReadUngriddedBase.__baseversion__ + + #: Name of dataset (OBS_ID) + DATA_ID = const.IPCFORESTS_NAME + + #: List of all datasets supported by this interface + SUPPORTED_DATASETS = [const.IPCFORESTS_NAME] + + TS_TYPE = "undefined" + + _FILEMASK = "dp_dem.csv" + + #: Temporal resolution codes that (so far) can be understood by pyaerocom + TS_TYPE_CODES = { + "1mn": "minutely", + "1h": "hourly", + "1d": "daily", + "1w": "weekly", + "1mo": "monthly", + "mn": "minutely", + "h": "hourly", + "d": "daily", + "w": "weekly", + "mo": "monthly", + } + + VAR_POSITION = { + "wetoxs": 20, + "wetoxn": 19, + "wetrdn": 17, + } + + def __init__(self, data_id=None, data_dir=None): + super().__init__(data_id, data_dir) + + self.metadata = None + # self.data_dir = data_dir + + if data_dir is not None: + self.metadata = MetadataReader(data_dir) + + def read(self, vars_to_retrieve=None, files=[], first_file=None, last_file=None): + """Method that reads list of files as instance of :class:`UngriddedData` + + Parameters + ---------- + vars_to_retrieve : :obj:`list` or similar, optional, + list containing variable IDs that are supposed to be read. If None, + all variables in :attr:`PROVIDES_VARIABLES` are loaded + files : :obj:`list`, optional + list of files to be read. If None, then the file list is used that + is returned on :func:`get_file_list`. + first_file : :obj:`int`, optional + index of first file in file list to read. If None, the very first + file in the list is used + last_file : :obj:`int`, optional + index of last file in list to read. If None, the very last file + in the list is used + + Returns + ------- + UngriddedData + instance of ungridded data object containing data from all files. + """ + ... + + def PROVIDES_VARIABLES(self): + """List of variables that are provided by this dataset + + Note + ---- + May be implemented as global constant in header + """ + pass + + def DEFAULT_VARS(self): + """List containing default variables to read""" + pass + + def read_file(self, filename, vars_to_retrieve=None): + """Read single file + + Parameters + ---------- + filename : str + string specifying filename + vars_to_retrieve : :obj:`list` or similar, optional, + list containing variable IDs that are supposed to be read. If None, + all variables in :attr:`PROVIDES_VARIABLES` are loaded + + Returns + ------- + :obj:`dict` or :obj:`StationData`, or other... + imported data in a suitable format that can be handled by + :func:`read` which is supposed to append the loaded results from + this method (which reads one datafile) to an instance of + :class:`UngriddedData` for all files. + """ + stations: dict[str, dict[str, Station]] = {} + if self.metadata is None: + raise ValueError(f"Metadata is not read yet") + + with open(filename, "r") as f: + f.readline() + for line_nr, line in tqdm(enumerate(f)): + words = line.split(";") + year = int(words[0]) + country_code = int(words[1]) + partner_code = int(words[2]) + plot_code = int(words[3]) + sampler_code = int(words[9]) + + period = int(words[6]) + + quantity = words[47] + if quantity == "" or quantity == "0": + continue + else: + quantity = float(quantity) + + try: + + self.metadata.plots.plots[country_code] + self.metadata.plots.plots[country_code][plot_code] + self.metadata.plots.plots[country_code][plot_code][sampler_code] + except KeyError: + logger.warning( + f"Some metadata is missing for {country_code=}, {plot_code=}, {sampler_code=}. Skipping" + ) + continue + + try: + self.metadata.plots.plots[country_code][plot_code][sampler_code].survey_years[ + year + ] + except KeyError as e: + logger.warning( + f"Year {year} can't be found for {country_code=}, {plot_code=}, {sampler_code=}. Only years found are {self.metadata.plots.plots[country_code][plot_code][sampler_code].survey_years.keys()}" + ) + continue + + try: + + days = self.metadata.plots.get_days( + year, country_code, plot_code, sampler_code + ) + except ValueError as e: + logger.warning(repr(e)) + continue + + try: + dtime = self.metadata.plots.get_date( + year, country_code, plot_code, sampler_code, period + ) + except ValueError: + continue + + ts_type = self.metadata.plots.get_ts_type( + year, country_code, plot_code, sampler_code + ) + station_name = Station.get_station_name(country_code, plot_code, sampler_code) + + if station_name not in stations: + stations[station_name] = {} + if ts_type not in stations[station_name]: + lat, lon, alt = self.metadata.plots.get_position( + year, country_code, plot_code, sampler_code + ) + stations[station_name][ts_type] = Station( + country_code, plot_code, sampler_code, lat, lon, alt, partner_code, ts_type + ) + + for species in self.VAR_POSITION: + conc = self._get_species_conc(words[self.VAR_POSITION[species]]) + + conc *= 1e-6 * quantity / days + + stations[station_name][ts_type].add_measurement( + species, dtime, conc, "mg m-2 d-1" + ) + + station_datas = [] + for station_name in stations: + for ts_type in stations[station_name]: + station = stations[station_name][ts_type] + station_data = StationData() + station_data.var_info = BrowseDict(**station.var_info) + for species in station.data.keys(): + station_data[species] = station.data[species] + station_data.dtime = station.dtime[species] + + station_data.country = station.country + + # Needs to convert coordinates to correct type! + station_data.station_coords = { + "latitude": station.lat, + "longitude": station.lon, + "altitude": station.alt, + } + + station_data.latitude = station.lat + station_data.longitude = station.lon + station_data.altitude = station.alt + + station_data.filename = filename + station_data.ts_type = station.ts_type + station_data.ts_type_src = station.ts_type + station_data.station_name = station.station_name + + station_datas.append(station_data) + + breakpoint() + return UngriddedData.from_station_data(station_datas) + + def _get_species_conc(self, conc_str: str) -> float: + return float(conc_str) if conc_str != "" else np.nan + + +if __name__ == "__main__": + reader = ReadIPCForest( + data_dir="/lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ipc-forests/dep" + ) + filename = "/lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ipc-forests/dep/dp_dem.csv" + data = reader.read_file(filename) From 4ee082ca07ffd2361c88533e31b6c997dcd2e11e Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Mon, 12 Sep 2022 11:44:36 +0000 Subject: [PATCH 03/82] First working version of reader --- pyaerocom/aeroval/coldatatojson_helpers.py | 10 +- pyaerocom/config.py | 6 +- pyaerocom/data/paths.ini | 2 + pyaerocom/io/ipcforests/__init__.py | 3 + pyaerocom/io/ipcforests/metadata.py | 42 +++++- pyaerocom/io/ipcforests/reader.py | 147 ++++++++++++++++----- pyaerocom/io/readungridded.py | 2 + 7 files changed, 171 insertions(+), 41 deletions(-) diff --git a/pyaerocom/aeroval/coldatatojson_helpers.py b/pyaerocom/aeroval/coldatatojson_helpers.py index d225b18c2..f77294fca 100644 --- a/pyaerocom/aeroval/coldatatojson_helpers.py +++ b/pyaerocom/aeroval/coldatatojson_helpers.py @@ -1078,10 +1078,12 @@ def _calc_temporal_corr(coldata): arr = coldata.data # Use only sites that contain at least 3 valid data points (otherwise # correlation will be 1). - obs_ok = arr[0].count(dim="time") > 2 - arr[0] = arr[0].where(obs_ok, drop=True) - arr[1] = arr[1].where(obs_ok, drop=True) - if np.prod(arr.shape) == 0: + obs_ok = coldata.data[0].count(dim="time") > 2 + arr = [] + arr.append(coldata.data[0].where(obs_ok, drop=True)) + arr.append(coldata.data[1].where(obs_ok, drop=True)) + + if np.prod(arr[0].shape) == 0 or np.prod(arr[1].shape) == 0: return np.nan, np.nan corr_time = xr.corr(arr[1], arr[0], dim="time") with ignore_warnings(RuntimeWarning, "Mean of empty slice", "All-NaN slice encountered"): diff --git a/pyaerocom/config.py b/pyaerocom/config.py index e323d5568..f480551eb 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -35,6 +35,9 @@ class Config: # default names of the different obs networks # might get overwritten from paths.ini see func read_config + #: IPC Forests + IPCFORESTS_NAME = "IPCFORESTS" + #: Aeronet Sun V2 access names AERONET_SUN_V2L15_AOD_DAILY_NAME = "AeronetSunV2Lev1.5.daily" AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME = "AeronetSun_2.0_NRT" @@ -88,9 +91,6 @@ class Config: #: DMS DMS_AMS_CVO_NAME = "DMS_AMS_CVO" - #: IPC Forests - IPCFORESTS_NAME = "IPCFOREST" - #: boolean specifying wheter EBAS DB is copied to local cache for faster #: access, defaults to True EBAS_DB_LOCAL_CACHE = True diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini index 9d85e81ea..924e24b55 100644 --- a/pyaerocom/data/paths.ini +++ b/pyaerocom/data/paths.ini @@ -149,6 +149,8 @@ EEA_V2 = EEAAQeRep.v2 AIR_NOW = AirNow MARCO_POLO = MarcoPolo +IPCFORESTS = IPCFORESTS + [parameters] #parameters definition ObsOnlyModelname = OBSERVATIONS-ONLY diff --git a/pyaerocom/io/ipcforests/__init__.py b/pyaerocom/io/ipcforests/__init__.py index eea436a37..c2f95f4c5 100644 --- a/pyaerocom/io/ipcforests/__init__.py +++ b/pyaerocom/io/ipcforests/__init__.py @@ -1,3 +1,6 @@ import logging +from . import metadata, reader +from .metadata import MetadataReader + logger = logging.getLogger(__name__) diff --git a/pyaerocom/io/ipcforests/metadata.py b/pyaerocom/io/ipcforests/metadata.py index 7d112d9b5..5a67f9ccc 100644 --- a/pyaerocom/io/ipcforests/metadata.py +++ b/pyaerocom/io/ipcforests/metadata.py @@ -4,6 +4,17 @@ import pandas as pd from pandas import date_range +DEP_TYPE = { + 1: "Throughfall", + 2: "Bulk deposition", + 3: "Wet-only deposition", + 4: "Stemflow", + 5: "Fog", + 6: "Frozen fog (rime)", + 9: "Other", + 8: "Through also - do not use", +} + COUNTRY_CODES = { 1: "FR", 2: "BE", @@ -113,6 +124,8 @@ def __init__( self.partner_code = partner_code self.ts_type = ts_type + self.sampler_type = DEP_TYPE[self.sampler_code] + self.data: dict[str, list[float]] = {} self.dtime: dict[str, list[datetime]] = {} @@ -135,6 +148,7 @@ def _add_species_to_var_info(self, species: str, unit: str) -> None: ts_type=self.ts_type, ts_type_src=self.ts_type, units=unit, + sampler_type=self.sampler_type, ) def add_measurement(self, species: str, time: datetime, measurement: float, unit: str) -> None: @@ -282,13 +296,33 @@ def get_days(self, year: int, country_code: int, plot_code: int, sampler_code: i def get_position( self, year: int, country_code: int, plot_code: int, sampler_code: int - ) -> Tuple[str, str, int]: - lat = self.plots[country_code][plot_code][sampler_code].lat - lon = self.plots[country_code][plot_code][sampler_code].lon + ) -> Tuple[float, float, int]: + lat = self._coord_to_desimal(self.plots[country_code][plot_code][sampler_code].lat) + lon = self._coord_to_desimal(self.plots[country_code][plot_code][sampler_code].lon) alt = self.plots[country_code][plot_code][sampler_code].alt return lat, lon, alt + def _coord_to_desimal(self, coord: str) -> float: + sign = 1 + if "-" in coord: + sign = -1 + coord = coord[1:] + + if coord == "0": + return 0 + + coord = coord[:-2] + if len(coord) >= 2: + minute = int(coord[-2:]) + coord = coord[:-2] + else: + return sign * ((int(coord) / 60.0)) + + degree = int(coord) + + return sign * (degree + (minute / 60.0)) + class MetadataReader: def __init__(self, dir: str) -> None: @@ -301,6 +335,8 @@ def __init__(self, dir: str) -> None: self.plots = Plots(self.dir + "/dp_pld.csv") self.plots.read_file(self.altitudes) + self.deposition_type = DEP_TYPE + def _get_altitude_dir(self) -> dict[str, int]: altitudes = {} with open(self.add_dir + "/dictionaries/d_altitude.csv") as f: diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index aecf38d40..69c4d0f55 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -2,15 +2,17 @@ import logging import os import re +from datetime import datetime from statistics import quantiles +from typing import Tuple import numpy as np from geonum.atmosphere import T0_STD, p0 -from metadata import MetadataReader, Station from tqdm import tqdm from pyaerocom import const from pyaerocom._lowlevel_helpers import BrowseDict +from pyaerocom.io.ipcforests.metadata import MetadataReader, Station from pyaerocom.io.readungriddedbase import ReadUngriddedBase from pyaerocom.molmasses import get_molmass from pyaerocom.stationdata import StationData @@ -23,7 +25,7 @@ class ReadIPCForest(ReadUngriddedBase): #: version log of this class (for caching) - __version__ = "0.1_" + ReadUngriddedBase.__baseversion__ + __version__ = "0.15_" + ReadUngriddedBase.__baseversion__ #: Name of dataset (OBS_ID) DATA_ID = const.IPCFORESTS_NAME @@ -55,15 +57,38 @@ class ReadIPCForest(ReadUngriddedBase): "wetrdn": 17, } + UNITS = { + "wetoxs": "mg S m-2 d-1", + "wetoxn": "mg N m-2 d-1", + "wetrdn": "mg N m-2 d-1", + } + + DEP_TYPES_TO_USE = ["Throughfall", "Bulk deposition", "Wet-only deposition"] + def __init__(self, data_id=None, data_dir=None): super().__init__(data_id, data_dir) self.metadata = None - # self.data_dir = data_dir + + self._file_dir = None if data_dir is not None: self.metadata = MetadataReader(data_dir) + @property + def file_dir(self): + """Directory containing EBAS NASA Ames files""" + if self._file_dir is not None: + return self._file_dir + return os.path.join(self.data_dir) + + @file_dir.setter + def file_dir(self, val): + if not isinstance(val, str) or not os.path.exists(val): + raise FileNotFoundError("Input directory does not exist") + + self._file_dir = val + def read(self, vars_to_retrieve=None, files=[], first_file=None, last_file=None): """Method that reads list of files as instance of :class:`UngriddedData` @@ -87,8 +112,11 @@ def read(self, vars_to_retrieve=None, files=[], first_file=None, last_file=None) UngriddedData instance of ungridded data object containing data from all files. """ - ... + data = self.read_file(self.data_dir + self._FILEMASK, vars_to_retrieve) + + return data + @property def PROVIDES_VARIABLES(self): """List of variables that are provided by this dataset @@ -96,11 +124,12 @@ def PROVIDES_VARIABLES(self): ---- May be implemented as global constant in header """ - pass + return list(self.VAR_POSITION.keys()) + @property def DEFAULT_VARS(self): """List containing default variables to read""" - pass + return list(self.VAR_POSITION.keys()) def read_file(self, filename, vars_to_retrieve=None): """Read single file @@ -123,7 +152,13 @@ def read_file(self, filename, vars_to_retrieve=None): """ stations: dict[str, dict[str, Station]] = {} if self.metadata is None: - raise ValueError(f"Metadata is not read yet") + if self.data_dir is None: + raise ValueError(f"Data Dir is not read yet") + + self.metadata = MetadataReader(self.data_dir) + + if vars_to_retrieve is None: + vars_to_retrieve = self.PROVIDES_VARIABLES with open(filename, "r") as f: f.readline() @@ -135,10 +170,17 @@ def read_file(self, filename, vars_to_retrieve=None): plot_code = int(words[3]) sampler_code = int(words[9]) + if self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE: + continue + + sampler_type = self.metadata.deposition_type[sampler_code] + period = int(words[6]) + start = words[4] + stop = words[5] quantity = words[47] - if quantity == "" or quantity == "0": + if quantity == "": # or quantity == "0": continue else: quantity = float(quantity) @@ -163,26 +205,12 @@ def read_file(self, filename, vars_to_retrieve=None): f"Year {year} can't be found for {country_code=}, {plot_code=}, {sampler_code=}. Only years found are {self.metadata.plots.plots[country_code][plot_code][sampler_code].survey_years.keys()}" ) continue + days, dtime, ts_type = self._get_days_date_ts_type( + year, country_code, plot_code, sampler_code, period, start, stop + ) - try: - - days = self.metadata.plots.get_days( - year, country_code, plot_code, sampler_code - ) - except ValueError as e: - logger.warning(repr(e)) + if days is None or dtime is None or ts_type is None: continue - - try: - dtime = self.metadata.plots.get_date( - year, country_code, plot_code, sampler_code, period - ) - except ValueError: - continue - - ts_type = self.metadata.plots.get_ts_type( - year, country_code, plot_code, sampler_code - ) station_name = Station.get_station_name(country_code, plot_code, sampler_code) if station_name not in stations: @@ -195,13 +223,13 @@ def read_file(self, filename, vars_to_retrieve=None): country_code, plot_code, sampler_code, lat, lon, alt, partner_code, ts_type ) - for species in self.VAR_POSITION: + for species in vars_to_retrieve: conc = self._get_species_conc(words[self.VAR_POSITION[species]]) - conc *= 1e-6 * quantity / days + conc *= quantity / days stations[station_name][ts_type].add_measurement( - species, dtime, conc, "mg m-2 d-1" + species, dtime, conc, self.UNITS[species] ) station_datas = [] @@ -232,14 +260,71 @@ def read_file(self, filename, vars_to_retrieve=None): station_data.ts_type_src = station.ts_type station_data.station_name = station.station_name - station_datas.append(station_data) + station_data.data_id = self.data_id - breakpoint() + station_datas.append(station_data) return UngriddedData.from_station_data(station_datas) def _get_species_conc(self, conc_str: str) -> float: return float(conc_str) if conc_str != "" else np.nan + def _get_days_date_ts_type( + self, + year: int, + country_code: int, + plot_code: int, + sampler_code: int, + period: int, + start: str | datetime, + stop: str | datetime, + ) -> Tuple[float | None, datetime | None, str | None]: + + if start != "" and stop != "": + if isinstance(start, str): + start = datetime.strptime(start, "%Y-%m-%d") + if isinstance(stop, str): + stop = datetime.strptime(stop, "%Y-%m-%d") + + if (stop - start).days <= 0: + return None, None, None + + return (stop - start).days, start, self._get_tstype(start, stop) + + if self.metadata is None: + raise ValueError(f"Metadata is not read yet") + + try: + + days = self.metadata.plots.get_days(year, country_code, plot_code, sampler_code) + except ValueError as e: + logger.warning(repr(e)) + return None, None, None + + if days == 0: + return None, None, None + + try: + dtime = self.metadata.plots.get_date( + year, country_code, plot_code, sampler_code, period + ) + except ValueError: + return None, None, None + + ts_type = self.metadata.plots.get_ts_type(year, country_code, plot_code, sampler_code) + + return days, dtime, ts_type + + def _get_tstype(self, start: datetime, stop: datetime) -> str: + + days = (stop - start).days + + if days >= 26: + return "monthly" + elif days >= 6: + return "weekly" + else: + return "daily" + if __name__ == "__main__": reader = ReadIPCForest( diff --git a/pyaerocom/io/readungridded.py b/pyaerocom/io/readungridded.py index 155932670..2844dca29 100755 --- a/pyaerocom/io/readungridded.py +++ b/pyaerocom/io/readungridded.py @@ -8,6 +8,7 @@ from pyaerocom.exceptions import DataRetrievalError, NetworkNotImplemented, NetworkNotSupported from pyaerocom.helpers import varlist_aerocom from pyaerocom.io.cachehandler_ungridded import CacheHandlerUngridded +from pyaerocom.io.ipcforests.reader import ReadIPCForest from pyaerocom.io.read_aasetal import ReadAasEtal from pyaerocom.io.read_aeronet_invv2 import ReadAeronetInvV2 from pyaerocom.io.read_aeronet_invv3 import ReadAeronetInvV3 @@ -59,6 +60,7 @@ class ReadUngridded: ReadMarcoPolo, ReadEEAAQEREP, ReadEEAAQEREP_V2, + ReadIPCForest, ] DONOTCACHE_NAME = "DONOTCACHE" From c4bf576ca20b4ee077ece48263241c8088a9d561 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Mon, 12 Sep 2022 12:40:04 +0000 Subject: [PATCH 04/82] Sampler type filter added --- pyaerocom/io/ipcforests/metadata.py | 8 ++++---- pyaerocom/io/ipcforests/reader.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pyaerocom/io/ipcforests/metadata.py b/pyaerocom/io/ipcforests/metadata.py index 5a67f9ccc..dbbb2ed95 100644 --- a/pyaerocom/io/ipcforests/metadata.py +++ b/pyaerocom/io/ipcforests/metadata.py @@ -6,13 +6,13 @@ DEP_TYPE = { 1: "Throughfall", - 2: "Bulk deposition", - 3: "Wet-only deposition", + 2: "Bulk", + 3: "Wet-only", 4: "Stemflow", 5: "Fog", - 6: "Frozen fog (rime)", + 6: "Frozen fog", 9: "Other", - 8: "Through also - do not use", + 8: "do not use", } COUNTRY_CODES = { diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index 69c4d0f55..9954cea54 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -25,7 +25,7 @@ class ReadIPCForest(ReadUngriddedBase): #: version log of this class (for caching) - __version__ = "0.15_" + ReadUngriddedBase.__baseversion__ + __version__ = "0.16_" + ReadUngriddedBase.__baseversion__ #: Name of dataset (OBS_ID) DATA_ID = const.IPCFORESTS_NAME @@ -63,7 +63,7 @@ class ReadIPCForest(ReadUngriddedBase): "wetrdn": "mg N m-2 d-1", } - DEP_TYPES_TO_USE = ["Throughfall", "Bulk deposition", "Wet-only deposition"] + DEP_TYPES_TO_USE = ["Throughfall", "Bulk", "Wet-only"] def __init__(self, data_id=None, data_dir=None): super().__init__(data_id, data_dir) @@ -170,7 +170,11 @@ def read_file(self, filename, vars_to_retrieve=None): plot_code = int(words[3]) sampler_code = int(words[9]) - if self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE: + # 8 is the code for "do not use" + if ( + self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE + or sampler_code == 8 + ): continue sampler_type = self.metadata.deposition_type[sampler_code] @@ -259,11 +263,12 @@ def read_file(self, filename, vars_to_retrieve=None): station_data.ts_type = station.ts_type station_data.ts_type_src = station.ts_type station_data.station_name = station.station_name + station_data._append_meta_item("sampler_type", station.sampler_type) station_data.data_id = self.data_id station_datas.append(station_data) - return UngriddedData.from_station_data(station_datas) + return UngriddedData.from_station_data(station_datas, add_meta_keys="sampler_type") def _get_species_conc(self, conc_str: str) -> float: return float(conc_str) if conc_str != "" else np.nan From 0381478f6d8afd37b6b40be9d76e4f25853ed4bd Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Thu, 15 Sep 2022 06:43:59 +0000 Subject: [PATCH 05/82] Adds sea salt correction to so4 dep --- pyaerocom/io/ipcforests/reader.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index 9954cea54..70dcb480e 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -25,7 +25,7 @@ class ReadIPCForest(ReadUngriddedBase): #: version log of this class (for caching) - __version__ = "0.16_" + ReadUngriddedBase.__baseversion__ + __version__ = "0.17_" + ReadUngriddedBase.__baseversion__ #: Name of dataset (OBS_ID) DATA_ID = const.IPCFORESTS_NAME @@ -55,12 +55,21 @@ class ReadIPCForest(ReadUngriddedBase): "wetoxs": 20, "wetoxn": 19, "wetrdn": 17, + "wetna": 16, } UNITS = { "wetoxs": "mg S m-2 d-1", "wetoxn": "mg N m-2 d-1", "wetrdn": "mg N m-2 d-1", + "wetna": "mg m-2 d-1", + } + + SEASALT_CORRECTION = { + "wetoxs": { + "ion": "wetna", + "ratio": 0.120, + } } DEP_TYPES_TO_USE = ["Throughfall", "Bulk", "Wet-only"] @@ -230,6 +239,20 @@ def read_file(self, filename, vars_to_retrieve=None): for species in vars_to_retrieve: conc = self._get_species_conc(words[self.VAR_POSITION[species]]) + # Sea-salt correction + if species in self.SEASALT_CORRECTION: + ion_name = self.SEASALT_CORRECTION[species]["ion"] + if ion_name not in self.VAR_POSITION: + raise ValueError( + f"Could not do sea salt correction for {species} with {ion_name}" + ) + + ion_conc = self._get_species_conc(words[self.VAR_POSITION[ion_name]]) + ratio = self.SEASALT_CORRECTION[species]["ratio"] + + conc -= ion_conc * ratio + + # Unit correction conc *= quantity / days stations[station_name][ts_type].add_measurement( From d5c78c1789d7d8c80b80ffa57b39d7e066793251 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Thu, 15 Sep 2022 07:00:46 +0000 Subject: [PATCH 06/82] Changes how and where ts_type is calculated --- pyaerocom/io/ipcforests/metadata.py | 10 +++++++--- pyaerocom/io/ipcforests/reader.py | 16 +++------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/pyaerocom/io/ipcforests/metadata.py b/pyaerocom/io/ipcforests/metadata.py index dbbb2ed95..551f66f7f 100644 --- a/pyaerocom/io/ipcforests/metadata.py +++ b/pyaerocom/io/ipcforests/metadata.py @@ -186,11 +186,15 @@ def get_date(self, period: int) -> datetime: def _get_tstype(self) -> str: - days = (self.stop - self.start).days / self.periods + return SurveyYear.get_tstype(self.days) - if self.days >= 26: + @staticmethod + def get_tstype(days: float) -> str: + if days >= 26: return "monthly" - elif self.days >= 6: + elif days >= 12: + return "2weekly" + elif days >= 6: return "weekly" else: return "daily" diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index 70dcb480e..e2ddf7c63 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -1,9 +1,6 @@ -import fnmatch import logging import os -import re from datetime import datetime -from statistics import quantiles from typing import Tuple import numpy as np @@ -12,11 +9,9 @@ from pyaerocom import const from pyaerocom._lowlevel_helpers import BrowseDict -from pyaerocom.io.ipcforests.metadata import MetadataReader, Station +from pyaerocom.io.ipcforests.metadata import MetadataReader, Station, SurveyYear from pyaerocom.io.readungriddedbase import ReadUngriddedBase -from pyaerocom.molmasses import get_molmass from pyaerocom.stationdata import StationData -from pyaerocom.tstype import TsType from pyaerocom.ungriddeddata import UngriddedData logger = logging.getLogger(__name__) @@ -25,7 +20,7 @@ class ReadIPCForest(ReadUngriddedBase): #: version log of this class (for caching) - __version__ = "0.17_" + ReadUngriddedBase.__baseversion__ + __version__ = "0.18_" + ReadUngriddedBase.__baseversion__ #: Name of dataset (OBS_ID) DATA_ID = const.IPCFORESTS_NAME @@ -346,12 +341,7 @@ def _get_tstype(self, start: datetime, stop: datetime) -> str: days = (stop - start).days - if days >= 26: - return "monthly" - elif days >= 6: - return "weekly" - else: - return "daily" + return SurveyYear.get_tstype(days) if __name__ == "__main__": From c047ba25cba26627487fb3fa6cee795902b47c3b Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Thu, 15 Sep 2022 10:47:16 +0000 Subject: [PATCH 07/82] Adds some docstrings to metadata.py --- pyaerocom/io/ipcforests/metadata.py | 163 ++++++++++++++++++++++++++-- 1 file changed, 155 insertions(+), 8 deletions(-) diff --git a/pyaerocom/io/ipcforests/metadata.py b/pyaerocom/io/ipcforests/metadata.py index 551f66f7f..7dcc61b87 100644 --- a/pyaerocom/io/ipcforests/metadata.py +++ b/pyaerocom/io/ipcforests/metadata.py @@ -98,12 +98,47 @@ } -class Variables: - def __init__(self) -> None: - pass - - class Station: + """Holds information for a singe IPC Forest station. The same station with different ts_types + are treated as seperate stations at this level + + Parameters + ---------- + country_code : int + int representing the country where the station is found + plot_code : int + The number of the station + sampler_code : int + int representing the type of sampler + lat : str + latitude + lon : str + longitude + alt: int + altitude + partner_code : int + number of the institude doing the measurement. Mostly used for metadata + ts_type : str + ts_type of the station + + Attributes + ---------- + sampler_type : str + The type of wetdep measurement + data : dict[str, list[float]] + dictionary holding a list of measurements for each species + dtime : dict[str, list[datetime]] + dictionary holding a list of timesteps for each species + country : str + name of country + station_name : str + name of station + var_info : dict[str, dict[str, str | list[float]]] + dict holder metadata for each species, used by StationData + + + """ + def __init__( self, country_code: int, @@ -141,9 +176,39 @@ def get_station_name( plot_code: int, sampler_code: int, ) -> str: + """ + Generates the station name based on country code, plot code and sampler code. Static method + that can be used without initiating the class + + Parameters + ---------- + country_code : int + int representing the country where the station is found + plot_code : int + The number of the station + sampler_code : int + int representing the type of sampler + + Returns + ------- + str + station name + + """ return f"{COUNTRY_CODES[country_code]}-{plot_code}-{sampler_code}" def _add_species_to_var_info(self, species: str, unit: str) -> None: + """ + Makes the var_info dict that is used later by StationData + + Parameters + ---------- + species : str + name of species + unit : str + unit + + """ self.var_info[species] = dict( ts_type=self.ts_type, ts_type_src=self.ts_type, @@ -152,6 +217,23 @@ def _add_species_to_var_info(self, species: str, unit: str) -> None: ) def add_measurement(self, species: str, time: datetime, measurement: float, unit: str) -> None: + + """ + Adds a single measurement to the data and time lists. If it is the first measurement, + a new var_info is created + + Parameters + ---------- + species : str + name of species + time : datetime + The timestamp of the measurement + measurement : float + The value of the measure ment + unit : str + unit + + """ if species not in self.var_info: self._add_species_to_var_info(species, unit) if species not in self.dtime: @@ -162,29 +244,94 @@ def add_measurement(self, species: str, time: datetime, measurement: float, unit self.data[species].append(measurement) def get_timeseries(self, species: str) -> pd.Series: + """ + Combines the data and timestamps of a given species to a pandas timeseries + + Parameters + ---------- + species : str + name of species + + Returns + ------- + pd.Series + Timeseries of the measurements of a given species for station + + """ return pd.Series(self.data[species], index=self.dtime[species]) class SurveyYear: + """ + Holds information about the start and stop dates, as well + as the number of periods for a single year at a single station + + + Parameters + ---------- + year: int + The year + start: datetime + start date + stop: datetime + end date + periods: int + number of periods in the survey year + + Attributes + ---------- + days : float + The number of days in each period + ts_type : str + The ts_type for the survey year + + + """ + def __init__(self, year: int, start: datetime, stop: datetime, periods: int) -> None: self.year = year self.start = start self.stop = stop self.periods = periods - self.daterange = date_range(start, stop, periods) + # self.daterange = date_range(start, stop, periods) self.days = (self.stop - self.start).days / self.periods self.ts_type = self._get_tstype() def get_date(self, period: int) -> datetime: + """ + Finds the date used to make each period in the year. + For now the function returns the first date in the period + + Parameters + ---------- + period : int + Which period to find the date for + + Returns + ------- + float + The first date in the period + + Raises + ------ + ValueError + If the period is not within the range of periods in the year + """ if period > self.periods or period <= 0: - raise ValueError(f"The period {period} needs to be in the range 1-{self.periods}") return self.start + timedelta(days=self.days * (period - 1)) - # return self.daterange[period - 1] def _get_tstype(self) -> str: + """ + Returns the ts_type by calling the static method + + Returns + ------- + str + the ts_type of the survey year + """ return SurveyYear.get_tstype(self.days) From 638316a8c56cd5b43b14ff3132504a103a07b5d0 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Mon, 26 Sep 2022 10:26:04 +0000 Subject: [PATCH 08/82] Adds total dep, check quality flags, and uses only the lowest sea salt correction --- pyaerocom/aeroval/glob_defaults.py | 3 + pyaerocom/data/variables.ini | 15 +++++ pyaerocom/io/ipcforests/metadata.py | 18 +++++- pyaerocom/io/ipcforests/reader.py | 91 +++++++++++++++++++++++------ pyaerocom/io/read_mscw_ctm.py | 6 ++ 5 files changed, 111 insertions(+), 22 deletions(-) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index 11d447058..32f233bdd 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -248,4 +248,7 @@ vmrco=["CO", "3D", "Volume mixing ratios"], vmrno2=["NO2", "3D", "Volume mixing ratios"], concco=["CO", "3D", "Particle concentration"], + depoxs=["TotDepOXS", "3D", "Total Deposition"], + depoxn=["TotDepOXN", "3D", "Total Deposition"], + deprdn=["TotDepRDN", "3D", "Total Deposition"], ) diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index 5829aa2a4..f7f5009ea 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -2049,6 +2049,21 @@ description=Wet deposition of total sulphur mass unit = mg S m-2 d-1 minimum=0 +[depoxn] +description=Total deposition of oxidized nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[deprdn] +description=Total deposition of reduced Nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[depoxs] +description=Total deposition of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + [dryoxn] description=dry deposition of oxidized nitrogen mass unit = mg N m-2 d-1 diff --git a/pyaerocom/io/ipcforests/metadata.py b/pyaerocom/io/ipcforests/metadata.py index 7dcc61b87..94435447b 100644 --- a/pyaerocom/io/ipcforests/metadata.py +++ b/pyaerocom/io/ipcforests/metadata.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta from typing import Tuple +import numpy as np import pandas as pd from pandas import date_range @@ -162,6 +163,7 @@ def __init__( self.sampler_type = DEP_TYPE[self.sampler_code] self.data: dict[str, list[float]] = {} + self.flags: dict[str, list[int]] = {} self.dtime: dict[str, list[datetime]] = {} self.country = COUNTRIES[country_code] @@ -216,7 +218,9 @@ def _add_species_to_var_info(self, species: str, unit: str) -> None: sampler_type=self.sampler_type, ) - def add_measurement(self, species: str, time: datetime, measurement: float, unit: str) -> None: + def add_measurement( + self, species: str, time: datetime, measurement: float, unit: str, flag: int + ) -> None: """ Adds a single measurement to the data and time lists. If it is the first measurement, @@ -239,11 +243,13 @@ def add_measurement(self, species: str, time: datetime, measurement: float, unit if species not in self.dtime: self.dtime[species] = [] self.data[species] = [] + self.flags[species] = [] self.dtime[species].append(time) self.data[species].append(measurement) + self.flags[species].append(flag) - def get_timeseries(self, species: str) -> pd.Series: + def get_timeseries(self, species: str, quality_limit: float = 0.5) -> pd.Series: """ Combines the data and timestamps of a given species to a pandas timeseries @@ -258,7 +264,13 @@ def get_timeseries(self, species: str) -> pd.Series: Timeseries of the measurements of a given species for station """ - return pd.Series(self.data[species], index=self.dtime[species]) + flags = np.array(self.flags[species]) + quality = np.sum(flags[np.where(flags == 0)]) / len(flags) + breakpoint() + if quality >= quality_limit: + return pd.Series(self.data[species], index=self.dtime[species]) + else: + return pd.Series(np.ones_like(self.data[species]) * np.nan, index=self.dtime[species]) class SurveyYear: diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index e2ddf7c63..be519990c 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -20,7 +20,7 @@ class ReadIPCForest(ReadUngriddedBase): #: version log of this class (for caching) - __version__ = "0.18_" + ReadUngriddedBase.__baseversion__ + __version__ = "0.2_" + ReadUngriddedBase.__baseversion__ #: Name of dataset (OBS_ID) DATA_ID = const.IPCFORESTS_NAME @@ -50,25 +50,36 @@ class ReadIPCForest(ReadUngriddedBase): "wetoxs": 20, "wetoxn": 19, "wetrdn": 17, + "depoxs": 20, + "depoxn": 19, + "deprdn": 17, "wetna": 16, + "wetcl": 18, } UNITS = { "wetoxs": "mg S m-2 d-1", "wetoxn": "mg N m-2 d-1", "wetrdn": "mg N m-2 d-1", + "depoxs": "mg S m-2 d-1", + "depoxn": "mg N m-2 d-1", + "deprdn": "mg N m-2 d-1", "wetna": "mg m-2 d-1", } SEASALT_CORRECTION = { - "wetoxs": { - "ion": "wetna", - "ratio": 0.120, - } + "wetoxs": 0.3338, + "depoxs": 0.3338, + } + SEASALT_FACTORS = { + "wetna": 0.120, + "wetcl": 0.103, } DEP_TYPES_TO_USE = ["Throughfall", "Bulk", "Wet-only"] + QUALITY_LIMIT = 0.5 + def __init__(self, data_id=None, data_dir=None): super().__init__(data_id, data_dir) @@ -173,6 +184,7 @@ def read_file(self, filename, vars_to_retrieve=None): partner_code = int(words[2]) plot_code = int(words[3]) sampler_code = int(words[9]) + q_flag = int(words[44]) if words[44] != "" else 0 # 8 is the code for "do not use" if ( @@ -232,26 +244,36 @@ def read_file(self, filename, vars_to_retrieve=None): ) for species in vars_to_retrieve: - conc = self._get_species_conc(words[self.VAR_POSITION[species]]) + conc = self._get_species_conc(words[self.VAR_POSITION[species]], species) # Sea-salt correction + # The factor self.SEASALT_CORRECTION[species] is the factor use to go from mg/L to mg S/L (for sulpher) if species in self.SEASALT_CORRECTION: - ion_name = self.SEASALT_CORRECTION[species]["ion"] - if ion_name not in self.VAR_POSITION: - raise ValueError( - f"Could not do sea salt correction for {species} with {ion_name}" - ) - ion_conc = self._get_species_conc(words[self.VAR_POSITION[ion_name]]) - ratio = self.SEASALT_CORRECTION[species]["ratio"] - - conc -= ion_conc * ratio + na_factor = ( + self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") + * self.SEASALT_FACTORS["wetna"] + * self.SEASALT_CORRECTION[species] + ) + cl_factor = ( + self._get_species_conc(words[self.VAR_POSITION["wetcl"]], "wetcl") + * self.SEASALT_FACTORS["wetcl"] + * self.SEASALT_CORRECTION[species] + ) + seasalt_correction = min(na_factor, cl_factor) + + if seasalt_correction > conc and conc > 0: + logger.warning( + f"Seasalt correction {seasalt_correction} is larger than concentration {conc} for {species}" + ) + if conc > 0: + conc -= seasalt_correction # Unit correction conc *= quantity / days stations[station_name][ts_type].add_measurement( - species, dtime, conc, self.UNITS[species] + species, dtime, conc, self.UNITS[species], q_flag ) station_datas = [] @@ -261,7 +283,12 @@ def read_file(self, filename, vars_to_retrieve=None): station_data = StationData() station_data.var_info = BrowseDict(**station.var_info) for species in station.data.keys(): - station_data[species] = station.data[species] + station_data[species] = self._clean_data_with_flags( + station.data[species], + station.dtime[species], + station.flags[species], + species, + ) station_data.dtime = station.dtime[species] station_data.country = station.country @@ -288,8 +315,12 @@ def read_file(self, filename, vars_to_retrieve=None): station_datas.append(station_data) return UngriddedData.from_station_data(station_datas, add_meta_keys="sampler_type") - def _get_species_conc(self, conc_str: str) -> float: - return float(conc_str) if conc_str != "" else np.nan + def _get_species_conc(self, conc_str: str, species: str) -> float: + conc = float(conc_str) if conc_str != "" else np.nan + if conc == -1: + logger.warning(f"Value for {species} found to be {conc}") + conc = np.nan + return conc def _get_days_date_ts_type( self, @@ -343,6 +374,28 @@ def _get_tstype(self, start: datetime, stop: datetime) -> str: return SurveyYear.get_tstype(days) + def _clean_data_with_flags( + self, + data: list[float], + time: list[datetime], + flags: list[int], + species: str, + ) -> list[float]: + + data_array = np.array(data) + flags_array = np.array(flags) + years = np.array([i.year for i in time]) + for year in range(1984, 2019): + yr_flags = flags_array[np.where(years == year)] + quality = np.sum(np.where(yr_flags == 0)) / len(yr_flags) + if quality < self.QUALITY_LIMIT: + logger.warning( + f"Quailty of {quality} found for {species} in year {year}. Setting data this year to NaN" + ) + data_array[np.where(years == year)] = np.nan + + return list(data_array) + if __name__ == "__main__": reader = ReadIPCForest( diff --git a/pyaerocom/io/read_mscw_ctm.py b/pyaerocom/io/read_mscw_ctm.py index 2b9f23df1..5d32e1a04 100755 --- a/pyaerocom/io/read_mscw_ctm.py +++ b/pyaerocom/io/read_mscw_ctm.py @@ -56,6 +56,9 @@ class ReadMscwCtm: # by the original data but computed on import) AUX_REQUIRES = { "depso4": ["dryso4", "wetso4"], + "depoxs": ["dryoxs", "wetoxs"], + "depoxn": ["dryoxn", "wetoxn"], + "deprdn": ["dryrdn", "wetrdn"], "concbc": ["concbcf", "concbcc"], "concno3": ["concno3c", "concno3f"], "concoa": ["concoac", "concoaf"], @@ -80,6 +83,9 @@ class ReadMscwCtm: # not iris.cube.Cube instance AUX_FUNS = { "depso4": add_dataarrays, + "depoxs": add_dataarrays, + "depoxn": add_dataarrays, + "deprdn": add_dataarrays, "concbc": add_dataarrays, "concno3": add_dataarrays, "concoa": add_dataarrays, From b4b14ee4dadc6a44642d0c29cab9c656b2a8e056 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Thu, 6 Oct 2022 07:23:54 +0000 Subject: [PATCH 09/82] Can now read new model data --- pyaerocom/aeroval/obsentry.py | 2 +- pyaerocom/colocation_auto.py | 2 +- pyaerocom/data/paths.ini | 3 +- pyaerocom/data/variables.ini | 54 +++++++++++++++++++++++++------ pyaerocom/io/fileconventions.py | 3 +- pyaerocom/io/ipcforests/reader.py | 6 ++++ pyaerocom/io/readgridded.py | 2 +- 7 files changed, 57 insertions(+), 15 deletions(-) diff --git a/pyaerocom/aeroval/obsentry.py b/pyaerocom/aeroval/obsentry.py index a02b7a608..337121867 100644 --- a/pyaerocom/aeroval/obsentry.py +++ b/pyaerocom/aeroval/obsentry.py @@ -51,7 +51,7 @@ class ObsEntry(BrowseDict): (c.g. :class:`pyaerocom.io.ReadUngridded`). """ - SUPPORTED_VERT_CODES = ["Column", "Profile", "Surface"] + SUPPORTED_VERT_CODES = ["Column", "Profile", "Surface", "2D"] ALT_NAMES_VERT_CODES = dict(ModelLevel="Profile") SUPPORTED_VERT_LOCS = DataSource.SUPPORTED_VERT_LOCS diff --git a/pyaerocom/colocation_auto.py b/pyaerocom/colocation_auto.py index b0dc026ca..06ac8c87d 100644 --- a/pyaerocom/colocation_auto.py +++ b/pyaerocom/colocation_auto.py @@ -292,7 +292,7 @@ class ColocationSetup(BrowseDict): #: file for ec550aer at the surface ('*ec550aer*Surface*.nc'), then, the #: colocation routine will look for '*ec550aer*ModelLevel*.nc' and if this #: exists, it will load it and extract the surface level. - OBS_VERT_TYPES_ALT = {"Surface": "ModelLevel"} + OBS_VERT_TYPES_ALT = {"Surface": "ModelLevel", "2D": "2D"} #: do not raise Exception if invalid item is attempted to be assigned #: (Overwritten from base class) diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini index 924e24b55..cb885cad6 100644 --- a/pyaerocom/data/paths.ini +++ b/pyaerocom/data/paths.ini @@ -49,7 +49,8 @@ dir= ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND3/, ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND2/, ${BASEDIR}/fou/kl/CAMS61/, - ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/ + ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/, + /lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/mod/deposition [obsfolders] #folders to for model data diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index f7f5009ea..ce3a89bc3 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -2354,16 +2354,16 @@ maximum = 10000 dimensions = time,lat,lon comments_and_purpose = Verification of the budget of atmospheric oxidizing agents -[wetpm10] -var_name = wetpm10 -description = Wet deposition of PM10 mass -standard_name = tendency_of_atmosphere_mass_content_of_pm10_dry_aerosol_particles_due_to_wet_deposition -var_type = dry deposition flux -unit = kg m-2 s-1 -minimum = 0 -maximum = 10000 -dimensions = time,lat,lon -comments_and_purpose = Verification of PM10 (major AQ metric) budget ; Consistent with AQMEII. +; [wetpm10] +; var_name = wetpm10 +; description = Wet deposition of PM10 mass +; standard_name = tendency_of_atmosphere_mass_content_of_pm10_dry_aerosol_particles_due_to_wet_deposition +; var_type = dry deposition flux +; unit = kg m-2 s-1 +; minimum = 0 +; maximum = 10000 +; dimensions = time,lat,lon +; comments_and_purpose = Verification of PM10 (major AQ metric) budget ; Consistent with AQMEII. [wetpm10ss] var_name = wetpm10ss @@ -5317,3 +5317,37 @@ unit = 1 minimum = -150 maximum = 150 dimensions = time,lat,lon + + + + + +; For CAMS2_40 Task4041 + + +; Gases + + +; PM + +; Deposition + +[wetpm10] +var_name = wetpm10 +description = Wet deposition of PM10 mass +standard_name = tendency_of_atmosphere_mass_content_of_pm10_dry_aerosol_particles_due_to_wet_deposition +var_type = dry deposition flux +unit = mg m-2 d-1 +minimum = 0 +maximum = 10000 +dimensions = time,lat,lon + +[wetpm25] +var_name = wetpm25 +description = Wet deposition of PM2.5 mass +standard_name = tendency_of_atmosphere_mass_content_of_pm25_dry_aerosol_particles_due_to_wet_deposition +var_type = dry deposition flux +unit = mg m-2 d-1 +minimum = 0 +maximum = 10000 +dimensions = time,lat,lon diff --git a/pyaerocom/io/fileconventions.py b/pyaerocom/io/fileconventions.py index fdd3ba163..14f91f23a 100644 --- a/pyaerocom/io/fileconventions.py +++ b/pyaerocom/io/fileconventions.py @@ -33,7 +33,7 @@ class FileConventionRead: _io_opts = const AEROCOM3_VERT_INFO = { - "2d": ["surface", "column", "modellevel"], + "2d": ["surface", "column", "modellevel", "2d"], "3d": ["modellevelatstations"], } @@ -158,6 +158,7 @@ def _info_from_aerocom3(self, file: str) -> dict: ) try: # include vars for the surface + if spl[self.vert_pos].lower() in self.AEROCOM3_VERT_INFO["2d"]: info["var_name"] = spl[self.var_pos] # also include 3d vars that provide station based data diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index be519990c..cdd558441 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -50,6 +50,9 @@ class ReadIPCForest(ReadUngriddedBase): "wetoxs": 20, "wetoxn": 19, "wetrdn": 17, + "dryoxs": 20, + "dryoxn": 19, + "dryrdn": 17, "depoxs": 20, "depoxn": 19, "deprdn": 17, @@ -61,6 +64,9 @@ class ReadIPCForest(ReadUngriddedBase): "wetoxs": "mg S m-2 d-1", "wetoxn": "mg N m-2 d-1", "wetrdn": "mg N m-2 d-1", + "dryoxs": "mg S m-2 d-1", + "dryoxn": "mg N m-2 d-1", + "dryrdn": "mg N m-2 d-1", "depoxs": "mg S m-2 d-1", "depoxn": "mg N m-2 d-1", "deprdn": "mg N m-2 d-1", diff --git a/pyaerocom/io/readgridded.py b/pyaerocom/io/readgridded.py index b33c02692..a6ba36bf6 100755 --- a/pyaerocom/io/readgridded.py +++ b/pyaerocom/io/readgridded.py @@ -167,7 +167,7 @@ class specifying details of the file naming convention for the model _data_dir = "" - VERT_ALT = {"Surface": "ModelLevel"} + VERT_ALT = {"Surface": "ModelLevel", "2D": "2D"} def __init__(self, data_id=None, data_dir=None, file_convention="aerocom3"): From 9b250a060dd47f14f05b02b2db2f299692f168eb Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Wed, 26 Oct 2022 11:36:03 +0000 Subject: [PATCH 10/82] More variables for task4041 --- pyaerocom/aeroval/obsentry.py | 2 +- pyaerocom/data/paths.ini | 3 ++- pyaerocom/io/aux_read_cubes.py | 8 ++++++++ pyaerocom/io/readgridded.py | 9 +++++++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/pyaerocom/aeroval/obsentry.py b/pyaerocom/aeroval/obsentry.py index 337121867..f1ba0256e 100644 --- a/pyaerocom/aeroval/obsentry.py +++ b/pyaerocom/aeroval/obsentry.py @@ -51,7 +51,7 @@ class ObsEntry(BrowseDict): (c.g. :class:`pyaerocom.io.ReadUngridded`). """ - SUPPORTED_VERT_CODES = ["Column", "Profile", "Surface", "2D"] + SUPPORTED_VERT_CODES = ["Column", "Profile", "Surface"] # , "2D"] ALT_NAMES_VERT_CODES = dict(ModelLevel="Profile") SUPPORTED_VERT_LOCS = DataSource.SUPPORTED_VERT_LOCS diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini index cb885cad6..a3c132784 100644 --- a/pyaerocom/data/paths.ini +++ b/pyaerocom/data/paths.ini @@ -50,7 +50,8 @@ dir= ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND2/, ${BASEDIR}/fou/kl/CAMS61/, ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/, - /lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/mod/deposition + /lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/mod/deposition, + /lustre/storeB/project/fou/kl/CAMS2_40/task4041/ [obsfolders] #folders to for model data diff --git a/pyaerocom/io/aux_read_cubes.py b/pyaerocom/io/aux_read_cubes.py index eb41cdc9c..ae3c3acde 100644 --- a/pyaerocom/io/aux_read_cubes.py +++ b/pyaerocom/io/aux_read_cubes.py @@ -323,3 +323,11 @@ def mmr_to_vmr_cube(data): cube.units = "nmole mole-1" cube.var_name = vmrvar return cube + + +def compute_sspm25(concssfine, concsscoarse): + mult_fun = CUBE_MATHS["multiply"] + concssfine, concsscoarse = _check_input_iscube(concssfine, concsscoarse) + concssfine, concsscoarse = _check_same_units(concssfine, concsscoarse) + + return add_cubes(concssfine, mult_fun(concsscoarse, 0.16)) diff --git a/pyaerocom/io/readgridded.py b/pyaerocom/io/readgridded.py index a6ba36bf6..ebb5464e6 100755 --- a/pyaerocom/io/readgridded.py +++ b/pyaerocom/io/readgridded.py @@ -27,6 +27,7 @@ from pyaerocom.io.aux_read_cubes import ( add_cubes, compute_angstrom_coeff_cubes, + compute_sspm25, divide_cubes, mmr_from_vmr, multiply_cubes, @@ -123,6 +124,8 @@ class specifying details of the file naming convention for the model "concprcpoxn": ("wetoxn", "pr"), "concprcpoxs": ("wetoxs", "pr"), "concprcprdn": ("wetrdn", "pr"), + "concsspm10": ("concss25", "concsscoarse"), + "concsspm25": ("concss25", "concsscoarse"), } AUX_ALT_VARS = { @@ -146,7 +149,9 @@ class specifying details of the file naming convention for the model "concno3": add_cubes, "concprcpoxn": compute_concprcp_from_pr_and_wetdep, "concprcpoxs": compute_concprcp_from_pr_and_wetdep, - "concprcprdn": compute_concprcp_from_pr_and_wetdep + "concprcprdn": compute_concprcp_from_pr_and_wetdep, + "concsspm10": add_cubes, + "concsspm25": compute_sspm25, #'mec550*' : divide_cubes, #'tau*' : lifetime_from_load_and_dep } @@ -167,7 +172,7 @@ class specifying details of the file naming convention for the model _data_dir = "" - VERT_ALT = {"Surface": "ModelLevel", "2D": "2D"} + VERT_ALT = {"Surface": "ModelLevel"} # , "2D": "2D"} def __init__(self, data_id=None, data_dir=None, file_convention="aerocom3"): From 6527c4f6d772c791b20c9f64b64390ea45498e72 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Wed, 26 Oct 2022 13:02:53 +0000 Subject: [PATCH 11/82] Adds vmr to conc for aerocom model data, in a compicated way --- pyaerocom/io/aux_components_fun.py | 119 +++++++++++++++++++++++++++++ pyaerocom/io/aux_read_cubes.py | 8 -- pyaerocom/io/readgridded.py | 17 ++++- 3 files changed, 134 insertions(+), 10 deletions(-) create mode 100644 pyaerocom/io/aux_components_fun.py diff --git a/pyaerocom/io/aux_components_fun.py b/pyaerocom/io/aux_components_fun.py new file mode 100644 index 000000000..0b708f97c --- /dev/null +++ b/pyaerocom/io/aux_components_fun.py @@ -0,0 +1,119 @@ +import logging +from traceback import format_exc + +import cf_units +import iris +import numpy as np +from geonum.atmosphere import T0_STD, p0 + +from pyaerocom._lowlevel_helpers import merge_dicts +from pyaerocom.helpers import copy_coords_cube +from pyaerocom.io.aux_read_cubes import ( + CUBE_MATHS, + _check_input_iscube, + _check_same_units, + add_cubes, +) +from pyaerocom.molmasses import get_mmr_to_vmr_fac, get_molmass +from pyaerocom.units_helpers import get_unit_conversion_fac + +logger = logging.getLogger(__name__) + + +single_component_mass = {"n": 14.0067, "c": 12.011, "s": 32.065} + + +def vmr_to_conc(data, vmr_unit, var_name, to_unit, component_unit=None): + """ + Convert volume mixing ratio (vmr) to mass concentration + + Parameters + ---------- + data : cube or GriddedData + array containing vmr values + + vmr_unit : str + unit of input data + var_name: str + name of variable, used to fund molar mass + to_unit : str, optional + Unit to which output data is converted. If None, output unit is + kg m-3. The default is None. + component_unit : str, optional + If none, the to_unit unit is returned. If, e.g. n or N, then ug m-3 -> ug N m-3 is returned + + Returns + ------- + cube + input data converted to mass concentration + + """ + + data = _check_input_iscube(data)[0] + + p_pascal = p0 # 1013 hPa (US standard atm) + T_kelvin = T0_STD # 15 deg celcius (US standard atm) + + mmol_air = get_molmass("air_dry") + mmol_var = get_molmass(var_name) + + if component_unit is not None and to_unit is not None: + component_mass = single_component_mass[component_unit.lower()] + component_unit_fac = component_mass / mmol_var + else: + component_unit_fac = 1 + Rspecific = 287.058 # J kg-1 K-1 + + conversion_fac = 1 / cf_units.Unit("mol mol-1").convert(1, vmr_unit) + + airdensity = p_pascal / (Rspecific * T_kelvin) # kg m-3 + mulfac = mmol_var / mmol_air * airdensity # kg m-3 + + mult_fun = CUBE_MATHS["multiply"] + conc = mult_fun(data, mulfac) # kg m-3 + if to_unit is not None: + conversion_fac *= cf_units.Unit("kg m-3").convert(1, to_unit) * component_unit_fac + if not np.isclose(conversion_fac, 1, rtol=1e-7): + conc = mult_fun(conc, conversion_fac) + + if to_unit is not None: + unit = to_unit + + if component_unit is not None: + unit_list = unit.split(" ") + unit = unit_list[0] + f" {component_unit.upper()} " + unit_list[1] + + conc.units = unit + else: + conc.units = "kg m-3" + + return conc + + +def calc_concNhno3_from_vmr(data): + + return vmr_to_conc( + data, vmr_unit="nmol mol-1", var_name="hno3", to_unit="ug m-3", component_unit="N" + ) + + +def calc_concno3pm25(concno3f, concno3c, fine_from_coarse_fraction: float = 0.134): + mult_fun = CUBE_MATHS["multiply"] + concno3pm25 = add_cubes(concno3f, mult_fun(concno3c, fine_from_coarse_fraction)) + + return concno3pm25 + + +def calc_concno3pm10(concno3f, concno3c): + mult_fun = CUBE_MATHS["multiply"] + concno3pm10 = add_cubes(concno3f, concno3c) + + return concno3pm10 + + +def calc_sspm25(concssfine, concsscoarse): + mult_fun = CUBE_MATHS["multiply"] + concssfine, concsscoarse = _check_input_iscube(concssfine, concsscoarse) + concssfine, concsscoarse = _check_same_units(concssfine, concsscoarse) + + return add_cubes(concssfine, mult_fun(concsscoarse, 0.16)) diff --git a/pyaerocom/io/aux_read_cubes.py b/pyaerocom/io/aux_read_cubes.py index ae3c3acde..eb41cdc9c 100644 --- a/pyaerocom/io/aux_read_cubes.py +++ b/pyaerocom/io/aux_read_cubes.py @@ -323,11 +323,3 @@ def mmr_to_vmr_cube(data): cube.units = "nmole mole-1" cube.var_name = vmrvar return cube - - -def compute_sspm25(concssfine, concsscoarse): - mult_fun = CUBE_MATHS["multiply"] - concssfine, concsscoarse = _check_input_iscube(concssfine, concsscoarse) - concssfine, concsscoarse = _check_same_units(concssfine, concsscoarse) - - return add_cubes(concssfine, mult_fun(concsscoarse, 0.16)) diff --git a/pyaerocom/io/readgridded.py b/pyaerocom/io/readgridded.py index ebb5464e6..cb3ef00e3 100755 --- a/pyaerocom/io/readgridded.py +++ b/pyaerocom/io/readgridded.py @@ -2,6 +2,7 @@ import logging import os import warnings +from functools import partial from glob import glob from pathlib import Path @@ -24,10 +25,16 @@ from pyaerocom.griddeddata import GriddedData from pyaerocom.helpers import get_highest_resolution, isnumeric, sort_ts_types, to_pandas_timestamp from pyaerocom.io import AerocomBrowser +from pyaerocom.io.aux_components_fun import ( + calc_concNhno3_from_vmr, + calc_concno3pm10, + calc_concno3pm25, + calc_sspm25, + vmr_to_conc, +) from pyaerocom.io.aux_read_cubes import ( add_cubes, compute_angstrom_coeff_cubes, - compute_sspm25, divide_cubes, mmr_from_vmr, multiply_cubes, @@ -126,6 +133,9 @@ class specifying details of the file naming convention for the model "concprcprdn": ("wetrdn", "pr"), "concsspm10": ("concss25", "concsscoarse"), "concsspm25": ("concss25", "concsscoarse"), + "concno3pm10": ("concno3f", "concno3c"), + "concno3pm25": ("concno3f", "concno3c"), + "concNhno3": ("vmrhno3",), } AUX_ALT_VARS = { @@ -151,7 +161,10 @@ class specifying details of the file naming convention for the model "concprcpoxs": compute_concprcp_from_pr_and_wetdep, "concprcprdn": compute_concprcp_from_pr_and_wetdep, "concsspm10": add_cubes, - "concsspm25": compute_sspm25, + "concsspm25": calc_sspm25, + "concno3pm10": calc_concno3pm10, + "concno3pm25": calc_concno3pm25, + "concNhno3": calc_concNhno3_from_vmr, #'mec550*' : divide_cubes, #'tau*' : lifetime_from_load_and_dep } From f7ec1a7e896033ce6b97861588cbe31a93a5f41f Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Mon, 31 Oct 2022 09:00:16 +0000 Subject: [PATCH 12/82] Adds more variables to task4041 --- pyaerocom/aeroval/glob_defaults.py | 6 +++ pyaerocom/io/aux_components_fun.py | 75 +++++++++++++++++++++++++++++- pyaerocom/io/readgridded.py | 18 +++++++ pyaerocom/molmasses.py | 1 + 4 files changed, 99 insertions(+), 1 deletion(-) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index 90c4208e3..a63f787a1 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -234,6 +234,11 @@ vmrox=["OX", "3D", "Gas volume mixing ratio"], concco=["CO", "3D", "Particle concentration"], vmrco=["CO", "3D", "Volume mixing ratios"], + vmrc2h2=["Ethyne", "3D", "Volume mixing ratios"], + vmrc2h4=["Ethylene", "3D", "Volume mixing ratios"], + vmrc2h6=["Ethane", "3D", "Volume mixing ratios"], + vmrhcho=["Formaldehyde", "3D", "Volume mixing ratios"], + vmrisop=["Isoprene", "3D", "Volume mixing ratios"], # PMs concpm10=["PM10", "3D", "Particle concentrations"], concpm25=["PM2.5", "3D", "Particle concentrations"], @@ -257,6 +262,7 @@ concCocpm25=["OC PM2.5", "3D", "Particle concentration"], concCecpm10=["EC PM10", "3D", "Particle concentration"], concCocpm10=["OC PM10", "3D", "Particle concentration"], + concCoc25=["OC PM2.5", "3D", "Particle concentration"], # Depositions drysox=["DryOXS", "3D", "Deposition"], dryoxs=["FakeDryOXS", "3D", "Deposition"], diff --git a/pyaerocom/io/aux_components_fun.py b/pyaerocom/io/aux_components_fun.py index 0b708f97c..74963928a 100644 --- a/pyaerocom/io/aux_components_fun.py +++ b/pyaerocom/io/aux_components_fun.py @@ -97,6 +97,35 @@ def calc_concNhno3_from_vmr(data): ) +def calc_concNnh3_from_vmr(data): + + return vmr_to_conc( + data, vmr_unit="nmol mol-1", var_name="nh3", to_unit="ug m-3", component_unit="N" + ) + + +def convert_to_ugN(data, var_name): + mult_fun = CUBE_MATHS["multiply"] + data = _check_input_iscube(data)[0] + mmol_var = get_molmass(var_name) + + component_mass = single_component_mass["n"] + component_unit_fac = component_mass / mmol_var + unit = data.units + unit_conversion = cf_units.Unit(str(unit)).convert(1, "ug m-3") + if not np.isclose(unit_conversion, 1, rtol=1e-7): + data = mult_fun(data, unit_conversion) + + data = mult_fun(data, component_unit_fac) + data.units = "ug N m-3" + + return data + + +def calc_concNnh4(concnh4): + return convert_to_ugN(concnh4, "nh4") + + def calc_concno3pm25(concno3f, concno3c, fine_from_coarse_fraction: float = 0.134): mult_fun = CUBE_MATHS["multiply"] concno3pm25 = add_cubes(concno3f, mult_fun(concno3c, fine_from_coarse_fraction)) @@ -105,15 +134,59 @@ def calc_concno3pm25(concno3f, concno3c, fine_from_coarse_fraction: float = 0.13 def calc_concno3pm10(concno3f, concno3c): - mult_fun = CUBE_MATHS["multiply"] concno3pm10 = add_cubes(concno3f, concno3c) return concno3pm10 +def calc_concNno3pm25(concno3f, concno3c, fine_from_coarse_fraction: float = 0.134): + M_N = 14.006 + M_O = 15.999 + M_H = 1.007 + + fac = M_N / (M_H + M_N + M_O * 3) + mult_fun = CUBE_MATHS["multiply"] + concno3f, concno3c = _check_input_iscube(concno3f, concno3c) + concno3f, concno3c = _check_same_units(concno3f, concno3c) + concno3pm25 = add_cubes(concno3f, mult_fun(concno3c, fine_from_coarse_fraction)) + concno3pm25.units = "ug N m-3" + return mult_fun(concno3pm25, fac) + + +def calc_concNno3pm10(concno3f, concno3c): + M_N = 14.006 + M_O = 15.999 + M_H = 1.007 + + fac = M_N / (M_H + M_N + M_O * 3) + mult_fun = CUBE_MATHS["multiply"] + concno3f, concno3c = _check_input_iscube(concno3f, concno3c) + concno3f, concno3c = _check_same_units(concno3f, concno3c) + concno3pm10 = add_cubes(concno3f, concno3c) + concno3pm10.units = "ug N m-3" + return mult_fun(concno3pm10, fac) + + def calc_sspm25(concssfine, concsscoarse): mult_fun = CUBE_MATHS["multiply"] concssfine, concsscoarse = _check_input_iscube(concssfine, concsscoarse) concssfine, concsscoarse = _check_same_units(concssfine, concsscoarse) return add_cubes(concssfine, mult_fun(concsscoarse, 0.16)) + + +def calc_concNtno3(concno3f, concno3c, vmrhno3): + concno3f, concno3c, vmrhno3 = _check_input_iscube(concno3f, concno3c, vmrhno3) + concno3f, concno3c = _check_same_units(concno3f, concno3c) + concNhno3 = calc_concNhno3_from_vmr(vmrhno3) + concNno3pm10 = calc_concNno3pm10(concno3f, concno3c) + + return add_cubes(concNhno3, concNno3pm10) + + +def calc_concNtnh(concnh4, vmrnh3): + concNnh3 = calc_concNnh3_from_vmr(vmrnh3) + concNnh4 = calc_concNnh4(concnh4) + concNnh3, concNnh4 = _check_same_units(concNnh3, concNnh4) + + return add_cubes(concNnh3, concNnh4) diff --git a/pyaerocom/io/readgridded.py b/pyaerocom/io/readgridded.py index cb3ef00e3..63aa164aa 100755 --- a/pyaerocom/io/readgridded.py +++ b/pyaerocom/io/readgridded.py @@ -27,8 +27,14 @@ from pyaerocom.io import AerocomBrowser from pyaerocom.io.aux_components_fun import ( calc_concNhno3_from_vmr, + calc_concNnh3_from_vmr, + calc_concNnh4, + calc_concNno3pm10, + calc_concNno3pm25, calc_concno3pm10, calc_concno3pm25, + calc_concNtnh, + calc_concNtno3, calc_sspm25, vmr_to_conc, ) @@ -135,7 +141,13 @@ class specifying details of the file naming convention for the model "concsspm25": ("concss25", "concsscoarse"), "concno3pm10": ("concno3f", "concno3c"), "concno3pm25": ("concno3f", "concno3c"), + "concNno3pm10": ("concno3f", "concno3c"), + "concNno3pm25": ("concno3f", "concno3c"), "concNhno3": ("vmrhno3",), + "concNtno3": ("concno3f", "concno3c", "vmrhno3"), + "concNnh3": ("vmrnh3",), + "concNnh4": ("concnh4",), + "concNtnh": ("concnh4", "vmrnh3"), } AUX_ALT_VARS = { @@ -164,7 +176,13 @@ class specifying details of the file naming convention for the model "concsspm25": calc_sspm25, "concno3pm10": calc_concno3pm10, "concno3pm25": calc_concno3pm25, + "concNno3pm10": calc_concNno3pm10, + "concNno3pm25": calc_concNno3pm25, "concNhno3": calc_concNhno3_from_vmr, + "concNtno3": calc_concNtno3, + "concNnh3": calc_concNnh3_from_vmr, + "concNnh4": calc_concNnh4, + "concNtnh": calc_concNtnh, #'mec550*' : divide_cubes, #'tau*' : lifetime_from_load_and_dep } diff --git a/pyaerocom/molmasses.py b/pyaerocom/molmasses.py index e17ec0034..cf562a31c 100644 --- a/pyaerocom/molmasses.py +++ b/pyaerocom/molmasses.py @@ -15,6 +15,7 @@ "glyoxal": 58.036, "glyox": 58.036, "hcho": 30.026, + "nh4": 18.04, } From a73f47322937135494dff9b04399995485d7c6f8 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Wed, 23 Nov 2022 10:32:06 +0000 Subject: [PATCH 13/82] Adds fakedrydep and totdep --- pyaerocom/aeroval/experiment_output.py | 3 + pyaerocom/aeroval/glob_defaults.py | 46 +++++++++ pyaerocom/aeroval/setupclasses.py | 1 + pyaerocom/aux_var_helpers.py | 10 ++ pyaerocom/data/ebas_config.ini | 60 +++++++++++ pyaerocom/data/variables.ini | 132 +++++++++++++++++++++++++ pyaerocom/io/aux_components_fun.py | 6 +- pyaerocom/io/ipcforests/reader.py | 7 ++ pyaerocom/io/read_ebas.py | 46 +++++++++ 9 files changed, 308 insertions(+), 3 deletions(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 133804654..5a131b70c 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -16,6 +16,7 @@ from pyaerocom.aeroval.glob_defaults import ( extended_statistics, statistics_defaults, + statistics_model_only, statistics_obs_only, statistics_trend, var_ranges_defaults, @@ -575,6 +576,8 @@ def _create_var_ranges_json(self): def _create_statistics_json(self): if self.cfg.statistics_opts.obs_only_stats: stats_info = statistics_obs_only + elif self.cfg.statistics_opts.model_only_stats: + stats_info = statistics_model_only else: stats_info = statistics_defaults stats_info.update(extended_statistics) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index c25bcd56f..cbd78f93d 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -58,6 +58,21 @@ "dryoxs": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, "dryoxn": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, "dryrdn": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, + "fakedryo3": {"scale": [0, 0.5, 1, 15, 20, 25, 0.30, 40, 50], "colmap": "coolwarm"}, + "fakedrypm10": {"scale": [0, 0.5, 1, 15, 20, 25, 0.30, 40, 50], "colmap": "coolwarm"}, + "fakedrypm25": {"scale": [0, 0.5, 1, 15, 20, 25, 0.30, 40, 50], "colmap": "coolwarm"}, + "fakedryoxs": { + "scale": [0, 0.05, 0.1, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40], + "colmap": "coolwarm", + }, + "fakedryoxn": { + "scale": [0, 0.05, 0.1, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40], + "colmap": "coolwarm", + }, + "fakedryrdn": { + "scale": [0, 0.05, 0.1, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40], + "colmap": "coolwarm", + }, } #: Default information for statistical parameters @@ -214,6 +229,18 @@ statistics_obs_only = { "refdata_mean": { "name": "Mean-Obs", + "longname": "Model Mean", + "scale": None, + "colmap": "coolwarm", + "unit": "1", + "decimals": 2, + }, +} + +# For experiments where only model data is interesting, as with fake drydep +statistics_model_only = { + "data_mean": { + "name": "Mean-Mod", "longname": "Observation Mean", "scale": None, "colmap": "coolwarm", @@ -250,6 +277,7 @@ concNno=["NO", "3D", "Concentration"], concno2=["NO2", "3D", "Gas concentrations"], concNno2=["NO2", "3D", "Gas concentrations"], + vmrno=["NO", "3D", "Volume mixing ratios"], vmrno2=["NO2", "3D", "Volume mixing ratios"], concno3=["NO3", "3D", "Gas concentrations"], conctno3=["tNO3", "3D", "Concentration"], @@ -312,4 +340,22 @@ wetoxn=["WetOXN", "3D", "Deposition"], wetrdn=["WetRDN", "3D", "Deposition"], prmm=["Precipitation", "3D", "Deposition"], + # Fake drydep + fakedryoxs=["FakeDryOXS", "3D", "Deposition"], + fakedryso2=["FakeDrySO2", "3D", "Deposition"], + fakedryso4=["FakeDrySO4", "3D", "Deposition"], + fakedryoxn=["FakeDryOXN", "3D", "Deposition"], + fakedryno2=["FakeDryNO2", "3D", "Deposition"], + fakedryno2no2=["FakeDryNO2NO2", "3D", "Deposition"], + fakedryhono=["FakeDryHONO", "3D", "Deposition"], + fakedryn2o5=["FakeDryN2O5", "3D", "Deposition"], + fakedryhno3=["FakeDryHNO3", "3D", "Deposition"], + fakedryno3c=["FakeDryNO3Coarse", "3D", "Deposition"], + fakedryno3f=["FakeDryNO3Fine", "3D", "Deposition"], + fakedryrdn=["FakeDryRDN", "3D", "Deposition"], + fakedrynh3=["FakeDryNH3", "3D", "Deposition"], + fakedrynh4=["FakeDryNH4", "3D", "Deposition"], + fakedryo3=["FakeDryO3", "3D", "Deposition"], + fakedrypm10=["FakeDryPM10", "3D", "Deposition"], + fakedrypm25=["FakeDryPM2.5", "3D", "Deposition"], ) diff --git a/pyaerocom/aeroval/setupclasses.py b/pyaerocom/aeroval/setupclasses.py index cb9578b2a..ae43e528b 100644 --- a/pyaerocom/aeroval/setupclasses.py +++ b/pyaerocom/aeroval/setupclasses.py @@ -155,6 +155,7 @@ def __init__(self, **kwargs): self.use_fairmode = False self.use_diurnal = False self.obs_only_stats = False + self.model_only_stats = False self.update(**kwargs) diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index 22d49fcaa..683c2d586 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -738,3 +738,13 @@ def calc_vmro3max(data): # print(data.var_info) # exit() return o3max + + +def identity(data): + return data + + +def make_fake_drydep_from_O3(data): + var_name = "vmro3" + data.var_info[var_name]["units"] = "mg m-2 d-1" + return data diff --git a/pyaerocom/data/ebas_config.ini b/pyaerocom/data/ebas_config.ini index 7be17d972..6333c4cfc 100644 --- a/pyaerocom/data/ebas_config.ini +++ b/pyaerocom/data/ebas_config.ini @@ -473,3 +473,63 @@ matrix=precip [wetso4] requires=concprcpso4 +#Fake Dry Dep + +# Sulpher Based dry dep +[fakedryoxs] +requires=concprcpoxs + +[fakedryso2] +requires=concprcpoxs + +[fakedryso4] +requires=concprcpoxs + + +# Oxidized nitrogen based dry dep +[fakedryoxn] +requires=concprcpoxn + +[fakedryno2] +requires=concprcpoxn + +[fakedryno2no2] +requires=concprcpoxn + +[fakedryhono] +requires=concprcpoxn + +[fakedryn2o5] +requires=concprcpoxn + +[fakedryhno3] +requires=concprcpoxn + +[fakedryno3c] +requires=concprcpoxn + +[fakedryno3f] +requires=concprcpoxn + + +# Reduced nitrogen based dry dep +[fakedryrdn] +requires=concprcprdn + +[fakedrynh3] +requires=concprcprdn + +[fakedrynh4] +requires=concprcprdn + + +# Other Fake dry dep + +[fakedryo3] +requires=vmro3 + +[fakedrypm10] +requires=concprcpoxs + +[fakedrypm25] +requires=concprcpoxs \ No newline at end of file diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index acbd2a30a..208d3b92e 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -5396,6 +5396,38 @@ unit = ug m-3 ; Deposition + +[dryoxnf] +description=Dry deposition in forest of oxidized nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[dryrdnf] +description=Dry deposition in forest of reduced Nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[dryoxsf] +description=Dry deposition in forest of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + + +[depoxnf] +description=Total deposition in forest of oxidized nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[deprdnf] +description=Total deposition in forest of reduced Nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[depoxsf] +description=Total deposition in forest of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + [wetno2] var_name = wetno2 description = Wet deposition of nitrogen oxide in total PM @@ -5439,3 +5471,103 @@ unit = mg m-2 d-1 minimum = 0 maximum = 10000 dimensions = time,lat,lon + +# Fake Dry Dep + +# Oxidized nitrogen based dry dep + +[fakedryoxn] +description=Fake dry deposition of oxidized nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[fakedryno2] +description=Fake dry deposition of NO2 +unit = mg N m-2 d-1 +minimum=0 + +[fakedryno2no2] +description=Fake dry deposition of NO2NO2 +unit = mg N m-2 d-1 +minimum=0 + +[fakedryhono] +description=Fake dry deposition of HONO +unit = mg N m-2 d-1 +minimum=0 + +[fakedryn2o5] +description=Fake dry deposition of N2O5 +unit = mg N m-2 d-1 +minimum=0 + +[fakedryhno3] +description=Fake dry deposition of HNO3 +unit = mg N m-2 d-1 +minimum=0 + +[fakedryno3c] +description=Fake dry deposition of NO3 Coarse +unit = mg N m-2 d-1 +minimum=0 + +[fakedryno3f] +description=Fake dry deposition of NO3 Fine +unit = mg N m-2 d-1 +minimum=0 +# Reduced nitrogen based dry dep + +[fakedryrdn] +description=Fake dry deposition of reduced Nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[fakedrynh3] +description=Fake dry deposition of NH3 +unit = mg N m-2 d-1 +minimum=0 + +[fakedrynh4] +description=Fake dry deposition of NH4 +unit = mg N m-2 d-1 +minimum=0 + +# Sulpher Based dry dep + +[fakedryoxs] +description=Fake dry deposition of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + +[fakedryso2] +description=Fake dry deposition of SO2 +unit = mg S m-2 d-1 +minimum=0 + +[fakedryso4] +description=Fake dry deposition of SO4 +unit = mg S m-2 d-1 +minimum=0 + +# Other Fake dry dep + +[fakedryo3] +description=Fake dry deposition of O3 +unit = mg m-2 d-1 +minimum=0 + +[fakedrypm10] +description=Fake dry deposition of PM10 +unit = mg m-2 d-1 +minimum=0 + + +[fakedrypm25] +description=Fake dry deposition of PM25 +unit = mg m-2 d-1 +minimum=0 + +[drypm25] +description=Dry deposition of PM25 +unit = mg m-2 d-1 +minimum=0 \ No newline at end of file diff --git a/pyaerocom/io/aux_components_fun.py b/pyaerocom/io/aux_components_fun.py index 74963928a..b6926aead 100644 --- a/pyaerocom/io/aux_components_fun.py +++ b/pyaerocom/io/aux_components_fun.py @@ -127,10 +127,10 @@ def calc_concNnh4(concnh4): def calc_concno3pm25(concno3f, concno3c, fine_from_coarse_fraction: float = 0.134): - mult_fun = CUBE_MATHS["multiply"] - concno3pm25 = add_cubes(concno3f, mult_fun(concno3c, fine_from_coarse_fraction)) + # mult_fun = CUBE_MATHS["multiply"] + # concno3pm25 = add_cubes(concno3f, mult_fun(concno3c, fine_from_coarse_fraction)) - return concno3pm25 + return concno3f def calc_concno3pm10(concno3f, concno3c): diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index 3b7c82eb0..e3969d487 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -56,6 +56,9 @@ class ReadIPCForest(ReadUngriddedBase): "depoxs": 20, "depoxn": 19, "deprdn": 17, + "depoxsf": 20, + "depoxnf": 19, + "deprdnf": 17, "wetso4": 20, "wetno3": 19, "wetnh4": 17, @@ -74,6 +77,9 @@ class ReadIPCForest(ReadUngriddedBase): "depoxs": "mg S m-2 d-1", "depoxn": "mg N m-2 d-1", "deprdn": "mg N m-2 d-1", + "depoxsf": "mg S m-2 d-1", + "depoxnf": "mg N m-2 d-1", + "deprdnf": "mg N m-2 d-1", "wetso4": "mg S m-2 d-1", "wetno3": "mg N m-2 d-1", "wetnh4": "mg N m-2 d-1", @@ -85,6 +91,7 @@ class ReadIPCForest(ReadUngriddedBase): "wetoxs": 0.3338, "wetso4": 0.3338, "depoxs": 0.3338, + "depoxsf": 0.3338, } SEASALT_FACTORS = { "wetna": 0.120, diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index a6029fb64..ca704f782 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -23,6 +23,8 @@ compute_wetrdn_from_concprcprdn, compute_wetso4_from_concprcpso4, concx_to_vmrx, + identity, + make_fake_drydep_from_O3, vmrx_to_concx, ) from pyaerocom.exceptions import ( @@ -239,6 +241,28 @@ class ReadEbas(ReadUngriddedBase): "wetno3": ["concprcpno3", "pr"], "wetnh4": ["concprcpnh4", "pr"], "vmro3max": ["vmro3"], + # Fake drydep + # Suphar based + "fakedryoxs": ["concprcpoxs", "pr"], + "fakedryso2": ["concprcpoxs", "pr"], + "fakedryso4": ["concprcpoxs", "pr"], + # Oxidized nitrogen based + "fakedryoxn": ["concprcpoxn", "pr"], + "fakedryno2": ["concprcpoxn", "pr"], + "fakedryno2no2": ["concprcpoxn", "pr"], + "fakedryhono": ["concprcpoxn", "pr"], + "fakedryn2o5": ["concprcpoxn", "pr"], + "fakedryhno3": ["concprcpoxn", "pr"], + "fakedryno3c": ["concprcpoxn", "pr"], + "fakedryno3f": ["concprcpoxn", "pr"], + # Reduced nitrogen based + "fakedryrdn": ["concprcprdn", "pr"], + "fakedrynh3": ["concprcprdn", "pr"], + "fakedrynh4": ["concprcprdn", "pr"], + # Other + "fakedryo3": ["vmro3"], + "fakedrypm10": ["concprcpoxs", "pr"], + "fakedrypm25": ["concprcpoxs", "pr"], } #: Meta information supposed to be migrated to computed variables @@ -262,6 +286,28 @@ class ReadEbas(ReadUngriddedBase): "wetno3": compute_wetno3_from_concprcpno3, "wetso4": compute_wetso4_from_concprcpso4, "vmro3max": calc_vmro3max, + # Fake dry dep + # Suphar based + "fakedryoxs": compute_wetoxs_from_concprcpoxs, + "fakedryso2": compute_wetoxs_from_concprcpoxs, + "fakedryso4": compute_wetoxs_from_concprcpoxs, + # Oxidized nitrogen based + "fakedryoxn": compute_wetoxn_from_concprcpoxn, + "fakedryno2": compute_wetoxn_from_concprcpoxn, + "fakedryno2no2": compute_wetoxn_from_concprcpoxn, + "fakedryhono": compute_wetoxn_from_concprcpoxn, + "fakedryn2o5": compute_wetoxn_from_concprcpoxn, + "fakedryhno3": compute_wetoxn_from_concprcpoxn, + "fakedryno3c": compute_wetoxn_from_concprcpoxn, + "fakedryno3f": compute_wetoxn_from_concprcpoxn, + # Reduced nitrogen based + "fakedryrdn": compute_wetrdn_from_concprcprdn, + "fakedrynh3": compute_wetrdn_from_concprcprdn, + "fakedrynh4": compute_wetrdn_from_concprcprdn, + # Other + "fakedryo3": make_fake_drydep_from_O3, + "fakedrypm10": compute_wetoxs_from_concprcpoxs, + "fakedrypm25": compute_wetoxs_from_concprcpoxs, } #: Custom reading options for individual variables. Keys need to be valid From 6c9092418a34610dad12a767f4a9802dcfba1903 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 2 Jan 2023 19:56:30 +0100 Subject: [PATCH 14/82] change path to standard aerocm location --- pyaerocom/data/paths.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini index a3c132784..25feec576 100644 --- a/pyaerocom/data/paths.ini +++ b/pyaerocom/data/paths.ini @@ -104,7 +104,8 @@ AIR_NOW = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/MACC_INSITU_AirNow MARCO_POLO = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/CHINA_MP_NRT #IPCForests -IPCFORESTS = /lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ipc-forests/dep/ +#IPCFORESTS = /lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ipc-forests/dep/ +IPCFORESTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/ipc-forests/dep/ [obsnames] #names of the different obs networks From bedee590c142d7013efdfc9344caafc00f161e72 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Wed, 11 Jan 2023 11:11:04 +0100 Subject: [PATCH 15/82] switch to storeB --- pyaerocom/data/paths.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini index 25feec576..e4ab90013 100644 --- a/pyaerocom/data/paths.ini +++ b/pyaerocom/data/paths.ini @@ -4,15 +4,15 @@ #created 20170824 by Jan Griesfeller for Met Norway [outputfolders] -CACHEDIR=/lustre/storeA/project/aerocom/user_data/pyaerocom_cache -OUTPUTDIR=/lustre/storeA/project/aerocom/aerocom2/pyaerocom_out +CACHEDIR=/lustre/storeB/project/aerocom/user_data/pyaerocom_cache +OUTPUTDIR=/lustre/storeB/project/aerocom/aerocom2/pyaerocom_out # The following is a local temp directory used for extraction of archive files # for maximum speed this should not be located on a network folder # used e.g for satellite level 2 reading when the input file is a tar file LOCALTMPDIR=/home/${USER}/MyPyaerocom/tmp [supplfolders] -BASEDIR=/lustre/storeA/project +BASEDIR=/lustre/storeB/project # Directory contining Etopo1 topography data files ETOPO1 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/topodata/etopo1 ERA5 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/ERA5 @@ -21,7 +21,7 @@ ERA5 = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/ERA5 [modelfolders] #these are the directories to search for the model data -BASEDIR=/lustre/storeA/project +BASEDIR=/lustre/storeB/project dir= ${BASEDIR}/aerocom/aerocom1/, ${BASEDIR}/aerocom/aerocom2/, @@ -55,7 +55,7 @@ dir= [obsfolders] #folders to for model data -BASEDIR=/lustre/storeA/project +BASEDIR=/lustre/storeB/project #Aeronet V2 AERONET_SUN_V2L15_AOD_DAILY = ${BASEDIR}/aerocom/ AERONET_SUN_V2L15_AOD_ALL_POINTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/AeronetSunNRT From 7ba24e0e57062ed0398a39b13bcd32b944066ec9 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Wed, 11 Jan 2023 11:22:20 +0100 Subject: [PATCH 16/82] remove another storeA location --- pyaerocom/config.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pyaerocom/config.py b/pyaerocom/config.py index 11408001b..1a09b87e5 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -5,7 +5,6 @@ from pathlib import Path import numpy as np - from pyaerocom import obs_io from pyaerocom._lowlevel_helpers import ( check_dir_access, @@ -182,7 +181,7 @@ class Config: # these are searched in preferred order both in root and home _DB_SEARCH_SUBDIRS = {} - _DB_SEARCH_SUBDIRS["lustre/storeA/project"] = "metno" + _DB_SEARCH_SUBDIRS["lustre/storeB/project"] = "metno" _DB_SEARCH_SUBDIRS["metno/aerocom_users_database"] = "users-db" _DB_SEARCH_SUBDIRS["MyPyaerocom/data"] = "local-db" @@ -585,14 +584,14 @@ def add_ungridded_obs(self, obs_id, data_dir, reader=None, check_read=False): self._check_obsreader(obs_id, data_dir, reader) def add_ungridded_post_dataset( - self, - obs_id, - obs_vars, - obs_aux_requires, - obs_merge_how, - obs_aux_funs=None, - obs_aux_units=None, - **kwargs, + self, + obs_id, + obs_vars, + obs_aux_requires, + obs_merge_how, + obs_aux_funs=None, + obs_aux_units=None, + **kwargs, ): """ Register new ungridded dataset @@ -719,7 +718,7 @@ def reload(self, keep_basedirs=True): self.read_config(self.last_config_file, keep_basedirs) def read_config( - self, config_file, basedir=None, init_obslocs_ungridded=False, init_data_search_dirs=False + self, config_file, basedir=None, init_obslocs_ungridded=False, init_data_search_dirs=False ): """ Import paths from one of the config ini files From 539a7f025e646c8e13d8cd1807877d0fc8a17db0 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 26 Jan 2023 13:46:33 +0100 Subject: [PATCH 17/82] make compatible to Python versions < 3.10 --- pyaerocom/io/ipcforests/metadata.py | 3 ++- pyaerocom/io/ipcforests/reader.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyaerocom/io/ipcforests/metadata.py b/pyaerocom/io/ipcforests/metadata.py index 94435447b..26b94ee34 100644 --- a/pyaerocom/io/ipcforests/metadata.py +++ b/pyaerocom/io/ipcforests/metadata.py @@ -1,9 +1,10 @@ +from __future__ import annotations + from datetime import datetime, timedelta from typing import Tuple import numpy as np import pandas as pd -from pandas import date_range DEP_TYPE = { 1: "Throughfall", diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index e3969d487..b3b004395 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -1,10 +1,11 @@ +from __future__ import annotations + import logging import os from datetime import datetime from typing import Tuple import numpy as np -from geonum.atmosphere import T0_STD, p0 from tqdm import tqdm from pyaerocom import const @@ -18,7 +19,6 @@ class ReadIPCForest(ReadUngriddedBase): - #: version log of this class (for caching) __version__ = "0.2_" + ReadUngriddedBase.__baseversion__ From 13ff5226e5f3d4d8c7f3cec4288141ffde0f6c2c Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 30 Jan 2023 16:54:42 +0100 Subject: [PATCH 18/82] added udunit for mg/m2/h --- pyaerocom/units_helpers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyaerocom/units_helpers.py b/pyaerocom/units_helpers.py index b17be3f03..eaac03c17 100644 --- a/pyaerocom/units_helpers.py +++ b/pyaerocom/units_helpers.py @@ -64,6 +64,7 @@ # may be used to specify alternative names for custom units defined # in UCONV_MUL_FACS + UALIASES = { # mass concentrations "ug S m-3": "ug S/m3", @@ -80,6 +81,7 @@ # deposition rates (explicit) ## sulphur species "mgS/m2/h": "mg S m-2 h-1", + "mg/m2/h": "mg m-2 h-1", "mgS/m**2/h": "mg S m-2 h-1", "mgSm-2h-1": "mg S m-2 h-1", "mgSm**-2h-1": "mg S m-2 h-1", @@ -280,10 +282,10 @@ def get_unit_conversion_fac(from_unit, to_unit, var_name=None, ts_type=None): return _get_unit_conversion_fac_helper(from_unit, to_unit, var_name) except UnitConversionError: if ( - ts_type is not None - and var_name is not None - and get_variable(var_name).is_rate - and rate_unit_implicit(from_unit) + ts_type is not None + and var_name is not None + and get_variable(var_name).is_rate + and rate_unit_implicit(from_unit) ): freq_si = TsType(ts_type).to_si() from_unit = f"{from_unit} {freq_si}-1" From d6671055656bec2fc916b0ddcc65cce88d8e0e5b Mon Sep 17 00:00:00 2001 From: lewisblake Date: Fri, 3 Feb 2023 09:19:25 +0000 Subject: [PATCH 19/82] implement color scales from Hilde --- pyaerocom/aeroval/glob_defaults.py | 43 +++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index 9818a005b..e9716b2da 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -81,10 +81,10 @@ "scale": [0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0], "colmap": "coolwarm", }, - "wetoxs": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, + "wetoxs": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, "wetoxn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, "wetrdn": { - "scale": [0, 0.75, 1.5, 2.25, 3.0, 3.75, 4.5, 5.25, 6.0, 6.75, 7.5, 8.25], + "scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm", }, "prmm": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, @@ -101,20 +101,51 @@ }, "ts": {"scale": [265, 270, 275, 280, 285, 290, 300, 305, 310, 315, 320], "colmap": "coolwarm"}, "fakedryo3": {"scale": [0, 0.5, 1, 15, 20, 25, 0.30, 40, 50], "colmap": "coolwarm"}, - "fakedrypm10": {"scale": [0, 0.5, 1, 15, 20, 25, 0.30, 40, 50], "colmap": "coolwarm"}, - "fakedrypm25": {"scale": [0, 0.5, 1, 15, 20, 25, 0.30, 40, 50], "colmap": "coolwarm"}, + "fakedrypm10": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "fakedrypm25": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "fakedryno2": {"scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], "colmap": "coolwarm"}, + "fakedryhono": {"scale": [0.001, 0.002, 0.003, 0.004, 0.005, 0.006], "colmap": "coolwarm"}, + "fakedryn2o5": {"scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm"}, + "fakedryhno3": { + "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], + "colmap": "coolwarm", + }, + "fakedryno3c": { + "scale": [0.01, 0.02, 0.03, 0.04, 0.05], + "colmap": "coolwarm", + }, + "fakedryno3f": {"scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm"}, + "fakedrynh3": { + "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80], + "colmap": "coolwarm", + }, + "fakedrynh4": { + "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], + "colmap": "coolwarm", + }, + "fakedryso2": { + "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], + "colmap": "coolwarm", + }, + "fakedryso4": { + "scale": [0.01, 0.02, 0.03, 0.04, 0.05], + "colmap": "coolwarm", + }, "fakedryoxs": { "scale": [0, 0.05, 0.1, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40], "colmap": "coolwarm", }, "fakedryoxn": { - "scale": [0, 0.05, 0.1, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40], + "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80], "colmap": "coolwarm", }, "fakedryrdn": { - "scale": [0, 0.05, 0.1, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40], + "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80], "colmap": "coolwarm", }, + "depoxs": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "depoxn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "deprdn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, } #: Default information for statistical parameters From f10a0aa4fe3208085b300ec0289c8dcefb18ea21 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Fri, 3 Feb 2023 09:22:41 +0000 Subject: [PATCH 20/82] run black --- pyaerocom/config.py | 18 +++++++++--------- pyaerocom/units_helpers.py | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pyaerocom/config.py b/pyaerocom/config.py index a8c6568d5..e0ea258a6 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -581,14 +581,14 @@ def add_ungridded_obs(self, obs_id, data_dir, reader=None, check_read=False): self._check_obsreader(obs_id, data_dir, reader) def add_ungridded_post_dataset( - self, - obs_id, - obs_vars, - obs_aux_requires, - obs_merge_how, - obs_aux_funs=None, - obs_aux_units=None, - **kwargs, + self, + obs_id, + obs_vars, + obs_aux_requires, + obs_merge_how, + obs_aux_funs=None, + obs_aux_units=None, + **kwargs, ): """ Register new ungridded dataset @@ -715,7 +715,7 @@ def reload(self, keep_basedirs=True): self.read_config(self.last_config_file, keep_basedirs) def read_config( - self, config_file, basedir=None, init_obslocs_ungridded=False, init_data_search_dirs=False + self, config_file, basedir=None, init_obslocs_ungridded=False, init_data_search_dirs=False ): """ Import paths from one of the config ini files diff --git a/pyaerocom/units_helpers.py b/pyaerocom/units_helpers.py index eaac03c17..e9371e4b0 100644 --- a/pyaerocom/units_helpers.py +++ b/pyaerocom/units_helpers.py @@ -282,10 +282,10 @@ def get_unit_conversion_fac(from_unit, to_unit, var_name=None, ts_type=None): return _get_unit_conversion_fac_helper(from_unit, to_unit, var_name) except UnitConversionError: if ( - ts_type is not None - and var_name is not None - and get_variable(var_name).is_rate - and rate_unit_implicit(from_unit) + ts_type is not None + and var_name is not None + and get_variable(var_name).is_rate + and rate_unit_implicit(from_unit) ): freq_si = TsType(ts_type).to_si() from_unit = f"{from_unit} {freq_si}-1" From 04fae581c41363c3b58885bc0a92adf14b3e998a Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 6 Feb 2023 14:27:33 +0100 Subject: [PATCH 21/82] added alias prmm = pr used in task4041 --- pyaerocom/data/aliases.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pyaerocom/data/aliases.ini b/pyaerocom/data/aliases.ini index 54aa19ac4..61405b4b9 100644 --- a/pyaerocom/data/aliases.ini +++ b/pyaerocom/data/aliases.ini @@ -49,6 +49,7 @@ dryoxs = drysox concss10 = concsscoarse concsscoarse=concss10 vmrglyoxal = vmrglyox +prmm = pr [alias_families] # so far only works with beginning of name From b3ab8f162d9fc52399deaceb641f286900175e5b Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 7 Feb 2023 15:36:44 +0100 Subject: [PATCH 22/82] added udunit for mg/m2/h --- pyaerocom/aux_var_helpers.py | 38 +++++++++++++++++++--------- pyaerocom/io/ipcforests/reader.py | 42 +++++++++++++++---------------- pyaerocom/units_helpers.py | 10 +++++--- 3 files changed, 53 insertions(+), 37 deletions(-) diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index 683c2d586..e3c615298 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -216,15 +216,15 @@ def compute_od_from_angstromexp(to_lambda, od_ref, lambda_ref, angstrom_coeff): def _calc_od_helper( - data, - var_name, - to_lambda, - od_ref, - lambda_ref, - od_ref_alt=None, - lambda_ref_alt=None, - use_angstrom_coeff="ang4487aer", - treshold_angstrom=None, + data, + var_name, + to_lambda, + od_ref, + lambda_ref, + od_ref_alt=None, + lambda_ref_alt=None, + use_angstrom_coeff="ang4487aer", + treshold_angstrom=None, ): """Helper method for computing ODs @@ -718,7 +718,6 @@ def concx_to_vmrx(data, p_pascal, T_kelvin, conc_unit, mmol_var, mmol_air=None, def calc_vmro3max(data): - var_name = "vmro3" new_var_name = "vmro3max" @@ -745,6 +744,21 @@ def identity(data): def make_fake_drydep_from_O3(data): + # sort of prototype to add a compted variable + # one has to extend the data structures of the station data object + # 'right', but has to return just the data array + # That concept is a bit confusing (why not do everything in data here?) var_name = "vmro3" - data.var_info[var_name]["units"] = "mg m-2 d-1" - return data + new_var_name = "fakedryo3" + + flags = data.data_flagged[var_name] + new_var_data = data[var_name] + units = data.var_info[var_name]["units"] + # data.var_info[new_var_name]["units"] = units + + if not new_var_name in data.var_info: + data.var_info[new_var_name] = {} + data.var_info[new_var_name] = data.var_info[var_name] + + data.data_flagged[new_var_name] = flags + return new_var_data diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index b3b004395..0fbb4574d 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -210,8 +210,8 @@ def read_file(self, filename, vars_to_retrieve=None): # 8 is the code for "do not use" if ( - self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE - or sampler_code == 8 + self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE + or sampler_code == 8 ): continue @@ -273,14 +273,14 @@ def read_file(self, filename, vars_to_retrieve=None): if species in self.SEASALT_CORRECTION: na_factor = ( - self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") - * self.SEASALT_FACTORS["wetna"] - * self.SEASALT_CORRECTION[species] + self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") + * self.SEASALT_FACTORS["wetna"] + * self.SEASALT_CORRECTION[species] ) cl_factor = ( - self._get_species_conc(words[self.VAR_POSITION["wetcl"]], "wetcl") - * self.SEASALT_FACTORS["wetcl"] - * self.SEASALT_CORRECTION[species] + self._get_species_conc(words[self.VAR_POSITION["wetcl"]], "wetcl") + * self.SEASALT_FACTORS["wetcl"] + * self.SEASALT_CORRECTION[species] ) seasalt_correction = min(na_factor, cl_factor) @@ -345,14 +345,14 @@ def _get_species_conc(self, conc_str: str, species: str) -> float: return conc def _get_days_date_ts_type( - self, - year: int, - country_code: int, - plot_code: int, - sampler_code: int, - period: int, - start: str | datetime, - stop: str | datetime, + self, + year: int, + country_code: int, + plot_code: int, + sampler_code: int, + period: int, + start: str | datetime, + stop: str | datetime, ) -> Tuple[float | None, datetime | None, str | None]: if start != "" and stop != "": @@ -397,11 +397,11 @@ def _get_tstype(self, start: datetime, stop: datetime) -> str: return SurveyYear.get_tstype(days) def _clean_data_with_flags( - self, - data: list[float], - time: list[datetime], - flags: list[int], - species: str, + self, + data: list[float], + time: list[datetime], + flags: list[int], + species: str, ) -> list[float]: data_array = np.array(data) diff --git a/pyaerocom/units_helpers.py b/pyaerocom/units_helpers.py index e9371e4b0..9532e237d 100644 --- a/pyaerocom/units_helpers.py +++ b/pyaerocom/units_helpers.py @@ -39,6 +39,8 @@ #: factor UCONV_MUL_FACS = pd.DataFrame( [ + # ["dryso4", "mg/m2/d", "mgS m-2 d-1", M_S / M_SO4], + # ["drynh4", "mg/m2/d", "mgN m-2 d-1", M_N/ M_NH4], # ["concso4", "ug S/m3", "ug m-3", M_SO4 / M_S], # ["SO4ugSm3", "ug/m3", "ug S m-3", M_S / M_SO4], # ["concso4pm25", "ug S/m3", "ug m-3", M_SO4 / M_S], @@ -282,10 +284,10 @@ def get_unit_conversion_fac(from_unit, to_unit, var_name=None, ts_type=None): return _get_unit_conversion_fac_helper(from_unit, to_unit, var_name) except UnitConversionError: if ( - ts_type is not None - and var_name is not None - and get_variable(var_name).is_rate - and rate_unit_implicit(from_unit) + ts_type is not None + and var_name is not None + and get_variable(var_name).is_rate + and rate_unit_implicit(from_unit) ): freq_si = TsType(ts_type).to_si() from_unit = f"{from_unit} {freq_si}-1" From e6471625ea6c8ccf3ff0fc7c8e5922f191912cdf Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Wed, 8 Feb 2023 14:33:43 +0100 Subject: [PATCH 23/82] create fake wetdep variables for EBASMC obs network --- pyaerocom/aux_var_helpers.py | 23 ++++++++ pyaerocom/data/ebas_config.ini | 61 +++++++++++++++++++++ pyaerocom/data/variables.ini | 97 +++++++++++++++++++++++++++++++++- pyaerocom/io/read_ebas.py | 45 ++++++++++++++++ 4 files changed, 225 insertions(+), 1 deletion(-) diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index e3c615298..ef81c5ced 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -718,6 +718,7 @@ def concx_to_vmrx(data, p_pascal, T_kelvin, conc_unit, mmol_var, mmol_air=None, def calc_vmro3max(data): + var_name = "vmro3" new_var_name = "vmro3max" @@ -759,6 +760,28 @@ def make_fake_drydep_from_O3(data): if not new_var_name in data.var_info: data.var_info[new_var_name] = {} data.var_info[new_var_name] = data.var_info[var_name] + data.var_info[new_var_name]["units"] = "mg m-2 d-1" + + data.data_flagged[new_var_name] = flags + return new_var_data + +def make_fake_wetdep_from_O3(data): + # sort of prototype to add a compted variable + # one has to extend the data structures of the station data object + # 'right', but has to return just the data array + # That concept is a bit confusing (why not do everything in data here?) + var_name = "vmro3" + new_var_name = "fakeweto3" + + flags = data.data_flagged[var_name] + new_var_data = data[var_name] + units = data.var_info[var_name]["units"] + # data.var_info[new_var_name]["units"] = units + + if not new_var_name in data.var_info: + data.var_info[new_var_name] = {} + data.var_info[new_var_name] = data.var_info[var_name] + data.var_info[new_var_name]["units"] = "mg m-2 d-1" data.data_flagged[new_var_name] = flags return new_var_data diff --git a/pyaerocom/data/ebas_config.ini b/pyaerocom/data/ebas_config.ini index b77716c90..dd58db9c5 100644 --- a/pyaerocom/data/ebas_config.ini +++ b/pyaerocom/data/ebas_config.ini @@ -536,4 +536,65 @@ requires=vmro3 requires=concprcpoxs [fakedrypm25] +requires=concprcpoxs + +#Fake wet Dep + +# Sulpher Based wet dep +[fakewetoxs] +requires=concprcpoxs + +[fakewetso2] +requires=concprcpoxs + +[fakewetso4] +requires=concprcpoxs + + +# Oxidized nitrogen based wet dep +[fakewetoxn] +requires=concprcpoxn + +[fakewetno2] +requires=concprcpoxn + +[fakewetno2no2] +requires=concprcpoxn + +[fakewethono] +requires=concprcpoxn + +[fakewetn2o5] +requires=concprcpoxn + +[fakewethno3] +requires=concprcpoxn + +[fakewetno3c] +requires=concprcpoxn + +[fakewetno3f] +requires=concprcpoxn + + +# Reduced nitrogen based wet dep +[fakewetrdn] +requires=concprcprdn + +[fakewetnh3] +requires=concprcprdn + +[fakewetnh4] +requires=concprcprdn + + +# Other Fake wet dep + +[fakeweto3] +requires=vmro3 + +[fakewetpm10] +requires=concprcpoxs + +[fakewetpm25] requires=concprcpoxs \ No newline at end of file diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index 208d3b92e..170401346 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -5570,4 +5570,99 @@ minimum=0 [drypm25] description=Dry deposition of PM25 unit = mg m-2 d-1 -minimum=0 \ No newline at end of file +minimum=0 + +# Fake wet Dep + +# Oxidized nitrogen based wet dep + +[fakewetoxn] +description=Fake wet deposition of oxidized nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[fakewetno2] +description=Fake wet deposition of NO2 +unit = mg N m-2 d-1 +minimum=0 + +[fakewetno2no2] +description=Fake wet deposition of NO2NO2 +unit = mg N m-2 d-1 +minimum=0 + +[fakewethono] +description=Fake wet deposition of HONO +unit = mg N m-2 d-1 +minimum=0 + +[fakewetn2o5] +description=Fake wet deposition of N2O5 +unit = mg N m-2 d-1 +minimum=0 + +[fakewethno3] +description=Fake wet deposition of HNO3 +unit = mg N m-2 d-1 +minimum=0 + +[fakewetno3c] +description=Fake wet deposition of NO3 Coarse +unit = mg N m-2 d-1 +minimum=0 + +[fakewetno3f] +description=Fake wet deposition of NO3 Fine +unit = mg N m-2 d-1 +minimum=0 +# Reduced nitrogen based wet dep + +[fakewetrdn] +description=Fake wet deposition of reduced Nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[fakewetnh3] +description=Fake wet deposition of NH3 +unit = mg N m-2 d-1 +minimum=0 + +[fakewetnh4] +description=Fake wet deposition of NH4 +unit = mg N m-2 d-1 +minimum=0 + +# Sulpher Based wet dep + +[fakewetoxs] +description=Fake wet deposition of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + +[fakewetso2] +description=Fake wet deposition of SO2 +unit = mg S m-2 d-1 +minimum=0 + +[fakewetso4] +description=Fake wet deposition of SO4 +unit = mg S m-2 d-1 +minimum=0 + +# Other Fake wet dep + +[fakeweto3] +description=Fake wet deposition of O3 +unit = mg m-2 d-1 +minimum=0 + +[fakewetpm10] +description=Fake wet deposition of PM10 +unit = mg m-2 d-1 +minimum=0 + + +[fakewetpm25] +description=Fake wet deposition of PM25 +unit = mg m-2 d-1 +minimum=0 diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index ca704f782..dd9ae6c44 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -25,6 +25,7 @@ concx_to_vmrx, identity, make_fake_drydep_from_O3, + make_fake_wetdep_from_O3, vmrx_to_concx, ) from pyaerocom.exceptions import ( @@ -263,6 +264,28 @@ class ReadEbas(ReadUngriddedBase): "fakedryo3": ["vmro3"], "fakedrypm10": ["concprcpoxs", "pr"], "fakedrypm25": ["concprcpoxs", "pr"], + # Fake wetdep + # Suphar based + "fakewetoxs": ["concprcpoxs", "pr"], + "fakewetso2": ["concprcpoxs", "pr"], + "fakewetso4": ["concprcpoxs", "pr"], + # Oxidized nitrogen based + "fakewetoxn": ["concprcpoxn", "pr"], + "fakewetno2": ["concprcpoxn", "pr"], + "fakewetno2no2": ["concprcpoxn", "pr"], + "fakewethono": ["concprcpoxn", "pr"], + "fakewetn2o5": ["concprcpoxn", "pr"], + "fakewethno3": ["concprcpoxn", "pr"], + "fakewetno3c": ["concprcpoxn", "pr"], + "fakewetno3f": ["concprcpoxn", "pr"], + # Reduced nitrogen based + "fakewetrdn": ["concprcprdn", "pr"], + "fakewetnh3": ["concprcprdn", "pr"], + "fakewetnh4": ["concprcprdn", "pr"], + # Other + "fakeweto3": ["vmro3"], + "fakewetpm10": ["concprcpoxs", "pr"], + "fakewetpm25": ["concprcpoxs", "pr"], } #: Meta information supposed to be migrated to computed variables @@ -308,6 +331,28 @@ class ReadEbas(ReadUngriddedBase): "fakedryo3": make_fake_drydep_from_O3, "fakedrypm10": compute_wetoxs_from_concprcpoxs, "fakedrypm25": compute_wetoxs_from_concprcpoxs, + # Fake wet dep + # Suphar based + "fakewetoxs": compute_wetoxs_from_concprcpoxs, + "fakewetso2": compute_wetoxs_from_concprcpoxs, + "fakewetso4": compute_wetoxs_from_concprcpoxs, + # Oxidized nitrogen based + "fakewetoxn": compute_wetoxn_from_concprcpoxn, + "fakewetno2": compute_wetoxn_from_concprcpoxn, + "fakewetno2no2": compute_wetoxn_from_concprcpoxn, + "fakewethono": compute_wetoxn_from_concprcpoxn, + "fakewetn2o5": compute_wetoxn_from_concprcpoxn, + "fakewethno3": compute_wetoxn_from_concprcpoxn, + "fakewetno3c": compute_wetoxn_from_concprcpoxn, + "fakewetno3f": compute_wetoxn_from_concprcpoxn, + # Reduced nitrogen based + "fakewetrdn": compute_wetrdn_from_concprcprdn, + "fakewetnh3": compute_wetrdn_from_concprcprdn, + "fakewetnh4": compute_wetrdn_from_concprcprdn, + # Other + "fakeweto3": make_fake_wetdep_from_O3, + "fakewetpm10": compute_wetoxs_from_concprcpoxs, + "fakewetpm25": compute_wetoxs_from_concprcpoxs, } #: Custom reading options for individual variables. Keys need to be valid From f96b93909c27c11a50bf998de347565363328172 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Fri, 10 Feb 2023 15:53:10 +0100 Subject: [PATCH 24/82] removed raise MetaDataError for data without flags. See comment for details --- pyaerocom/ungriddeddata.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pyaerocom/ungriddeddata.py b/pyaerocom/ungriddeddata.py index 8578b84d8..917b40035 100644 --- a/pyaerocom/ungriddeddata.py +++ b/pyaerocom/ungriddeddata.py @@ -1883,12 +1883,20 @@ def apply_filters(self, var_outlier_ranges=None, **filter_attributes): ) if set_flags_nan: if not data.has_flag_data: - raise MetaDataError( + # jgriesfeller 20230210 + # not sure if raising this exception is the right thing to do + # the fake variables (vars computed from other variables) might not have + # and do not need flags (because that has been done during the read of the + # variable they are computed from) + # disabling and logging it for now + # raise MetaDataError( + logger.info( 'Cannot apply filter "set_flags_nan" to ' "UngriddedData object, since it does not " "contain flag information" ) - data = data.set_flags_nan(inplace=True) + else: + data = data.set_flags_nan(inplace=True) if region_id: data = data.filter_region(region_id) return data From 9c92244f556e97c21e621f80721b886de5c619bd Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Wed, 15 Feb 2023 14:51:57 +0100 Subject: [PATCH 25/82] updated version --- pyaerocom/io/ipcforests/reader.py | 44 +++++++++++++++---------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/io/ipcforests/reader.py index 0fbb4574d..e333cc619 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/io/ipcforests/reader.py @@ -20,7 +20,7 @@ class ReadIPCForest(ReadUngriddedBase): #: version log of this class (for caching) - __version__ = "0.2_" + ReadUngriddedBase.__baseversion__ + __version__ = "0.3_" + ReadUngriddedBase.__baseversion__ #: Name of dataset (OBS_ID) DATA_ID = const.IPCFORESTS_NAME @@ -210,8 +210,8 @@ def read_file(self, filename, vars_to_retrieve=None): # 8 is the code for "do not use" if ( - self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE - or sampler_code == 8 + self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE + or sampler_code == 8 ): continue @@ -273,14 +273,14 @@ def read_file(self, filename, vars_to_retrieve=None): if species in self.SEASALT_CORRECTION: na_factor = ( - self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") - * self.SEASALT_FACTORS["wetna"] - * self.SEASALT_CORRECTION[species] + self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") + * self.SEASALT_FACTORS["wetna"] + * self.SEASALT_CORRECTION[species] ) cl_factor = ( - self._get_species_conc(words[self.VAR_POSITION["wetcl"]], "wetcl") - * self.SEASALT_FACTORS["wetcl"] - * self.SEASALT_CORRECTION[species] + self._get_species_conc(words[self.VAR_POSITION["wetcl"]], "wetcl") + * self.SEASALT_FACTORS["wetcl"] + * self.SEASALT_CORRECTION[species] ) seasalt_correction = min(na_factor, cl_factor) @@ -345,14 +345,14 @@ def _get_species_conc(self, conc_str: str, species: str) -> float: return conc def _get_days_date_ts_type( - self, - year: int, - country_code: int, - plot_code: int, - sampler_code: int, - period: int, - start: str | datetime, - stop: str | datetime, + self, + year: int, + country_code: int, + plot_code: int, + sampler_code: int, + period: int, + start: str | datetime, + stop: str | datetime, ) -> Tuple[float | None, datetime | None, str | None]: if start != "" and stop != "": @@ -397,11 +397,11 @@ def _get_tstype(self, start: datetime, stop: datetime) -> str: return SurveyYear.get_tstype(days) def _clean_data_with_flags( - self, - data: list[float], - time: list[datetime], - flags: list[int], - species: str, + self, + data: list[float], + time: list[datetime], + flags: list[int], + species: str, ) -> list[float]: data_array = np.array(data) From 6776152426e3006100db0d063548019f8c3b7fa8 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 23 Mar 2023 14:19:06 +0100 Subject: [PATCH 26/82] add apt update to get recent IPs --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 455805baf..a6e708afc 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -46,6 +46,8 @@ jobs: run: | which python python --version + - name: Update apt IPs + run: sudo apt update - name: Install system pacakges run: sudo apt install libudunits2-dev libgeos-dev libproj-dev proj-data proj-bin pandoc - name: Install Tox From 8921f825fa45daa651d65ad7cf1985f031e67373 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 23 Mar 2023 14:25:29 +0100 Subject: [PATCH 27/82] move apt update from docs yo venv --- .github/workflows/CI.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a6e708afc..6ac1f28f3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -21,6 +21,8 @@ jobs: run: | which python python --version + - name: Update apt IPs + run: sudo apt update - name: Install system pacakges run: sudo apt install libudunits2-dev libgeos-dev libproj-dev proj-data proj-bin - name: Install Tox @@ -46,9 +48,7 @@ jobs: run: | which python python --version - - name: Update apt IPs - run: sudo apt update - - name: Install system pacakges + - name: Install system packages run: sudo apt install libudunits2-dev libgeos-dev libproj-dev proj-data proj-bin pandoc - name: Install Tox run: | @@ -79,7 +79,9 @@ jobs: run: | which python python --version - - name: Install system pacakges + - name: Update apt IPs + run: sudo apt update + - name: Install system packages run: sudo apt install libudunits2-dev libgeos-dev libproj-dev proj-data proj-bin - name: Cache pip and tox uses: actions/cache@v3 From 7a639f2cafa44d0082aeaabc2712260d778c226f Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 23 Mar 2023 14:32:55 +0100 Subject: [PATCH 28/82] linting --- pyaerocom/aux_var_helpers.py | 19 ++++++++++--------- pyaerocom/config.py | 1 + pyaerocom/units_helpers.py | 8 ++++---- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index ef81c5ced..66853624d 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -216,15 +216,15 @@ def compute_od_from_angstromexp(to_lambda, od_ref, lambda_ref, angstrom_coeff): def _calc_od_helper( - data, - var_name, - to_lambda, - od_ref, - lambda_ref, - od_ref_alt=None, - lambda_ref_alt=None, - use_angstrom_coeff="ang4487aer", - treshold_angstrom=None, + data, + var_name, + to_lambda, + od_ref, + lambda_ref, + od_ref_alt=None, + lambda_ref_alt=None, + use_angstrom_coeff="ang4487aer", + treshold_angstrom=None, ): """Helper method for computing ODs @@ -765,6 +765,7 @@ def make_fake_drydep_from_O3(data): data.data_flagged[new_var_name] = flags return new_var_data + def make_fake_wetdep_from_O3(data): # sort of prototype to add a compted variable # one has to extend the data structures of the station data object diff --git a/pyaerocom/config.py b/pyaerocom/config.py index e0ea258a6..5e7ffb9c0 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -5,6 +5,7 @@ from pathlib import Path import numpy as np + from pyaerocom import obs_io from pyaerocom._lowlevel_helpers import ( check_dir_access, diff --git a/pyaerocom/units_helpers.py b/pyaerocom/units_helpers.py index 9532e237d..f62f73a07 100644 --- a/pyaerocom/units_helpers.py +++ b/pyaerocom/units_helpers.py @@ -284,10 +284,10 @@ def get_unit_conversion_fac(from_unit, to_unit, var_name=None, ts_type=None): return _get_unit_conversion_fac_helper(from_unit, to_unit, var_name) except UnitConversionError: if ( - ts_type is not None - and var_name is not None - and get_variable(var_name).is_rate - and rate_unit_implicit(from_unit) + ts_type is not None + and var_name is not None + and get_variable(var_name).is_rate + and rate_unit_implicit(from_unit) ): freq_si = TsType(ts_type).to_si() from_unit = f"{from_unit} {freq_si}-1" From 30ef25960778db4b457c2203bb5f55754bf89c9f Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 23 Mar 2023 16:29:42 +0100 Subject: [PATCH 29/82] move code to plugins --- pyaerocom/{io => plugins}/ipcforests/__init__.py | 0 pyaerocom/{io => plugins}/ipcforests/metadata.py | 0 pyaerocom/{io => plugins}/ipcforests/reader.py | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename pyaerocom/{io => plugins}/ipcforests/__init__.py (100%) rename pyaerocom/{io => plugins}/ipcforests/metadata.py (100%) rename pyaerocom/{io => plugins}/ipcforests/reader.py (99%) diff --git a/pyaerocom/io/ipcforests/__init__.py b/pyaerocom/plugins/ipcforests/__init__.py similarity index 100% rename from pyaerocom/io/ipcforests/__init__.py rename to pyaerocom/plugins/ipcforests/__init__.py diff --git a/pyaerocom/io/ipcforests/metadata.py b/pyaerocom/plugins/ipcforests/metadata.py similarity index 100% rename from pyaerocom/io/ipcforests/metadata.py rename to pyaerocom/plugins/ipcforests/metadata.py diff --git a/pyaerocom/io/ipcforests/reader.py b/pyaerocom/plugins/ipcforests/reader.py similarity index 99% rename from pyaerocom/io/ipcforests/reader.py rename to pyaerocom/plugins/ipcforests/reader.py index e333cc619..99bab7cc8 100644 --- a/pyaerocom/io/ipcforests/reader.py +++ b/pyaerocom/plugins/ipcforests/reader.py @@ -10,7 +10,7 @@ from pyaerocom import const from pyaerocom._lowlevel_helpers import BrowseDict -from pyaerocom.io.ipcforests.metadata import MetadataReader, Station, SurveyYear +from pyaerocom.plugins.ipcforests.metadata import MetadataReader, Station, SurveyYear from pyaerocom.io.readungriddedbase import ReadUngriddedBase from pyaerocom.stationdata import StationData from pyaerocom.ungriddeddata import UngriddedData From 2f34f03392435fb6d811c1e5b28d826dd32ba37d Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 23 Mar 2023 16:30:25 +0100 Subject: [PATCH 30/82] move code location --- pyaerocom/io/readungridded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyaerocom/io/readungridded.py b/pyaerocom/io/readungridded.py index 53e7b231c..44494b103 100755 --- a/pyaerocom/io/readungridded.py +++ b/pyaerocom/io/readungridded.py @@ -14,7 +14,7 @@ from pyaerocom.exceptions import DataRetrievalError, NetworkNotImplemented, NetworkNotSupported from pyaerocom.helpers import varlist_aerocom from pyaerocom.io.cachehandler_ungridded import CacheHandlerUngridded -from pyaerocom.io.ipcforests.reader import ReadIPCForest +from pyaerocom.plugins.ipcforests.reader import ReadIPCForest from pyaerocom.io.read_aasetal import ReadAasEtal from pyaerocom.io.read_aeronet_invv2 import ReadAeronetInvV2 from pyaerocom.io.read_aeronet_invv3 import ReadAeronetInvV3 From a9ed9822d6edcbae77720688efdee4851382cb47 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 23 Mar 2023 16:32:15 +0100 Subject: [PATCH 31/82] linting --- pyaerocom/io/readungridded.py | 2 +- pyaerocom/plugins/ipcforests/reader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom/io/readungridded.py b/pyaerocom/io/readungridded.py index 44494b103..a4d257661 100755 --- a/pyaerocom/io/readungridded.py +++ b/pyaerocom/io/readungridded.py @@ -14,7 +14,6 @@ from pyaerocom.exceptions import DataRetrievalError, NetworkNotImplemented, NetworkNotSupported from pyaerocom.helpers import varlist_aerocom from pyaerocom.io.cachehandler_ungridded import CacheHandlerUngridded -from pyaerocom.plugins.ipcforests.reader import ReadIPCForest from pyaerocom.io.read_aasetal import ReadAasEtal from pyaerocom.io.read_aeronet_invv2 import ReadAeronetInvV2 from pyaerocom.io.read_aeronet_invv3 import ReadAeronetInvV3 @@ -28,6 +27,7 @@ from pyaerocom.io.read_eea_aqerep import ReadEEAAQEREP from pyaerocom.io.read_eea_aqerep_v2 import ReadEEAAQEREP_V2 from pyaerocom.io.read_marcopolo import ReadMarcoPolo +from pyaerocom.plugins.ipcforests.reader import ReadIPCForest from pyaerocom.ungriddeddata import UngriddedData from pyaerocom.variable import get_aliases diff --git a/pyaerocom/plugins/ipcforests/reader.py b/pyaerocom/plugins/ipcforests/reader.py index 99bab7cc8..70801c898 100644 --- a/pyaerocom/plugins/ipcforests/reader.py +++ b/pyaerocom/plugins/ipcforests/reader.py @@ -10,8 +10,8 @@ from pyaerocom import const from pyaerocom._lowlevel_helpers import BrowseDict -from pyaerocom.plugins.ipcforests.metadata import MetadataReader, Station, SurveyYear from pyaerocom.io.readungriddedbase import ReadUngriddedBase +from pyaerocom.plugins.ipcforests.metadata import MetadataReader, Station, SurveyYear from pyaerocom.stationdata import StationData from pyaerocom.ungriddeddata import UngriddedData From 59fc53e43a0421348e29f7eed59f605b4da2b2ae Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Fri, 24 Mar 2023 15:29:09 +0100 Subject: [PATCH 32/82] adjust tests to new variables --- tests/io/test_read_ebas.py | 47 +++++++++++++++++++++++++++++++++++++ tests/test_varcollection.py | 4 ++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/tests/io/test_read_ebas.py b/tests/io/test_read_ebas.py index 1e7869546..36d54e332 100644 --- a/tests/io/test_read_ebas.py +++ b/tests/io/test_read_ebas.py @@ -311,6 +311,53 @@ def test_PROVIDES_VARIABLES(reader: ReadEbas): "concso4coarse", "concnh4coarse", "concno3pm25", + 'vmrnh3', + 'fakedryoxn', + 'fakewetpm25', + 'concss25', + 'concprcpno3', + 'concprcpso4', + 'concCoc25', + 'concom25', + 'concprcpnh4', + 'concsscoarse', + 'fakedryhno3', + 'fakedryhono', + 'fakedryn2o5', + 'fakedrynh3', + 'fakedrynh4', + 'fakedryno2', + 'fakedryno2no2', + 'fakedryno3c', + 'fakedryno3f', + 'fakedryo3', + 'fakedryoxs', + 'fakedrypm10', + 'fakedrypm25', + 'fakedryrdn', + 'fakedryso2', + 'fakedryso4', + 'fakewethno3', + 'fakewethono', + 'fakewetn2o5', + 'fakewetnh3', + 'fakewetnh4', + 'fakewetno2', + 'fakewetno2no2', + 'fakewetno3c', + 'fakewetno3f', + 'fakeweto3', + 'fakewetoxn', + 'fakewetoxs', + 'fakewetpm10', + 'fakewetrdn', + 'fakewetso2', + 'fakewetso4', + 'vmrhno3', + 'vmrtp', + 'wetnh4', + 'wetno3', + 'wetso4', } assert set(reader.PROVIDES_VARIABLES) == (PROVIDES_VARIABLES) diff --git a/tests/test_varcollection.py b/tests/test_varcollection.py index 2390c82ca..d5fcc2ab0 100644 --- a/tests/test_varcollection.py +++ b/tests/test_varcollection.py @@ -81,9 +81,9 @@ def test_VarCollection_get_var_error(collection: VarCollection): "search_pattern,num", [ ("*blaaaaaaa*", 0), - ("dep*", 0), + ("dep*", 6), ("od*", 26), - ("conc*", 82), + ("conc*", 87), ], ) def test_VarCollection_find(collection: VarCollection, search_pattern: str, num: int): From 157b96939f31430845c294c6132f9acaa43b1334 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Fri, 24 Mar 2023 16:03:12 +0100 Subject: [PATCH 33/82] remove private path --- pyaerocom/data/paths.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini index e4ab90013..094448264 100644 --- a/pyaerocom/data/paths.ini +++ b/pyaerocom/data/paths.ini @@ -50,7 +50,6 @@ dir= ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND2/, ${BASEDIR}/fou/kl/CAMS61/, ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/, - /lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/mod/deposition, /lustre/storeB/project/fou/kl/CAMS2_40/task4041/ [obsfolders] From 74d47b71b66ac5bcc85061bbadef4fde1b1439b7 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Fri, 24 Mar 2023 16:03:37 +0100 Subject: [PATCH 34/82] remove private clutter --- pyaerocom/plugins/ipcforests/reader.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyaerocom/plugins/ipcforests/reader.py b/pyaerocom/plugins/ipcforests/reader.py index 70801c898..61f164f31 100644 --- a/pyaerocom/plugins/ipcforests/reader.py +++ b/pyaerocom/plugins/ipcforests/reader.py @@ -419,9 +419,3 @@ def _clean_data_with_flags( return list(data_array) -if __name__ == "__main__": - reader = ReadIPCForest( - data_dir="/lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ipc-forests/dep" - ) - filename = "/lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ipc-forests/dep/dp_dem.csv" - data = reader.read_file(filename) From 2fa2dbc53e6596c13bd7f70159feeb6c70836a08 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Fri, 24 Mar 2023 16:04:06 +0100 Subject: [PATCH 35/82] linting --- tests/io/test_read_ebas.py | 94 +++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/tests/io/test_read_ebas.py b/tests/io/test_read_ebas.py index 36d54e332..fc5a83009 100644 --- a/tests/io/test_read_ebas.py +++ b/tests/io/test_read_ebas.py @@ -311,53 +311,53 @@ def test_PROVIDES_VARIABLES(reader: ReadEbas): "concso4coarse", "concnh4coarse", "concno3pm25", - 'vmrnh3', - 'fakedryoxn', - 'fakewetpm25', - 'concss25', - 'concprcpno3', - 'concprcpso4', - 'concCoc25', - 'concom25', - 'concprcpnh4', - 'concsscoarse', - 'fakedryhno3', - 'fakedryhono', - 'fakedryn2o5', - 'fakedrynh3', - 'fakedrynh4', - 'fakedryno2', - 'fakedryno2no2', - 'fakedryno3c', - 'fakedryno3f', - 'fakedryo3', - 'fakedryoxs', - 'fakedrypm10', - 'fakedrypm25', - 'fakedryrdn', - 'fakedryso2', - 'fakedryso4', - 'fakewethno3', - 'fakewethono', - 'fakewetn2o5', - 'fakewetnh3', - 'fakewetnh4', - 'fakewetno2', - 'fakewetno2no2', - 'fakewetno3c', - 'fakewetno3f', - 'fakeweto3', - 'fakewetoxn', - 'fakewetoxs', - 'fakewetpm10', - 'fakewetrdn', - 'fakewetso2', - 'fakewetso4', - 'vmrhno3', - 'vmrtp', - 'wetnh4', - 'wetno3', - 'wetso4', + "vmrnh3", + "fakedryoxn", + "fakewetpm25", + "concss25", + "concprcpno3", + "concprcpso4", + "concCoc25", + "concom25", + "concprcpnh4", + "concsscoarse", + "fakedryhno3", + "fakedryhono", + "fakedryn2o5", + "fakedrynh3", + "fakedrynh4", + "fakedryno2", + "fakedryno2no2", + "fakedryno3c", + "fakedryno3f", + "fakedryo3", + "fakedryoxs", + "fakedrypm10", + "fakedrypm25", + "fakedryrdn", + "fakedryso2", + "fakedryso4", + "fakewethno3", + "fakewethono", + "fakewetn2o5", + "fakewetnh3", + "fakewetnh4", + "fakewetno2", + "fakewetno2no2", + "fakewetno3c", + "fakewetno3f", + "fakeweto3", + "fakewetoxn", + "fakewetoxs", + "fakewetpm10", + "fakewetrdn", + "fakewetso2", + "fakewetso4", + "vmrhno3", + "vmrtp", + "wetnh4", + "wetno3", + "wetso4", } assert set(reader.PROVIDES_VARIABLES) == (PROVIDES_VARIABLES) From 052a514ffb2e3519633b5d364b771806407c3afb Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 27 Mar 2023 10:01:29 +0200 Subject: [PATCH 36/82] linting --- pyaerocom/plugins/ipcforests/reader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyaerocom/plugins/ipcforests/reader.py b/pyaerocom/plugins/ipcforests/reader.py index 61f164f31..860054fbd 100644 --- a/pyaerocom/plugins/ipcforests/reader.py +++ b/pyaerocom/plugins/ipcforests/reader.py @@ -417,5 +417,3 @@ def _clean_data_with_flags( data_array[np.where(years == year)] = np.nan return list(data_array) - - From bb6b8cb133ee0976eb3dea1fba7a97e20e608cde Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 27 Mar 2023 10:52:38 +0200 Subject: [PATCH 37/82] remove clutter --- pyaerocom/plugins/ipcforests/metadata.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pyaerocom/plugins/ipcforests/metadata.py b/pyaerocom/plugins/ipcforests/metadata.py index 26b94ee34..912bde80d 100644 --- a/pyaerocom/plugins/ipcforests/metadata.py +++ b/pyaerocom/plugins/ipcforests/metadata.py @@ -510,11 +510,3 @@ def _get_altitude_dir(self) -> dict[str, int]: altitudes[words[0]] = int(words[4]) + (int(words[5]) - int(words[4])) // 2 return altitudes - - -if __name__ == "__main__": - metadata = MetadataReader( - "/lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ipc-forests/dep" - ) - breakpoint() - # print(metadata.altitudes) From 99a73a8d84485ca824c93a7c0066dd6228c286d5 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 27 Mar 2023 13:32:39 +0200 Subject: [PATCH 38/82] added data_searchdirno; there's one absolute path in path.ini now --- tests/test_config.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index f61cffae8..acfca356c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -115,21 +115,21 @@ def test_Config_has_access_users_database(): @lustre_avail @pytest.mark.parametrize( - "cfg_id,basedir,init_obslocs_ungridded,init_data_search_dirs", + "cfg_id,basedir,init_obslocs_ungridded,init_data_search_dirs,data_searchdirno", [ - ("metno", None, False, False), - ("metno", None, True, False), - ("metno", None, True, True), - ("metno", f"/home/{USER}", True, True), - ("users-db", None, False, False), + ("metno", None, False, False, 0), + ("metno", None, True, False, 0), + ("metno", None, True, True, 0), + ("metno", f"/home/{USER}", True, True, 1), + ("users-db", None, False, False, 0), ], ) -def test_Config_read_config(cfg_id, basedir, init_obslocs_ungridded, init_data_search_dirs): +def test_Config_read_config(cfg_id, basedir, init_obslocs_ungridded, init_data_search_dirs, data_searchdirno): cfg = testmod.Config(try_infer_environment=False) cfg_file = cfg._config_files[cfg_id] assert Path(cfg_file).exists() cfg.read_config(cfg_file, basedir, init_obslocs_ungridded, init_data_search_dirs) - assert len(cfg.DATA_SEARCH_DIRS) == 0 + assert len(cfg.DATA_SEARCH_DIRS) == data_searchdirno assert len(cfg.OBSLOCS_UNGRIDDED) == 0 assert Path(cfg.OUTPUTDIR).exists() assert Path(cfg.COLOCATEDDATADIR).exists() From 9ca9a8609843445ff1fede8f24d8a272ca170d2e Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 27 Mar 2023 13:33:34 +0200 Subject: [PATCH 39/82] linting --- tests/test_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index acfca356c..9c3d60b6f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -124,7 +124,9 @@ def test_Config_has_access_users_database(): ("users-db", None, False, False, 0), ], ) -def test_Config_read_config(cfg_id, basedir, init_obslocs_ungridded, init_data_search_dirs, data_searchdirno): +def test_Config_read_config( + cfg_id, basedir, init_obslocs_ungridded, init_data_search_dirs, data_searchdirno +): cfg = testmod.Config(try_infer_environment=False) cfg_file = cfg._config_files[cfg_id] assert Path(cfg_file).exists() From b12a6e866858dc2409a81df7b2063e84b999fa5d Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 27 Mar 2023 14:42:49 +0200 Subject: [PATCH 40/82] start ipcforests testing --- tests/plugins/ipcforests/__init__.py | 0 tests/plugins/ipcforests/test_reader.py | 104 ++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 tests/plugins/ipcforests/__init__.py create mode 100644 tests/plugins/ipcforests/test_reader.py diff --git a/tests/plugins/ipcforests/__init__.py b/tests/plugins/ipcforests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/plugins/ipcforests/test_reader.py b/tests/plugins/ipcforests/test_reader.py new file mode 100644 index 000000000..fb752b60f --- /dev/null +++ b/tests/plugins/ipcforests/test_reader.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from pyaerocom import const +from tests.conftest import lustre_unavail + +try: + MEP_PATH = Path(const.OBSLOCS_UNGRIDDED[const.IPCFORESTS]) +except KeyError: + pytest.mark.skip(reason=f"IPCForests path not initialised due to non existence in CI") + +# stationnames are not consistent betweem variables!\ +# depoxn +# STATION_NAMES = ("DE-604-1", "NO-19-1", "UK-718-1") +# fakedrypm10 +STATION_NAMES = ("Birkenes II", "La Coulonche", "Jarczew") + +VARS_DEFAULT = {"fakedrypm10"} +VARS_PROVIDED = VARS_DEFAULT + + +@lustre_unavail +@pytest.fixture(scope="module") +def reader() -> ReadIPC: + return ReadMEP(data_dir=str(MEP_PATH)) + + +@lustre_unavail +@pytest.fixture() +def station_files(station: str) -> list[Path]: + files = sorted(MEP_PATH.rglob(f"mep-rd-{station}-*.nc")) + assert files, f"no files for {station}" + return files + + +@lustre_unavail +def test_DATASET_NAME(reader: ReadMEP): + assert reader.DATASET_NAME == "MEP" + + +@lustre_unavail +def test_DEFAULT_VARS(reader: ReadMEP): + assert set(reader.DEFAULT_VARS) >= VARS_DEFAULT + + +@lustre_unavail +def test_files(reader: ReadMEP): + assert reader.files, "no stations files found" + assert len(reader.files) >= 3, "found less files than expected" + + +@lustre_unavail +def test_FOUND_FILES(reader: ReadMEP): + assert reader.FOUND_FILES, "no stations files found" + assert len(reader.FOUND_FILES) >= 3, "found less files than expected" + + +@lustre_unavail +@pytest.mark.parametrize("station", STATION_NAMES) +def test_stations(reader: ReadMEP, station: str): + assert reader.stations()[station], f"no {station} station files" + + +@lustre_unavail +def test_PROVIDES_VARIABLES(reader: ReadMEP): + return set(reader.PROVIDES_VARIABLES) >= VARS_PROVIDED + + +@lustre_unavail +@pytest.mark.parametrize("station", STATION_NAMES) +def test_read_file(reader: ReadMEP, station_files: list[str]): + data = reader.read_file(station_files[-1]) + assert set(data.contains_vars) == VARS_DEFAULT + + +@lustre_unavail +def test_read_file_error(reader: ReadMEP): + bad_station_file = "not-a-file.nc" + with pytest.raises(ValueError) as e: + reader.read_file(bad_station_file) + assert str(e.value) == f"missing {bad_station_file}" + + +@lustre_unavail +@pytest.mark.parametrize("station", STATION_NAMES) +def test_read(reader: ReadMEP, station_files: list[str]): + data = reader.read(VARS_PROVIDED, station_files, first_file=0, last_file=5) + assert set(data.contains_vars) == VARS_PROVIDED + + +@lustre_unavail +def test_read_error(reader: ReadMEP): + bad_variable_name = "not-a-variable" + with pytest.raises(ValueError) as e: + reader.read((bad_variable_name,)) + assert str(e.value) == f"Unsupported variables: {bad_variable_name}" + + +@lustre_unavail +def test_reader_gives_correct_mep_path(reader: ReadMEP): + assert str(MEP_PATH) == reader.data_dir From c621108ceee8d9f600e7cb8a38d4e32b76d4f405 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 27 Mar 2023 16:52:33 +0200 Subject: [PATCH 41/82] some more testing --- pyaerocom/plugins/ipcforests/reader.py | 56 +++++++++------- tests/plugins/ipcforests/test_reader.py | 89 ++++++++++--------------- 2 files changed, 68 insertions(+), 77 deletions(-) diff --git a/pyaerocom/plugins/ipcforests/reader.py b/pyaerocom/plugins/ipcforests/reader.py index 860054fbd..b9767fee2 100644 --- a/pyaerocom/plugins/ipcforests/reader.py +++ b/pyaerocom/plugins/ipcforests/reader.py @@ -3,6 +3,7 @@ import logging import os from datetime import datetime +from pathlib import Path from typing import Tuple import numpy as np @@ -85,6 +86,8 @@ class ReadIPCForest(ReadUngriddedBase): "wetnh4": "mg N m-2 d-1", "wetno2": "mg N m-2 d-1", "wetna": "mg m-2 d-1", + # unverified + "wetcl": "mg m-2 d-1", } SEASALT_CORRECTION = { @@ -108,6 +111,7 @@ def __init__(self, data_id=None, data_dir=None): self.metadata = None self._file_dir = None + self.files = None if data_dir is not None: self.metadata = MetadataReader(data_dir) @@ -149,7 +153,8 @@ def read(self, vars_to_retrieve=None, files=[], first_file=None, last_file=None) UngriddedData instance of ungridded data object containing data from all files. """ - data = self.read_file(self.data_dir + self._FILEMASK, vars_to_retrieve) + self.files = Path(self.data_dir).joinpath(self._FILEMASK) + data = self.read_file(str(self.files), vars_to_retrieve, last_line=None) return data @@ -168,7 +173,7 @@ def DEFAULT_VARS(self): """List containing default variables to read""" return list(self.VAR_POSITION.keys()) - def read_file(self, filename, vars_to_retrieve=None): + def read_file(self, filename, vars_to_retrieve=None, last_line=None): """Read single file Parameters @@ -178,6 +183,9 @@ def read_file(self, filename, vars_to_retrieve=None): vars_to_retrieve : :obj:`list` or similar, optional, list containing variable IDs that are supposed to be read. If None, all variables in :attr:`PROVIDES_VARIABLES` are loaded + last_line : int + last line number to read + Used to speed up testing Returns ------- @@ -197,8 +205,10 @@ def read_file(self, filename, vars_to_retrieve=None): if vars_to_retrieve is None: vars_to_retrieve = self.PROVIDES_VARIABLES + lineno = 0 with open(filename, "r") as f: f.readline() + for line_nr, line in tqdm(enumerate(f)): words = line.split(";") year = int(words[0]) @@ -210,8 +220,8 @@ def read_file(self, filename, vars_to_retrieve=None): # 8 is the code for "do not use" if ( - self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE - or sampler_code == 8 + self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE + or sampler_code == 8 ): continue @@ -273,14 +283,14 @@ def read_file(self, filename, vars_to_retrieve=None): if species in self.SEASALT_CORRECTION: na_factor = ( - self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") - * self.SEASALT_FACTORS["wetna"] - * self.SEASALT_CORRECTION[species] + self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") + * self.SEASALT_FACTORS["wetna"] + * self.SEASALT_CORRECTION[species] ) cl_factor = ( - self._get_species_conc(words[self.VAR_POSITION["wetcl"]], "wetcl") - * self.SEASALT_FACTORS["wetcl"] - * self.SEASALT_CORRECTION[species] + self._get_species_conc(words[self.VAR_POSITION["wetcl"]], "wetcl") + * self.SEASALT_FACTORS["wetcl"] + * self.SEASALT_CORRECTION[species] ) seasalt_correction = min(na_factor, cl_factor) @@ -345,14 +355,14 @@ def _get_species_conc(self, conc_str: str, species: str) -> float: return conc def _get_days_date_ts_type( - self, - year: int, - country_code: int, - plot_code: int, - sampler_code: int, - period: int, - start: str | datetime, - stop: str | datetime, + self, + year: int, + country_code: int, + plot_code: int, + sampler_code: int, + period: int, + start: str | datetime, + stop: str | datetime, ) -> Tuple[float | None, datetime | None, str | None]: if start != "" and stop != "": @@ -397,11 +407,11 @@ def _get_tstype(self, start: datetime, stop: datetime) -> str: return SurveyYear.get_tstype(days) def _clean_data_with_flags( - self, - data: list[float], - time: list[datetime], - flags: list[int], - species: str, + self, + data: list[float], + time: list[datetime], + flags: list[int], + species: str, ) -> list[float]: data_array = np.array(data) diff --git a/tests/plugins/ipcforests/test_reader.py b/tests/plugins/ipcforests/test_reader.py index fb752b60f..fa58b6c7e 100644 --- a/tests/plugins/ipcforests/test_reader.py +++ b/tests/plugins/ipcforests/test_reader.py @@ -5,100 +5,81 @@ import pytest from pyaerocom import const +from pyaerocom.plugins.ipcforests.reader import ReadIPCForest from tests.conftest import lustre_unavail try: - MEP_PATH = Path(const.OBSLOCS_UNGRIDDED[const.IPCFORESTS]) + IPC_PATH = Path(const.OBSLOCS_UNGRIDDED[const.IPCFORESTS_NAME]) except KeyError: pytest.mark.skip(reason=f"IPCForests path not initialised due to non existence in CI") -# stationnames are not consistent betweem variables!\ -# depoxn -# STATION_NAMES = ("DE-604-1", "NO-19-1", "UK-718-1") +# stationnames are not consistent between variables! +# wetoxn +STATION_NAMES = ("DE-604-2", "NO-7-2", "UK-718-2") # fakedrypm10 -STATION_NAMES = ("Birkenes II", "La Coulonche", "Jarczew") +# STATION_NAMES = ("Birkenes II", "La Coulonche", "Jarczew") -VARS_DEFAULT = {"fakedrypm10"} +VARS_DEFAULT = {"wetoxn"} VARS_PROVIDED = VARS_DEFAULT @lustre_unavail @pytest.fixture(scope="module") -def reader() -> ReadIPC: - return ReadMEP(data_dir=str(MEP_PATH)) +def reader() -> ReadIPCForest: + return ReadIPCForest(data_dir=str(IPC_PATH)) @lustre_unavail @pytest.fixture() def station_files(station: str) -> list[Path]: - files = sorted(MEP_PATH.rglob(f"mep-rd-{station}-*.nc")) - assert files, f"no files for {station}" + p = IPC_PATH.glob("dp_dem.csv") + files = [x for x in p if x.is_file()] + # assert files, f"no files for {station}" + assert files return files @lustre_unavail -def test_DATASET_NAME(reader: ReadMEP): - assert reader.DATASET_NAME == "MEP" +def test_DATASET_NAME(reader: ReadIPCForest): + assert reader.DATA_ID == const.IPCFORESTS_NAME @lustre_unavail -def test_DEFAULT_VARS(reader: ReadMEP): +def test_DEFAULT_VARS(reader: ReadIPCForest): assert set(reader.DEFAULT_VARS) >= VARS_DEFAULT -@lustre_unavail -def test_files(reader: ReadMEP): - assert reader.files, "no stations files found" - assert len(reader.files) >= 3, "found less files than expected" - - -@lustre_unavail -def test_FOUND_FILES(reader: ReadMEP): - assert reader.FOUND_FILES, "no stations files found" - assert len(reader.FOUND_FILES) >= 3, "found less files than expected" +# @lustre_unavail +# def test_FOUND_FILES(reader: ReadIPCForest): +# assert reader.FOUND_FILES, "no stations files found" +# assert len(reader.FOUND_FILES) >= 3, "found less files than expected" -@lustre_unavail -@pytest.mark.parametrize("station", STATION_NAMES) -def test_stations(reader: ReadMEP, station: str): - assert reader.stations()[station], f"no {station} station files" +# @lustre_unavail +# @pytest.mark.parametrize("station", STATION_NAMES) +# def test_stations(reader: ReadIPCForest, station: str): +# assert reader.stations()[station], f"no {station} station files" @lustre_unavail -def test_PROVIDES_VARIABLES(reader: ReadMEP): +def test_PROVIDES_VARIABLES(reader: ReadIPCForest): return set(reader.PROVIDES_VARIABLES) >= VARS_PROVIDED @lustre_unavail -@pytest.mark.parametrize("station", STATION_NAMES) -def test_read_file(reader: ReadMEP, station_files: list[str]): - data = reader.read_file(station_files[-1]) +# @pytest.mark.parametrize("station", STATION_NAMES) +def test_read_file(reader: ReadIPCForest, ): + data = reader.read(vars_to_retrieve=VARS_DEFAULT) assert set(data.contains_vars) == VARS_DEFAULT -@lustre_unavail -def test_read_file_error(reader: ReadMEP): - bad_station_file = "not-a-file.nc" - with pytest.raises(ValueError) as e: - reader.read_file(bad_station_file) - assert str(e.value) == f"missing {bad_station_file}" - - -@lustre_unavail -@pytest.mark.parametrize("station", STATION_NAMES) -def test_read(reader: ReadMEP, station_files: list[str]): - data = reader.read(VARS_PROVIDED, station_files, first_file=0, last_file=5) - assert set(data.contains_vars) == VARS_PROVIDED - - -@lustre_unavail -def test_read_error(reader: ReadMEP): - bad_variable_name = "not-a-variable" - with pytest.raises(ValueError) as e: - reader.read((bad_variable_name,)) - assert str(e.value) == f"Unsupported variables: {bad_variable_name}" +# @lustre_unavail +# @pytest.mark.parametrize("station", STATION_NAMES) +# def test_read(reader: ReadIPCForest, station_files: list[str]): +# data = reader.read(VARS_PROVIDED, station_files, first_file=0, last_file=5) +# assert set(data.contains_vars) == VARS_PROVIDED @lustre_unavail -def test_reader_gives_correct_mep_path(reader: ReadMEP): - assert str(MEP_PATH) == reader.data_dir +def test_reader_gives_correct_IPC_PATH(reader: ReadIPCForest): + assert str(IPC_PATH) == reader.data_dir From 6aeef68c527601349092a5ef86b2e73b2f1bb30d Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 28 Mar 2023 09:57:03 +0200 Subject: [PATCH 42/82] linting --- pyaerocom/plugins/ipcforests/reader.py | 42 ++++++++++++------------- tests/plugins/ipcforests/test_reader.py | 4 ++- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/pyaerocom/plugins/ipcforests/reader.py b/pyaerocom/plugins/ipcforests/reader.py index b9767fee2..258a49148 100644 --- a/pyaerocom/plugins/ipcforests/reader.py +++ b/pyaerocom/plugins/ipcforests/reader.py @@ -220,8 +220,8 @@ def read_file(self, filename, vars_to_retrieve=None, last_line=None): # 8 is the code for "do not use" if ( - self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE - or sampler_code == 8 + self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE + or sampler_code == 8 ): continue @@ -283,14 +283,14 @@ def read_file(self, filename, vars_to_retrieve=None, last_line=None): if species in self.SEASALT_CORRECTION: na_factor = ( - self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") - * self.SEASALT_FACTORS["wetna"] - * self.SEASALT_CORRECTION[species] + self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") + * self.SEASALT_FACTORS["wetna"] + * self.SEASALT_CORRECTION[species] ) cl_factor = ( - self._get_species_conc(words[self.VAR_POSITION["wetcl"]], "wetcl") - * self.SEASALT_FACTORS["wetcl"] - * self.SEASALT_CORRECTION[species] + self._get_species_conc(words[self.VAR_POSITION["wetcl"]], "wetcl") + * self.SEASALT_FACTORS["wetcl"] + * self.SEASALT_CORRECTION[species] ) seasalt_correction = min(na_factor, cl_factor) @@ -355,14 +355,14 @@ def _get_species_conc(self, conc_str: str, species: str) -> float: return conc def _get_days_date_ts_type( - self, - year: int, - country_code: int, - plot_code: int, - sampler_code: int, - period: int, - start: str | datetime, - stop: str | datetime, + self, + year: int, + country_code: int, + plot_code: int, + sampler_code: int, + period: int, + start: str | datetime, + stop: str | datetime, ) -> Tuple[float | None, datetime | None, str | None]: if start != "" and stop != "": @@ -407,11 +407,11 @@ def _get_tstype(self, start: datetime, stop: datetime) -> str: return SurveyYear.get_tstype(days) def _clean_data_with_flags( - self, - data: list[float], - time: list[datetime], - flags: list[int], - species: str, + self, + data: list[float], + time: list[datetime], + flags: list[int], + species: str, ) -> list[float]: data_array = np.array(data) diff --git a/tests/plugins/ipcforests/test_reader.py b/tests/plugins/ipcforests/test_reader.py index fa58b6c7e..48941835e 100644 --- a/tests/plugins/ipcforests/test_reader.py +++ b/tests/plugins/ipcforests/test_reader.py @@ -68,7 +68,9 @@ def test_PROVIDES_VARIABLES(reader: ReadIPCForest): @lustre_unavail # @pytest.mark.parametrize("station", STATION_NAMES) -def test_read_file(reader: ReadIPCForest, ): +def test_read_file( + reader: ReadIPCForest, +): data = reader.read(vars_to_retrieve=VARS_DEFAULT) assert set(data.contains_vars) == VARS_DEFAULT From c007b977c1ca6940084562a2cd40f9e839699fdd Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 28 Mar 2023 10:15:14 +0200 Subject: [PATCH 43/82] take years without data for a species into account --- pyaerocom/plugins/ipcforests/reader.py | 30 ++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/pyaerocom/plugins/ipcforests/reader.py b/pyaerocom/plugins/ipcforests/reader.py index 258a49148..4290a5a0a 100644 --- a/pyaerocom/plugins/ipcforests/reader.py +++ b/pyaerocom/plugins/ipcforests/reader.py @@ -105,6 +105,9 @@ class ReadIPCForest(ReadUngriddedBase): QUALITY_LIMIT = 0.5 + MIN_YEAR = 1984 + MAX_YEAR = 2019 + def __init__(self, data_id=None, data_dir=None): super().__init__(data_id, data_dir) @@ -154,7 +157,10 @@ def read(self, vars_to_retrieve=None, files=[], first_file=None, last_file=None) instance of ungridded data object containing data from all files. """ self.files = Path(self.data_dir).joinpath(self._FILEMASK) - data = self.read_file(str(self.files), vars_to_retrieve, last_line=None) + data = self.read_file( + str(self.files), + vars_to_retrieve, + ) return data @@ -205,7 +211,6 @@ def read_file(self, filename, vars_to_retrieve=None, last_line=None): if vars_to_retrieve is None: vars_to_retrieve = self.PROVIDES_VARIABLES - lineno = 0 with open(filename, "r") as f: f.readline() @@ -417,13 +422,20 @@ def _clean_data_with_flags( data_array = np.array(data) flags_array = np.array(flags) years = np.array([i.year for i in time]) - for year in range(1984, 2019): + + for year in range(self.MIN_YEAR, self.MAX_YEAR): yr_flags = flags_array[np.where(years == year)] - quality = np.sum(np.where(yr_flags == 0)) / len(yr_flags) - if quality < self.QUALITY_LIMIT: - logger.warning( - f"Quailty of {quality} found for {species} in year {year}. Setting data this year to NaN" - ) - data_array[np.where(years == year)] = np.nan + # yr_flags can have 0 length + try: + quality = np.sum(np.where(yr_flags == 0)) / len(yr_flags) + if quality < self.QUALITY_LIMIT: + logger.warning( + f"Quailty of {quality} found for {species} in year {year}. Setting data this year to NaN" + ) + data_array[np.where(years == year)] = np.nan + + except RuntimeWarning: + # yr_flags has 0 length + logger.warning(f"No data for species {species} in year {year}.") return list(data_array) From cc22095b23452ea4b5f9a3c9bfbc27d0aaddb232 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 28 Mar 2023 10:49:09 +0200 Subject: [PATCH 44/82] added a simple metadata test --- tests/plugins/ipcforests/test_reader.py | 33 +++++++++++-------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/tests/plugins/ipcforests/test_reader.py b/tests/plugins/ipcforests/test_reader.py index 48941835e..568b422d2 100644 --- a/tests/plugins/ipcforests/test_reader.py +++ b/tests/plugins/ipcforests/test_reader.py @@ -6,6 +6,7 @@ from pyaerocom import const from pyaerocom.plugins.ipcforests.reader import ReadIPCForest +from pyaerocom.plugins.ipcforests.metadata import MetadataReader as ReadIPCForestMeta from tests.conftest import lustre_unavail try: @@ -22,12 +23,15 @@ VARS_DEFAULT = {"wetoxn"} VARS_PROVIDED = VARS_DEFAULT - @lustre_unavail @pytest.fixture(scope="module") def reader() -> ReadIPCForest: return ReadIPCForest(data_dir=str(IPC_PATH)) +@lustre_unavail +@pytest.fixture(scope="module") +def meta_reader() -> ReadIPCForestMeta: + return ReadIPCForestMeta(str(IPC_PATH)) @lustre_unavail @pytest.fixture() @@ -38,7 +42,6 @@ def station_files(station: str) -> list[Path]: assert files return files - @lustre_unavail def test_DATASET_NAME(reader: ReadIPCForest): assert reader.DATA_ID == const.IPCFORESTS_NAME @@ -49,16 +52,9 @@ def test_DEFAULT_VARS(reader: ReadIPCForest): assert set(reader.DEFAULT_VARS) >= VARS_DEFAULT -# @lustre_unavail -# def test_FOUND_FILES(reader: ReadIPCForest): -# assert reader.FOUND_FILES, "no stations files found" -# assert len(reader.FOUND_FILES) >= 3, "found less files than expected" - - -# @lustre_unavail -# @pytest.mark.parametrize("station", STATION_NAMES) -# def test_stations(reader: ReadIPCForest, station: str): -# assert reader.stations()[station], f"no {station} station files" +@lustre_unavail +def test_METADATA(meta_reader: ReadIPCForestMeta): + assert len(meta_reader.deposition_type) >= 3, "found less deposition types than expected" @lustre_unavail @@ -67,7 +63,6 @@ def test_PROVIDES_VARIABLES(reader: ReadIPCForest): @lustre_unavail -# @pytest.mark.parametrize("station", STATION_NAMES) def test_read_file( reader: ReadIPCForest, ): @@ -75,11 +70,13 @@ def test_read_file( assert set(data.contains_vars) == VARS_DEFAULT -# @lustre_unavail -# @pytest.mark.parametrize("station", STATION_NAMES) -# def test_read(reader: ReadIPCForest, station_files: list[str]): -# data = reader.read(VARS_PROVIDED, station_files, first_file=0, last_file=5) -# assert set(data.contains_vars) == VARS_PROVIDED +@lustre_unavail +def test_read_station(reader: ReadIPCForest,): + # IPCForest reader does not support partial read of stations at this time + # not easy to implement due to being a single file dataset + data = reader.read(vars_to_retrieve=VARS_DEFAULT, ) + for station in STATION_NAMES: + assert station in data.unique_station_names @lustre_unavail From 71601812f9b55148bd9a2d23727dca9745e5e46a Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 28 Mar 2023 10:49:46 +0200 Subject: [PATCH 45/82] linting --- tests/plugins/ipcforests/test_reader.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/plugins/ipcforests/test_reader.py b/tests/plugins/ipcforests/test_reader.py index 568b422d2..2033fb05d 100644 --- a/tests/plugins/ipcforests/test_reader.py +++ b/tests/plugins/ipcforests/test_reader.py @@ -5,8 +5,8 @@ import pytest from pyaerocom import const -from pyaerocom.plugins.ipcforests.reader import ReadIPCForest from pyaerocom.plugins.ipcforests.metadata import MetadataReader as ReadIPCForestMeta +from pyaerocom.plugins.ipcforests.reader import ReadIPCForest from tests.conftest import lustre_unavail try: @@ -23,16 +23,19 @@ VARS_DEFAULT = {"wetoxn"} VARS_PROVIDED = VARS_DEFAULT + @lustre_unavail @pytest.fixture(scope="module") def reader() -> ReadIPCForest: return ReadIPCForest(data_dir=str(IPC_PATH)) + @lustre_unavail @pytest.fixture(scope="module") def meta_reader() -> ReadIPCForestMeta: return ReadIPCForestMeta(str(IPC_PATH)) + @lustre_unavail @pytest.fixture() def station_files(station: str) -> list[Path]: @@ -42,6 +45,7 @@ def station_files(station: str) -> list[Path]: assert files return files + @lustre_unavail def test_DATASET_NAME(reader: ReadIPCForest): assert reader.DATA_ID == const.IPCFORESTS_NAME @@ -71,10 +75,14 @@ def test_read_file( @lustre_unavail -def test_read_station(reader: ReadIPCForest,): +def test_read_station( + reader: ReadIPCForest, +): # IPCForest reader does not support partial read of stations at this time # not easy to implement due to being a single file dataset - data = reader.read(vars_to_retrieve=VARS_DEFAULT, ) + data = reader.read( + vars_to_retrieve=VARS_DEFAULT, + ) for station in STATION_NAMES: assert station in data.unique_station_names From 80119c7942acb7c4bac2620598082eadd0fa79e6 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Wed, 29 Mar 2023 10:28:49 +0200 Subject: [PATCH 46/82] replace fake with proxy in all variable names; linting --- pyaerocom/aeroval/glob_defaults.py | 79 +++++++-------- pyaerocom/aux_var_helpers.py | 9 +- pyaerocom/data/ebas_config.ini | 76 +++++++-------- pyaerocom/data/variables.ini | 144 +++++++++++++-------------- pyaerocom/io/read_ebas.py | 151 ++++++++++++++--------------- tests/io/test_read_ebas.py | 68 ++++++------- 6 files changed, 263 insertions(+), 264 deletions(-) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index e9716b2da..f99d442ca 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -100,46 +100,49 @@ "colmap": "coolwarm", }, "ts": {"scale": [265, 270, 275, 280, 285, 290, 300, 305, 310, 315, 320], "colmap": "coolwarm"}, - "fakedryo3": {"scale": [0, 0.5, 1, 15, 20, 25, 0.30, 40, 50], "colmap": "coolwarm"}, - "fakedrypm10": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, - "fakedrypm25": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, - "fakedryno2": {"scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], "colmap": "coolwarm"}, - "fakedryhono": {"scale": [0.001, 0.002, 0.003, 0.004, 0.005, 0.006], "colmap": "coolwarm"}, - "fakedryn2o5": {"scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm"}, - "fakedryhno3": { + "proxydryo3": {"scale": [0, 0.5, 1, 15, 20, 25, 0.30, 40, 50], "colmap": "coolwarm"}, + "proxydrypm10": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "proxydrypm25": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "proxydryno2": { "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], "colmap": "coolwarm", }, - "fakedryno3c": { + "proxydryhono": {"scale": [0.001, 0.002, 0.003, 0.004, 0.005, 0.006], "colmap": "coolwarm"}, + "proxydryn2o5": {"scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm"}, + "proxydryhno3": { + "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], + "colmap": "coolwarm", + }, + "proxydryno3c": { "scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm", }, - "fakedryno3f": {"scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm"}, - "fakedrynh3": { + "proxydryno3f": {"scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm"}, + "proxydrynh3": { "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80], "colmap": "coolwarm", }, - "fakedrynh4": { + "proxydrynh4": { "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], "colmap": "coolwarm", }, - "fakedryso2": { + "proxydryso2": { "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], "colmap": "coolwarm", }, - "fakedryso4": { + "proxydryso4": { "scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm", }, - "fakedryoxs": { + "proxydryoxs": { "scale": [0, 0.05, 0.1, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40], "colmap": "coolwarm", }, - "fakedryoxn": { + "proxydryoxn": { "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80], "colmap": "coolwarm", }, - "fakedryrdn": { + "proxydryrdn": { "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80], "colmap": "coolwarm", }, @@ -310,7 +313,7 @@ }, } -# For experiments where only model data is interesting, as with fake drydep +# For experiments where only model data is interesting, as with proxy drydep statistics_model_only = { "data_mean": { "name": "Mean-Mod", @@ -404,9 +407,9 @@ concCoc25=["OC PM2.5", "3D", "Particle concentration"], # Depositions drysox=["DryOXS", "3D", "Deposition"], - dryoxs=["FakeDryOXS", "3D", "Deposition"], - dryoxn=["FakeDryOXN", "3D", "Deposition"], - dryrdn=["FakeDryRDN", "3D", "Deposition"], + dryoxs=["proxyDryOXS", "3D", "Deposition"], + dryoxn=["proxyDryOXN", "3D", "Deposition"], + dryrdn=["proxyDryRDN", "3D", "Deposition"], depoxs=["TotDepOXS", "3D", "Total Deposition"], depoxn=["TotDepOXN", "3D", "Total Deposition"], deprdn=["TotDepRDN", "3D", "Total Deposition"], @@ -416,22 +419,22 @@ prmm=["Precipitation", "3D", "Deposition"], # Temperature ts=["Surface Temperature", "3D", "Temperature"], - # Fake drydep - fakedryoxs=["FakeDryOXS", "3D", "Deposition"], - fakedryso2=["FakeDrySO2", "3D", "Deposition"], - fakedryso4=["FakeDrySO4", "3D", "Deposition"], - fakedryoxn=["FakeDryOXN", "3D", "Deposition"], - fakedryno2=["FakeDryNO2", "3D", "Deposition"], - fakedryno2no2=["FakeDryNO2NO2", "3D", "Deposition"], - fakedryhono=["FakeDryHONO", "3D", "Deposition"], - fakedryn2o5=["FakeDryN2O5", "3D", "Deposition"], - fakedryhno3=["FakeDryHNO3", "3D", "Deposition"], - fakedryno3c=["FakeDryNO3Coarse", "3D", "Deposition"], - fakedryno3f=["FakeDryNO3Fine", "3D", "Deposition"], - fakedryrdn=["FakeDryRDN", "3D", "Deposition"], - fakedrynh3=["FakeDryNH3", "3D", "Deposition"], - fakedrynh4=["FakeDryNH4", "3D", "Deposition"], - fakedryo3=["FakeDryO3", "3D", "Deposition"], - fakedrypm10=["FakeDryPM10", "3D", "Deposition"], - fakedrypm25=["FakeDryPM2.5", "3D", "Deposition"], + # proxy drydep + proxydryoxs=["proxyDryOXS", "3D", "Deposition"], + proxydryso2=["proxyDrySO2", "3D", "Deposition"], + proxydryso4=["proxyDrySO4", "3D", "Deposition"], + proxydryoxn=["proxyDryOXN", "3D", "Deposition"], + proxydryno2=["proxyDryNO2", "3D", "Deposition"], + proxydryno2no2=["proxyDryNO2NO2", "3D", "Deposition"], + proxydryhono=["proxyDryHONO", "3D", "Deposition"], + proxydryn2o5=["proxyDryN2O5", "3D", "Deposition"], + proxydryhno3=["proxyDryHNO3", "3D", "Deposition"], + proxydryno3c=["proxyDryNO3Coarse", "3D", "Deposition"], + proxydryno3f=["proxyDryNO3Fine", "3D", "Deposition"], + proxydryrdn=["proxyDryRDN", "3D", "Deposition"], + proxydrynh3=["proxyDryNH3", "3D", "Deposition"], + proxydrynh4=["proxyDryNH4", "3D", "Deposition"], + proxydryo3=["proxyDryO3", "3D", "Deposition"], + proxydrypm10=["proxyDryPM10", "3D", "Deposition"], + proxydrypm25=["proxyDryPM2.5", "3D", "Deposition"], ) diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index 66853624d..9cae84ee7 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -718,7 +718,6 @@ def concx_to_vmrx(data, p_pascal, T_kelvin, conc_unit, mmol_var, mmol_air=None, def calc_vmro3max(data): - var_name = "vmro3" new_var_name = "vmro3max" @@ -744,13 +743,13 @@ def identity(data): return data -def make_fake_drydep_from_O3(data): +def make_proxy_drydep_from_O3(data): # sort of prototype to add a compted variable # one has to extend the data structures of the station data object # 'right', but has to return just the data array # That concept is a bit confusing (why not do everything in data here?) var_name = "vmro3" - new_var_name = "fakedryo3" + new_var_name = "proxydryo3" flags = data.data_flagged[var_name] new_var_data = data[var_name] @@ -766,13 +765,13 @@ def make_fake_drydep_from_O3(data): return new_var_data -def make_fake_wetdep_from_O3(data): +def make_proxy_wetdep_from_O3(data): # sort of prototype to add a compted variable # one has to extend the data structures of the station data object # 'right', but has to return just the data array # That concept is a bit confusing (why not do everything in data here?) var_name = "vmro3" - new_var_name = "fakeweto3" + new_var_name = "proxyweto3" flags = data.data_flagged[var_name] new_var_data = data[var_name] diff --git a/pyaerocom/data/ebas_config.ini b/pyaerocom/data/ebas_config.ini index dd58db9c5..0647c27cf 100644 --- a/pyaerocom/data/ebas_config.ini +++ b/pyaerocom/data/ebas_config.ini @@ -477,124 +477,124 @@ matrix=precip [wetso4] requires=concprcpso4 -#Fake Dry Dep +#proxy Dry Dep # Sulpher Based dry dep -[fakedryoxs] +[proxydryoxs] requires=concprcpoxs -[fakedryso2] +[proxydryso2] requires=concprcpoxs -[fakedryso4] +[proxydryso4] requires=concprcpoxs # Oxidized nitrogen based dry dep -[fakedryoxn] +[proxydryoxn] requires=concprcpoxn -[fakedryno2] +[proxydryno2] requires=concprcpoxn -[fakedryno2no2] +[proxydryno2no2] requires=concprcpoxn -[fakedryhono] +[proxydryhono] requires=concprcpoxn -[fakedryn2o5] +[proxydryn2o5] requires=concprcpoxn -[fakedryhno3] +[proxydryhno3] requires=concprcpoxn -[fakedryno3c] +[proxydryno3c] requires=concprcpoxn -[fakedryno3f] +[proxydryno3f] requires=concprcpoxn # Reduced nitrogen based dry dep -[fakedryrdn] +[proxydryrdn] requires=concprcprdn -[fakedrynh3] +[proxydrynh3] requires=concprcprdn -[fakedrynh4] +[proxydrynh4] requires=concprcprdn -# Other Fake dry dep +# Other proxy dry dep -[fakedryo3] +[proxydryo3] requires=vmro3 -[fakedrypm10] +[proxydrypm10] requires=concprcpoxs -[fakedrypm25] +[proxydrypm25] requires=concprcpoxs -#Fake wet Dep +#proxy wet Dep # Sulpher Based wet dep -[fakewetoxs] +[proxywetoxs] requires=concprcpoxs -[fakewetso2] +[proxywetso2] requires=concprcpoxs -[fakewetso4] +[proxywetso4] requires=concprcpoxs # Oxidized nitrogen based wet dep -[fakewetoxn] +[proxywetoxn] requires=concprcpoxn -[fakewetno2] +[proxywetno2] requires=concprcpoxn -[fakewetno2no2] +[proxywetno2no2] requires=concprcpoxn -[fakewethono] +[proxywethono] requires=concprcpoxn -[fakewetn2o5] +[proxywetn2o5] requires=concprcpoxn -[fakewethno3] +[proxywethno3] requires=concprcpoxn -[fakewetno3c] +[proxywetno3c] requires=concprcpoxn -[fakewetno3f] +[proxywetno3f] requires=concprcpoxn # Reduced nitrogen based wet dep -[fakewetrdn] +[proxywetrdn] requires=concprcprdn -[fakewetnh3] +[proxywetnh3] requires=concprcprdn -[fakewetnh4] +[proxywetnh4] requires=concprcprdn -# Other Fake wet dep +# Other proxy wet dep -[fakeweto3] +[proxyweto3] requires=vmro3 -[fakewetpm10] +[proxywetpm10] requires=concprcpoxs -[fakewetpm25] +[proxywetpm25] requires=concprcpoxs \ No newline at end of file diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index 170401346..a8c22fd93 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -5472,98 +5472,98 @@ minimum = 0 maximum = 10000 dimensions = time,lat,lon -# Fake Dry Dep +# proxy Dry Dep # Oxidized nitrogen based dry dep -[fakedryoxn] -description=Fake dry deposition of oxidized nitrogen mass +[proxydryoxn] +description=proxy dry deposition of oxidized nitrogen mass unit = mg N m-2 d-1 minimum=0 -[fakedryno2] -description=Fake dry deposition of NO2 +[proxydryno2] +description=proxy dry deposition of NO2 unit = mg N m-2 d-1 minimum=0 -[fakedryno2no2] -description=Fake dry deposition of NO2NO2 +[proxydryno2no2] +description=proxy dry deposition of NO2NO2 unit = mg N m-2 d-1 minimum=0 -[fakedryhono] -description=Fake dry deposition of HONO +[proxydryhono] +description=proxy dry deposition of HONO unit = mg N m-2 d-1 minimum=0 -[fakedryn2o5] -description=Fake dry deposition of N2O5 +[proxydryn2o5] +description=proxy dry deposition of N2O5 unit = mg N m-2 d-1 minimum=0 -[fakedryhno3] -description=Fake dry deposition of HNO3 +[proxydryhno3] +description=proxy dry deposition of HNO3 unit = mg N m-2 d-1 minimum=0 -[fakedryno3c] -description=Fake dry deposition of NO3 Coarse +[proxydryno3c] +description=proxy dry deposition of NO3 Coarse unit = mg N m-2 d-1 minimum=0 -[fakedryno3f] -description=Fake dry deposition of NO3 Fine +[proxydryno3f] +description=proxy dry deposition of NO3 Fine unit = mg N m-2 d-1 minimum=0 # Reduced nitrogen based dry dep -[fakedryrdn] -description=Fake dry deposition of reduced Nitrogen mass +[proxydryrdn] +description=proxy dry deposition of reduced Nitrogen mass unit = mg N m-2 d-1 minimum=0 -[fakedrynh3] -description=Fake dry deposition of NH3 +[proxydrynh3] +description=proxy dry deposition of NH3 unit = mg N m-2 d-1 minimum=0 -[fakedrynh4] -description=Fake dry deposition of NH4 +[proxydrynh4] +description=proxy dry deposition of NH4 unit = mg N m-2 d-1 minimum=0 # Sulpher Based dry dep -[fakedryoxs] -description=Fake dry deposition of total sulphur mass +[proxydryoxs] +description=proxy dry deposition of total sulphur mass unit = mg S m-2 d-1 minimum=0 -[fakedryso2] -description=Fake dry deposition of SO2 +[proxydryso2] +description=proxy dry deposition of SO2 unit = mg S m-2 d-1 minimum=0 -[fakedryso4] -description=Fake dry deposition of SO4 +[proxydryso4] +description=proxy dry deposition of SO4 unit = mg S m-2 d-1 minimum=0 -# Other Fake dry dep +# Other proxy dry dep -[fakedryo3] -description=Fake dry deposition of O3 +[proxydryo3] +description=proxy dry deposition of O3 unit = mg m-2 d-1 minimum=0 -[fakedrypm10] -description=Fake dry deposition of PM10 +[proxydrypm10] +description=proxy dry deposition of PM10 unit = mg m-2 d-1 minimum=0 -[fakedrypm25] -description=Fake dry deposition of PM25 +[proxydrypm25] +description=proxy dry deposition of PM25 unit = mg m-2 d-1 minimum=0 @@ -5572,97 +5572,97 @@ description=Dry deposition of PM25 unit = mg m-2 d-1 minimum=0 -# Fake wet Dep +# proxy wet Dep # Oxidized nitrogen based wet dep -[fakewetoxn] -description=Fake wet deposition of oxidized nitrogen mass +[proxywetoxn] +description=proxy wet deposition of oxidized nitrogen mass unit = mg N m-2 d-1 minimum=0 -[fakewetno2] -description=Fake wet deposition of NO2 +[proxywetno2] +description=proxy wet deposition of NO2 unit = mg N m-2 d-1 minimum=0 -[fakewetno2no2] -description=Fake wet deposition of NO2NO2 +[proxywetno2no2] +description=proxy wet deposition of NO2NO2 unit = mg N m-2 d-1 minimum=0 -[fakewethono] -description=Fake wet deposition of HONO +[proxywethono] +description=proxy wet deposition of HONO unit = mg N m-2 d-1 minimum=0 -[fakewetn2o5] -description=Fake wet deposition of N2O5 +[proxywetn2o5] +description=proxy wet deposition of N2O5 unit = mg N m-2 d-1 minimum=0 -[fakewethno3] -description=Fake wet deposition of HNO3 +[proxywethno3] +description=proxy wet deposition of HNO3 unit = mg N m-2 d-1 minimum=0 -[fakewetno3c] -description=Fake wet deposition of NO3 Coarse +[proxywetno3c] +description=proxy wet deposition of NO3 Coarse unit = mg N m-2 d-1 minimum=0 -[fakewetno3f] -description=Fake wet deposition of NO3 Fine +[proxywetno3f] +description=proxy wet deposition of NO3 Fine unit = mg N m-2 d-1 minimum=0 # Reduced nitrogen based wet dep -[fakewetrdn] -description=Fake wet deposition of reduced Nitrogen mass +[proxywetrdn] +description=proxy wet deposition of reduced Nitrogen mass unit = mg N m-2 d-1 minimum=0 -[fakewetnh3] -description=Fake wet deposition of NH3 +[proxywetnh3] +description=proxy wet deposition of NH3 unit = mg N m-2 d-1 minimum=0 -[fakewetnh4] -description=Fake wet deposition of NH4 +[proxywetnh4] +description=proxy wet deposition of NH4 unit = mg N m-2 d-1 minimum=0 # Sulpher Based wet dep -[fakewetoxs] -description=Fake wet deposition of total sulphur mass +[proxywetoxs] +description=proxy wet deposition of total sulphur mass unit = mg S m-2 d-1 minimum=0 -[fakewetso2] -description=Fake wet deposition of SO2 +[proxywetso2] +description=proxy wet deposition of SO2 unit = mg S m-2 d-1 minimum=0 -[fakewetso4] -description=Fake wet deposition of SO4 +[proxywetso4] +description=proxy wet deposition of SO4 unit = mg S m-2 d-1 minimum=0 -# Other Fake wet dep +# Other proxy wet dep -[fakeweto3] -description=Fake wet deposition of O3 +[proxyweto3] +description=proxy wet deposition of O3 unit = mg m-2 d-1 minimum=0 -[fakewetpm10] -description=Fake wet deposition of PM10 +[proxywetpm10] +description=proxy wet deposition of PM10 unit = mg m-2 d-1 minimum=0 -[fakewetpm25] -description=Fake wet deposition of PM25 +[proxywetpm25] +description=proxy wet deposition of PM25 unit = mg m-2 d-1 minimum=0 diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index dd9ae6c44..b6701a8be 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -24,8 +24,8 @@ compute_wetso4_from_concprcpso4, concx_to_vmrx, identity, - make_fake_drydep_from_O3, - make_fake_wetdep_from_O3, + make_proxy_drydep_from_O3, + make_proxy_wetdep_from_O3, vmrx_to_concx, ) from pyaerocom.exceptions import ( @@ -133,7 +133,6 @@ class ReadEbasOptions(BrowseDict): _FILTER_IDS = ["prefer_statistics", "wavelength_tol_nm"] def __init__(self, **args): - self.prefer_statistics = ["arithmetic mean", "median"] self.ignore_statistics = ["percentile:15.87", "percentile:84.13"] @@ -242,50 +241,50 @@ class ReadEbas(ReadUngriddedBase): "wetno3": ["concprcpno3", "pr"], "wetnh4": ["concprcpnh4", "pr"], "vmro3max": ["vmro3"], - # Fake drydep + # proxy drydep # Suphar based - "fakedryoxs": ["concprcpoxs", "pr"], - "fakedryso2": ["concprcpoxs", "pr"], - "fakedryso4": ["concprcpoxs", "pr"], + "proxydryoxs": ["concprcpoxs", "pr"], + "proxydryso2": ["concprcpoxs", "pr"], + "proxydryso4": ["concprcpoxs", "pr"], # Oxidized nitrogen based - "fakedryoxn": ["concprcpoxn", "pr"], - "fakedryno2": ["concprcpoxn", "pr"], - "fakedryno2no2": ["concprcpoxn", "pr"], - "fakedryhono": ["concprcpoxn", "pr"], - "fakedryn2o5": ["concprcpoxn", "pr"], - "fakedryhno3": ["concprcpoxn", "pr"], - "fakedryno3c": ["concprcpoxn", "pr"], - "fakedryno3f": ["concprcpoxn", "pr"], + "proxydryoxn": ["concprcpoxn", "pr"], + "proxydryno2": ["concprcpoxn", "pr"], + "proxydryno2no2": ["concprcpoxn", "pr"], + "proxydryhono": ["concprcpoxn", "pr"], + "proxydryn2o5": ["concprcpoxn", "pr"], + "proxydryhno3": ["concprcpoxn", "pr"], + "proxydryno3c": ["concprcpoxn", "pr"], + "proxydryno3f": ["concprcpoxn", "pr"], # Reduced nitrogen based - "fakedryrdn": ["concprcprdn", "pr"], - "fakedrynh3": ["concprcprdn", "pr"], - "fakedrynh4": ["concprcprdn", "pr"], + "proxydryrdn": ["concprcprdn", "pr"], + "proxydrynh3": ["concprcprdn", "pr"], + "proxydrynh4": ["concprcprdn", "pr"], # Other - "fakedryo3": ["vmro3"], - "fakedrypm10": ["concprcpoxs", "pr"], - "fakedrypm25": ["concprcpoxs", "pr"], - # Fake wetdep + "proxydryo3": ["vmro3"], + "proxydrypm10": ["concprcpoxs", "pr"], + "proxydrypm25": ["concprcpoxs", "pr"], + # proxy wetdep # Suphar based - "fakewetoxs": ["concprcpoxs", "pr"], - "fakewetso2": ["concprcpoxs", "pr"], - "fakewetso4": ["concprcpoxs", "pr"], + "proxywetoxs": ["concprcpoxs", "pr"], + "proxywetso2": ["concprcpoxs", "pr"], + "proxywetso4": ["concprcpoxs", "pr"], # Oxidized nitrogen based - "fakewetoxn": ["concprcpoxn", "pr"], - "fakewetno2": ["concprcpoxn", "pr"], - "fakewetno2no2": ["concprcpoxn", "pr"], - "fakewethono": ["concprcpoxn", "pr"], - "fakewetn2o5": ["concprcpoxn", "pr"], - "fakewethno3": ["concprcpoxn", "pr"], - "fakewetno3c": ["concprcpoxn", "pr"], - "fakewetno3f": ["concprcpoxn", "pr"], + "proxywetoxn": ["concprcpoxn", "pr"], + "proxywetno2": ["concprcpoxn", "pr"], + "proxywetno2no2": ["concprcpoxn", "pr"], + "proxywethono": ["concprcpoxn", "pr"], + "proxywetn2o5": ["concprcpoxn", "pr"], + "proxywethno3": ["concprcpoxn", "pr"], + "proxywetno3c": ["concprcpoxn", "pr"], + "proxywetno3f": ["concprcpoxn", "pr"], # Reduced nitrogen based - "fakewetrdn": ["concprcprdn", "pr"], - "fakewetnh3": ["concprcprdn", "pr"], - "fakewetnh4": ["concprcprdn", "pr"], + "proxywetrdn": ["concprcprdn", "pr"], + "proxywetnh3": ["concprcprdn", "pr"], + "proxywetnh4": ["concprcprdn", "pr"], # Other - "fakeweto3": ["vmro3"], - "fakewetpm10": ["concprcpoxs", "pr"], - "fakewetpm25": ["concprcpoxs", "pr"], + "proxyweto3": ["vmro3"], + "proxywetpm10": ["concprcpoxs", "pr"], + "proxywetpm25": ["concprcpoxs", "pr"], } #: Meta information supposed to be migrated to computed variables @@ -309,50 +308,50 @@ class ReadEbas(ReadUngriddedBase): "wetno3": compute_wetno3_from_concprcpno3, "wetso4": compute_wetso4_from_concprcpso4, "vmro3max": calc_vmro3max, - # Fake dry dep + # proxy dry dep # Suphar based - "fakedryoxs": compute_wetoxs_from_concprcpoxs, - "fakedryso2": compute_wetoxs_from_concprcpoxs, - "fakedryso4": compute_wetoxs_from_concprcpoxs, + "proxydryoxs": compute_wetoxs_from_concprcpoxs, + "proxydryso2": compute_wetoxs_from_concprcpoxs, + "proxydryso4": compute_wetoxs_from_concprcpoxs, # Oxidized nitrogen based - "fakedryoxn": compute_wetoxn_from_concprcpoxn, - "fakedryno2": compute_wetoxn_from_concprcpoxn, - "fakedryno2no2": compute_wetoxn_from_concprcpoxn, - "fakedryhono": compute_wetoxn_from_concprcpoxn, - "fakedryn2o5": compute_wetoxn_from_concprcpoxn, - "fakedryhno3": compute_wetoxn_from_concprcpoxn, - "fakedryno3c": compute_wetoxn_from_concprcpoxn, - "fakedryno3f": compute_wetoxn_from_concprcpoxn, + "proxydryoxn": compute_wetoxn_from_concprcpoxn, + "proxydryno2": compute_wetoxn_from_concprcpoxn, + "proxydryno2no2": compute_wetoxn_from_concprcpoxn, + "proxydryhono": compute_wetoxn_from_concprcpoxn, + "proxydryn2o5": compute_wetoxn_from_concprcpoxn, + "proxydryhno3": compute_wetoxn_from_concprcpoxn, + "proxydryno3c": compute_wetoxn_from_concprcpoxn, + "proxydryno3f": compute_wetoxn_from_concprcpoxn, # Reduced nitrogen based - "fakedryrdn": compute_wetrdn_from_concprcprdn, - "fakedrynh3": compute_wetrdn_from_concprcprdn, - "fakedrynh4": compute_wetrdn_from_concprcprdn, + "proxydryrdn": compute_wetrdn_from_concprcprdn, + "proxydrynh3": compute_wetrdn_from_concprcprdn, + "proxydrynh4": compute_wetrdn_from_concprcprdn, # Other - "fakedryo3": make_fake_drydep_from_O3, - "fakedrypm10": compute_wetoxs_from_concprcpoxs, - "fakedrypm25": compute_wetoxs_from_concprcpoxs, - # Fake wet dep + "proxydryo3": make_proxy_drydep_from_O3, + "proxydrypm10": compute_wetoxs_from_concprcpoxs, + "proxydrypm25": compute_wetoxs_from_concprcpoxs, + # proxy wet dep # Suphar based - "fakewetoxs": compute_wetoxs_from_concprcpoxs, - "fakewetso2": compute_wetoxs_from_concprcpoxs, - "fakewetso4": compute_wetoxs_from_concprcpoxs, + "proxywetoxs": compute_wetoxs_from_concprcpoxs, + "proxywetso2": compute_wetoxs_from_concprcpoxs, + "proxywetso4": compute_wetoxs_from_concprcpoxs, # Oxidized nitrogen based - "fakewetoxn": compute_wetoxn_from_concprcpoxn, - "fakewetno2": compute_wetoxn_from_concprcpoxn, - "fakewetno2no2": compute_wetoxn_from_concprcpoxn, - "fakewethono": compute_wetoxn_from_concprcpoxn, - "fakewetn2o5": compute_wetoxn_from_concprcpoxn, - "fakewethno3": compute_wetoxn_from_concprcpoxn, - "fakewetno3c": compute_wetoxn_from_concprcpoxn, - "fakewetno3f": compute_wetoxn_from_concprcpoxn, + "proxywetoxn": compute_wetoxn_from_concprcpoxn, + "proxywetno2": compute_wetoxn_from_concprcpoxn, + "proxywetno2no2": compute_wetoxn_from_concprcpoxn, + "proxywethono": compute_wetoxn_from_concprcpoxn, + "proxywetn2o5": compute_wetoxn_from_concprcpoxn, + "proxywethno3": compute_wetoxn_from_concprcpoxn, + "proxywetno3c": compute_wetoxn_from_concprcpoxn, + "proxywetno3f": compute_wetoxn_from_concprcpoxn, # Reduced nitrogen based - "fakewetrdn": compute_wetrdn_from_concprcprdn, - "fakewetnh3": compute_wetrdn_from_concprcprdn, - "fakewetnh4": compute_wetrdn_from_concprcprdn, + "proxywetrdn": compute_wetrdn_from_concprcprdn, + "proxywetnh3": compute_wetrdn_from_concprcprdn, + "proxywetnh4": compute_wetrdn_from_concprcprdn, # Other - "fakeweto3": make_fake_wetdep_from_O3, - "fakewetpm10": compute_wetoxs_from_concprcpoxs, - "fakewetpm25": compute_wetoxs_from_concprcpoxs, + "proxyweto3": make_proxy_wetdep_from_O3, + "proxywetpm10": compute_wetoxs_from_concprcpoxs, + "proxywetpm25": compute_wetoxs_from_concprcpoxs, } #: Custom reading options for individual variables. Keys need to be valid @@ -387,7 +386,6 @@ class ReadEbas(ReadUngriddedBase): #: by auxiliary variables on class init, for details see __init__ method of #: base class ReadUngriddedBase) def __init__(self, data_id=None, data_dir=None): - super().__init__(data_id=data_id, data_dir=data_dir) self._opts = {"default": ReadEbasOptions()} @@ -1523,7 +1521,6 @@ def _check_correct_freq(self, file, freq_ebas): ) def _flag_incorrect_frequencies(self, filedata): - # time diffs in units of s for each measurement dt = (filedata.stop_meas - filedata.start_meas).astype(float) # frequency in file (supposedly) diff --git a/tests/io/test_read_ebas.py b/tests/io/test_read_ebas.py index fc5a83009..3ff387eee 100644 --- a/tests/io/test_read_ebas.py +++ b/tests/io/test_read_ebas.py @@ -312,8 +312,8 @@ def test_PROVIDES_VARIABLES(reader: ReadEbas): "concnh4coarse", "concno3pm25", "vmrnh3", - "fakedryoxn", - "fakewetpm25", + "proxydryoxn", + "proxywetpm25", "concss25", "concprcpno3", "concprcpso4", @@ -321,38 +321,38 @@ def test_PROVIDES_VARIABLES(reader: ReadEbas): "concom25", "concprcpnh4", "concsscoarse", - "fakedryhno3", - "fakedryhono", - "fakedryn2o5", - "fakedrynh3", - "fakedrynh4", - "fakedryno2", - "fakedryno2no2", - "fakedryno3c", - "fakedryno3f", - "fakedryo3", - "fakedryoxs", - "fakedrypm10", - "fakedrypm25", - "fakedryrdn", - "fakedryso2", - "fakedryso4", - "fakewethno3", - "fakewethono", - "fakewetn2o5", - "fakewetnh3", - "fakewetnh4", - "fakewetno2", - "fakewetno2no2", - "fakewetno3c", - "fakewetno3f", - "fakeweto3", - "fakewetoxn", - "fakewetoxs", - "fakewetpm10", - "fakewetrdn", - "fakewetso2", - "fakewetso4", + "proxydryhno3", + "proxydryhono", + "proxydryn2o5", + "proxydrynh3", + "proxydrynh4", + "proxydryno2", + "proxydryno2no2", + "proxydryno3c", + "proxydryno3f", + "proxydryo3", + "proxydryoxs", + "proxydrypm10", + "proxydrypm25", + "proxydryrdn", + "proxydryso2", + "proxydryso4", + "proxywethno3", + "proxywethono", + "proxywetn2o5", + "proxywetnh3", + "proxywetnh4", + "proxywetno2", + "proxywetno2no2", + "proxywetno3c", + "proxywetno3f", + "proxyweto3", + "proxywetoxn", + "proxywetoxs", + "proxywetpm10", + "proxywetrdn", + "proxywetso2", + "proxywetso4", "vmrhno3", "vmrtp", "wetnh4", From 2be6358f8a3825473e2001f3e73a4e1b47ce9510 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Wed, 29 Mar 2023 10:29:10 +0200 Subject: [PATCH 47/82] linting --- pyaerocom/_concprcp_units_helpers.py | 1 + pyaerocom/aeroval/coldatatojson_helpers.py | 4 +--- pyaerocom/aeroval/experiment_output.py | 3 --- pyaerocom/aeroval/helpers.py | 1 - pyaerocom/aeroval/modelmaps_engine.py | 1 - pyaerocom/aeroval/modelmaps_helpers.py | 1 - pyaerocom/aeroval/obsentry.py | 1 - pyaerocom/aeroval/setupclasses.py | 2 -- pyaerocom/aeroval/superobs_engine.py | 2 -- pyaerocom/colocateddata.py | 2 +- pyaerocom/colocation_auto.py | 5 ----- pyaerocom/combine_vardata_ungridded.py | 4 ---- pyaerocom/config.py | 2 -- pyaerocom/extras/satellite_l2/aeolus_l2a.py | 4 ---- pyaerocom/extras/satellite_l2/base_reader.py | 2 -- pyaerocom/extras/satellite_l2/sentinel5p.py | 5 ----- pyaerocom/griddeddata.py | 17 ++++------------- pyaerocom/helpers.py | 3 --- pyaerocom/io/aerocom_browser.py | 1 - pyaerocom/io/aux_components_fun.py | 2 -- pyaerocom/io/cachehandler_ungridded.py | 2 +- pyaerocom/io/ebas_nasa_ames.py | 6 +++--- pyaerocom/io/fileconventions.py | 1 - pyaerocom/io/iris_io.py | 2 -- pyaerocom/io/read_aasetal.py | 1 - pyaerocom/io/read_aeronet_invv3.py | 1 - pyaerocom/io/read_aeronet_sdav2.py | 1 - pyaerocom/io/read_airnow.py | 1 - pyaerocom/io/read_marcopolo.py | 3 --- pyaerocom/io/readgridded.py | 3 --- pyaerocom/io/readungridded.py | 3 --- pyaerocom/io/readungriddedbase.py | 1 + pyaerocom/io/utils.py | 1 - pyaerocom/metastandards.py | 3 --- pyaerocom/obs_io.py | 1 - pyaerocom/plugins/ghost/reader.py | 1 - pyaerocom/plugins/ipcforests/metadata.py | 1 - pyaerocom/plugins/ipcforests/reader.py | 6 ------ .../plugins/mscw_ctm/additional_variables.py | 4 ---- pyaerocom/plugins/mscw_ctm/reader.py | 1 - pyaerocom/region.py | 2 -- .../testdata-minimal/create_subset_ebas.py | 3 --- .../testdata-minimal/create_subsets_aeronet.py | 4 +--- pyaerocom/stationdata.py | 2 -- pyaerocom/time_resampler.py | 1 - pyaerocom/trends_engine.py | 1 - pyaerocom/trends_helpers.py | 1 - pyaerocom/ungriddeddata.py | 4 ---- pyaerocom/variable.py | 1 - pyaerocom/vert_coords.py | 5 ----- pyaerocom/vertical_profile.py | 2 -- scripts/aeolus2netcdf.py | 1 - scripts/create_var_csv_from_htap2_exceltab.py | 1 - tests/aeroval/test_aeroval_HIGHLEV.py | 1 - tests/aeroval/test_coldatatojson_helpers.py | 2 -- tests/aeroval/test_experiment_output.py | 1 - tests/fixtures/tm5.py | 1 - tests/io/test_read_aasetal.py | 4 ---- tests/io/test_read_airnow.py | 2 -- tests/plot/test_config.py | 1 - tests/plot/test_mapping.py | 1 - tests/plugins/gaw/test_dms.py | 1 - .../mscw_ctm/test_additional_variables.py | 5 ----- tests/plugins/mscw_ctm/test_reader.py | 1 - tests/test_colocation.py | 1 - tests/test_combine_vardata_ungridded.py | 2 -- tests/test_filter.py | 2 -- tests/test_griddeddata.py | 1 - tests/test_helpers.py | 1 - tests/test_metastandards.py | 2 -- tests/test_ungriddeddata.py | 1 - 71 files changed, 13 insertions(+), 151 deletions(-) diff --git a/pyaerocom/_concprcp_units_helpers.py b/pyaerocom/_concprcp_units_helpers.py index 3f79c3eb7..1621db58d 100644 --- a/pyaerocom/_concprcp_units_helpers.py +++ b/pyaerocom/_concprcp_units_helpers.py @@ -12,6 +12,7 @@ PR_IMPLICIT_UNITS = [Unit("mm")] DEP_TEST_UNIT = "kg m-2 s-1" + # ToDo: check if still needed def translate_rate_units_implicit(unit_implicit, ts_type): unit = Unit(unit_implicit) diff --git a/pyaerocom/aeroval/coldatatojson_helpers.py b/pyaerocom/aeroval/coldatatojson_helpers.py index 65b982639..981a9811a 100644 --- a/pyaerocom/aeroval/coldatatojson_helpers.py +++ b/pyaerocom/aeroval/coldatatojson_helpers.py @@ -634,7 +634,7 @@ def _init_site_coord_arrays(data): def _get_stat_regions(lats, lons, regions): regs = [] - for (lat, lon) in zip(lats, lons): + for lat, lon in zip(lats, lons): reg = find_closest_region_coord(lat, lon, regions=regions) regs.append(reg) return regs @@ -857,11 +857,9 @@ def _process_map_and_scat( # Code for the calculation of trends if add_trends and freq != "daily": - (start, stop) = _get_min_max_year_periods([per]) if stop - start >= trends_min_yrs: - try: time = subset.data.time.values (obs_trend, mod_trend) = _make_trends( diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 5a131b70c..90c0ed51a 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -445,7 +445,6 @@ def _check_clean_ts_file(self, fp): return modified def _clean_modelmap_files(self): - # Note: to be called after cleanup of files in map subdir json_files = self._get_json_output_files("contour") rm = [] @@ -627,7 +626,6 @@ def _init_menu_entry(self, var: str) -> dict: return {"type": tp, "cat": cat, "name": name, "longname": lname, "obs": {}} def _check_ovar_mvar_entry(self, mcfg, mod_var, ocfg, obs_var): - muv = mcfg.model_use_vars mrv = mcfg.model_rename_vars @@ -750,7 +748,6 @@ def _create_menu_dict(self): (obs_name, obs_var, vert_code, mod_name, mod_var, per) = self._info_from_map_file(file) if self._is_part_of_experiment(obs_name, obs_var, mod_name, mod_var): - mcfg = self.cfg.model_cfg.get_entry(mod_name) var = mcfg.get_varname_web(mod_var, obs_var) if not var in new: diff --git a/pyaerocom/aeroval/helpers.py b/pyaerocom/aeroval/helpers.py index ae5c7b387..ccde12deb 100644 --- a/pyaerocom/aeroval/helpers.py +++ b/pyaerocom/aeroval/helpers.py @@ -151,7 +151,6 @@ def _get_min_max_year_periods(statistics_periods): def make_dummy_model(obs_list: list, cfg) -> str: - # Sets up variable for the model register tmpdir = const.LOCAL_TMP_DIR const.add_data_search_dir(tmpdir) diff --git a/pyaerocom/aeroval/modelmaps_engine.py b/pyaerocom/aeroval/modelmaps_engine.py index 1ba8da1ce..025d349ce 100644 --- a/pyaerocom/aeroval/modelmaps_engine.py +++ b/pyaerocom/aeroval/modelmaps_engine.py @@ -49,7 +49,6 @@ def run(self, **kwargs): return files def _get_vars_to_process(self, model_name, var_list): - mvars = self.cfg.model_cfg.get_entry(model_name).get_vars_to_process( self.cfg.obs_cfg.get_all_vars() )[1] diff --git a/pyaerocom/aeroval/modelmaps_helpers.py b/pyaerocom/aeroval/modelmaps_helpers.py index c0543004e..2db9b9272 100644 --- a/pyaerocom/aeroval/modelmaps_helpers.py +++ b/pyaerocom/aeroval/modelmaps_helpers.py @@ -67,7 +67,6 @@ def griddeddata_to_jsondict(data, lat_res_deg=5, lon_res_deg=5): if isinstance(nparr, dask.array.core.Array): nparr = nparr.compute() for i, (lat, lon) in enumerate(stacked.station_name.values): - coord = lat, lon vals = nparr[:, i] dd[str(coord)] = sd = {} diff --git a/pyaerocom/aeroval/obsentry.py b/pyaerocom/aeroval/obsentry.py index f1ba0256e..81806c646 100644 --- a/pyaerocom/aeroval/obsentry.py +++ b/pyaerocom/aeroval/obsentry.py @@ -60,7 +60,6 @@ class ObsEntry(BrowseDict): obs_vert_type = StrType() def __init__(self, **kwargs): - self.obs_id = "" self.obs_vars = [] diff --git a/pyaerocom/aeroval/setupclasses.py b/pyaerocom/aeroval/setupclasses.py index ae43e528b..35935c9e6 100644 --- a/pyaerocom/aeroval/setupclasses.py +++ b/pyaerocom/aeroval/setupclasses.py @@ -207,7 +207,6 @@ def _get_all_period_strings(self): class WebDisplaySetup(ConstrainedContainer): - map_zoom = EitherOf(["World", "Europe"]) regions_how = EitherOf(["default", "aerocom", "htap", "country"]) @@ -385,7 +384,6 @@ def from_json(filepath: str) -> "EvalSetup": return EvalSetup(**settings) def _import_aux_funs(self): - h = ReadAuxHandler(self.io_aux_file) self._aux_funs.update(**h.import_all()) diff --git a/pyaerocom/aeroval/superobs_engine.py b/pyaerocom/aeroval/superobs_engine.py index 8b011aa3a..5478018a2 100644 --- a/pyaerocom/aeroval/superobs_engine.py +++ b/pyaerocom/aeroval/superobs_engine.py @@ -18,7 +18,6 @@ class SuperObsEngine(ProcessingEngine, HasColocator): """ def run(self, model_name, obs_name, var_list, try_colocate_if_missing=True): - self._process_entry( model_name=model_name, obs_name=obs_name, @@ -27,7 +26,6 @@ def run(self, model_name, obs_name, var_list, try_colocate_if_missing=True): ) def _process_entry(self, model_name, obs_name, var_list, try_colocate_if_missing): - sobs_cfg = self.cfg.obs_cfg.get_entry(obs_name) if var_list is None: diff --git a/pyaerocom/colocateddata.py b/pyaerocom/colocateddata.py index 5f0cdacf4..f9bd86c6f 100644 --- a/pyaerocom/colocateddata.py +++ b/pyaerocom/colocateddata.py @@ -1715,7 +1715,7 @@ def apply_region_mask(self, region_id, inplace=False): drop_idx = [] nstats = len(arr.station_name) - for (lat, lon, stat) in data._iter_stats(): + for lat, lon, stat in data._iter_stats(): if get_mask_value(lat, lon, mask) < 1: drop_idx.append(stat) diff --git a/pyaerocom/colocation_auto.py b/pyaerocom/colocation_auto.py index 06ac8c87d..6edb40934 100644 --- a/pyaerocom/colocation_auto.py +++ b/pyaerocom/colocation_auto.py @@ -319,7 +319,6 @@ def __init__( save_coldata=False, **kwargs, ): - self.model_id = model_id self.obs_id = obs_id self.obs_vars = obs_vars @@ -1029,7 +1028,6 @@ def _get_gridded_reader_class(self, what): return reader def _check_add_model_read_aux(self, model_var): - if not model_var in self.model_read_aux: return False info = self.model_read_aux[model_var] @@ -1078,7 +1076,6 @@ def _check_obs_vars_available(self): self.obs_vars = avail def _print_processing_status(self): - mname = self.get_model_name() oname = self.get_obs_name() logger.info(f"Colocation processing status for {mname} vs. {oname}") @@ -1168,7 +1165,6 @@ def _get_ts_type_read(self, var_name, is_model): return tst def _read_gridded(self, var_name, is_model): - start, stop = self.start, self.stop ts_type_read = self._get_ts_type_read(var_name, is_model) kwargs = {} @@ -1367,7 +1363,6 @@ def _colocation_func(self): return colocate_gridded_gridded def _prepare_colocation_args(self, model_var, obs_var): - model_data = self.get_model_data(model_var) obs_data = self.get_obs_data(obs_var) rshow = self._eval_resample_how(model_var, obs_var) diff --git a/pyaerocom/combine_vardata_ungridded.py b/pyaerocom/combine_vardata_ungridded.py index fda3af7bf..b593021c2 100644 --- a/pyaerocom/combine_vardata_ungridded.py +++ b/pyaerocom/combine_vardata_ungridded.py @@ -23,7 +23,6 @@ def _check_input_data_ids_and_vars(data_ids_and_vars): def _map_same_stations(stats_short, stats_long, match_stats_how, match_stats_tol_km): - long_coords = list(zip(stats_long["latitude"], stats_long["longitude"])) # index matches and corresponding station name matches @@ -218,7 +217,6 @@ def _combine_2_sites( add_ts = df.mean(axis=1) elif merge_how == "eval": - func = merge_eval_fun.replace(col_order[0], col_names[0]) func = func.replace(col_order[1], col_names[1]) if "=" in merge_eval_fun: @@ -233,7 +231,6 @@ def _combine_2_sites( var_name_out = var_name_out.replace(f"{stat_other.data_id};", "") if add_ts is not None: - var_info = {"ts_type": to_ts_type, "units": var_unit_out} var_info.update(merge_info_vars) @@ -458,7 +455,6 @@ def combine_vardata_ungridded( var_short, var_long = short["var_name"], long["var_name"] for idx_short, idx_long in zip(_index_short, _index_long): - stat_short = short["stats"][idx_short] stat_short.check_var_unit_aerocom(var_short) stat_long = long["stats"][idx_long] diff --git a/pyaerocom/config.py b/pyaerocom/config.py index 5e7ffb9c0..aa74fdf7a 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -193,7 +193,6 @@ class Config: _LUSTRE_CHECK_PATH = "/project/aerocom/aerocom1/" def __init__(self, config_file=None, try_infer_environment=True): - # Directories self._outputdir = None self._cache_basedir = None @@ -278,7 +277,6 @@ def _basedirs_search_db(self): return [self.ROOTDIR, self.HOMEDIR] def _infer_config_from_basedir(self, basedir): - basedir = os.path.normpath(basedir) for env_id, chk_sub in self._check_subdirs_cfg.items(): chkdir = os.path.join(basedir, chk_sub) diff --git a/pyaerocom/extras/satellite_l2/aeolus_l2a.py b/pyaerocom/extras/satellite_l2/aeolus_l2a.py index 0a9d1f57c..594dff534 100755 --- a/pyaerocom/extras/satellite_l2/aeolus_l2a.py +++ b/pyaerocom/extras/satellite_l2/aeolus_l2a.py @@ -1339,7 +1339,6 @@ def to_netcdf_simple( data_to_write=None, gridded=False, ): - """method to store the file contents in a very basic netcdf file Parameters: @@ -2397,7 +2396,6 @@ def plot_profile_v2( interpolated = f(target_heights) out_arr[time_step_idx, :] = interpolated except ValueError: - # this happens when height_data and var_data have only one entry # set out_arr[time_step_idx,:] to NaN in this case for now # breakpoint() @@ -2724,7 +2722,6 @@ def plot_profile_v3( interpolated = f(target_heights) out_arr[time_step_idx, :] = interpolated except ValueError: - # this happens when height_data and var_data have only one entry # set out_arr[time_step_idx,:] to NaN in this case for now # breakpoint() @@ -3089,7 +3086,6 @@ def _to_grid_grid_init( if levelno is None or levelno == 1 or levelno == 0: super()._to_grid_grid_init(gridtype=gridtype, vars=vars, init_time=init_time) else: - import time start_time = time.perf_counter() diff --git a/pyaerocom/extras/satellite_l2/base_reader.py b/pyaerocom/extras/satellite_l2/base_reader.py index 936867c2e..b18632e01 100644 --- a/pyaerocom/extras/satellite_l2/base_reader.py +++ b/pyaerocom/extras/satellite_l2/base_reader.py @@ -307,7 +307,6 @@ def read( # and extract the files with supported suffixes to const.CACHEDIR non_archive_files = [] for idx, _file in enumerate(sorted(files)): - self.logger.info(f"file: {_file}") suffix = pathlib.Path(_file).suffix if suffix in self.SUPPORTED_ARCHIVE_SUFFIXES: @@ -496,7 +495,6 @@ def to_netcdf_simple( data_to_write=None, gridded=False, ): - """method to store the file contents in a very basic netcdf file Parameters: diff --git a/pyaerocom/extras/satellite_l2/sentinel5p.py b/pyaerocom/extras/satellite_l2/sentinel5p.py index 6091e0d2b..9594f4b3f 100755 --- a/pyaerocom/extras/satellite_l2/sentinel5p.py +++ b/pyaerocom/extras/satellite_l2/sentinel5p.py @@ -574,7 +574,6 @@ def read_file( elif len(groups) == 2: file_data[var] = np.squeeze(coda.fetch(product, groups[0], groups[1])) elif len(groups) == 4: - file_data[var] = np.squeeze( coda.fetch(product, groups[0], groups[1], groups[2], groups[3]) ) @@ -659,7 +658,6 @@ def to_netcdf_simple( gridded=False, apply_quality_flag=0.0, ): - if data_to_write is None: _data = self.data else: @@ -986,7 +984,6 @@ def to_grid( return if engine == "python": - data_for_gridding, gridded_var_data = self._to_grid_grid_init( gridtype=gridtype, vars=vars, init_time=_data["time"].mean() ) @@ -1074,7 +1071,6 @@ def to_grid( pass if isinstance(data, xr.Dataset): - pass else: super().to_grid( @@ -1151,5 +1147,4 @@ def select_bbox(self, data=None, vars=None, bbox=None): return _data else: - super().select_bbox(_data, bbox) diff --git a/pyaerocom/griddeddata.py b/pyaerocom/griddeddata.py index 2db3ba189..da48f9a73 100644 --- a/pyaerocom/griddeddata.py +++ b/pyaerocom/griddeddata.py @@ -135,7 +135,6 @@ class GriddedData: def __init__( self, input=None, var_name=None, check_unit=True, convert_unit_on_init=True, **meta ): - if input is None: input = iris.cube.Cube([]) @@ -742,7 +741,6 @@ def _check_invalid_unit_alias(self): """ cube = self.grid if "invalid_units" in cube.attributes and cube.attributes["invalid_units"] in UALIASES: - from_unit = cube.attributes["invalid_units"] to_unit = UALIASES[from_unit] logger.info(f"Updating invalid unit in {repr(cube)} from {from_unit} to {to_unit}") @@ -825,7 +823,6 @@ def _try_convert_custom_unit(self, new_unit): self._apply_unit_mulfac(new_unit, mulfac) def _apply_unit_mulfac(self, new_unit, mulfac): - if mulfac != 1: new_cube = self._grid * mulfac new_cube.attributes.update(self._grid.attributes) @@ -1042,7 +1039,6 @@ def mean_at_coords(self, latitude=None, longitude=None, time_resample_kwargs=Non return np.nanmean(mean) def _coords_to_iris_sample_points(self, **coords): - sample_points = [] num = None for cname, vals in coords.items(): @@ -1057,7 +1053,7 @@ def _coords_to_iris_sample_points(self, **coords): def _iris_sample_points_to_coords(self, sample_points): lats, lons = None, None - for (name, vals) in sample_points: + for name, vals in sample_points: if isnumeric(vals): vals = [vals] if name in ("lat", "latitude"): @@ -1082,7 +1078,6 @@ def to_time_series( use_iris=False, **coords, ): - """Extract time-series for provided input coordinates (lon, lat) Extract time series for each lon / lat coordinate in this cube or at @@ -1176,7 +1171,6 @@ def to_time_series( ) def _to_time_series_xarray(self, scheme="nearest", add_meta=None, ts_type=None, **coords): - try: self.check_dimcoords_tseries() except DimensionOrderError: @@ -1223,7 +1217,6 @@ def _to_time_series_xarray(self, scheme="nearest", add_meta=None, ts_type=None, lats = subset[lat_id].data lons = subset[lon_id].data for sidx in range(subset.shape[-1]): - data = StationData( latitude=lats[sidx], longitude=lons[sidx], @@ -1320,7 +1313,6 @@ def _to_timeseries_2D( def _to_timeseries_3D( self, sample_points, scheme, collapse_scalar, vert_scheme, add_meta=None ): - # Data contains vertical dimension data = self._apply_vert_scheme(sample_points, vert_scheme) @@ -1646,7 +1638,6 @@ def _resample_time_iris(self, to_ts_type): return data def _resample_time_xarray(self, to_ts_type, how, min_num_obs): - arr = xr.DataArray.from_iris(self.cube) from_ts_type = self.ts_type try: @@ -1654,7 +1645,9 @@ def _resample_time_xarray(self, to_ts_type, how, min_num_obs): arr_out = rs.resample( to_ts_type, from_ts_type=from_ts_type, how=how, min_num_obs=min_num_obs ) - except ValueError: # likely non-standard datetime objects in array (cf https://github.com/pydata/xarray/issues/3426) + except ( + ValueError + ): # likely non-standard datetime objects in array (cf https://github.com/pydata/xarray/issues/3426) arr["time"] = self.time_stamps() rs = TimeResampler(arr) arr_out = rs.resample( @@ -2087,7 +2080,6 @@ def _check_meta_netcdf(self): self.cube.attributes = meta_out def _to_netcdf_aerocom(self, out_dir, **kwargs): - years = self.years_avail() outpaths = [] for subset in self.split_years(years): @@ -2562,7 +2554,6 @@ def delete_aux_vars(self): """Delete auxiliary variables and iris AuxFactories""" c = self.cube for aux_fac in c.aux_factories: - c.remove_aux_factory(aux_fac) for coord in c.coords(): diff --git a/pyaerocom/helpers.py b/pyaerocom/helpers.py index 79e590d67..858a754e2 100644 --- a/pyaerocom/helpers.py +++ b/pyaerocom/helpers.py @@ -63,7 +63,6 @@ def varlist_aerocom(varlist): - if isinstance(varlist, str): varlist = [varlist] elif not isinstance(varlist, list): @@ -348,7 +347,6 @@ def numpy_to_cube(data, dims=None, var_name=None, units=None, **attrs): sh = data.shape if dims is not None: if not len(dims) == data.ndim: - raise DataDimensionError("Input number of dimensios must match array dimension number") for i, dim in enumerate(dims): if not isinstance(dim, iris.coords.DimCoord): @@ -1403,7 +1401,6 @@ def datetime2str(time, ts_type=None): def start_stop_str(start, stop=None, ts_type=None): - conv = TS_TYPE_DATETIME_CONV[ts_type] if is_year(start) and stop is None: return str(start) diff --git a/pyaerocom/io/aerocom_browser.py b/pyaerocom/io/aerocom_browser.py index 3531789c0..18e51c10d 100644 --- a/pyaerocom/io/aerocom_browser.py +++ b/pyaerocom/io/aerocom_browser.py @@ -112,7 +112,6 @@ def _browse(self, name_or_pattern, ignorecase=True, return_if_match=True): # now check if it is actually an exact match, if # applicable if return_if_match: - if ignorecase: match = name_or_pattern.lower() == subdir.lower() else: diff --git a/pyaerocom/io/aux_components_fun.py b/pyaerocom/io/aux_components_fun.py index b6926aead..408b2aeb9 100644 --- a/pyaerocom/io/aux_components_fun.py +++ b/pyaerocom/io/aux_components_fun.py @@ -91,14 +91,12 @@ def vmr_to_conc(data, vmr_unit, var_name, to_unit, component_unit=None): def calc_concNhno3_from_vmr(data): - return vmr_to_conc( data, vmr_unit="nmol mol-1", var_name="hno3", to_unit="ug m-3", component_unit="N" ) def calc_concNnh3_from_vmr(data): - return vmr_to_conc( data, vmr_unit="nmol mol-1", var_name="nh3", to_unit="ug m-3", component_unit="N" ) diff --git a/pyaerocom/io/cachehandler_ungridded.py b/pyaerocom/io/cachehandler_ungridded.py index fc33b0d13..53174a017 100644 --- a/pyaerocom/io/cachehandler_ungridded.py +++ b/pyaerocom/io/cachehandler_ungridded.py @@ -12,6 +12,7 @@ logger = logging.getLogger(__name__) + # TODO: Write data attribute list contains_vars in header of pickled file and # check if variables match the request class CacheHandlerUngridded: @@ -140,7 +141,6 @@ def file_path(self, var_or_file_name, cache_dir=None): return os.path.join(cache_dir, var_or_file_name) def _check_pkl_head_vs_database(self, in_handle): - current = self.cache_meta_info() head = pickle.load(in_handle) diff --git a/pyaerocom/io/ebas_nasa_ames.py b/pyaerocom/io/ebas_nasa_ames.py index 51d25b027..429104b56 100644 --- a/pyaerocom/io/ebas_nasa_ames.py +++ b/pyaerocom/io/ebas_nasa_ames.py @@ -487,7 +487,7 @@ def all_cols_contain(self, colnums, what): def assign_flagcols(self): _prev = 0 - for (idx, item) in enumerate(self.var_defs): + for idx, item in enumerate(self.var_defs): if item.is_flag: for _idx in range(_prev, idx): self.var_defs[_idx].flag_col = idx @@ -495,7 +495,7 @@ def assign_flagcols(self): def init_flags(self, evaluate=True): """Decode flag columns and store info in :attr:`flags`""" - for (idx, item) in enumerate(self.var_defs): + for idx, item in enumerate(self.var_defs): if item.is_flag: data = self.data[:, idx] flag = EbasFlagCol(raw_data=data, interpret_on_init=evaluate) @@ -760,7 +760,7 @@ def _data_short_str(self): def print_col_info(self): """Print information about individual columns""" - for (idx, coldef) in enumerate(self.var_defs): + for idx, coldef in enumerate(self.var_defs): print(f"Column {idx}\n{coldef}") def __str__(self): diff --git a/pyaerocom/io/fileconventions.py b/pyaerocom/io/fileconventions.py index b9322e585..360d67cd7 100644 --- a/pyaerocom/io/fileconventions.py +++ b/pyaerocom/io/fileconventions.py @@ -48,7 +48,6 @@ def __init__( data_id_pos=None, from_file=None, ): - self.name = name self.file_sep = file_sep diff --git a/pyaerocom/io/iris_io.py b/pyaerocom/io/iris_io.py index 3373e8cdc..d54360e4a 100644 --- a/pyaerocom/io/iris_io.py +++ b/pyaerocom/io/iris_io.py @@ -213,7 +213,6 @@ def check_and_regrid_lons_cube(cube): def check_dim_coord_names_cube(cube): - from pyaerocom import const coords = dict( @@ -536,7 +535,6 @@ def correct_time_coord(cube, ts_type, year): def _check_correct_dtypes_timedim_cube_list(cubes): - try: dtypes = np.unique([cube.coord("time").points.dtype for cube in cubes]) except iris.exceptions.CoordinateNotFoundError: diff --git a/pyaerocom/io/read_aasetal.py b/pyaerocom/io/read_aasetal.py index f1d11fa89..1dad32ec2 100644 --- a/pyaerocom/io/read_aasetal.py +++ b/pyaerocom/io/read_aasetal.py @@ -285,7 +285,6 @@ def read(self, vars_to_retrieve=None): metadata[meta_key]["var_info"] = {} for var_count, var in enumerate(temp_vars): - values = stat[var] start = idx + var_count * num_times stop = start + num_times diff --git a/pyaerocom/io/read_aeronet_invv3.py b/pyaerocom/io/read_aeronet_invv3.py index 7e3a3d2cf..377faf7a4 100644 --- a/pyaerocom/io/read_aeronet_invv3.py +++ b/pyaerocom/io/read_aeronet_invv3.py @@ -126,7 +126,6 @@ def read_file(self, filename, vars_to_retrieve=None, vars_as_series=False): self.logger.debug(f"Reading file {filename}") with open(filename) as in_file: - data_out["dataset_info"] = in_file.readline().strip() self.logger.debug(f"Skipping line: {in_file.readline()}") data_out["algorithm_info"] = in_file.readline().strip() diff --git a/pyaerocom/io/read_aeronet_sdav2.py b/pyaerocom/io/read_aeronet_sdav2.py index b227e1396..38df02c10 100755 --- a/pyaerocom/io/read_aeronet_sdav2.py +++ b/pyaerocom/io/read_aeronet_sdav2.py @@ -205,7 +205,6 @@ def read_file(self, filename, vars_to_retrieve=None, vars_as_series=False): self.logger.info(f"Reading file {filename}") # Iterate over the lines of the file with open(filename) as in_file: - c_head_line = in_file.readline() c_algorithm = in_file.readline() diff --git a/pyaerocom/io/read_airnow.py b/pyaerocom/io/read_airnow.py index 1b2f9b756..e9c7b86e3 100644 --- a/pyaerocom/io/read_airnow.py +++ b/pyaerocom/io/read_airnow.py @@ -362,7 +362,6 @@ def _filedata_to_statlist(self, arrs, vars_to_retrieve): dtime = self.make_datetime64_array(data[:, 0], data[:, 1]) stats = [] for var in vars_to_retrieve: - # extract only variable data (should speed things up) var_in_file = self.VAR_MAP[var] mask = data[:, varcol] == var_in_file diff --git a/pyaerocom/io/read_marcopolo.py b/pyaerocom/io/read_marcopolo.py index 9c44ce21d..bb19c124b 100644 --- a/pyaerocom/io/read_marcopolo.py +++ b/pyaerocom/io/read_marcopolo.py @@ -198,7 +198,6 @@ def _read_file(self, file): def _make_station_data( self, var, var_stats_unique, stat_ids, var_stats, var_dtime, var_values, stat_meta ): - units = self.VAR_UNITS stats = [] for stat_id in var_stats_unique: @@ -226,7 +225,6 @@ def _make_station_data( return stats def _read_files(self, files, vars_to_retrieve): - filecols = self.FILE_COLS stat_meta = self._init_station_metadata() @@ -291,7 +289,6 @@ def read_file(self): raise NotImplementedError("Not needed for these data since the format is unsuitable...") def compute_additional_vars(self, statlist_from_file, vars_to_compute): - for var in vars_to_compute: fun = self.AUX_FUNS[var] requires = self.AUX_REQUIRES[var] diff --git a/pyaerocom/io/readgridded.py b/pyaerocom/io/readgridded.py index 63aa164aa..ef7b77cea 100755 --- a/pyaerocom/io/readgridded.py +++ b/pyaerocom/io/readgridded.py @@ -206,7 +206,6 @@ class specifying details of the file naming convention for the model VERT_ALT = {"Surface": "ModelLevel"} # , "2D": "2D"} def __init__(self, data_id=None, data_dir=None, file_convention="aerocom3"): - self._data_dir = None #: data_id of gridded dataset @@ -480,7 +479,6 @@ def _check_var_match_pattern(self, var_name): if fnmatch.fnmatch(var_name, pattern): vars_required = self.AUX_REQUIRES[pattern] for addvar in vars_required: - if not "*" in addvar: vars_found.append(addvar) else: @@ -1699,7 +1697,6 @@ def read_var( ) if constraints is not None: - if isinstance(constraints, dict): constraints = [constraints] for constraint in constraints: diff --git a/pyaerocom/io/readungridded.py b/pyaerocom/io/readungridded.py index a4d257661..ad7deb340 100755 --- a/pyaerocom/io/readungridded.py +++ b/pyaerocom/io/readungridded.py @@ -71,7 +71,6 @@ class ReadUngridded: DONOTCACHE_NAME = "DONOTCACHE" def __init__(self, data_ids=None, ignore_cache=False, data_dirs=None): - # will be assigned in setter method of data_ids self._data_ids = [] self._data_dirs = {} @@ -387,7 +386,6 @@ def read_dataset( data_read = None if len(vars_to_read) > 0: - _loglevel = logger.level logger.setLevel(logging.INFO) data_read = reader.read(vars_to_read, **kwargs) @@ -529,7 +527,6 @@ def read_dataset_post( for aux_var in aux_vars: input_data_ids_vars.append((aux_data, aux_id, aux_var)) else: - # read variables individually, so filter_post is more # flexible if some post filters are specified for # individual variables... diff --git a/pyaerocom/io/readungriddedbase.py b/pyaerocom/io/readungriddedbase.py index 24e822d4a..6d1211441 100644 --- a/pyaerocom/io/readungriddedbase.py +++ b/pyaerocom/io/readungriddedbase.py @@ -15,6 +15,7 @@ logger = logging.getLogger(__name__) + # TODO: Proposal: include attribute ts_type that is by default undefined but # may be set to either of the defined class ReadUngriddedBase(abc.ABC): diff --git a/pyaerocom/io/utils.py b/pyaerocom/io/utils.py index 1da4f64d7..eaaa88dbe 100644 --- a/pyaerocom/io/utils.py +++ b/pyaerocom/io/utils.py @@ -9,7 +9,6 @@ def get_ungridded_reader(obs_id): - for reader in ReadUngridded.SUPPORTED_READERS: if obs_id in reader.SUPPORTED_DATASETS: return reader diff --git a/pyaerocom/metastandards.py b/pyaerocom/metastandards.py index bc2aa16f2..8eb679d11 100644 --- a/pyaerocom/metastandards.py +++ b/pyaerocom/metastandards.py @@ -57,7 +57,6 @@ class DataSource(BrowseDict): _ini_file_name = "data_sources.ini" def __init__(self, **info): - self.data_id = None self.dataset_name = None self.data_product = None @@ -170,7 +169,6 @@ class StationMetaData(DataSource): """ def __init__(self, **info): - self.filename = None self.station_id = None @@ -215,7 +213,6 @@ class AerocomDataID: KEYS = ["model_name", "meteo", "experiment", "perturbation"] def __init__(self, data_id=None, **meta_info): - self._data_id = None self._values = None diff --git a/pyaerocom/obs_io.py b/pyaerocom/obs_io.py index b68f51cdf..d5c66fd02 100644 --- a/pyaerocom/obs_io.py +++ b/pyaerocom/obs_io.py @@ -35,7 +35,6 @@ class AuxInfoUngridded: def __init__( self, data_id, vars_supported, aux_requires, aux_merge_how, aux_funs=None, aux_units=None ): - self.data_id = data_id self.vars_supported = vars_supported diff --git a/pyaerocom/plugins/ghost/reader.py b/pyaerocom/plugins/ghost/reader.py index f7dee073a..facd2f322 100644 --- a/pyaerocom/plugins/ghost/reader.py +++ b/pyaerocom/plugins/ghost/reader.py @@ -380,7 +380,6 @@ def read_file(self, filename, var_to_read=None, invalidate_flags=None, var_to_wr invalid = self._eval_flags(vardata, invalidate_flags, ds) for idx in ds.station.values: - stat = {} meta = StationMetaData() meta["ts_type"] = self.TS_TYPE diff --git a/pyaerocom/plugins/ipcforests/metadata.py b/pyaerocom/plugins/ipcforests/metadata.py index 912bde80d..966836e40 100644 --- a/pyaerocom/plugins/ipcforests/metadata.py +++ b/pyaerocom/plugins/ipcforests/metadata.py @@ -222,7 +222,6 @@ def _add_species_to_var_info(self, species: str, unit: str) -> None: def add_measurement( self, species: str, time: datetime, measurement: float, unit: str, flag: int ) -> None: - """ Adds a single measurement to the data and time lists. If it is the first measurement, a new var_info is created diff --git a/pyaerocom/plugins/ipcforests/reader.py b/pyaerocom/plugins/ipcforests/reader.py index 4290a5a0a..6182ed1ca 100644 --- a/pyaerocom/plugins/ipcforests/reader.py +++ b/pyaerocom/plugins/ipcforests/reader.py @@ -243,7 +243,6 @@ def read_file(self, filename, vars_to_retrieve=None, last_line=None): quantity = float(quantity) try: - self.metadata.plots.plots[country_code] self.metadata.plots.plots[country_code][plot_code] self.metadata.plots.plots[country_code][plot_code][sampler_code] @@ -286,7 +285,6 @@ def read_file(self, filename, vars_to_retrieve=None, last_line=None): # Sea-salt correction # The factor self.SEASALT_CORRECTION[species] is the factor use to go from mg/L to mg S/L (for sulpher) if species in self.SEASALT_CORRECTION: - na_factor = ( self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") * self.SEASALT_FACTORS["wetna"] @@ -369,7 +367,6 @@ def _get_days_date_ts_type( start: str | datetime, stop: str | datetime, ) -> Tuple[float | None, datetime | None, str | None]: - if start != "" and stop != "": if isinstance(start, str): start = datetime.strptime(start, "%Y-%m-%d") @@ -385,7 +382,6 @@ def _get_days_date_ts_type( raise ValueError(f"Metadata is not read yet") try: - days = self.metadata.plots.get_days(year, country_code, plot_code, sampler_code) except ValueError as e: logger.warning(repr(e)) @@ -406,7 +402,6 @@ def _get_days_date_ts_type( return days, dtime, ts_type def _get_tstype(self, start: datetime, stop: datetime) -> str: - days = (stop - start).days return SurveyYear.get_tstype(days) @@ -418,7 +413,6 @@ def _clean_data_with_flags( flags: list[int], species: str, ) -> list[float]: - data_array = np.array(data) flags_array = np.array(flags) years = np.array([i.year for i in time]) diff --git a/pyaerocom/plugins/mscw_ctm/additional_variables.py b/pyaerocom/plugins/mscw_ctm/additional_variables.py index 422762f57..07ffcc74e 100644 --- a/pyaerocom/plugins/mscw_ctm/additional_variables.py +++ b/pyaerocom/plugins/mscw_ctm/additional_variables.py @@ -98,7 +98,6 @@ def calc_concNno3pm25( # ToDo: add docstring def calc_concno3pm10(concno3f, concno3c): - concno3pm10 = concno3f + concno3c concno3pm10.attrs["var_name"] = "concno3pm10" @@ -109,7 +108,6 @@ def calc_concno3pm10(concno3f, concno3c): # ToDo: add docstring def calc_concno3pm25(concno3f, concno3c, fine_from_coarse_fraction=0.134): - concno3pm25 = concno3f + fine_from_coarse_fraction * concno3c concno3pm25.attrs["var_name"] = "concno3pm25" @@ -280,7 +278,6 @@ def calc_vmrox_from_conc(concno2, conco3): def calc_vmrno2(concno2: xr.DataArray) -> xr.DataArray: - vmrno2 = concx_to_vmrx( data=concno2, p_pascal=p0, # 1013 hPa (US standard atm) @@ -352,7 +349,6 @@ def calc_concNno(concno): def calc_vmro3(conco3): - vmro3 = concx_to_vmrx( data=conco3, p_pascal=p0, # 1013 hPa (US standard atm) diff --git a/pyaerocom/plugins/mscw_ctm/reader.py b/pyaerocom/plugins/mscw_ctm/reader.py index 93d7bec17..a3593fabe 100755 --- a/pyaerocom/plugins/mscw_ctm/reader.py +++ b/pyaerocom/plugins/mscw_ctm/reader.py @@ -244,7 +244,6 @@ def _get_yrs_from_filepaths(self): fps = self.filepaths yrs = [] for fp in fps: - try: yr = re.search(r".*(20\d\d).*", fp).group(1) except: # pragma: no cover diff --git a/pyaerocom/region.py b/pyaerocom/region.py index d65d49a4b..09c6663b5 100644 --- a/pyaerocom/region.py +++ b/pyaerocom/region.py @@ -48,7 +48,6 @@ class Region(BrowseDict): """ def __init__(self, region_id=None, **kwargs): - if region_id is None: region_id = ALL_REGION_NAME @@ -174,7 +173,6 @@ def get_mask_data(self): return self._mask_data def plot_mask(self, ax, color, alpha=0.2): - mask = self.get_mask_data() # import numpy as np data = mask.data diff --git a/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py b/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py index 67270ee54..3944e4160 100755 --- a/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py +++ b/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py @@ -74,7 +74,6 @@ def check_outdated(filedir): files_valid = [] with open(JSON_FILE, "r") as f: - data = simplejson.load(f) for var, stats in data.items(): @@ -156,7 +155,6 @@ def get_files_var_statnum(data, var, statnum): def main(): - # reader = pya.io.ReadUngridded(NAME, data_dir=EBAS_BASE_DIR) reader = pya.io.ReadUngridded( NAME, @@ -245,7 +243,6 @@ def main(): if not UPDATE: print("NOTHING WILL BE COPIED TO TEST DATA") else: - src = Path(EBAS_BASE_DIR).joinpath("data") print(f"updating test data @ {r_lowlev.DATASET_PATH}") diff --git a/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py b/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py index e3f9ab38c..0e712db51 100755 --- a/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py +++ b/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py @@ -37,7 +37,6 @@ revision_files = {} if __name__ == "__main__": - loaded = {} for name, varlist in NETWORKS.items(): reader = pya.io.ReadUngridded() @@ -48,12 +47,11 @@ use_stats = [] - for (attr, val, maxnum) in filters: + for attr, val, maxnum in filters: subsets = {} statnames = [] for name, data in loaded.items(): - subset = data.apply_filters(**{attr: val}) subsets[name] = subset diff --git a/pyaerocom/stationdata.py b/pyaerocom/stationdata.py index c92410a8f..7a7c8f2bd 100644 --- a/pyaerocom/stationdata.py +++ b/pyaerocom/stationdata.py @@ -83,7 +83,6 @@ class StationData(StationMetaData): ] def __init__(self, **meta_info): - self.dtime = [] self.var_info = BrowseDict() @@ -720,7 +719,6 @@ def _check_ts_types_for_merge(self, other, var_name): return ts_type def _update_var_timeinfo(self): - for var, info in self.var_info.items(): data = self[var] if not isinstance(data, pd.Series): diff --git a/pyaerocom/time_resampler.py b/pyaerocom/time_resampler.py index b61544415..f10a69903 100644 --- a/pyaerocom/time_resampler.py +++ b/pyaerocom/time_resampler.py @@ -78,7 +78,6 @@ def _get_resample_how(self, fr, to, how): return val def _get_idx_entry(self, fr, to, min_num_obs, how): - min_num = fr.get_min_num_obs(to, min_num_obs) _how = self._get_resample_how(fr, to, how) diff --git a/pyaerocom/trends_engine.py b/pyaerocom/trends_engine.py index b0ed6d921..16611b1fc 100644 --- a/pyaerocom/trends_engine.py +++ b/pyaerocom/trends_engine.py @@ -207,7 +207,6 @@ def plot(self, season="all", period=None, ax=None): ax.plot(self.monthly, label="monthly", c="#4d4d4d") ax.plot(self.get_yearly(season), " ok", label="yearly") if period in self.periods_avail: - (s_data, s_period, td, tp, tdstr, tpstr) = self._get_trend_data(season, period) ax.plot(s_data, "-", color=self.get_trend_color(td), label="trend", lw=2) diff --git a/pyaerocom/trends_helpers.py b/pyaerocom/trends_helpers.py index afcfb28d9..779100f42 100644 --- a/pyaerocom/trends_helpers.py +++ b/pyaerocom/trends_helpers.py @@ -77,7 +77,6 @@ def _compute_trend_error(m, m_err, v0, v0_err): def _get_season(mon): - for seas, months in SEASONS.items(): if mon in months: return seas diff --git a/pyaerocom/ungriddeddata.py b/pyaerocom/ungriddeddata.py index 917b40035..7173aa314 100644 --- a/pyaerocom/ungriddeddata.py +++ b/pyaerocom/ungriddeddata.py @@ -137,7 +137,6 @@ def _ROWNO(self): return self._data.shape[0] def __init__(self, num_points=None, add_cols=None): - if num_points is None: num_points = self._CHUNKSIZE @@ -1068,7 +1067,6 @@ def _metablock_to_stationdata( # for at least one of the input variables FOUND_ONE = False for var in vars_avail: - # get indices of this variable var_idx = self.meta_idx[meta_idx][var] @@ -1229,7 +1227,6 @@ def to_station_data_all( _iter = self._generate_station_index(by_station_name, ignore_index) for idx in _iter: - try: data = self.to_station_data( idx, @@ -2232,7 +2229,6 @@ def _find_common_meta(self, ignore_keys=None): for meta_key, meta in self.metadata.items(): found = False for idx, meta_reg in enumerate(meta_registered): - if same_meta_dict(meta_reg, meta, ignore_keys=ignore_keys): same_indices[idx].append(meta_key) found = True diff --git a/pyaerocom/variable.py b/pyaerocom/variable.py index 71379341b..bcf16ab06 100644 --- a/pyaerocom/variable.py +++ b/pyaerocom/variable.py @@ -378,7 +378,6 @@ def read_config(): @property def var_name_info(self): - return VarNameInfo(self.var_name) @property diff --git a/pyaerocom/vert_coords.py b/pyaerocom/vert_coords.py index 067d27f17..d8e78dcd7 100644 --- a/pyaerocom/vert_coords.py +++ b/pyaerocom/vert_coords.py @@ -185,7 +185,6 @@ def is_supported(standard_name): class VerticalCoordinate: - NAMES_SUPPORTED = { "altitude": "z", "air_pressure": "pres", @@ -287,7 +286,6 @@ def calc_pressure(self, lev, **kwargs): """ if not self.var_name in self.NAMES_SUPPORTED: - raise CoordinateNameError( f"Variable {self.var_name} cannot be converted to pressure levels. " f"Conversion is only possible for supported variables:\n{self.vars_supported_str}" @@ -319,7 +317,6 @@ def pressure2altitude(self, p, **kwargs): class AltitudeAccess: - #: Additional variable names (in AEROCOM convention) that are used #: to search for additional files that can be used to access or compute #: the altitude levels at each grid point @@ -479,7 +476,6 @@ def _check_vars_in_data_obj(self): # ToDo: check alias names def _check_var_in_data_obj(self, var_name): - c = VerticalCoordinate(var_name) if c.var_name in self.data_obj: @@ -515,7 +511,6 @@ def check_altitude_access(self, **coord_info): return False def _check_altitude_access_helper(self, coord_name, **coord_info): - cstd_name = const.COORDINFO[coord_name].standard_name if not self.search_aux_coords(coord_name): diff --git a/pyaerocom/vertical_profile.py b/pyaerocom/vertical_profile.py index d493725f4..67422b298 100644 --- a/pyaerocom/vertical_profile.py +++ b/pyaerocom/vertical_profile.py @@ -9,7 +9,6 @@ class VerticalProfile: """Object representing single variable profile data""" def __init__(self, data, altitude, dtime, var_name, data_err, var_unit, altitude_unit): - self.var_name = var_name self.dtime = dtime self.data = data @@ -93,7 +92,6 @@ def plot( if whole_alt_range: ax.set_ylim([np.min([0, self.altitude.min()]), self.altitude.max()]) if plot_errs: - lower = self.data - self.data_err upper = self.data + self.data_err if errs_shaded: diff --git a/scripts/aeolus2netcdf.py b/scripts/aeolus2netcdf.py index bc97423fe..2dd30f467 100644 --- a/scripts/aeolus2netcdf.py +++ b/scripts/aeolus2netcdf.py @@ -533,7 +533,6 @@ def main(): # write L3 gridded data if "gridfile" in options: - vars_to_copy = [obj._ALTITUDENAME, obj._LONGITUDENAME, obj._LATITUDENAME, ""] vars_to_read = options["variables"].copy() netcdf_indir = options["modelindir"] diff --git a/scripts/create_var_csv_from_htap2_exceltab.py b/scripts/create_var_csv_from_htap2_exceltab.py index ba009417b..1ba19b5a0 100644 --- a/scripts/create_var_csv_from_htap2_exceltab.py +++ b/scripts/create_var_csv_from_htap2_exceltab.py @@ -51,7 +51,6 @@ def main(): CTRL_COL = "C" for i, item in enumerate(sheet["A"]): - if sheet[CTRL_COL][i].value is None: continue diff --git a/tests/aeroval/test_aeroval_HIGHLEV.py b/tests/aeroval/test_aeroval_HIGHLEV.py index cb0a7b0fd..8385d7a19 100644 --- a/tests/aeroval/test_aeroval_HIGHLEV.py +++ b/tests/aeroval/test_aeroval_HIGHLEV.py @@ -56,7 +56,6 @@ ], ) def test_ExperimentOutput__FILES(eval_config: dict, chk_files: dict): - cfg = EvalSetup(**eval_config) proc = ExperimentProcessor(cfg) proc.exp_output.delete_experiment_data(also_coldata=True) diff --git a/tests/aeroval/test_coldatatojson_helpers.py b/tests/aeroval/test_coldatatojson_helpers.py index 5cd899fd9..cfb461ab8 100644 --- a/tests/aeroval/test_coldatatojson_helpers.py +++ b/tests/aeroval/test_coldatatojson_helpers.py @@ -29,7 +29,6 @@ def test__write_stationdata_json(tmp_path: Path): - data = dict(station_name="stat1", obs_name="obs1", var_name_web="var1", vert_code="Column") path: Path = tmp_path / get_stationfile_name(**data) assert not path.exists() @@ -49,7 +48,6 @@ def test__write_stationdata_json(tmp_path: Path): def test__write_site_data(tmp_path: Path): - data = [ dict( model_name=f"model{n}", diff --git a/tests/aeroval/test_experiment_output.py b/tests/aeroval/test_experiment_output.py index fbc545fde..81d1826e1 100644 --- a/tests/aeroval/test_experiment_output.py +++ b/tests/aeroval/test_experiment_output.py @@ -310,7 +310,6 @@ def test_Experiment_Output_clean_json_files_CFG1_INVALIDOBS(eval_config: dict): def test_ExperimentOutput_reorder_experiments( dummy_expout: ExperimentOutput, add_names, order, result ): - path = Path(dummy_expout.experiments_file) data = dict().fromkeys(add_names, dict(public=True)) diff --git a/tests/fixtures/tm5.py b/tests/fixtures/tm5.py index e60244c4b..c55b80cb5 100644 --- a/tests/fixtures/tm5.py +++ b/tests/fixtures/tm5.py @@ -31,7 +31,6 @@ def data_tm5() -> GriddedData: def load_coldata_tm5_aeronet_from_scratch(path: Path) -> ColocatedData: - arr = open_dataarray(path) if "_min_num_obs" in arr.attrs: info = {} diff --git a/tests/io/test_read_aasetal.py b/tests/io/test_read_aasetal.py index 95002f03d..dade4b83f 100644 --- a/tests/io/test_read_aasetal.py +++ b/tests/io/test_read_aasetal.py @@ -107,11 +107,9 @@ def test_aasetal_data(aasetal_data: UngriddedData): @lustre_unavail @pytest.mark.xfail(raises=UnitConversionError) def test_aasetal_data_correct_units(aasetal_data: UngriddedData): - tested = [] stats = [] for meta_key, meta in aasetal_data.metadata.items(): - for var, info in meta["var_info"].items(): if var in tested: # test each variable only once @@ -120,7 +118,6 @@ def test_aasetal_data_correct_units(aasetal_data: UngriddedData): tested.append(var) stats.append(meta["station_name"]) if len(tested) == len(VARS): - break assert meta_key == 520 @@ -147,7 +144,6 @@ def test_aasetal_data_correct_units(aasetal_data: UngriddedData): def test_reading_routines( aasetal_data: UngriddedData, data_paths: list[Path], filenum, station_name, colname, var_name ): - UNITCONVERSION = ReadAasEtal().UNITCONVERSION df = pd.read_csv(data_paths[filenum], sep=",", low_memory=False) diff --git a/tests/io/test_read_airnow.py b/tests/io/test_read_airnow.py index 60d2c3e1f..b49c0b2f7 100644 --- a/tests/io/test_read_airnow.py +++ b/tests/io/test_read_airnow.py @@ -336,7 +336,6 @@ def test__read_files_single_var( first_vals: list[float], unit: str, ): - files = reader.get_file_list() data = reader._read_files(files, [var_name]) assert isinstance(data, list) @@ -361,7 +360,6 @@ def test__read_files_single_var( def test__read_files_single_var_error(reader: ReadAirNow): - files = reader.get_file_list() # NH3 not available in selected 3 test files with pytest.raises(DataRetrievalError) as e: diff --git a/tests/plot/test_config.py b/tests/plot/test_config.py index 0a66f799d..13e4fa798 100644 --- a/tests/plot/test_config.py +++ b/tests/plot/test_config.py @@ -25,7 +25,6 @@ def test__COLOR_THEMES(): ], ) def test_ColorTheme___init__(name, theme_name): - assert ColorTheme(name).name == theme_name diff --git a/tests/plot/test_mapping.py b/tests/plot/test_mapping.py index 6881133e0..ce30ba118 100644 --- a/tests/plot/test_mapping.py +++ b/tests/plot/test_mapping.py @@ -81,7 +81,6 @@ def test_init_map(kwargs: dict): ], ) def test_init_map_error(kwargs: dict, error: str): - with pytest.raises(ValueError) as e: init_map(**kwargs) assert str(e.value) == error diff --git a/tests/plugins/gaw/test_dms.py b/tests/plugins/gaw/test_dms.py index cfa6cde97..92cf7ae56 100644 --- a/tests/plugins/gaw/test_dms.py +++ b/tests/plugins/gaw/test_dms.py @@ -66,7 +66,6 @@ def test_vmrdms_ams(data_vmrdms_ams_cvo): @lustre_unavail def test_vmrdms_ams_subset(data_vmrdms_ams_cvo): - stat = data_vmrdms_ams_cvo.to_station_data(meta_idx=0, start=2000, stop=2008, freq="monthly") assert_array_equal( diff --git a/tests/plugins/mscw_ctm/test_additional_variables.py b/tests/plugins/mscw_ctm/test_additional_variables.py index b65fe1135..fd0bba758 100644 --- a/tests/plugins/mscw_ctm/test_additional_variables.py +++ b/tests/plugins/mscw_ctm/test_additional_variables.py @@ -12,7 +12,6 @@ def test_calc_concNhno3(): - conchno3 = create_fake_MSCWCtm_data() concNhno3_from_func = calc_concNhno3(conchno3) @@ -27,7 +26,6 @@ def test_calc_concNhno3(): def test_calc_concNno3pm10(): - concno3c = create_fake_MSCWCtm_data() concno3f = create_fake_MSCWCtm_data() @@ -45,7 +43,6 @@ def test_calc_concNno3pm10(): def test_calc_concNno3pm25(): - concno3c = create_fake_MSCWCtm_data() concno3f = create_fake_MSCWCtm_data() @@ -63,7 +60,6 @@ def test_calc_concNno3pm25(): def test_calc_conNtno3(): - conchno3 = create_fake_MSCWCtm_data() concno3f = create_fake_MSCWCtm_data() concno3c = create_fake_MSCWCtm_data() @@ -121,7 +117,6 @@ def test_calc_concNnh4(): def test_update_EC_units(): - concecpm25 = create_fake_MSCWCtm_data() concCecpm25_from_func = update_EC_units(concecpm25) diff --git a/tests/plugins/mscw_ctm/test_reader.py b/tests/plugins/mscw_ctm/test_reader.py index 800dcddac..9ddee6875 100644 --- a/tests/plugins/mscw_ctm/test_reader.py +++ b/tests/plugins/mscw_ctm/test_reader.py @@ -395,7 +395,6 @@ def test_ReadEMEP__init__(): def emep_data_path(tmp_path: Path, freq: str | list[str], vars_and_units: dict[str, str]) -> Path: - reader = ReadMscwCtm() varmap = reader.var_map diff --git a/tests/test_colocation.py b/tests/test_colocation.py index 9e1575af0..5d48aebe8 100644 --- a/tests/test_colocation.py +++ b/tests/test_colocation.py @@ -195,7 +195,6 @@ def test_colocate_gridded_ungridded_new_var(data_tm5, aeronetsunv3lev2_subset): def test_colocate_gridded_ungridded( data_tm5, aeronetsunv3lev2_subset, addargs, ts_type, shape, obsmean, modmean ): - coldata = colocate_gridded_ungridded(data_tm5, aeronetsunv3lev2_subset, **addargs) assert isinstance(coldata, ColocatedData) diff --git a/tests/test_combine_vardata_ungridded.py b/tests/test_combine_vardata_ungridded.py index 22ff24a25..d870dc756 100644 --- a/tests/test_combine_vardata_ungridded.py +++ b/tests/test_combine_vardata_ungridded.py @@ -60,7 +60,6 @@ def stats_sda_fineaod(SDA_DATA): def test_combine_vardata_ungridded_single_ungridded( SUN_DATA, var1, var2, kwargs, numst, mean_first ): - input_data = [(SUN_DATA, SUN_ID, var1), (SUN_DATA, SUN_ID, var2)] stats = combine_vardata_ungridded(input_data, **kwargs) @@ -251,7 +250,6 @@ def test__map_same_stations_samedata(stats_sun_aod, match_stats_how, match_stats def test__map_same_stations( stats_sun_aod, stats_sda_aod, match_stats_how, match_stats_tol_km, num_matches, diff_idx ): - index_short, index_long, statnames_short, statnames_long = _map_same_stations( stats_sun_aod, stats_sda_aod, match_stats_how, match_stats_tol_km ) diff --git a/tests/test_filter.py b/tests/test_filter.py index adb2c6de5..d4e0313a7 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -28,7 +28,6 @@ def test_filter_attributes(): ], ) def test_filter_griddeddata(data_tm5, filter_name, mean): - # use copy so that this fixture can be used elsewhere without being c # changed by this method globally model = data_tm5.copy() @@ -43,7 +42,6 @@ def test_filter_griddeddata(data_tm5, filter_name, mean): "filter_name,num_sites", [(f"{ALL_REGION_NAME}-wMOUNTAINS", 22), ("OCN", 8), ("EUROPE", 7)] ) def test_filter_ungriddeddata(aeronetsunv3lev2_subset, filter_name, num_sites): - obs_data = aeronetsunv3lev2_subset f = Filter(filter_name) diff --git a/tests/test_griddeddata.py b/tests/test_griddeddata.py index c81d457ed..d8a83f9a9 100644 --- a/tests/test_griddeddata.py +++ b/tests/test_griddeddata.py @@ -138,7 +138,6 @@ def test_GriddedData_interpolate(data_tm5: GriddedData): def test_GriddedData_to_time_series(data_tm5: GriddedData): - stats = data_tm5.to_time_series(latitude=TESTLATS, longitude=TESTLONS) assert [stat.latitude for stat in stats] == [-9, 21] assert [stat.longitude for stat in stats] == [-118.5, 70.5] diff --git a/tests/test_helpers.py b/tests/test_helpers.py index aa58b8d92..1ed5fa6f4 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -124,7 +124,6 @@ def fake_hourly_ts(): ) @pytest.mark.filterwarnings("ignore:Mean of empty slice:RuntimeWarning") def test_resample_timeseries(fake_hourly_ts, freq, how, min_num_obs, num, avg): - s1 = helpers.resample_timeseries(fake_hourly_ts, freq=freq, how=how, min_num_obs=min_num_obs) assert len(s1) == num assert np.nanmean(s1) == pytest.approx(avg, abs=1e-2, nan_ok=True) diff --git a/tests/test_metastandards.py b/tests/test_metastandards.py index 7521c150a..83dfa4744 100644 --- a/tests/test_metastandards.py +++ b/tests/test_metastandards.py @@ -41,7 +41,6 @@ def test_datasource( revision_date, stat_merge_pref_attr, ): - ds = DataSource(data_id=data_id) assert ds["dataset_name"] == dataset_name assert ds["data_product"] == data_product @@ -91,7 +90,6 @@ def test_stationmetadata(): ], ) def test_aerocomdataid(data_id, values, test_addstuff): - data_id = AerocomDataID(data_id) assert data_id.values == values diff --git a/tests/test_ungriddeddata.py b/tests/test_ungriddeddata.py index 6615ea707..5b7556429 100644 --- a/tests/test_ungriddeddata.py +++ b/tests/test_ungriddeddata.py @@ -214,7 +214,6 @@ def test_check_unit(data_scat_jungfraujoch): @pytest.mark.filterwarnings("ignore:invalid value encountered in .*divide:RuntimeWarning") def test_check_convert_var_units(data_scat_jungfraujoch): - out = data_scat_jungfraujoch.check_convert_var_units("sc550aer", "m-1", inplace=False) fac = 1e-6 From d425203afe91c7bd8fe0946f758ca62c57f864d3 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 11 Apr 2023 14:19:01 +0200 Subject: [PATCH 48/82] updated due to PR review --- pyaerocom/io/readgridded.py | 10 ++++------ pyaerocom/plugins/ipcforests/__init__.py | 4 ---- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/pyaerocom/io/readgridded.py b/pyaerocom/io/readgridded.py index 63aa164aa..40cfaae6a 100755 --- a/pyaerocom/io/readgridded.py +++ b/pyaerocom/io/readgridded.py @@ -2,7 +2,6 @@ import logging import os import warnings -from functools import partial from glob import glob from pathlib import Path @@ -36,7 +35,6 @@ calc_concNtnh, calc_concNtno3, calc_sspm25, - vmr_to_conc, ) from pyaerocom.io.aux_read_cubes import ( add_cubes, @@ -183,8 +181,8 @@ class specifying details of the file naming convention for the model "concNnh3": calc_concNnh3_from_vmr, "concNnh4": calc_concNnh4, "concNtnh": calc_concNtnh, - #'mec550*' : divide_cubes, - #'tau*' : lifetime_from_load_and_dep + # 'mec550*' : divide_cubes, + # 'tau*' : lifetime_from_load_and_dep } #: Additional arguments passed to computation methods for auxiliary data @@ -203,7 +201,7 @@ class specifying details of the file naming convention for the model _data_dir = "" - VERT_ALT = {"Surface": "ModelLevel"} # , "2D": "2D"} + VERT_ALT = {"Surface": "ModelLevel"} def __init__(self, data_id=None, data_dir=None, file_convention="aerocom3"): @@ -2244,7 +2242,7 @@ def __repr__(self): def __str__(self): head = f"Pyaerocom {type(self).__name__}" s = ( - f"\n{head}\n{len(head)*'-'}\n" + f"\n{head}\n{len(head) * '-'}\n" f"Data ID: {self.data_id}\n" f"Data directory: {self.data_dir}\n" f"Available experiments: {self.experiments}\n" diff --git a/pyaerocom/plugins/ipcforests/__init__.py b/pyaerocom/plugins/ipcforests/__init__.py index c2f95f4c5..96378059c 100644 --- a/pyaerocom/plugins/ipcforests/__init__.py +++ b/pyaerocom/plugins/ipcforests/__init__.py @@ -1,6 +1,2 @@ -import logging - from . import metadata, reader from .metadata import MetadataReader - -logger = logging.getLogger(__name__) From 8a845a7a018728463742bcc86b816121ae4af106 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Fri, 14 Apr 2023 10:19:34 +0200 Subject: [PATCH 49/82] add some deepcopy calls --- pyaerocom/aeroval/coldatatojson_helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyaerocom/aeroval/coldatatojson_helpers.py b/pyaerocom/aeroval/coldatatojson_helpers.py index 981a9811a..bb9c87eea 100644 --- a/pyaerocom/aeroval/coldatatojson_helpers.py +++ b/pyaerocom/aeroval/coldatatojson_helpers.py @@ -385,7 +385,7 @@ def _get_period_keys(resolution): if resolution == "seasonal": period_keys = ["DJF", "MAM", "JJA", "SON"] elif resolution == "yearly": - period_keys = ["Annual"] + period_keys = ["All"] return period_keys @@ -426,8 +426,8 @@ def _process_one_station_weekly(stat_name, i, repw_res, meta_glob, time): ts_data = { "time": time, - "seasonal": {"obs": yeardict, "mod": yeardict}, - "yearly": {"obs": yeardict, "mod": yeardict}, + "seasonal": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)}, + "yearly": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)}, } ts_data["station_name"] = stat_name ts_data.update(meta_glob) @@ -526,8 +526,8 @@ def _process_weekly_object_to_country_time_series(repw_res, meta_glob, regions_h for regid, regname in region_ids.items(): ts_data = { "time": time, - "seasonal": {"obs": yeardict, "mod": yeardict}, - "yearly": {"obs": yeardict, "mod": yeardict}, + "seasonal": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)}, + "yearly": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)}, } ts_data["station_name"] = regname ts_data.update(meta_glob) From 9857f49c8a5314aa913ed97fb4b36300d4812570 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 8 May 2023 16:51:45 +0200 Subject: [PATCH 50/82] WIP on correcting the diurnal cycle calculation --- pyaerocom/aeroval/coldatatojson_helpers.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pyaerocom/aeroval/coldatatojson_helpers.py b/pyaerocom/aeroval/coldatatojson_helpers.py index bb9c87eea..4c26e3f8a 100644 --- a/pyaerocom/aeroval/coldatatojson_helpers.py +++ b/pyaerocom/aeroval/coldatatojson_helpers.py @@ -2,13 +2,12 @@ Helpers for conversion of ColocatedData to JSON files for web interface. """ import logging -import os -from copy import deepcopy -from datetime import datetime - import numpy as np +import os import pandas as pd import xarray as xr +from copy import deepcopy +from datetime import datetime from pyaerocom._lowlevel_helpers import read_json, write_json from pyaerocom._warnings import ignore_warnings @@ -598,6 +597,16 @@ def _process_sites_weekly_ts(coldata, regions_how, region_ids, meta_glob): ), } + # try to merge repw_res together into one dataarray + repw_res_v2 = xr.concat( + [ + _create_diurnal_weekly_data_object(coldata, "seasonal")["rep_week"], + _create_diurnal_weekly_data_object(coldata, "yearly")["rep_week"].expand_dims( + "period", axis=0 + ), + ], + dim="period", + ) ts_objs = _process_weekly_object_to_station_time_series(repw_res, meta_glob) ts_objs_reg = _process_weekly_object_to_country_time_series( repw_res, meta_glob, regions_how, region_ids From 6b9b90bd2095620d29059f4350b1c9076aa7b61f Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 3 Jul 2023 11:37:01 +0200 Subject: [PATCH 51/82] corrected wetso4 definition --- pyaerocom/data/ebas_config.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyaerocom/data/ebas_config.ini b/pyaerocom/data/ebas_config.ini index 0647c27cf..ee26af738 100644 --- a/pyaerocom/data/ebas_config.ini +++ b/pyaerocom/data/ebas_config.ini @@ -470,7 +470,8 @@ requires=concprcpno3 [concprcpso4] -component=sulphate_corrected#,sulphate_total +#component=sulphate_corrected#,sulphate_total +component=sulphate_corrected # after discussion with Wenche matrix=precip From c70fc313fc6f925e43e401e4423f59fba449bd08 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 4 Jul 2023 10:06:37 +0200 Subject: [PATCH 52/82] introduced wetoxst and wetoxsc as variables --- pyaerocom/aux_var_helpers.py | 64 +++++++++++++++++++++++++++++----- pyaerocom/data/ebas_config.ini | 14 ++++++++ pyaerocom/io/read_ebas.py | 23 ++++++------ 3 files changed, 82 insertions(+), 19 deletions(-) diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index 9cae84ee7..442f98ea2 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -216,15 +216,15 @@ def compute_od_from_angstromexp(to_lambda, od_ref, lambda_ref, angstrom_coeff): def _calc_od_helper( - data, - var_name, - to_lambda, - od_ref, - lambda_ref, - od_ref_alt=None, - lambda_ref_alt=None, - use_angstrom_coeff="ang4487aer", - treshold_angstrom=None, + data, + var_name, + to_lambda, + od_ref, + lambda_ref, + od_ref_alt=None, + lambda_ref_alt=None, + use_angstrom_coeff="ang4487aer", + treshold_angstrom=None, ): """Helper method for computing ODs @@ -563,6 +563,52 @@ def compute_wetoxs_from_concprcpoxs(data): return _compute_wdep_from_concprcp_helper(data, "wetoxs", "concprcpoxs", "pr") +def compute_wetoxs_from_concprcpoxst(data): + """Compute wdep from conc in precip and precip data + + Note + ---- + In addition to the returned numpy array, the input instance of + :class:`StationData` is modified by additional metadata and flags for + the new variable. See also :func:`_compute_wdep_from_concprcp_helper`. + + Parameters + ---------- + StationData + data object containing concprcp and precip data + + Returns + ------- + numpy.ndarray + array with wet deposition values + + """ + return _compute_wdep_from_concprcp_helper(data, "wetoxs", "concprcpoxst", "pr") + + +def compute_wetoxs_from_concprcpoxsc(data): + """Compute wdep from conc in precip and precip data + + Note + ---- + In addition to the returned numpy array, the input instance of + :class:`StationData` is modified by additional metadata and flags for + the new variable. See also :func:`_compute_wdep_from_concprcp_helper`. + + Parameters + ---------- + StationData + data object containing concprcp and precip data + + Returns + ------- + numpy.ndarray + array with wet deposition values + + """ + return _compute_wdep_from_concprcp_helper(data, "wetoxs", "concprcpoxsc", "pr") + + def compute_wetoxn_from_concprcpoxn(data): """Compute wdep from conc in precip and precip data diff --git a/pyaerocom/data/ebas_config.ini b/pyaerocom/data/ebas_config.ini index ee26af738..29c736906 100644 --- a/pyaerocom/data/ebas_config.ini +++ b/pyaerocom/data/ebas_config.ini @@ -370,6 +370,14 @@ matrix=air component=sulphate_corrected,sulphate_total matrix=precip +[concprcpoxsc] +component=sulphate_corrected +matrix=precip + +[concprcpoxst] +component=sulphate_total +matrix=precip + [concprcpoxn] component=nitrate matrix=precip @@ -384,6 +392,12 @@ matrix=precip [wetoxs] requires=concprcpoxs +[wetoxst] +requires=concprcpoxst + +[wetoxsc] +requires=concprcpoxsc + [wetrdn] requires=concprcprdn diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index b6701a8be..71590cf60 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -23,7 +23,6 @@ compute_wetrdn_from_concprcprdn, compute_wetso4_from_concprcpso4, concx_to_vmrx, - identity, make_proxy_drydep_from_O3, make_proxy_wetdep_from_O3, vmrx_to_concx, @@ -212,7 +211,7 @@ class ReadEbas(ReadUngriddedBase): "Vavihill": "Hallahus", "Virolahti II": "Virolahti III", } - #'Trollhaugen' : 'Troll'} + # 'Trollhaugen' : 'Troll'} #: Temporal resolution codes that (so far) can be understood by pyaerocom TS_TYPE_CODES = { "1mn": "minutely", @@ -235,6 +234,8 @@ class ReadEbas(ReadUngriddedBase): "ac550dryaer": ["ac550aer", "acrh"], "ang4470dryaer": ["sc440dryaer", "sc700dryaer"], "wetoxs": ["concprcpoxs", "pr"], + "wetoxsc": ["concprcpoxsc", "pr"], + "wetoxst": ["concprcpoxst", "pr"], "wetoxn": ["concprcpoxn", "pr"], "wetrdn": ["concprcprdn", "pr"], "wetso4": ["concprcpso4", "pr"], @@ -302,6 +303,8 @@ class ReadEbas(ReadUngriddedBase): "ac550dryaer": compute_ac550dryaer, "ang4470dryaer": compute_ang4470dryaer_from_dry_scat, "wetoxs": compute_wetoxs_from_concprcpoxs, + "wetoxsc": compute_wetoxs_from_concprcpoxsc, + "wetoxst": compute_wetoxs_from_concprcpoxst, "wetoxn": compute_wetoxn_from_concprcpoxn, "wetrdn": compute_wetrdn_from_concprcprdn, "wetnh4": compute_wetnh4_from_concprcpnh4, @@ -1383,7 +1386,7 @@ def get_ebas_var(self, var_name): return self._loaded_ebas_vars[var_name] def read_file( - self, filename, vars_to_retrieve=None, _vars_to_read=None, _vars_to_compute=None + self, filename, vars_to_retrieve=None, _vars_to_read=None, _vars_to_compute=None ): """Read EBAS NASA Ames file @@ -1541,9 +1544,9 @@ def _flag_incorrect_frequencies(self, filedata): opts = self.get_read_opts(var) if opts.freq_min_cov > frac_valid: raise TemporalSamplingError( - f"Only {frac_valid*100:.2f}% of measuerements are in " + f"Only {frac_valid * 100:.2f}% of measuerements are in " f"{tst} resolution. Minimum requirement for {var} is " - f"{opts.freq_min_cov*100:.2f}%" + f"{opts.freq_min_cov * 100:.2f}%" ) if not var in filedata.data_flagged: filedata.data_flagged[var] = np.zeros(num).astype(bool) @@ -1728,7 +1731,7 @@ def _check_keep_aux_vars(self, vars_to_retrieve): return vars_to_retrieve + add def read( - self, vars_to_retrieve=None, first_file=None, last_file=None, files=None, **constraints + self, vars_to_retrieve=None, first_file=None, last_file=None, files=None, **constraints ): """Method that reads list of files as instance of :class:`UngriddedData` @@ -1825,10 +1828,10 @@ def _read_files(self, files, vars_to_retrieve, files_contain, constraints): station_data = self.read_file(_file, vars_to_retrieve=contains) except ( - NotInFileError, - EbasFileError, - TemporalResolutionError, - TemporalSamplingError, + NotInFileError, + EbasFileError, + TemporalResolutionError, + TemporalSamplingError, ) as e: self.files_failed.append(_file) self.logger.warning( From e662d4ccd8b8cd4bb94355616ccc4b763497bb5a Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 4 Jul 2023 10:24:33 +0200 Subject: [PATCH 53/82] introduced wetoxst and wetoxsc as variables --- pyaerocom/data/variables.ini | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index 6ca0c0cee..53be09f23 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -2097,6 +2097,16 @@ description=Wet deposition of total sulphur mass unit = mg S m-2 d-1 minimum=0 +[wetoxsc] +description=Wet deposition of total sulphur corrected mass +unit = mg S m-2 d-1 +minimum=0 + +[wetoxst] +description=Wet deposition of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + [depoxn] description=Total deposition of oxidized nitrogen mass unit = mg N m-2 d-1 @@ -2606,6 +2616,16 @@ description = Mass concentration of total sulphur in precipitation unit = ug S m-3 var_type = mass concentration +[concprcpoxsc] +description = Mass concentration of total sulphur in precipitation +unit = ug S m-3 +var_type = mass concentration + +[concprcpoxst] +description = Mass concentration of total sulphur in precipitation +unit = ug S m-3 +var_type = mass concentration + [concprcprdn] description = Mass concentration of reduced nitrogen in precipitation unit = ug N m-3 From e4fcfe6b2c677393108bc3eecbecaf13529e03be Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 4 Jul 2023 10:36:09 +0200 Subject: [PATCH 54/82] introduced wetoxst and wetoxsc as variables --- pyaerocom/io/read_ebas.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index 71590cf60..661e93d34 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -20,6 +20,8 @@ compute_wetno3_from_concprcpno3, compute_wetoxn_from_concprcpoxn, compute_wetoxs_from_concprcpoxs, + compute_wetoxs_from_concprcpoxsc, + compute_wetoxs_from_concprcpoxst, compute_wetrdn_from_concprcprdn, compute_wetso4_from_concprcpso4, concx_to_vmrx, @@ -1386,7 +1388,7 @@ def get_ebas_var(self, var_name): return self._loaded_ebas_vars[var_name] def read_file( - self, filename, vars_to_retrieve=None, _vars_to_read=None, _vars_to_compute=None + self, filename, vars_to_retrieve=None, _vars_to_read=None, _vars_to_compute=None ): """Read EBAS NASA Ames file @@ -1731,7 +1733,7 @@ def _check_keep_aux_vars(self, vars_to_retrieve): return vars_to_retrieve + add def read( - self, vars_to_retrieve=None, first_file=None, last_file=None, files=None, **constraints + self, vars_to_retrieve=None, first_file=None, last_file=None, files=None, **constraints ): """Method that reads list of files as instance of :class:`UngriddedData` @@ -1828,10 +1830,10 @@ def _read_files(self, files, vars_to_retrieve, files_contain, constraints): station_data = self.read_file(_file, vars_to_retrieve=contains) except ( - NotInFileError, - EbasFileError, - TemporalResolutionError, - TemporalSamplingError, + NotInFileError, + EbasFileError, + TemporalResolutionError, + TemporalSamplingError, ) as e: self.files_failed.append(_file) self.logger.warning( From 19e7a8d9548eacbd5bf31f152e40c1f370913052 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 4 Jul 2023 10:38:57 +0200 Subject: [PATCH 55/82] add DOMOS path --- pyaerocom/data/paths.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini index b7fc39805..6c777d8ca 100644 --- a/pyaerocom/data/paths.ini +++ b/pyaerocom/data/paths.ini @@ -50,7 +50,8 @@ dir= ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND2/, ${BASEDIR}/fou/kl/CAMS61/, ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/, - /lustre/storeB/project/fou/kl/CAMS2_40/task4041/ + /lustre/storeB/project/fou/kl/CAMS2_40/task4041/, + /lustre/storeB/project/aerocom/aerocom-users-database/DOMOS/ [obsfolders] #folders to for model data From 14e27781e0cd84808ccb18739e8ae53945aa021f Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 4 Jul 2023 12:55:11 +0200 Subject: [PATCH 56/82] addes wetsoxsc and wetoxst definition --- pyaerocom/aeroval/glob_defaults.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index 4ebde240b..bc9f2e77b 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -596,6 +596,8 @@ depoxn=["TotDepOXN", "3D", "Total Deposition"], deprdn=["TotDepRDN", "3D", "Total Deposition"], wetoxs=["WetOXS", "3D", "Deposition"], + wetoxsc=["WetOXScorr", "3D", "Deposition"], + wetoxst=["WetOXStot", "3D", "Deposition"], wetoxn=["WetOXN", "3D", "Deposition"], wetrdn=["WetRDN", "3D", "Deposition"], prmm=["Precipitation", "3D", "Deposition"], From d69f9adff17971f4ee2b573e758df1bf5e0dd3bc Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 28 Aug 2023 13:34:01 +0200 Subject: [PATCH 57/82] adjusted some colorbars --- pyaerocom/aeroval/glob_defaults.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index bc9f2e77b..b24f0238c 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -205,10 +205,10 @@ "scale": [0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0], "colmap": "coolwarm", }, - "wetoxs": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, + "wetoxs": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, "wetoxn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, "wetrdn": { - "scale": [0, 0.75, 1.5, 2.25, 3.0, 3.75, 4.5, 5.25, 6.0, 6.75, 7.5, 8.25], + "scale": [0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0], "colmap": "coolwarm", }, "prmm": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, From 0eaf4e1022942250d1a66d4530d740b26ae62703 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 28 Sep 2023 13:23:53 +0200 Subject: [PATCH 58/82] linting --- pyaerocom/aux_var_helpers.py | 18 +++++++++--------- pyaerocom/io/readungridded.py | 2 -- .../testdata-minimal/create_subset_ebas.py | 3 --- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index 442f98ea2..2ab01ad46 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -216,15 +216,15 @@ def compute_od_from_angstromexp(to_lambda, od_ref, lambda_ref, angstrom_coeff): def _calc_od_helper( - data, - var_name, - to_lambda, - od_ref, - lambda_ref, - od_ref_alt=None, - lambda_ref_alt=None, - use_angstrom_coeff="ang4487aer", - treshold_angstrom=None, + data, + var_name, + to_lambda, + od_ref, + lambda_ref, + od_ref_alt=None, + lambda_ref_alt=None, + use_angstrom_coeff="ang4487aer", + treshold_angstrom=None, ): """Helper method for computing ODs diff --git a/pyaerocom/io/readungridded.py b/pyaerocom/io/readungridded.py index 3bd79508c..496458b62 100755 --- a/pyaerocom/io/readungridded.py +++ b/pyaerocom/io/readungridded.py @@ -69,7 +69,6 @@ class ReadUngridded: DONOTCACHE_NAME = "DONOTCACHE" def __init__(self, data_ids=None, ignore_cache=False, data_dirs=None): - # will be assigned in setter method of data_ids self._data_ids = [] self._data_dirs = {} @@ -526,7 +525,6 @@ def read_dataset_post( for aux_var in aux_vars: input_data_ids_vars.append((aux_data, aux_id, aux_var)) else: - # read variables individually, so filter_post is more # flexible if some post filters are specified for # individual variables... diff --git a/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py b/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py index ca0eca422..a7deed21b 100755 --- a/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py +++ b/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py @@ -73,7 +73,6 @@ def check_outdated(filedir): files_valid = [] with open(JSON_FILE) as f: - data = simplejson.load(f, allow_nan=True) for var, stats in data.items(): @@ -155,7 +154,6 @@ def get_files_var_statnum(data, var, statnum): def main(): - # reader = pya.io.ReadUngridded(NAME, data_dir=EBAS_BASE_DIR) reader = pya.io.ReadUngridded( NAME, @@ -244,7 +242,6 @@ def main(): if not UPDATE: print("NOTHING WILL BE COPIED TO TEST DATA") else: - src = Path(EBAS_BASE_DIR).joinpath("data") print(f"updating test data @ {r_lowlev.DATASET_PATH}") From 5d3742ba4cc09726db3a0a400646467d57fa1dad Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 28 Sep 2023 14:17:04 +0200 Subject: [PATCH 59/82] try to solve CI linting --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d4c10f94d..1d8769cdf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,9 +142,10 @@ exclude_lines = [ ] [tool.black] -target-version = ['py38'] +target-version = ['py39'] extend_skip = ["pyaerocom-tutorials"] line-length = 99 +required-version = "23" [tool.isort] py_version = "39" From 419a82db2f004b1e2ecbf442fab777f512ef8b33 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 28 Sep 2023 14:25:36 +0200 Subject: [PATCH 60/82] try to solve CI linting --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1d8769cdf..2d48c21ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,7 +145,7 @@ exclude_lines = [ target-version = ['py39'] extend_skip = ["pyaerocom-tutorials"] line-length = 99 -required-version = "23" +required-version = "22" [tool.isort] py_version = "39" From 97ae41a0d4053d570d5e9049646d9f12d705b2f1 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 28 Sep 2023 14:40:59 +0200 Subject: [PATCH 61/82] remove black version --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2d48c21ad..01b530759 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,7 +145,6 @@ exclude_lines = [ target-version = ['py39'] extend_skip = ["pyaerocom-tutorials"] line-length = 99 -required-version = "22" [tool.isort] py_version = "39" From e9ee2f63371542db748dd66bcb0b8ed0f7ed472e Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 28 Sep 2023 14:41:52 +0200 Subject: [PATCH 62/82] update black version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80967fcea..520bffe59 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.9.1 hooks: - id: black - repo: https://github.com/pycqa/isort From 75a2b5d2a51688beb26bd4611dc0457e5938acf8 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 28 Sep 2023 14:47:44 +0200 Subject: [PATCH 63/82] linting (isort) --- pyaerocom/aeroval/coldatatojson_helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyaerocom/aeroval/coldatatojson_helpers.py b/pyaerocom/aeroval/coldatatojson_helpers.py index 4c26e3f8a..bbec6ccf9 100644 --- a/pyaerocom/aeroval/coldatatojson_helpers.py +++ b/pyaerocom/aeroval/coldatatojson_helpers.py @@ -2,13 +2,14 @@ Helpers for conversion of ColocatedData to JSON files for web interface. """ import logging -import numpy as np import os -import pandas as pd -import xarray as xr from copy import deepcopy from datetime import datetime +import numpy as np +import pandas as pd +import xarray as xr + from pyaerocom._lowlevel_helpers import read_json, write_json from pyaerocom._warnings import ignore_warnings from pyaerocom.aeroval.fairmode_stats import fairmode_stats From 01a28b24b77d44d006efee60b3e91d3138015de4 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 28 Sep 2023 22:19:53 +0200 Subject: [PATCH 64/82] linting --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 01b530759..e5e75f0d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,7 +142,7 @@ exclude_lines = [ ] [tool.black] -target-version = ['py39'] +target-version = ["py39", "py310"] extend_skip = ["pyaerocom-tutorials"] line-length = 99 From 0c5aef3c5f1ae24b61b103c81657b8de3c4648de Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Fri, 29 Sep 2023 15:59:24 +0200 Subject: [PATCH 65/82] added variable wetna --- pyaerocom/data/variables.ini | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index 74b4e4e1d..da7a4b621 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -2291,6 +2291,17 @@ maximum = 10000 dimensions = time,lat,lon comments_and_purpose = Verification of aerosol budget and speciation +[wetna] +var_name = wetna +description = Wet deposition of sodium +standard_name = tendency_of_atmosphere_mass_content_of_seasalt_dry_aerosol_particles_due_to_wet_deposition +var_type = wet deposition flux +unit = kg m-2 s-1 +minimum = 0 +maximum = 10000 +dimensions = time,lat,lon +comments_and_purpose = Verification of aerosol budget and speciation + [wetdust] var_name = wetdust description = Wet deposition of dust From c13c2c229e421e9f0be4a7f8957956cb01d2b799 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 5 Oct 2023 13:01:48 +0200 Subject: [PATCH 66/82] adjusted wetoxs, wetrdn and wetoxn color bars --- pyaerocom/aeroval/glob_defaults.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index b24f0238c..673f1e538 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -205,10 +205,10 @@ "scale": [0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0], "colmap": "coolwarm", }, - "wetoxs": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, - "wetoxn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "wetoxs": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, + "wetoxn": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, "wetrdn": { - "scale": [0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0], + "scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm", }, "prmm": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, From 2c325fd31106c1aeeaeebdb0f3cd136b0e3d5a7d Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Fri, 6 Oct 2023 15:56:36 +0200 Subject: [PATCH 67/82] added variable wetna --- pyaerocom/aux_var_helpers.py | 4 ++++ pyaerocom/data/ebas_config.ini | 7 +++++++ pyaerocom/data/variables.ini | 5 +++++ pyaerocom/io/read_ebas.py | 6 ++++-- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index 2ab01ad46..a7baead6d 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -667,6 +667,10 @@ def compute_wetso4_from_concprcpso4(data): return _compute_wdep_from_concprcp_helper(data, "wetso4", "concprcpso4", "pr") +def compute_wetna_from_concprcpna(data): + return _compute_wdep_from_concprcp_helper(data, "wetna", "concprcpna", "pr") + + def vmrx_to_concx(data, p_pascal, T_kelvin, vmr_unit, mmol_var, mmol_air=None, to_unit=None): """ Convert volume mixing ratio (vmr) to mass concentration diff --git a/pyaerocom/data/ebas_config.ini b/pyaerocom/data/ebas_config.ini index 29c736906..79e8ab710 100644 --- a/pyaerocom/data/ebas_config.ini +++ b/pyaerocom/data/ebas_config.ini @@ -386,6 +386,10 @@ matrix=precip component=ammonium matrix=precip +[concprcpna] +component=sodium +matrix=precip + # 3. Deposition rates # 3.1. Wet deposition @@ -404,6 +408,9 @@ requires=concprcprdn [wetoxn] requires=concprcpoxn +[wetna] +requires=concprcpna + # 4. Precipitation [pr] # pyaerocom unit kg m-2 s-1 component=precipitation_amount_off,precipitation_amount diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index da7a4b621..77535b8db 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -2607,6 +2607,11 @@ description = Mass concentration of sulphate in precipitation unit = ug S m-3 var_type = mass concentration +[concprcpna] +description = Mass concentration of sodium in precipitation +unit = ug m-3 +var_type = mass concentration + [concprcpno3] description = Mass concentration of NO3 in precipitation unit = ug N m-3 diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index 661e93d34..df22c0608 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -5,8 +5,6 @@ import numpy as np from geonum.atmosphere import T0_STD, p0 -from tqdm import tqdm - from pyaerocom import const from pyaerocom._lowlevel_helpers import BrowseDict from pyaerocom.aux_var_helpers import ( @@ -24,6 +22,7 @@ compute_wetoxs_from_concprcpoxst, compute_wetrdn_from_concprcprdn, compute_wetso4_from_concprcpso4, + compute_wetna_from_concprcpna, concx_to_vmrx, make_proxy_drydep_from_O3, make_proxy_wetdep_from_O3, @@ -47,6 +46,7 @@ from pyaerocom.tstype import TsType from pyaerocom.ungriddeddata import UngriddedData from pyaerocom.units_helpers import get_unit_conversion_fac +from tqdm import tqdm logger = logging.getLogger(__name__) @@ -241,6 +241,7 @@ class ReadEbas(ReadUngriddedBase): "wetoxn": ["concprcpoxn", "pr"], "wetrdn": ["concprcprdn", "pr"], "wetso4": ["concprcpso4", "pr"], + "wetna": ["concprcpna", "pr"], "wetno3": ["concprcpno3", "pr"], "wetnh4": ["concprcpnh4", "pr"], "vmro3max": ["vmro3"], @@ -312,6 +313,7 @@ class ReadEbas(ReadUngriddedBase): "wetnh4": compute_wetnh4_from_concprcpnh4, "wetno3": compute_wetno3_from_concprcpno3, "wetso4": compute_wetso4_from_concprcpso4, + "wetna": compute_wetna_from_concprcpna, "vmro3max": calc_vmro3max, # proxy dry dep # Suphar based From f5d950414319b771c45ee3560283791eed47f40f Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Wed, 11 Oct 2023 15:18:03 +0200 Subject: [PATCH 68/82] various changes to variable wetna for aeroval --- pyaerocom/aeroval/glob_defaults.py | 2 ++ pyaerocom/data/variables.ini | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index 673f1e538..8e964f3f6 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -206,6 +206,7 @@ "colmap": "coolwarm", }, "wetoxs": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, + "wetna": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, "wetoxn": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, "wetrdn": { "scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], @@ -595,6 +596,7 @@ depoxs=["TotDepOXS", "3D", "Total Deposition"], depoxn=["TotDepOXN", "3D", "Total Deposition"], deprdn=["TotDepRDN", "3D", "Total Deposition"], + wetna=["WetNa", "3D", "Deposition"], wetoxs=["WetOXS", "3D", "Deposition"], wetoxsc=["WetOXScorr", "3D", "Deposition"], wetoxst=["WetOXStot", "3D", "Deposition"], diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index ef1597f3e..2eb591162 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -2324,7 +2324,7 @@ var_name = wetna description = Wet deposition of sodium standard_name = tendency_of_atmosphere_mass_content_of_seasalt_dry_aerosol_particles_due_to_wet_deposition var_type = wet deposition flux -unit = kg m-2 s-1 +unit = mg m-2 d-1 minimum = 0 maximum = 10000 dimensions = time,lat,lon From 41b46697fed9601e7d127cf7706d0a718d079c44 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 12 Oct 2023 13:52:20 +0200 Subject: [PATCH 69/82] added deposition to forrest variables --- pyaerocom/aeroval/glob_defaults.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index 673f1e538..4a0721e5b 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -211,6 +211,12 @@ "scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm", }, + "wetoxsf": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, + "wetoxnf": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, + "wetrdnf": { + "scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], + "colmap": "coolwarm", + }, "prmm": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, "dryoxs": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, "dryoxn": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, @@ -292,6 +298,9 @@ "depoxs": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, "depoxn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, "deprdn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "depoxsf": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "depoxnf": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "deprdnf": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, } #: Default information for statistical parameters @@ -595,6 +604,9 @@ depoxs=["TotDepOXS", "3D", "Total Deposition"], depoxn=["TotDepOXN", "3D", "Total Deposition"], deprdn=["TotDepRDN", "3D", "Total Deposition"], + depoxsf=["TotDepOXSforr", "3D", "Total Deposition"], + depoxnf=["TotDepOXNforr", "3D", "Total Deposition"], + deprdnf=["TotDepRDNforr", "3D", "Total Deposition"], wetoxs=["WetOXS", "3D", "Deposition"], wetoxsc=["WetOXScorr", "3D", "Deposition"], wetoxst=["WetOXStot", "3D", "Deposition"], From 7ee8042d99d915c6e33677f355036bbc8ba366a4 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 12 Oct 2023 14:16:30 +0200 Subject: [PATCH 70/82] add statistics_model_only again that was accidently removed during merge --- pyaerocom/aeroval/glob_defaults.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index 132eb194a..7407945af 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -498,6 +498,18 @@ }, } +# For experiments where only model data is interesting, as with proxy drydep +statistics_model_only = { + "data_mean": { + "name": "Mean-Mod", + "longname": "Model Mean", + "scale": None, + "colmap": "coolwarm", + "unit": "1", + "decimals": 2, + }, +} + #: Mapping of pyaerocom variable names to web naming conventions ## Note: A 2D variable is defined under Column on the website, 3D is defined under Surface var_web_info = dict( From ec52c40706a179a10e8a023aeae2c50d63b2a35d Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 12 Oct 2023 14:56:21 +0200 Subject: [PATCH 71/82] added proxy wetdeps --- pyaerocom/aeroval/glob_defaults.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index 7407945af..3bcdcf9e9 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -636,4 +636,22 @@ proxydryo3=["proxyDryO3", "3D", "Deposition"], proxydrypm10=["proxyDryPM10", "3D", "Deposition"], proxydrypm25=["proxyDryPM2.5", "3D", "Deposition"], + # proxy wetdep + proxywetoxs=["proxyWetOXS", "3D", "Deposition"], + proxywetso2=["proxyWetSO2", "3D", "Deposition"], + proxywetso4=["proxyWetSO4", "3D", "Deposition"], + proxywetoxn=["proxyWetOXN", "3D", "Deposition"], + proxywetno2=["proxyWetNO2", "3D", "Deposition"], + proxywetno2no2=["proxyWetNO2NO2", "3D", "Deposition"], + proxywethono=["proxyWetHONO", "3D", "Deposition"], + proxywetn2o5=["proxyWetN2O5", "3D", "Deposition"], + proxywethno3=["proxyWetHNO3", "3D", "Deposition"], + proxywetno3c=["proxyWetNO3Coarse", "3D", "Deposition"], + proxywetno3f=["proxyWetNO3Fine", "3D", "Deposition"], + proxywetrdn=["proxyWetRDN", "3D", "Deposition"], + proxywetnh3=["proxyWetNH3", "3D", "Deposition"], + proxywetnh4=["proxyWetNH4", "3D", "Deposition"], + proxyweto3=["proxyWetO3", "3D", "Deposition"], + proxywetpm10=["proxyWetPM10", "3D", "Deposition"], + proxywetpm25=["proxyWetPM2.5", "3D", "Deposition"], ) From 671f417544a45bcc9ecac7115a9fd20f2493f70e Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 16 Oct 2023 14:36:49 +0200 Subject: [PATCH 72/82] linting --- pyaerocom/io/read_ebas.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index df22c0608..ae36bf358 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -5,6 +5,8 @@ import numpy as np from geonum.atmosphere import T0_STD, p0 +from tqdm import tqdm + from pyaerocom import const from pyaerocom._lowlevel_helpers import BrowseDict from pyaerocom.aux_var_helpers import ( @@ -14,6 +16,7 @@ compute_sc440dryaer, compute_sc550dryaer, compute_sc700dryaer, + compute_wetna_from_concprcpna, compute_wetnh4_from_concprcpnh4, compute_wetno3_from_concprcpno3, compute_wetoxn_from_concprcpoxn, @@ -22,7 +25,6 @@ compute_wetoxs_from_concprcpoxst, compute_wetrdn_from_concprcprdn, compute_wetso4_from_concprcpso4, - compute_wetna_from_concprcpna, concx_to_vmrx, make_proxy_drydep_from_O3, make_proxy_wetdep_from_O3, @@ -46,7 +48,6 @@ from pyaerocom.tstype import TsType from pyaerocom.ungriddeddata import UngriddedData from pyaerocom.units_helpers import get_unit_conversion_fac -from tqdm import tqdm logger = logging.getLogger(__name__) From f3b8b3dcb928e9e89bacbdac9807ea15496a3a7b Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 16 Oct 2023 15:16:27 +0200 Subject: [PATCH 73/82] adjust to new plugin structure for obs reading --- pyaerocom/plugins/ipcforests/__init__.py | 2 -- pyaerocom/plugins/ipcforests/reader.py | 2 +- pyproject.toml | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyaerocom/plugins/ipcforests/__init__.py b/pyaerocom/plugins/ipcforests/__init__.py index 96378059c..e69de29bb 100644 --- a/pyaerocom/plugins/ipcforests/__init__.py +++ b/pyaerocom/plugins/ipcforests/__init__.py @@ -1,2 +0,0 @@ -from . import metadata, reader -from .metadata import MetadataReader diff --git a/pyaerocom/plugins/ipcforests/reader.py b/pyaerocom/plugins/ipcforests/reader.py index 6182ed1ca..18f9678d3 100644 --- a/pyaerocom/plugins/ipcforests/reader.py +++ b/pyaerocom/plugins/ipcforests/reader.py @@ -21,7 +21,7 @@ class ReadIPCForest(ReadUngriddedBase): #: version log of this class (for caching) - __version__ = "0.3_" + ReadUngriddedBase.__baseversion__ + __version__ = "0.4_" + ReadUngriddedBase.__baseversion__ #: Name of dataset (OBS_ID) DATA_ID = const.IPCFORESTS_NAME diff --git a/pyproject.toml b/pyproject.toml index 9a9e87521..d4b486da3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ ReadGAW = "pyaerocom.plugins.gaw.reader:ReadGAW" ReadGhost = "pyaerocom.plugins.ghost.reader:ReadGhost" ReadMEP = "pyaerocom.plugins.mep.reader:ReadMEP" ReadICOS = "pyaerocom.plugins.icos.reader:ReadICOS" +ReadIPCForest = "pyaerocom.plugins.ipcforests.reader:ReadIPCForest" [tool.flit.sdist] include = ["LICENSE", "README.*", "pyaerocom_env.yml", "tests"] From 8e0ab9de337358168ea09aacebca3a8195ecbdb7 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Wed, 18 Oct 2023 14:14:37 +0200 Subject: [PATCH 74/82] adjust tests --- tests/aeroval/test_experiment_output.py | 29 ++++++++++++++++++++++++- tests/io/test_read_ebas.py | 6 +++++ tests/plugins/test_entry_points.py | 2 +- tests/test_varcollection.py | 2 +- 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/aeroval/test_experiment_output.py b/tests/aeroval/test_experiment_output.py index 9cf917611..72c3cb356 100644 --- a/tests/aeroval/test_experiment_output.py +++ b/tests/aeroval/test_experiment_output.py @@ -232,7 +232,34 @@ def test_ExperimentOutput_delete_experiment_data(tmp_path: Path, also_coldata: b [ ( "ang4487aer", - {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + { + "scale": [ + -0.2, + -0.1, + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0, + 1.1, + 1.2, + 1.3, + 1.4, + 1.5, + 1.6, + 1.7, + 1.8, + 1.9, + 2.0, + ], + "colmap": "coolwarm", + }, ), ( "concprcpso4", diff --git a/tests/io/test_read_ebas.py b/tests/io/test_read_ebas.py index 81dd36413..3cd7ff507 100644 --- a/tests/io/test_read_ebas.py +++ b/tests/io/test_read_ebas.py @@ -357,6 +357,12 @@ def test_PROVIDES_VARIABLES(reader: ReadEbas): "wetnh4", "wetno3", "wetso4", + "wetna", + "concprcpoxst", + 'wetoxsc', + 'concprcpoxsc', + 'wetoxst', + 'concprcpna', } assert set(reader.PROVIDES_VARIABLES) == (PROVIDES_VARIABLES) diff --git a/tests/plugins/test_entry_points.py b/tests/plugins/test_entry_points.py index 390ee10c1..3e397cf0d 100644 --- a/tests/plugins/test_entry_points.py +++ b/tests/plugins/test_entry_points.py @@ -15,4 +15,4 @@ def test_gridded(): def test_ungridded(): names = {ep.name for ep in metadata.entry_points(group="pyaerocom.ungridded")} assert names, "no entry points found" - assert names == {"ReadGAW", "ReadGhost", "ReadMEP", "ReadICOS"} + assert names == {"ReadGAW", "ReadGhost", "ReadMEP", "ReadICOS", "ReadIPCForest"} diff --git a/tests/test_varcollection.py b/tests/test_varcollection.py index d5fcc2ab0..248021926 100644 --- a/tests/test_varcollection.py +++ b/tests/test_varcollection.py @@ -83,7 +83,7 @@ def test_VarCollection_get_var_error(collection: VarCollection): ("*blaaaaaaa*", 0), ("dep*", 6), ("od*", 26), - ("conc*", 87), + ("conc*", 90), ], ) def test_VarCollection_find(collection: VarCollection, search_pattern: str, num: int): From 322d9f15ee082b5e144c0ba212b8468a3b87c944 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 19 Oct 2023 10:00:08 +0200 Subject: [PATCH 75/82] adjust tests --- tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 2cc991c41..3adfba53d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -119,7 +119,7 @@ def test_Config_has_access_users_database(): ("metno", None, False, False, 0), ("metno", None, True, False, 0), ("metno", None, True, True, 0), - ("metno", f"/home/{USER}", True, True, 1), + ("metno", f"/home/{USER}", True, True, 2), ("users-db", None, False, False, 0), ], ) From 7862fcd9efb6275ca0618573b3d65e1262de4808 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Thu, 19 Oct 2023 10:14:53 +0200 Subject: [PATCH 76/82] linting --- tests/io/test_read_ebas.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/io/test_read_ebas.py b/tests/io/test_read_ebas.py index 3cd7ff507..c5acbdb2b 100644 --- a/tests/io/test_read_ebas.py +++ b/tests/io/test_read_ebas.py @@ -359,10 +359,10 @@ def test_PROVIDES_VARIABLES(reader: ReadEbas): "wetso4", "wetna", "concprcpoxst", - 'wetoxsc', - 'concprcpoxsc', - 'wetoxst', - 'concprcpna', + "wetoxsc", + "concprcpoxsc", + "wetoxst", + "concprcpna", } assert set(reader.PROVIDES_VARIABLES) == (PROVIDES_VARIABLES) From 47529cbb23ae4b6a2449db22dffb692215472981 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 31 Oct 2023 11:40:07 +0100 Subject: [PATCH 77/82] added more variables --- tests/io/test_ebas_varinfo.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/io/test_ebas_varinfo.py b/tests/io/test_ebas_varinfo.py index b1fe67cce..73b049837 100644 --- a/tests/io/test_ebas_varinfo.py +++ b/tests/io/test_ebas_varinfo.py @@ -168,6 +168,16 @@ ("wetoxs", None, None, None, None, ["concprcpoxs"], 1), ("wetoxn", None, None, None, None, ["concprcpoxn"], 1), ("wetrdn", None, None, None, None, ["concprcprdn"], 1), + ("wetoxsc", None, None, None, None, ["concprcpoxsc"], 1), + ("wetoxst", None, None, None, None, ["concprcpoxst"], 1), + ("wetna", None, None, None, None, ["concprcpna"], 1), + ("proxydryoxs", None, None, None, None, ["concprcpoxs"], 1), + ("proxydryoxn", None, None, None, None, ["concprcpoxn"], 1), + ("proxydryrdn", None, None, None, None, ["concprcprdn"], 1), + ("proxydryo3", None, None, None, None, ["vmro3"], 1), + ("wetno3", None, None, None, None, ["concprcpno3"], 1), + ("wetnh4", None, None, None, None, ["concprcpnh4"], 1), + ("wetso4", None, None, None, None, ["concprcpso4"], 1), ( "prmm", ["precipitation_amount_off", "precipitation_amount"], From d0b18cd996536f5af87ba0c492e86979024fc536 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 31 Oct 2023 11:40:45 +0100 Subject: [PATCH 78/82] removed duplicate wetpm10 entry introduced by merging mein-dev --- pyaerocom/data/variables.ini | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index 8df9ee904..dbe323b4b 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -2451,17 +2451,6 @@ maximum = 10000 dimensions = time,lat,lon comments_and_purpose = Verification of the budget of atmospheric oxidizing agents -[wetpm10] -var_name = wetpm10 -description = Wet deposition of PM10 mass -standard_name = tendency_of_atmosphere_mass_content_of_pm10_dry_aerosol_particles_due_to_wet_deposition -var_type = dry deposition flux -unit = kg m-2 s-1 -minimum = 0 -maximum = 10000 -dimensions = time,lat,lon -comments_and_purpose = Verification of PM10 (major AQ metric) budget ; Consistent with AQMEII. - [wetpm10ss] var_name = wetpm10ss description = Wet deposition of PM10 seasalt @@ -5588,8 +5577,6 @@ minimum = 0 maximum = 10000 dimensions = time,lat,lon - - [wetpm10] var_name = wetpm10 description = Wet deposition of PM10 mass @@ -5599,6 +5586,7 @@ unit = mg m-2 d-1 minimum = 0 maximum = 10000 dimensions = time,lat,lon +comments_and_purpose = unit according to EBAS DB, not according to CF convention (uses kg m-2 s-1) [wetpm25] var_name = wetpm25 From 288de77b7f1ec56631dd248e3dfed92958624be4 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 31 Oct 2023 11:56:02 +0100 Subject: [PATCH 79/82] removed unused code --- pyaerocom/aeroval/coldatatojson_helpers.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pyaerocom/aeroval/coldatatojson_helpers.py b/pyaerocom/aeroval/coldatatojson_helpers.py index 96a675afa..90dd21849 100644 --- a/pyaerocom/aeroval/coldatatojson_helpers.py +++ b/pyaerocom/aeroval/coldatatojson_helpers.py @@ -598,16 +598,6 @@ def _process_sites_weekly_ts(coldata, regions_how, region_ids, meta_glob): ), } - # try to merge repw_res together into one dataarray - repw_res_v2 = xr.concat( - [ - _create_diurnal_weekly_data_object(coldata, "seasonal")["rep_week"], - _create_diurnal_weekly_data_object(coldata, "yearly")["rep_week"].expand_dims( - "period", axis=0 - ), - ], - dim="period", - ) ts_objs = _process_weekly_object_to_station_time_series(repw_res, meta_glob) ts_objs_reg = _process_weekly_object_to_country_time_series( repw_res, meta_glob, regions_how, region_ids From e978e28e0d9c19f3b6442887559fab0f5e08d7eb Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 31 Oct 2023 12:02:24 +0100 Subject: [PATCH 80/82] adjusted to review --- pyaerocom/aeroval/coldatatojson_helpers.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pyaerocom/aeroval/coldatatojson_helpers.py b/pyaerocom/aeroval/coldatatojson_helpers.py index 90dd21849..7f6480dd4 100644 --- a/pyaerocom/aeroval/coldatatojson_helpers.py +++ b/pyaerocom/aeroval/coldatatojson_helpers.py @@ -5,6 +5,7 @@ import os from copy import deepcopy from datetime import datetime +from typing import Literal import numpy as np import pandas as pd @@ -381,13 +382,16 @@ def _create_diurnal_weekly_data_object(coldata, resolution): return output_array -def _get_period_keys(resolution): - if resolution == "seasonal": - period_keys = ["DJF", "MAM", "JJA", "SON"] - elif resolution == "yearly": - period_keys = ["All"] - return period_keys +def _get_period_keys(resolution: Literal["seasonal", "yearly"]): + period_keys = dict( + seasonal=["DJF", "MAM", "JJA", "SON"], + yearly=["All"], + ) + + if resolution not in period_keys: + raise ValueError(f"Unknown {resolution=}") + return period_keys[resolution] def _process_one_station_weekly(stat_name, i, repw_res, meta_glob, time): """ From da622bf1cdb17907ab4b6f195d6c30545ec2d50f Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 31 Oct 2023 13:00:30 +0100 Subject: [PATCH 81/82] adjusted to review --- pyaerocom/aeroval/coldatatojson_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyaerocom/aeroval/coldatatojson_helpers.py b/pyaerocom/aeroval/coldatatojson_helpers.py index 7f6480dd4..a29d372f3 100644 --- a/pyaerocom/aeroval/coldatatojson_helpers.py +++ b/pyaerocom/aeroval/coldatatojson_helpers.py @@ -393,6 +393,7 @@ def _get_period_keys(resolution: Literal["seasonal", "yearly"]): return period_keys[resolution] + def _process_one_station_weekly(stat_name, i, repw_res, meta_glob, time): """ Processes one station data set for all supported averaging windows into a From 1016ee727deb3d8eee796a66a1edffa1e641c5a4 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 31 Oct 2023 13:00:39 +0100 Subject: [PATCH 82/82] adjusted test --- tests/test_varcollection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_varcollection.py b/tests/test_varcollection.py index 248021926..fff03d390 100644 --- a/tests/test_varcollection.py +++ b/tests/test_varcollection.py @@ -81,7 +81,7 @@ def test_VarCollection_get_var_error(collection: VarCollection): "search_pattern,num", [ ("*blaaaaaaa*", 0), - ("dep*", 6), + ("dep*", 7), ("od*", 26), ("conc*", 90), ],