diff --git a/pyaerocom/aeroval/coldatatojson_helpers.py b/pyaerocom/aeroval/coldatatojson_helpers.py index 3af9f6852..a29d372f3 100644 --- a/pyaerocom/aeroval/coldatatojson_helpers.py +++ b/pyaerocom/aeroval/coldatatojson_helpers.py @@ -5,6 +5,7 @@ import os from copy import deepcopy from datetime import datetime +from typing import Literal import numpy as np import pandas as pd @@ -381,12 +382,16 @@ def _create_diurnal_weekly_data_object(coldata, resolution): return output_array -def _get_period_keys(resolution): - if resolution == "seasonal": - period_keys = ["DJF", "MAM", "JJA", "SON"] - elif resolution == "yearly": - period_keys = ["Annual"] - return period_keys +def _get_period_keys(resolution: Literal["seasonal", "yearly"]): + period_keys = dict( + seasonal=["DJF", "MAM", "JJA", "SON"], + yearly=["All"], + ) + + if resolution not in period_keys: + raise ValueError(f"Unknown {resolution=}") + + return period_keys[resolution] def _process_one_station_weekly(stat_name, i, repw_res, meta_glob, time): @@ -426,8 +431,8 @@ def _process_one_station_weekly(stat_name, i, repw_res, meta_glob, time): ts_data = { "time": time, - "seasonal": {"obs": yeardict, "mod": yeardict}, - "yearly": {"obs": yeardict, "mod": yeardict}, + "seasonal": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)}, + "yearly": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)}, } ts_data["station_name"] = stat_name ts_data.update(meta_glob) @@ -526,8 +531,8 @@ def _process_weekly_object_to_country_time_series(repw_res, meta_glob, regions_h for regid, regname in region_ids.items(): ts_data = { "time": time, - "seasonal": {"obs": yeardict, "mod": yeardict}, - "yearly": {"obs": yeardict, "mod": yeardict}, + "seasonal": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)}, + "yearly": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)}, } ts_data["station_name"] = regname ts_data.update(meta_glob) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 4e1564389..bbfc8b471 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -16,6 +16,7 @@ from pyaerocom.aeroval.glob_defaults import ( extended_statistics, statistics_defaults, + statistics_model_only, statistics_obs_only, statistics_trend, var_ranges_defaults, @@ -574,6 +575,8 @@ def _create_var_ranges_json(self): def _create_statistics_json(self): if self.cfg.statistics_opts.obs_only_stats: stats_info = statistics_obs_only + elif self.cfg.statistics_opts.model_only_stats: + stats_info = statistics_model_only else: stats_info = statistics_defaults stats_info.update(extended_statistics) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index cb9e7faf3..00f57cd62 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -200,10 +200,17 @@ "scale": [0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0], "colmap": "coolwarm", }, - "wetoxs": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, - "wetoxn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "wetoxs": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, + "wetna": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, + "wetoxn": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, "wetrdn": { - "scale": [0, 0.75, 1.5, 2.25, 3.0, 3.75, 4.5, 5.25, 6.0, 6.75, 7.5, 8.25], + "scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], + "colmap": "coolwarm", + }, + "wetoxsf": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, + "wetoxnf": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"}, + "wetrdnf": { + "scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm", }, "prmm": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"}, @@ -215,6 +222,7 @@ "colmap": "coolwarm", }, "drydust": { + # "scale": [0.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0], "scale": [0.0, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0], "colmap": "coolwarm", }, @@ -255,6 +263,58 @@ "colmap": "coolwarm", }, "ts": {"scale": [265, 270, 275, 280, 285, 290, 300, 305, 310, 315, 320], "colmap": "coolwarm"}, + "proxydryo3": {"scale": [0, 0.5, 1, 15, 20, 25, 0.30, 40, 50], "colmap": "coolwarm"}, + "proxydrypm10": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "proxydrypm25": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "proxydryno2": { + "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], + "colmap": "coolwarm", + }, + "proxydryhono": {"scale": [0.001, 0.002, 0.003, 0.004, 0.005, 0.006], "colmap": "coolwarm"}, + "proxydryn2o5": {"scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm"}, + "proxydryhno3": { + "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], + "colmap": "coolwarm", + }, + "proxydryno3c": { + "scale": [0.01, 0.02, 0.03, 0.04, 0.05], + "colmap": "coolwarm", + }, + "proxydryno3f": {"scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm"}, + "proxydrynh3": { + "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80], + "colmap": "coolwarm", + }, + "proxydrynh4": { + "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], + "colmap": "coolwarm", + }, + "proxydryso2": { + "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4], + "colmap": "coolwarm", + }, + "proxydryso4": { + "scale": [0.01, 0.02, 0.03, 0.04, 0.05], + "colmap": "coolwarm", + }, + "proxydryoxs": { + "scale": [0, 0.05, 0.1, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40], + "colmap": "coolwarm", + }, + "proxydryoxn": { + "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80], + "colmap": "coolwarm", + }, + "proxydryrdn": { + "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80], + "colmap": "coolwarm", + }, + "depoxs": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "depoxn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "deprdn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "depoxsf": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "depoxnf": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, + "deprdnf": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"}, } #: Default information for statistical parameters @@ -451,6 +511,18 @@ }, } +# For experiments where only model data is interesting, as with proxy drydep +statistics_model_only = { + "data_mean": { + "name": "Mean-Mod", + "longname": "Model Mean", + "scale": None, + "colmap": "coolwarm", + "unit": "1", + "decimals": 2, + }, +} + #: Mapping of pyaerocom variable names to web naming conventions ## Note: A 2D variable is defined under Column on the website, 3D is defined under Surface var_web_info = dict( @@ -487,6 +559,7 @@ concNno=["NO", "3D", "Concentration"], concno2=["NO2", "3D", "Gas concentrations"], concNno2=["NO2", "3D", "Gas concentrations"], + vmrno=["NO", "3D", "Volume mixing ratios"], vmrno2=["NO2", "3D", "Volume mixing ratios"], concno3=["NO3", "3D", "Gas concentrations"], conctno3=["tNO3", "3D", "Concentration"], @@ -509,6 +582,11 @@ concco=["CO", "3D", "Particle concentration"], vmrco=["CO", "3D", "Volume mixing ratios"], vmrco2=["CO2", "3D", "Volume mixing ratios"], + vmrc2h2=["Ethyne", "3D", "Volume mixing ratios"], + vmrc2h4=["Ethylene", "3D", "Volume mixing ratios"], + vmrc2h6=["Ethane", "3D", "Volume mixing ratios"], + vmrhcho=["Formaldehyde", "3D", "Volume mixing ratios"], + vmrisop=["Isoprene", "3D", "Volume mixing ratios"], vmrch4=["CH4", "3D", "Volume mixing ratios"], # PMs concpm10=["PM10", "3D", "Particle concentrations"], @@ -543,9 +621,47 @@ drydust=["DryDustDep", "3D", "Deposition"], wetdust=["WetDustDep", "3D", "Deposition"], wetoxs=["WetOXS", "3D", "Deposition"], + wetoxsc=["WetOXScorr", "3D", "Deposition"], + wetoxst=["WetOXStot", "3D", "Deposition"], wetoxn=["WetOXN", "3D", "Deposition"], wetrdn=["WetRDN", "3D", "Deposition"], prmm=["Precipitation", "3D", "Deposition"], # Temperature ts=["Surface Temperature", "3D", "Temperature"], + # proxy drydep + proxydryoxs=["proxyDryOXS", "3D", "Deposition"], + proxydryso2=["proxyDrySO2", "3D", "Deposition"], + proxydryso4=["proxyDrySO4", "3D", "Deposition"], + proxydryoxn=["proxyDryOXN", "3D", "Deposition"], + proxydryno2=["proxyDryNO2", "3D", "Deposition"], + proxydryno2no2=["proxyDryNO2NO2", "3D", "Deposition"], + proxydryhono=["proxyDryHONO", "3D", "Deposition"], + proxydryn2o5=["proxyDryN2O5", "3D", "Deposition"], + proxydryhno3=["proxyDryHNO3", "3D", "Deposition"], + proxydryno3c=["proxyDryNO3Coarse", "3D", "Deposition"], + proxydryno3f=["proxyDryNO3Fine", "3D", "Deposition"], + proxydryrdn=["proxyDryRDN", "3D", "Deposition"], + proxydrynh3=["proxyDryNH3", "3D", "Deposition"], + proxydrynh4=["proxyDryNH4", "3D", "Deposition"], + proxydryo3=["proxyDryO3", "3D", "Deposition"], + proxydrypm10=["proxyDryPM10", "3D", "Deposition"], + proxydrypm25=["proxyDryPM2.5", "3D", "Deposition"], + # proxy wetdep + proxywetoxs=["proxyWetOXS", "3D", "Deposition"], + proxywetso2=["proxyWetSO2", "3D", "Deposition"], + proxywetso4=["proxyWetSO4", "3D", "Deposition"], + proxywetoxn=["proxyWetOXN", "3D", "Deposition"], + proxywetno2=["proxyWetNO2", "3D", "Deposition"], + proxywetno2no2=["proxyWetNO2NO2", "3D", "Deposition"], + proxywethono=["proxyWetHONO", "3D", "Deposition"], + proxywetn2o5=["proxyWetN2O5", "3D", "Deposition"], + proxywethno3=["proxyWetHNO3", "3D", "Deposition"], + proxywetno3c=["proxyWetNO3Coarse", "3D", "Deposition"], + proxywetno3f=["proxyWetNO3Fine", "3D", "Deposition"], + proxywetrdn=["proxyWetRDN", "3D", "Deposition"], + proxywetnh3=["proxyWetNH3", "3D", "Deposition"], + proxywetnh4=["proxyWetNH4", "3D", "Deposition"], + proxyweto3=["proxyWetO3", "3D", "Deposition"], + proxywetpm10=["proxyWetPM10", "3D", "Deposition"], + proxywetpm25=["proxyWetPM2.5", "3D", "Deposition"], ) diff --git a/pyaerocom/aeroval/obsentry.py b/pyaerocom/aeroval/obsentry.py index 19615de8c..49fd50103 100644 --- a/pyaerocom/aeroval/obsentry.py +++ b/pyaerocom/aeroval/obsentry.py @@ -51,7 +51,7 @@ class ObsEntry(BrowseDict): (c.g. :class:`pyaerocom.io.ReadUngridded`). """ - SUPPORTED_VERT_CODES = ["Column", "Profile", "Surface"] + SUPPORTED_VERT_CODES = ["Column", "Profile", "Surface"] # , "2D"] ALT_NAMES_VERT_CODES = dict(ModelLevel="Profile") SUPPORTED_VERT_LOCS = DataSource.SUPPORTED_VERT_LOCS diff --git a/pyaerocom/aeroval/setupclasses.py b/pyaerocom/aeroval/setupclasses.py index a8870ca52..a54177a2d 100644 --- a/pyaerocom/aeroval/setupclasses.py +++ b/pyaerocom/aeroval/setupclasses.py @@ -155,6 +155,7 @@ def __init__(self, **kwargs): self.use_fairmode = False self.use_diurnal = False self.obs_only_stats = False + self.model_only_stats = False self.update(**kwargs) diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index 91fb6ca16..a7baead6d 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -563,6 +563,52 @@ def compute_wetoxs_from_concprcpoxs(data): return _compute_wdep_from_concprcp_helper(data, "wetoxs", "concprcpoxs", "pr") +def compute_wetoxs_from_concprcpoxst(data): + """Compute wdep from conc in precip and precip data + + Note + ---- + In addition to the returned numpy array, the input instance of + :class:`StationData` is modified by additional metadata and flags for + the new variable. See also :func:`_compute_wdep_from_concprcp_helper`. + + Parameters + ---------- + StationData + data object containing concprcp and precip data + + Returns + ------- + numpy.ndarray + array with wet deposition values + + """ + return _compute_wdep_from_concprcp_helper(data, "wetoxs", "concprcpoxst", "pr") + + +def compute_wetoxs_from_concprcpoxsc(data): + """Compute wdep from conc in precip and precip data + + Note + ---- + In addition to the returned numpy array, the input instance of + :class:`StationData` is modified by additional metadata and flags for + the new variable. See also :func:`_compute_wdep_from_concprcp_helper`. + + Parameters + ---------- + StationData + data object containing concprcp and precip data + + Returns + ------- + numpy.ndarray + array with wet deposition values + + """ + return _compute_wdep_from_concprcp_helper(data, "wetoxs", "concprcpoxsc", "pr") + + def compute_wetoxn_from_concprcpoxn(data): """Compute wdep from conc in precip and precip data @@ -609,6 +655,22 @@ def compute_wetrdn_from_concprcprdn(data): return _compute_wdep_from_concprcp_helper(data, "wetrdn", "concprcprdn", "pr") +def compute_wetnh4_from_concprcpnh4(data): + return _compute_wdep_from_concprcp_helper(data, "wetnh4", "concprcpnh4", "pr") + + +def compute_wetno3_from_concprcpno3(data): + return _compute_wdep_from_concprcp_helper(data, "wetno3", "concprcpno3", "pr") + + +def compute_wetso4_from_concprcpso4(data): + return _compute_wdep_from_concprcp_helper(data, "wetso4", "concprcpso4", "pr") + + +def compute_wetna_from_concprcpna(data): + return _compute_wdep_from_concprcp_helper(data, "wetna", "concprcpna", "pr") + + def vmrx_to_concx(data, p_pascal, T_kelvin, vmr_unit, mmol_var, mmol_air=None, to_unit=None): """ Convert volume mixing ratio (vmr) to mass concentration @@ -725,3 +787,51 @@ def calc_vmro3max(data): # print(data.var_info) # exit() return o3max + + +def identity(data): + return data + + +def make_proxy_drydep_from_O3(data): + # sort of prototype to add a compted variable + # one has to extend the data structures of the station data object + # 'right', but has to return just the data array + # That concept is a bit confusing (why not do everything in data here?) + var_name = "vmro3" + new_var_name = "proxydryo3" + + flags = data.data_flagged[var_name] + new_var_data = data[var_name] + units = data.var_info[var_name]["units"] + # data.var_info[new_var_name]["units"] = units + + if not new_var_name in data.var_info: + data.var_info[new_var_name] = {} + data.var_info[new_var_name] = data.var_info[var_name] + data.var_info[new_var_name]["units"] = "mg m-2 d-1" + + data.data_flagged[new_var_name] = flags + return new_var_data + + +def make_proxy_wetdep_from_O3(data): + # sort of prototype to add a compted variable + # one has to extend the data structures of the station data object + # 'right', but has to return just the data array + # That concept is a bit confusing (why not do everything in data here?) + var_name = "vmro3" + new_var_name = "proxyweto3" + + flags = data.data_flagged[var_name] + new_var_data = data[var_name] + units = data.var_info[var_name]["units"] + # data.var_info[new_var_name]["units"] = units + + if not new_var_name in data.var_info: + data.var_info[new_var_name] = {} + data.var_info[new_var_name] = data.var_info[var_name] + data.var_info[new_var_name]["units"] = "mg m-2 d-1" + + data.data_flagged[new_var_name] = flags + return new_var_data diff --git a/pyaerocom/colocation_auto.py b/pyaerocom/colocation_auto.py index 001beb8bf..9d9f45658 100644 --- a/pyaerocom/colocation_auto.py +++ b/pyaerocom/colocation_auto.py @@ -295,7 +295,7 @@ class ColocationSetup(BrowseDict): #: file for ec550aer at the surface ('*ec550aer*Surface*.nc'), then, the #: colocation routine will look for '*ec550aer*ModelLevel*.nc' and if this #: exists, it will load it and extract the surface level. - OBS_VERT_TYPES_ALT = {"Surface": "ModelLevel"} + OBS_VERT_TYPES_ALT = {"Surface": "ModelLevel", "2D": "2D"} #: do not raise Exception if invalid item is attempted to be assigned #: (Overwritten from base class) diff --git a/pyaerocom/config.py b/pyaerocom/config.py index 70988e92f..77a373129 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -35,6 +35,9 @@ class Config: # default names of the different obs networks # might get overwritten from paths.ini see func read_config + #: IPC Forests + IPCFORESTS_NAME = "IPCFORESTS" + #: Aeronet Sun V2 access names AERONET_SUN_V2L15_AOD_DAILY_NAME = "AeronetSunV2Lev1.5.daily" AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME = "AeronetSun_2.0_NRT" diff --git a/pyaerocom/data/aliases.ini b/pyaerocom/data/aliases.ini index 54aa19ac4..61405b4b9 100644 --- a/pyaerocom/data/aliases.ini +++ b/pyaerocom/data/aliases.ini @@ -49,6 +49,7 @@ dryoxs = drysox concss10 = concsscoarse concsscoarse=concss10 vmrglyoxal = vmrglyox +prmm = pr [alias_families] # so far only works with beginning of name diff --git a/pyaerocom/data/ebas_config.ini b/pyaerocom/data/ebas_config.ini index 9622f1c64..79e8ab710 100644 --- a/pyaerocom/data/ebas_config.ini +++ b/pyaerocom/data/ebas_config.ini @@ -370,6 +370,14 @@ matrix=air component=sulphate_corrected,sulphate_total matrix=precip +[concprcpoxsc] +component=sulphate_corrected +matrix=precip + +[concprcpoxst] +component=sulphate_total +matrix=precip + [concprcpoxn] component=nitrate matrix=precip @@ -378,18 +386,31 @@ matrix=precip component=ammonium matrix=precip +[concprcpna] +component=sodium +matrix=precip + # 3. Deposition rates # 3.1. Wet deposition [wetoxs] requires=concprcpoxs +[wetoxst] +requires=concprcpoxst + +[wetoxsc] +requires=concprcpoxsc + [wetrdn] requires=concprcprdn [wetoxn] requires=concprcpoxn +[wetna] +requires=concprcpna + # 4. Precipitation [pr] # pyaerocom unit kg m-2 s-1 component=precipitation_amount_off,precipitation_amount @@ -405,4 +426,197 @@ matrix=pm10 [concCecpm10] component=elemental_carbon -matrix=pm10 \ No newline at end of file +matrix=pm10 + +# CAMS2_40 Task4041 + +# Gases + +[vmrhno3] +component=nitric_acid +matrix=air + +[vmrnh3] +component=ammonia +matrix=air + +[vmrtp] +component=monoterpenes +matrix=air + +; [vmrpan] +; component=methanal +; matrix=air + +; [vmroh] +; component=methanal +; matrix=air + +# PM + +[concCoc25] +component=organic_carbon +matrix=pm25,pm1 + +[concom25] +component=organic_mass +matrix=pm25,pm1 + + +[concsscoarse] +component=sodium +matrix=pm10 +scale_factor=3.27 + +[concss25] +component=sodium +matrix=pm25 +scale_factor=3.27 + + +# Deposition +[concprcpnh4] +component=ammonium +matrix=precip + +[wetnh4] +requires=concprcpnh4 + +[concprcpno3] +component=nitrate +matrix=precip + +[wetno3] +requires=concprcpno3 + + +[concprcpso4] +#component=sulphate_corrected#,sulphate_total +component=sulphate_corrected +# after discussion with Wenche +matrix=precip + +[wetso4] +requires=concprcpso4 + +#proxy Dry Dep + +# Sulpher Based dry dep +[proxydryoxs] +requires=concprcpoxs + +[proxydryso2] +requires=concprcpoxs + +[proxydryso4] +requires=concprcpoxs + + +# Oxidized nitrogen based dry dep +[proxydryoxn] +requires=concprcpoxn + +[proxydryno2] +requires=concprcpoxn + +[proxydryno2no2] +requires=concprcpoxn + +[proxydryhono] +requires=concprcpoxn + +[proxydryn2o5] +requires=concprcpoxn + +[proxydryhno3] +requires=concprcpoxn + +[proxydryno3c] +requires=concprcpoxn + +[proxydryno3f] +requires=concprcpoxn + + +# Reduced nitrogen based dry dep +[proxydryrdn] +requires=concprcprdn + +[proxydrynh3] +requires=concprcprdn + +[proxydrynh4] +requires=concprcprdn + + +# Other proxy dry dep + +[proxydryo3] +requires=vmro3 + +[proxydrypm10] +requires=concprcpoxs + +[proxydrypm25] +requires=concprcpoxs + +#proxy wet Dep + +# Sulpher Based wet dep +[proxywetoxs] +requires=concprcpoxs + +[proxywetso2] +requires=concprcpoxs + +[proxywetso4] +requires=concprcpoxs + + +# Oxidized nitrogen based wet dep +[proxywetoxn] +requires=concprcpoxn + +[proxywetno2] +requires=concprcpoxn + +[proxywetno2no2] +requires=concprcpoxn + +[proxywethono] +requires=concprcpoxn + +[proxywetn2o5] +requires=concprcpoxn + +[proxywethno3] +requires=concprcpoxn + +[proxywetno3c] +requires=concprcpoxn + +[proxywetno3f] +requires=concprcpoxn + + +# Reduced nitrogen based wet dep +[proxywetrdn] +requires=concprcprdn + +[proxywetnh3] +requires=concprcprdn + +[proxywetnh4] +requires=concprcprdn + + +# Other proxy wet dep + +[proxyweto3] +requires=vmro3 + +[proxywetpm10] +requires=concprcpoxs + +[proxywetpm25] +requires=concprcpoxs \ No newline at end of file diff --git a/pyaerocom/data/paths.ini b/pyaerocom/data/paths.ini index 24fe72f78..1a5766c1b 100644 --- a/pyaerocom/data/paths.ini +++ b/pyaerocom/data/paths.ini @@ -50,7 +50,9 @@ dir= ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND3/, ${BASEDIR}/aerocom/aerocom-users-database/AEROCOM-PHASE-II-IND2/, ${BASEDIR}/fou/kl/CAMS61/, - ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/ + ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/PYAEROCOM/, + /lustre/storeB/project/fou/kl/CAMS2_40/task4041/, + /lustre/storeB/project/aerocom/aerocom-users-database/DOMOS/ [obsfolders] #folders to for model data @@ -104,6 +106,10 @@ MARCO_POLO = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/CHINA_MP_NRT MEP = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/MEP/aggregated/ ICOS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/ICOS/aggregated/ +#IPCForests +#IPCFORESTS = /lustre/storeB/project/fou/kl/emep/People/danielh/projects/pyaerocom/obs/ipc-forests/dep/ +IPCFORESTS = ${BASEDIR}/aerocom/aerocom1/AEROCOM_OBSDATA/ipc-forests/dep/ + [obsnames] #names of the different obs networks #Aeronet V2 @@ -151,6 +157,8 @@ MARCO_POLO = MarcoPolo MEP = MEP ICOS = ICOS +IPCFORESTS = IPCFORESTS + [parameters] #parameters definition ObsOnlyModelname = OBSERVATIONS-ONLY diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index d6f081203..dbe323b4b 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -2125,6 +2125,31 @@ description=Wet deposition of total sulphur mass unit = mg S m-2 d-1 minimum=0 +[wetoxsc] +description=Wet deposition of total sulphur corrected mass +unit = mg S m-2 d-1 +minimum=0 + +[wetoxst] +description=Wet deposition of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + +[depoxn] +description=Total deposition of oxidized nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[deprdn] +description=Total deposition of reduced Nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[depoxs] +description=Total deposition of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + [dryoxn] description=dry deposition of oxidized nitrogen mass unit = mg N m-2 d-1 @@ -2167,7 +2192,7 @@ var_name = wetno3 description = Wet deposition of nitrate ion in total PM standard_name = tendency_of_atmosphere_mass_content_of_nitrate_dry_aerosol_particles_due_to_wet_deposition var_type = wet deposition flux -unit = kg m-2 s-1 +unit = mg m-2 d-1 minimum = 0 maximum = 10000 dimensions = time,lat,lon @@ -2211,7 +2236,7 @@ var_name = wetnh3 description = Wet deposition of NH3 standard_name = tendency_of_atmosphere_mass_content_of_ammonia_due_to_wet_deposition var_type = wet deposition flux -unit = kg m-2 s-1 +unit = mg m-2 d-1 minimum = 0 maximum = 10000 dimensions = time,lat,lon @@ -2222,7 +2247,7 @@ var_name = wetnh4 description = Wet deposition of NH4 standard_name = tendency_of_atmosphere_mass_content_of_ammonium_dry_aerosol_particles_due_to_wet_deposition var_type = wet deposition flux -unit = kg m-2 s-1 +unit = mg m-2 d-1 minimum = 0 maximum = 10000 dimensions = time,lat,lon @@ -2233,7 +2258,7 @@ var_name = wetso2 description = Wet deposition of SO2 standard_name = tendency_of_atmosphere_mass_content_of_sulfur_dioxide_due_to_wet_deposition var_type = wet deposition flux -unit = kg m-2 s-1 +unit = mg m-2 d-1 minimum = 0 maximum = 10000 dimensions = time,lat,lon @@ -2244,7 +2269,7 @@ var_name = wetso4 description = Wet deposition of SO4 standard_name = tendency_of_atmosphere_mass_content_of_sulfate_dry_aerosol_particles_due_to_wet_deposition var_type = wet deposition flux -unit = kg m-2 s-1 +unit = mg m-2 d-1 minimum = 0 maximum = 10000 dimensions = time,lat,lon @@ -2294,12 +2319,23 @@ maximum = 10000 dimensions = time,lat,lon comments_and_purpose = Verification of aerosol budget and speciation +[wetna] +var_name = wetna +description = Wet deposition of sodium +standard_name = tendency_of_atmosphere_mass_content_of_seasalt_dry_aerosol_particles_due_to_wet_deposition +var_type = wet deposition flux +unit = mg m-2 d-1 +minimum = 0 +maximum = 10000 +dimensions = time,lat,lon +comments_and_purpose = Verification of aerosol budget and speciation + [wetdust] var_name = wetdust description = Wet deposition of dust standard_name = tendency_of_atmosphere_mass_content_of_dust_dry_aerosol_particles_due_to_wet_deposition var_type = wet deposition flux -unit = mg m-2 d-1 +unit = kg m-2 s-1 minimum = 0 maximum = 10000 dimensions = time,lat,lon @@ -2415,17 +2451,6 @@ maximum = 10000 dimensions = time,lat,lon comments_and_purpose = Verification of the budget of atmospheric oxidizing agents -[wetpm10] -var_name = wetpm10 -description = Wet deposition of PM10 mass -standard_name = tendency_of_atmosphere_mass_content_of_pm10_dry_aerosol_particles_due_to_wet_deposition -var_type = dry deposition flux -unit = kg m-2 s-1 -minimum = 0 -maximum = 10000 -dimensions = time,lat,lon -comments_and_purpose = Verification of PM10 (major AQ metric) budget ; Consistent with AQMEII. - [wetpm10ss] var_name = wetpm10ss description = Wet deposition of PM10 seasalt @@ -2599,6 +2624,11 @@ description = Mass concentration of sulphate in precipitation unit = ug S m-3 var_type = mass concentration +[concprcpna] +description = Mass concentration of sodium in precipitation +unit = ug m-3 +var_type = mass concentration + [concprcpno3] description = Mass concentration of NO3 in precipitation unit = ug N m-3 @@ -2619,6 +2649,16 @@ description = Mass concentration of total sulphur in precipitation unit = ug S m-3 var_type = mass concentration +[concprcpoxsc] +description = Mass concentration of total sulphur in precipitation +unit = ug S m-3 +var_type = mass concentration + +[concprcpoxst] +description = Mass concentration of total sulphur in precipitation +unit = ug S m-3 +var_type = mass concentration + [concprcprdn] description = Mass concentration of reduced nitrogen in precipitation unit = ug N m-3 @@ -3774,41 +3814,6 @@ maximum = 0.1 dimensions = time,lat,lon comments_and_purpose = Speciation of size partitioning the total aerosol ; Sources attribution and impact analysis - -[mmrco2] -var_name = mmrco2 -description = Carbon Dioxide -standard_name = mass_fraction_of_carbon_dioxide_in_air -var_type = mass mixing ratio -unit = kg kg-1 -minimum = 0 -maximum = 1000000 -dimensions = time,lat,lon - -[mmrch4] -var_name = mmrco2 -description = Methane -standard_name = mass_fraction_of_methane_in_air -var_type = mass mixing ratio -unit = kg kg-1 -minimum = 0 -maximum = 1000000 -dimensions = time,lat,lon - -[mmrco] -var_name = mmrco -description = Carbon Monoxide -standard_name = mass_fraction_of_carbon_monoxide_in_air -var_type = mass mixing ratio -unit = kg kg-1 -minimum = 0 -maximum = 1000000 -dimensions = time,lat,lon - - - - - [vmrhg0] var_name = vmrhg0 description = Hg0(g) Volume Mixing Ratio @@ -5483,4 +5488,307 @@ dimensions = time,lat,lon description=total deposition of dust unit = mg m-2 d-1 minimum = 0 -maximum = 1000 \ No newline at end of file +maximum = 1000 + + + + +; For CAMS2_40 Task4041 + + +; Gases + + +; PM + +[concCoc25] +description=Mass concentration of organic carbon in PM2.5 +unit = ug C m-3 + +[concom25] +description=Mass concentration of organic matter in PM2.5 +unit = ug m-3 + +[concss25] +description=Mass concentration of seasalt pm25 +unit = ug m-3 + +[concdust25] +description=Mass concentration of dust in PM2.5 +unit = ug m-3 + +[concdustcoarse] +description=Mass concentration of coarse dust +unit = ug m-3 + +; Deposition + + +[dryoxnf] +description=Dry deposition in forest of oxidized nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[dryrdnf] +description=Dry deposition in forest of reduced Nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[dryoxsf] +description=Dry deposition in forest of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + + +[depoxnf] +description=Total deposition in forest of oxidized nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[deprdnf] +description=Total deposition in forest of reduced Nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[depoxsf] +description=Total deposition in forest of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + +[wetno2] +var_name = wetno2 +description = Wet deposition of nitrogen oxide in total PM +standard_name = tendency_of_atmosphere_mass_content_of_nitrogen_oxide_dry_aerosol_particles_due_to_wet_deposition +var_type = wet deposition flux +unit = mg m-2 d-1 +minimum = 0 +maximum = 10000 +dimensions = time,lat,lon +comments_and_purpose = Verification of nitrogen budget. Verification of aerosol budget and speciation. + + +[wetho2no2] +var_name = wetho2no2 +description = Wet deposition of Peroxynitric acid in total PM +standard_name = tendency_of_atmosphere_mass_content_of_peroxynitric_acid_dry_aerosol_particles_due_to_wet_deposition +var_type = wet deposition flux +unit = mg m-2 d-1 +minimum = 0 +maximum = 10000 +dimensions = time,lat,lon + +[wetpm10] +var_name = wetpm10 +description = Wet deposition of PM10 mass +standard_name = tendency_of_atmosphere_mass_content_of_pm10_dry_aerosol_particles_due_to_wet_deposition +var_type = dry deposition flux +unit = mg m-2 d-1 +minimum = 0 +maximum = 10000 +dimensions = time,lat,lon +comments_and_purpose = unit according to EBAS DB, not according to CF convention (uses kg m-2 s-1) + +[wetpm25] +var_name = wetpm25 +description = Wet deposition of PM2.5 mass +standard_name = tendency_of_atmosphere_mass_content_of_pm25_dry_aerosol_particles_due_to_wet_deposition +var_type = dry deposition flux +unit = mg m-2 d-1 +minimum = 0 +maximum = 10000 +dimensions = time,lat,lon + +# proxy Dry Dep + +# Oxidized nitrogen based dry dep + +[proxydryoxn] +description=proxy dry deposition of oxidized nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[proxydryno2] +description=proxy dry deposition of NO2 +unit = mg N m-2 d-1 +minimum=0 + +[proxydryno2no2] +description=proxy dry deposition of NO2NO2 +unit = mg N m-2 d-1 +minimum=0 + +[proxydryhono] +description=proxy dry deposition of HONO +unit = mg N m-2 d-1 +minimum=0 + +[proxydryn2o5] +description=proxy dry deposition of N2O5 +unit = mg N m-2 d-1 +minimum=0 + +[proxydryhno3] +description=proxy dry deposition of HNO3 +unit = mg N m-2 d-1 +minimum=0 + +[proxydryno3c] +description=proxy dry deposition of NO3 Coarse +unit = mg N m-2 d-1 +minimum=0 + +[proxydryno3f] +description=proxy dry deposition of NO3 Fine +unit = mg N m-2 d-1 +minimum=0 +# Reduced nitrogen based dry dep + +[proxydryrdn] +description=proxy dry deposition of reduced Nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[proxydrynh3] +description=proxy dry deposition of NH3 +unit = mg N m-2 d-1 +minimum=0 + +[proxydrynh4] +description=proxy dry deposition of NH4 +unit = mg N m-2 d-1 +minimum=0 + +# Sulpher Based dry dep + +[proxydryoxs] +description=proxy dry deposition of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + +[proxydryso2] +description=proxy dry deposition of SO2 +unit = mg S m-2 d-1 +minimum=0 + +[proxydryso4] +description=proxy dry deposition of SO4 +unit = mg S m-2 d-1 +minimum=0 + +# Other proxy dry dep + +[proxydryo3] +description=proxy dry deposition of O3 +unit = mg m-2 d-1 +minimum=0 + +[proxydrypm10] +description=proxy dry deposition of PM10 +unit = mg m-2 d-1 +minimum=0 + + +[proxydrypm25] +description=proxy dry deposition of PM25 +unit = mg m-2 d-1 +minimum=0 + +[drypm25] +description=Dry deposition of PM25 +unit = mg m-2 d-1 +minimum=0 + +# proxy wet Dep + +# Oxidized nitrogen based wet dep + +[proxywetoxn] +description=proxy wet deposition of oxidized nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[proxywetno2] +description=proxy wet deposition of NO2 +unit = mg N m-2 d-1 +minimum=0 + +[proxywetno2no2] +description=proxy wet deposition of NO2NO2 +unit = mg N m-2 d-1 +minimum=0 + +[proxywethono] +description=proxy wet deposition of HONO +unit = mg N m-2 d-1 +minimum=0 + +[proxywetn2o5] +description=proxy wet deposition of N2O5 +unit = mg N m-2 d-1 +minimum=0 + +[proxywethno3] +description=proxy wet deposition of HNO3 +unit = mg N m-2 d-1 +minimum=0 + +[proxywetno3c] +description=proxy wet deposition of NO3 Coarse +unit = mg N m-2 d-1 +minimum=0 + +[proxywetno3f] +description=proxy wet deposition of NO3 Fine +unit = mg N m-2 d-1 +minimum=0 +# Reduced nitrogen based wet dep + +[proxywetrdn] +description=proxy wet deposition of reduced Nitrogen mass +unit = mg N m-2 d-1 +minimum=0 + +[proxywetnh3] +description=proxy wet deposition of NH3 +unit = mg N m-2 d-1 +minimum=0 + +[proxywetnh4] +description=proxy wet deposition of NH4 +unit = mg N m-2 d-1 +minimum=0 + +# Sulpher Based wet dep + +[proxywetoxs] +description=proxy wet deposition of total sulphur mass +unit = mg S m-2 d-1 +minimum=0 + +[proxywetso2] +description=proxy wet deposition of SO2 +unit = mg S m-2 d-1 +minimum=0 + +[proxywetso4] +description=proxy wet deposition of SO4 +unit = mg S m-2 d-1 +minimum=0 + +# Other proxy wet dep + +[proxyweto3] +description=proxy wet deposition of O3 +unit = mg m-2 d-1 +minimum=0 + +[proxywetpm10] +description=proxy wet deposition of PM10 +unit = mg m-2 d-1 +minimum=0 + + +[proxywetpm25] +description=proxy wet deposition of PM25 +unit = mg m-2 d-1 +minimum=0 diff --git a/pyaerocom/io/aux_components_fun.py b/pyaerocom/io/aux_components_fun.py new file mode 100644 index 000000000..408b2aeb9 --- /dev/null +++ b/pyaerocom/io/aux_components_fun.py @@ -0,0 +1,190 @@ +import logging +from traceback import format_exc + +import cf_units +import iris +import numpy as np +from geonum.atmosphere import T0_STD, p0 + +from pyaerocom._lowlevel_helpers import merge_dicts +from pyaerocom.helpers import copy_coords_cube +from pyaerocom.io.aux_read_cubes import ( + CUBE_MATHS, + _check_input_iscube, + _check_same_units, + add_cubes, +) +from pyaerocom.molmasses import get_mmr_to_vmr_fac, get_molmass +from pyaerocom.units_helpers import get_unit_conversion_fac + +logger = logging.getLogger(__name__) + + +single_component_mass = {"n": 14.0067, "c": 12.011, "s": 32.065} + + +def vmr_to_conc(data, vmr_unit, var_name, to_unit, component_unit=None): + """ + Convert volume mixing ratio (vmr) to mass concentration + + Parameters + ---------- + data : cube or GriddedData + array containing vmr values + + vmr_unit : str + unit of input data + var_name: str + name of variable, used to fund molar mass + to_unit : str, optional + Unit to which output data is converted. If None, output unit is + kg m-3. The default is None. + component_unit : str, optional + If none, the to_unit unit is returned. If, e.g. n or N, then ug m-3 -> ug N m-3 is returned + + Returns + ------- + cube + input data converted to mass concentration + + """ + + data = _check_input_iscube(data)[0] + + p_pascal = p0 # 1013 hPa (US standard atm) + T_kelvin = T0_STD # 15 deg celcius (US standard atm) + + mmol_air = get_molmass("air_dry") + mmol_var = get_molmass(var_name) + + if component_unit is not None and to_unit is not None: + component_mass = single_component_mass[component_unit.lower()] + component_unit_fac = component_mass / mmol_var + else: + component_unit_fac = 1 + Rspecific = 287.058 # J kg-1 K-1 + + conversion_fac = 1 / cf_units.Unit("mol mol-1").convert(1, vmr_unit) + + airdensity = p_pascal / (Rspecific * T_kelvin) # kg m-3 + mulfac = mmol_var / mmol_air * airdensity # kg m-3 + + mult_fun = CUBE_MATHS["multiply"] + conc = mult_fun(data, mulfac) # kg m-3 + if to_unit is not None: + conversion_fac *= cf_units.Unit("kg m-3").convert(1, to_unit) * component_unit_fac + if not np.isclose(conversion_fac, 1, rtol=1e-7): + conc = mult_fun(conc, conversion_fac) + + if to_unit is not None: + unit = to_unit + + if component_unit is not None: + unit_list = unit.split(" ") + unit = unit_list[0] + f" {component_unit.upper()} " + unit_list[1] + + conc.units = unit + else: + conc.units = "kg m-3" + + return conc + + +def calc_concNhno3_from_vmr(data): + return vmr_to_conc( + data, vmr_unit="nmol mol-1", var_name="hno3", to_unit="ug m-3", component_unit="N" + ) + + +def calc_concNnh3_from_vmr(data): + return vmr_to_conc( + data, vmr_unit="nmol mol-1", var_name="nh3", to_unit="ug m-3", component_unit="N" + ) + + +def convert_to_ugN(data, var_name): + mult_fun = CUBE_MATHS["multiply"] + data = _check_input_iscube(data)[0] + mmol_var = get_molmass(var_name) + + component_mass = single_component_mass["n"] + component_unit_fac = component_mass / mmol_var + unit = data.units + unit_conversion = cf_units.Unit(str(unit)).convert(1, "ug m-3") + if not np.isclose(unit_conversion, 1, rtol=1e-7): + data = mult_fun(data, unit_conversion) + + data = mult_fun(data, component_unit_fac) + data.units = "ug N m-3" + + return data + + +def calc_concNnh4(concnh4): + return convert_to_ugN(concnh4, "nh4") + + +def calc_concno3pm25(concno3f, concno3c, fine_from_coarse_fraction: float = 0.134): + # mult_fun = CUBE_MATHS["multiply"] + # concno3pm25 = add_cubes(concno3f, mult_fun(concno3c, fine_from_coarse_fraction)) + + return concno3f + + +def calc_concno3pm10(concno3f, concno3c): + concno3pm10 = add_cubes(concno3f, concno3c) + + return concno3pm10 + + +def calc_concNno3pm25(concno3f, concno3c, fine_from_coarse_fraction: float = 0.134): + M_N = 14.006 + M_O = 15.999 + M_H = 1.007 + + fac = M_N / (M_H + M_N + M_O * 3) + mult_fun = CUBE_MATHS["multiply"] + concno3f, concno3c = _check_input_iscube(concno3f, concno3c) + concno3f, concno3c = _check_same_units(concno3f, concno3c) + concno3pm25 = add_cubes(concno3f, mult_fun(concno3c, fine_from_coarse_fraction)) + concno3pm25.units = "ug N m-3" + return mult_fun(concno3pm25, fac) + + +def calc_concNno3pm10(concno3f, concno3c): + M_N = 14.006 + M_O = 15.999 + M_H = 1.007 + + fac = M_N / (M_H + M_N + M_O * 3) + mult_fun = CUBE_MATHS["multiply"] + concno3f, concno3c = _check_input_iscube(concno3f, concno3c) + concno3f, concno3c = _check_same_units(concno3f, concno3c) + concno3pm10 = add_cubes(concno3f, concno3c) + concno3pm10.units = "ug N m-3" + return mult_fun(concno3pm10, fac) + + +def calc_sspm25(concssfine, concsscoarse): + mult_fun = CUBE_MATHS["multiply"] + concssfine, concsscoarse = _check_input_iscube(concssfine, concsscoarse) + concssfine, concsscoarse = _check_same_units(concssfine, concsscoarse) + + return add_cubes(concssfine, mult_fun(concsscoarse, 0.16)) + + +def calc_concNtno3(concno3f, concno3c, vmrhno3): + concno3f, concno3c, vmrhno3 = _check_input_iscube(concno3f, concno3c, vmrhno3) + concno3f, concno3c = _check_same_units(concno3f, concno3c) + concNhno3 = calc_concNhno3_from_vmr(vmrhno3) + concNno3pm10 = calc_concNno3pm10(concno3f, concno3c) + + return add_cubes(concNhno3, concNno3pm10) + + +def calc_concNtnh(concnh4, vmrnh3): + concNnh3 = calc_concNnh3_from_vmr(vmrnh3) + concNnh4 = calc_concNnh4(concnh4) + concNnh3, concNnh4 = _check_same_units(concNnh3, concNnh4) + + return add_cubes(concNnh3, concNnh4) diff --git a/pyaerocom/io/fileconventions.py b/pyaerocom/io/fileconventions.py index 789441f4a..5ef5476d0 100644 --- a/pyaerocom/io/fileconventions.py +++ b/pyaerocom/io/fileconventions.py @@ -33,7 +33,7 @@ class FileConventionRead: _io_opts = const AEROCOM3_VERT_INFO = { - "2d": ["surface", "column", "modellevel"], + "2d": ["surface", "column", "modellevel", "2d"], "3d": ["modellevelatstations"], } @@ -162,6 +162,7 @@ def _info_from_aerocom3(self, file: str) -> dict: ) try: # include vars for the surface + if spl[self.vert_pos].lower() in self.AEROCOM3_VERT_INFO["2d"]: info["var_name"] = spl[self.var_pos] # also include 3d vars that provide station based data diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index 7b30f6b88..ae36bf358 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -16,10 +16,18 @@ compute_sc440dryaer, compute_sc550dryaer, compute_sc700dryaer, + compute_wetna_from_concprcpna, + compute_wetnh4_from_concprcpnh4, + compute_wetno3_from_concprcpno3, compute_wetoxn_from_concprcpoxn, compute_wetoxs_from_concprcpoxs, + compute_wetoxs_from_concprcpoxsc, + compute_wetoxs_from_concprcpoxst, compute_wetrdn_from_concprcprdn, + compute_wetso4_from_concprcpso4, concx_to_vmrx, + make_proxy_drydep_from_O3, + make_proxy_wetdep_from_O3, vmrx_to_concx, ) from pyaerocom.exceptions import ( @@ -206,7 +214,7 @@ class ReadEbas(ReadUngriddedBase): "Vavihill": "Hallahus", "Virolahti II": "Virolahti III", } - #'Trollhaugen' : 'Troll'} + # 'Trollhaugen' : 'Troll'} #: Temporal resolution codes that (so far) can be understood by pyaerocom TS_TYPE_CODES = { "1mn": "minutely", @@ -229,9 +237,59 @@ class ReadEbas(ReadUngriddedBase): "ac550dryaer": ["ac550aer", "acrh"], "ang4470dryaer": ["sc440dryaer", "sc700dryaer"], "wetoxs": ["concprcpoxs", "pr"], + "wetoxsc": ["concprcpoxsc", "pr"], + "wetoxst": ["concprcpoxst", "pr"], "wetoxn": ["concprcpoxn", "pr"], "wetrdn": ["concprcprdn", "pr"], + "wetso4": ["concprcpso4", "pr"], + "wetna": ["concprcpna", "pr"], + "wetno3": ["concprcpno3", "pr"], + "wetnh4": ["concprcpnh4", "pr"], "vmro3max": ["vmro3"], + # proxy drydep + # Suphar based + "proxydryoxs": ["concprcpoxs", "pr"], + "proxydryso2": ["concprcpoxs", "pr"], + "proxydryso4": ["concprcpoxs", "pr"], + # Oxidized nitrogen based + "proxydryoxn": ["concprcpoxn", "pr"], + "proxydryno2": ["concprcpoxn", "pr"], + "proxydryno2no2": ["concprcpoxn", "pr"], + "proxydryhono": ["concprcpoxn", "pr"], + "proxydryn2o5": ["concprcpoxn", "pr"], + "proxydryhno3": ["concprcpoxn", "pr"], + "proxydryno3c": ["concprcpoxn", "pr"], + "proxydryno3f": ["concprcpoxn", "pr"], + # Reduced nitrogen based + "proxydryrdn": ["concprcprdn", "pr"], + "proxydrynh3": ["concprcprdn", "pr"], + "proxydrynh4": ["concprcprdn", "pr"], + # Other + "proxydryo3": ["vmro3"], + "proxydrypm10": ["concprcpoxs", "pr"], + "proxydrypm25": ["concprcpoxs", "pr"], + # proxy wetdep + # Suphar based + "proxywetoxs": ["concprcpoxs", "pr"], + "proxywetso2": ["concprcpoxs", "pr"], + "proxywetso4": ["concprcpoxs", "pr"], + # Oxidized nitrogen based + "proxywetoxn": ["concprcpoxn", "pr"], + "proxywetno2": ["concprcpoxn", "pr"], + "proxywetno2no2": ["concprcpoxn", "pr"], + "proxywethono": ["concprcpoxn", "pr"], + "proxywetn2o5": ["concprcpoxn", "pr"], + "proxywethno3": ["concprcpoxn", "pr"], + "proxywetno3c": ["concprcpoxn", "pr"], + "proxywetno3f": ["concprcpoxn", "pr"], + # Reduced nitrogen based + "proxywetrdn": ["concprcprdn", "pr"], + "proxywetnh3": ["concprcprdn", "pr"], + "proxywetnh4": ["concprcprdn", "pr"], + # Other + "proxyweto3": ["vmro3"], + "proxywetpm10": ["concprcpoxs", "pr"], + "proxywetpm25": ["concprcpoxs", "pr"], } #: Meta information supposed to be migrated to computed variables @@ -249,9 +307,59 @@ class ReadEbas(ReadUngriddedBase): "ac550dryaer": compute_ac550dryaer, "ang4470dryaer": compute_ang4470dryaer_from_dry_scat, "wetoxs": compute_wetoxs_from_concprcpoxs, + "wetoxsc": compute_wetoxs_from_concprcpoxsc, + "wetoxst": compute_wetoxs_from_concprcpoxst, "wetoxn": compute_wetoxn_from_concprcpoxn, "wetrdn": compute_wetrdn_from_concprcprdn, + "wetnh4": compute_wetnh4_from_concprcpnh4, + "wetno3": compute_wetno3_from_concprcpno3, + "wetso4": compute_wetso4_from_concprcpso4, + "wetna": compute_wetna_from_concprcpna, "vmro3max": calc_vmro3max, + # proxy dry dep + # Suphar based + "proxydryoxs": compute_wetoxs_from_concprcpoxs, + "proxydryso2": compute_wetoxs_from_concprcpoxs, + "proxydryso4": compute_wetoxs_from_concprcpoxs, + # Oxidized nitrogen based + "proxydryoxn": compute_wetoxn_from_concprcpoxn, + "proxydryno2": compute_wetoxn_from_concprcpoxn, + "proxydryno2no2": compute_wetoxn_from_concprcpoxn, + "proxydryhono": compute_wetoxn_from_concprcpoxn, + "proxydryn2o5": compute_wetoxn_from_concprcpoxn, + "proxydryhno3": compute_wetoxn_from_concprcpoxn, + "proxydryno3c": compute_wetoxn_from_concprcpoxn, + "proxydryno3f": compute_wetoxn_from_concprcpoxn, + # Reduced nitrogen based + "proxydryrdn": compute_wetrdn_from_concprcprdn, + "proxydrynh3": compute_wetrdn_from_concprcprdn, + "proxydrynh4": compute_wetrdn_from_concprcprdn, + # Other + "proxydryo3": make_proxy_drydep_from_O3, + "proxydrypm10": compute_wetoxs_from_concprcpoxs, + "proxydrypm25": compute_wetoxs_from_concprcpoxs, + # proxy wet dep + # Suphar based + "proxywetoxs": compute_wetoxs_from_concprcpoxs, + "proxywetso2": compute_wetoxs_from_concprcpoxs, + "proxywetso4": compute_wetoxs_from_concprcpoxs, + # Oxidized nitrogen based + "proxywetoxn": compute_wetoxn_from_concprcpoxn, + "proxywetno2": compute_wetoxn_from_concprcpoxn, + "proxywetno2no2": compute_wetoxn_from_concprcpoxn, + "proxywethono": compute_wetoxn_from_concprcpoxn, + "proxywetn2o5": compute_wetoxn_from_concprcpoxn, + "proxywethno3": compute_wetoxn_from_concprcpoxn, + "proxywetno3c": compute_wetoxn_from_concprcpoxn, + "proxywetno3f": compute_wetoxn_from_concprcpoxn, + # Reduced nitrogen based + "proxywetrdn": compute_wetrdn_from_concprcprdn, + "proxywetnh3": compute_wetrdn_from_concprcprdn, + "proxywetnh4": compute_wetrdn_from_concprcprdn, + # Other + "proxyweto3": make_proxy_wetdep_from_O3, + "proxywetpm10": compute_wetoxs_from_concprcpoxs, + "proxywetpm25": compute_wetoxs_from_concprcpoxs, } #: Custom reading options for individual variables. Keys need to be valid @@ -1441,9 +1549,9 @@ def _flag_incorrect_frequencies(self, filedata): opts = self.get_read_opts(var) if opts.freq_min_cov > frac_valid: raise TemporalSamplingError( - f"Only {frac_valid*100:.2f}% of measuerements are in " + f"Only {frac_valid * 100:.2f}% of measuerements are in " f"{tst} resolution. Minimum requirement for {var} is " - f"{opts.freq_min_cov*100:.2f}%" + f"{opts.freq_min_cov * 100:.2f}%" ) if not var in filedata.data_flagged: filedata.data_flagged[var] = np.zeros(num).astype(bool) diff --git a/pyaerocom/io/readgridded.py b/pyaerocom/io/readgridded.py index f4a1307d3..a514a7283 100755 --- a/pyaerocom/io/readgridded.py +++ b/pyaerocom/io/readgridded.py @@ -24,6 +24,18 @@ from pyaerocom.griddeddata import GriddedData from pyaerocom.helpers import get_highest_resolution, isnumeric, sort_ts_types, to_pandas_timestamp from pyaerocom.io import AerocomBrowser +from pyaerocom.io.aux_components_fun import ( + calc_concNhno3_from_vmr, + calc_concNnh3_from_vmr, + calc_concNnh4, + calc_concNno3pm10, + calc_concNno3pm25, + calc_concno3pm10, + calc_concno3pm25, + calc_concNtnh, + calc_concNtno3, + calc_sspm25, +) from pyaerocom.io.aux_read_cubes import ( add_cubes, compute_angstrom_coeff_cubes, @@ -123,6 +135,17 @@ class specifying details of the file naming convention for the model "concprcpoxn": ("wetoxn", "pr"), "concprcpoxs": ("wetoxs", "pr"), "concprcprdn": ("wetrdn", "pr"), + "concsspm10": ("concss25", "concsscoarse"), + "concsspm25": ("concss25", "concsscoarse"), + "concno3pm10": ("concno3f", "concno3c"), + "concno3pm25": ("concno3f", "concno3c"), + "concNno3pm10": ("concno3f", "concno3c"), + "concNno3pm25": ("concno3f", "concno3c"), + "concNhno3": ("vmrhno3",), + "concNtno3": ("concno3f", "concno3c", "vmrhno3"), + "concNnh3": ("vmrnh3",), + "concNnh4": ("concnh4",), + "concNtnh": ("concnh4", "vmrnh3"), } AUX_ALT_VARS = { @@ -146,9 +169,20 @@ class specifying details of the file naming convention for the model "concno3": add_cubes, "concprcpoxn": compute_concprcp_from_pr_and_wetdep, "concprcpoxs": compute_concprcp_from_pr_and_wetdep, - "concprcprdn": compute_concprcp_from_pr_and_wetdep - #'mec550*' : divide_cubes, - #'tau*' : lifetime_from_load_and_dep + "concprcprdn": compute_concprcp_from_pr_and_wetdep, + "concsspm10": add_cubes, + "concsspm25": calc_sspm25, + "concno3pm10": calc_concno3pm10, + "concno3pm25": calc_concno3pm25, + "concNno3pm10": calc_concNno3pm10, + "concNno3pm25": calc_concNno3pm25, + "concNhno3": calc_concNhno3_from_vmr, + "concNtno3": calc_concNtno3, + "concNnh3": calc_concNnh3_from_vmr, + "concNnh4": calc_concNnh4, + "concNtnh": calc_concNtnh, + # 'mec550*' : divide_cubes, + # 'tau*' : lifetime_from_load_and_dep } #: Additional arguments passed to computation methods for auxiliary data @@ -2205,7 +2239,7 @@ def __repr__(self): def __str__(self): head = f"Pyaerocom {type(self).__name__}" s = ( - f"\n{head}\n{len(head)*'-'}\n" + f"\n{head}\n{len(head) * '-'}\n" f"Data ID: {self.data_id}\n" f"Data directory: {self.data_dir}\n" f"Available experiments: {self.experiments}\n" diff --git a/pyaerocom/io/readungridded.py b/pyaerocom/io/readungridded.py index 67c0d1c8a..aa29ff947 100755 --- a/pyaerocom/io/readungridded.py +++ b/pyaerocom/io/readungridded.py @@ -26,6 +26,7 @@ from pyaerocom.io.read_ebas import ReadEbas from pyaerocom.io.read_eea_aqerep import ReadEEAAQEREP from pyaerocom.io.read_eea_aqerep_v2 import ReadEEAAQEREP_V2 +from pyaerocom.plugins.ipcforests.reader import ReadIPCForest from pyaerocom.ungriddeddata import UngriddedData from pyaerocom.variable import get_aliases @@ -59,6 +60,7 @@ class ReadUngridded: ReadAirNow, ReadEEAAQEREP, ReadEEAAQEREP_V2, + ReadIPCForest, ] SUPPORTED_READERS.extend( ep.load() for ep in metadata.entry_points(group="pyaerocom.ungridded") diff --git a/pyaerocom/plugins/ipcforests/__init__.py b/pyaerocom/plugins/ipcforests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyaerocom/plugins/ipcforests/metadata.py b/pyaerocom/plugins/ipcforests/metadata.py new file mode 100644 index 000000000..966836e40 --- /dev/null +++ b/pyaerocom/plugins/ipcforests/metadata.py @@ -0,0 +1,511 @@ +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import Tuple + +import numpy as np +import pandas as pd + +DEP_TYPE = { + 1: "Throughfall", + 2: "Bulk", + 3: "Wet-only", + 4: "Stemflow", + 5: "Fog", + 6: "Frozen fog", + 9: "Other", + 8: "do not use", +} + +COUNTRY_CODES = { + 1: "FR", + 2: "BE", + 3: "NL", + 4: "DE", + 5: "IT", + 6: "UK", + 7: "IE", + 8: "DK", + 9: "GR", + 10: "PT", + 11: "ES", + 12: "LU", + 13: "SE", + 14: "AT", + 15: "FI", + 50: "CH", + 51: "HU", + 52: "RO", + 53: "PL", + 54: "SK", + 55: "NO", + 56: "LT", + 57: "HR", + 58: "CZ", + 59: "EE", + 60: "SI", + 61: "MD", + 62: "RU", + 63: "BG", + 64: "LV", + 65: "BY", + 66: "CY", + 67: "CS", + 68: "AD", + 95: "cn", + 80: "ME", + 96: "AZ", + 72: "TR", +} + +COUNTRIES = { + 1: "France", + 2: "Belgium", + 3: "Netherlands", + 4: "Germany", + 5: "Italy", + 6: "United Kingdom", + 7: "Ireland", + 8: "Denmark", + 9: "Greece", + 10: "Portugal", + 11: "Spain", + 12: "Luxembourg", + 13: "Sweden", + 14: "Austria", + 15: "Finland", + 50: "Switzerland", + 51: "Hungary", + 52: "Romania", + 53: "Poland", + 54: "Slovak Republic", + 55: "Norway", + 56: "Lithuania", + 57: "Croatia", + 58: "Czech Republic", + 59: "Estonia", + 60: "Slovenia", + 61: "Republic of Moldova", + 62: "Russia", + 63: "Bulgaria", + 64: "Latvia", + 65: "Belarus", + 66: "Cyprus", + 67: "Serbia", + 68: "Andorra", + 95: "Canaries (Spain)", + 80: "Montenegro", + 96: "Azores (Portugal)", + 72: "Türkiye", +} + + +class Station: + """Holds information for a singe IPC Forest station. The same station with different ts_types + are treated as seperate stations at this level + + Parameters + ---------- + country_code : int + int representing the country where the station is found + plot_code : int + The number of the station + sampler_code : int + int representing the type of sampler + lat : str + latitude + lon : str + longitude + alt: int + altitude + partner_code : int + number of the institude doing the measurement. Mostly used for metadata + ts_type : str + ts_type of the station + + Attributes + ---------- + sampler_type : str + The type of wetdep measurement + data : dict[str, list[float]] + dictionary holding a list of measurements for each species + dtime : dict[str, list[datetime]] + dictionary holding a list of timesteps for each species + country : str + name of country + station_name : str + name of station + var_info : dict[str, dict[str, str | list[float]]] + dict holder metadata for each species, used by StationData + + + """ + + def __init__( + self, + country_code: int, + plot_code: int, + sampler_code: int, + lat: str, + lon: str, + alt: int, + partner_code: int, + ts_type: str, + ) -> None: + self.country_code = country_code + self.plot_code = plot_code + self.sampler_code = sampler_code + self.lat = lat + self.lon = lon + self.alt = alt + self.partner_code = partner_code + self.ts_type = ts_type + + self.sampler_type = DEP_TYPE[self.sampler_code] + + self.data: dict[str, list[float]] = {} + self.flags: dict[str, list[int]] = {} + self.dtime: dict[str, list[datetime]] = {} + + self.country = COUNTRIES[country_code] + + self.station_name = self.get_station_name(country_code, plot_code, sampler_code) + + self.var_info: dict[str, dict[str, str | list[float]]] = {} + + @staticmethod + def get_station_name( + country_code: int, + plot_code: int, + sampler_code: int, + ) -> str: + """ + Generates the station name based on country code, plot code and sampler code. Static method + that can be used without initiating the class + + Parameters + ---------- + country_code : int + int representing the country where the station is found + plot_code : int + The number of the station + sampler_code : int + int representing the type of sampler + + Returns + ------- + str + station name + + """ + return f"{COUNTRY_CODES[country_code]}-{plot_code}-{sampler_code}" + + def _add_species_to_var_info(self, species: str, unit: str) -> None: + """ + Makes the var_info dict that is used later by StationData + + Parameters + ---------- + species : str + name of species + unit : str + unit + + """ + self.var_info[species] = dict( + ts_type=self.ts_type, + ts_type_src=self.ts_type, + units=unit, + sampler_type=self.sampler_type, + ) + + def add_measurement( + self, species: str, time: datetime, measurement: float, unit: str, flag: int + ) -> None: + """ + Adds a single measurement to the data and time lists. If it is the first measurement, + a new var_info is created + + Parameters + ---------- + species : str + name of species + time : datetime + The timestamp of the measurement + measurement : float + The value of the measure ment + unit : str + unit + + """ + if species not in self.var_info: + self._add_species_to_var_info(species, unit) + if species not in self.dtime: + self.dtime[species] = [] + self.data[species] = [] + self.flags[species] = [] + + self.dtime[species].append(time) + self.data[species].append(measurement) + self.flags[species].append(flag) + + def get_timeseries(self, species: str, quality_limit: float = 0.5) -> pd.Series: + """ + Combines the data and timestamps of a given species to a pandas timeseries + + Parameters + ---------- + species : str + name of species + + Returns + ------- + pd.Series + Timeseries of the measurements of a given species for station + + """ + flags = np.array(self.flags[species]) + quality = np.sum(flags[np.where(flags == 0)]) / len(flags) + breakpoint() + if quality >= quality_limit: + return pd.Series(self.data[species], index=self.dtime[species]) + else: + return pd.Series(np.ones_like(self.data[species]) * np.nan, index=self.dtime[species]) + + +class SurveyYear: + """ + Holds information about the start and stop dates, as well + as the number of periods for a single year at a single station + + + Parameters + ---------- + year: int + The year + start: datetime + start date + stop: datetime + end date + periods: int + number of periods in the survey year + + Attributes + ---------- + days : float + The number of days in each period + ts_type : str + The ts_type for the survey year + + + """ + + def __init__(self, year: int, start: datetime, stop: datetime, periods: int) -> None: + self.year = year + self.start = start + self.stop = stop + self.periods = periods + + # self.daterange = date_range(start, stop, periods) + self.days = (self.stop - self.start).days / self.periods + + self.ts_type = self._get_tstype() + + def get_date(self, period: int) -> datetime: + """ + Finds the date used to make each period in the year. + For now the function returns the first date in the period + + Parameters + ---------- + period : int + Which period to find the date for + + Returns + ------- + float + The first date in the period + + Raises + ------ + ValueError + If the period is not within the range of periods in the year + """ + if period > self.periods or period <= 0: + raise ValueError(f"The period {period} needs to be in the range 1-{self.periods}") + return self.start + timedelta(days=self.days * (period - 1)) + + def _get_tstype(self) -> str: + """ + Returns the ts_type by calling the static method + + Returns + ------- + str + the ts_type of the survey year + """ + + return SurveyYear.get_tstype(self.days) + + @staticmethod + def get_tstype(days: float) -> str: + if days >= 26: + return "monthly" + elif days >= 12: + return "2weekly" + elif days >= 6: + return "weekly" + else: + return "daily" + + +class Plot: + def __init__( + self, + country_code: int, + plot_code: int, + sampler_code: int, + lat: str, + lon: str, + alt: int, + partner_code: int, + ) -> None: + self.country_code = country_code + self.plot_code = plot_code + self.sampler_code = sampler_code + self.lat = lat + self.lon = lon + self.alt = alt + self.partner_code = partner_code + + self.periods = {} + self.survey_years: dict[int, SurveyYear] = {} + + def add_survey_year(self, year: int, start: str, stop: str, periods: int) -> None: + start_dt = datetime.strptime(start, "%Y-%m-%d") + stop_dt = datetime.strptime(stop, "%Y-%m-%d") + self.survey_years[year] = SurveyYear(year, start_dt, stop_dt, periods) + + def get_date(self, year: int, period: int) -> datetime: + return self.survey_years[year].get_date(period) + + +class Plots: + def __init__(self, plot_file: str) -> None: + self.plot_file: str = plot_file + self.plots: dict[int, dict[int, dict[int, Plot]]] = {} + + def read_file(self, altitudes: dict[str, int]) -> dict[int, dict[int, dict[int, Plot]]]: + plots: dict[int, dict[int, dict[int, Plot]]] = {} + print(f"Starting to read plot metadata") + with open(self.plot_file, "r") as f: + f.readline() + for line in f: + words = line.split(";") + if words[0] == "": + continue + + survey_year = int(words[0]) + country_code = int(words[1]) + partner_code = int(words[2]) + plot_code = int(words[3]) + sampler_code = int(words[4]) + + lat = words[6] + lon = words[7] + alt_code = words[8] + start = words[9] + stop = words[10] + periods = int(words[11]) + + alt = altitudes[alt_code] + + if start == "" or stop == "": + continue + + if country_code not in plots: + plots[country_code] = {} + if plot_code not in plots[country_code]: + plots[country_code][plot_code] = {} + if sampler_code not in plots[country_code][plot_code]: + plots[country_code][plot_code][sampler_code] = Plot( + country_code, plot_code, sampler_code, lat, lon, alt, partner_code + ) + + plots[country_code][plot_code][sampler_code].add_survey_year( + survey_year, start, stop, periods + ) + print(f"Done read plot metadata") + self.plots = plots + return plots + + def get_ts_type(self, year: int, country_code: int, plot_code: int, sampler_code: int) -> str: + return self.plots[country_code][plot_code][sampler_code].survey_years[year].ts_type + + def get_date( + self, year: int, country_code: int, plot_code: int, sampler_code: int, period: int + ) -> datetime: + return self.plots[country_code][plot_code][sampler_code].get_date(year, period) + + def get_days(self, year: int, country_code: int, plot_code: int, sampler_code: int) -> float: + start = self.plots[country_code][plot_code][sampler_code].survey_years[year].start + stop = self.plots[country_code][plot_code][sampler_code].survey_years[year].stop + days = self.plots[country_code][plot_code][sampler_code].survey_years[year].days + + if start == stop: + raise ValueError(f"start {start} is the same as stop {stop}") + + return days + + def get_position( + self, year: int, country_code: int, plot_code: int, sampler_code: int + ) -> Tuple[float, float, int]: + lat = self._coord_to_desimal(self.plots[country_code][plot_code][sampler_code].lat) + lon = self._coord_to_desimal(self.plots[country_code][plot_code][sampler_code].lon) + alt = self.plots[country_code][plot_code][sampler_code].alt + + return lat, lon, alt + + def _coord_to_desimal(self, coord: str) -> float: + sign = 1 + if "-" in coord: + sign = -1 + coord = coord[1:] + + if coord == "0": + return 0 + + coord = coord[:-2] + if len(coord) >= 2: + minute = int(coord[-2:]) + coord = coord[:-2] + else: + return sign * ((int(coord) / 60.0)) + + degree = int(coord) + + return sign * (degree + (minute / 60.0)) + + +class MetadataReader: + def __init__(self, dir: str) -> None: + self.dir = dir + + self.add_dir = dir + "/adds" + + self.altitudes = self._get_altitude_dir() + + self.plots = Plots(self.dir + "/dp_pld.csv") + self.plots.read_file(self.altitudes) + + self.deposition_type = DEP_TYPE + + def _get_altitude_dir(self) -> dict[str, int]: + altitudes = {} + with open(self.add_dir + "/dictionaries/d_altitude.csv") as f: + f.readline() + for line in f: + words = line.split(";") + altitudes[words[0]] = int(words[4]) + (int(words[5]) - int(words[4])) // 2 + + return altitudes diff --git a/pyaerocom/plugins/ipcforests/reader.py b/pyaerocom/plugins/ipcforests/reader.py new file mode 100644 index 000000000..18f9678d3 --- /dev/null +++ b/pyaerocom/plugins/ipcforests/reader.py @@ -0,0 +1,435 @@ +from __future__ import annotations + +import logging +import os +from datetime import datetime +from pathlib import Path +from typing import Tuple + +import numpy as np +from tqdm import tqdm + +from pyaerocom import const +from pyaerocom._lowlevel_helpers import BrowseDict +from pyaerocom.io.readungriddedbase import ReadUngriddedBase +from pyaerocom.plugins.ipcforests.metadata import MetadataReader, Station, SurveyYear +from pyaerocom.stationdata import StationData +from pyaerocom.ungriddeddata import UngriddedData + +logger = logging.getLogger(__name__) + + +class ReadIPCForest(ReadUngriddedBase): + #: version log of this class (for caching) + __version__ = "0.4_" + ReadUngriddedBase.__baseversion__ + + #: Name of dataset (OBS_ID) + DATA_ID = const.IPCFORESTS_NAME + + #: List of all datasets supported by this interface + SUPPORTED_DATASETS = [const.IPCFORESTS_NAME] + + TS_TYPE = "undefined" + + _FILEMASK = "dp_dem.csv" + + #: Temporal resolution codes that (so far) can be understood by pyaerocom + TS_TYPE_CODES = { + "1mn": "minutely", + "1h": "hourly", + "1d": "daily", + "1w": "weekly", + "1mo": "monthly", + "mn": "minutely", + "h": "hourly", + "d": "daily", + "w": "weekly", + "mo": "monthly", + } + + VAR_POSITION = { + "wetoxs": 20, + "wetoxn": 19, + "wetrdn": 17, + "dryoxs": 20, + "dryoxn": 19, + "dryrdn": 17, + "depoxs": 20, + "depoxn": 19, + "deprdn": 17, + "depoxsf": 20, + "depoxnf": 19, + "deprdnf": 17, + "wetso4": 20, + "wetno3": 19, + "wetnh4": 17, + "wetno2": 41, + "wetna": 16, + "wetcl": 18, + } + + UNITS = { + "wetoxs": "mg S m-2 d-1", + "wetoxn": "mg N m-2 d-1", + "wetrdn": "mg N m-2 d-1", + "dryoxs": "mg S m-2 d-1", + "dryoxn": "mg N m-2 d-1", + "dryrdn": "mg N m-2 d-1", + "depoxs": "mg S m-2 d-1", + "depoxn": "mg N m-2 d-1", + "deprdn": "mg N m-2 d-1", + "depoxsf": "mg S m-2 d-1", + "depoxnf": "mg N m-2 d-1", + "deprdnf": "mg N m-2 d-1", + "wetso4": "mg S m-2 d-1", + "wetno3": "mg N m-2 d-1", + "wetnh4": "mg N m-2 d-1", + "wetno2": "mg N m-2 d-1", + "wetna": "mg m-2 d-1", + # unverified + "wetcl": "mg m-2 d-1", + } + + SEASALT_CORRECTION = { + "wetoxs": 0.3338, + "wetso4": 0.3338, + "depoxs": 0.3338, + "depoxsf": 0.3338, + } + SEASALT_FACTORS = { + "wetna": 0.120, + "wetcl": 0.103, + } + + DEP_TYPES_TO_USE = ["Throughfall", "Bulk", "Wet-only"] + + QUALITY_LIMIT = 0.5 + + MIN_YEAR = 1984 + MAX_YEAR = 2019 + + def __init__(self, data_id=None, data_dir=None): + super().__init__(data_id, data_dir) + + self.metadata = None + + self._file_dir = None + self.files = None + + if data_dir is not None: + self.metadata = MetadataReader(data_dir) + + @property + def file_dir(self): + """Directory containing EBAS NASA Ames files""" + if self._file_dir is not None: + return self._file_dir + return os.path.join(self.data_dir) + + @file_dir.setter + def file_dir(self, val): + if not isinstance(val, str) or not os.path.exists(val): + raise FileNotFoundError("Input directory does not exist") + + self._file_dir = val + + def read(self, vars_to_retrieve=None, files=[], first_file=None, last_file=None): + """Method that reads list of files as instance of :class:`UngriddedData` + + Parameters + ---------- + vars_to_retrieve : :obj:`list` or similar, optional, + list containing variable IDs that are supposed to be read. If None, + all variables in :attr:`PROVIDES_VARIABLES` are loaded + files : :obj:`list`, optional + list of files to be read. If None, then the file list is used that + is returned on :func:`get_file_list`. + first_file : :obj:`int`, optional + index of first file in file list to read. If None, the very first + file in the list is used + last_file : :obj:`int`, optional + index of last file in list to read. If None, the very last file + in the list is used + + Returns + ------- + UngriddedData + instance of ungridded data object containing data from all files. + """ + self.files = Path(self.data_dir).joinpath(self._FILEMASK) + data = self.read_file( + str(self.files), + vars_to_retrieve, + ) + + return data + + @property + def PROVIDES_VARIABLES(self): + """List of variables that are provided by this dataset + + Note + ---- + May be implemented as global constant in header + """ + return list(self.VAR_POSITION.keys()) + + @property + def DEFAULT_VARS(self): + """List containing default variables to read""" + return list(self.VAR_POSITION.keys()) + + def read_file(self, filename, vars_to_retrieve=None, last_line=None): + """Read single file + + Parameters + ---------- + filename : str + string specifying filename + vars_to_retrieve : :obj:`list` or similar, optional, + list containing variable IDs that are supposed to be read. If None, + all variables in :attr:`PROVIDES_VARIABLES` are loaded + last_line : int + last line number to read + Used to speed up testing + + Returns + ------- + :obj:`dict` or :obj:`StationData`, or other... + imported data in a suitable format that can be handled by + :func:`read` which is supposed to append the loaded results from + this method (which reads one datafile) to an instance of + :class:`UngriddedData` for all files. + """ + stations: dict[str, dict[str, Station]] = {} + if self.metadata is None: + if self.data_dir is None: + raise ValueError(f"Data Dir is not read yet") + + self.metadata = MetadataReader(self.data_dir) + + if vars_to_retrieve is None: + vars_to_retrieve = self.PROVIDES_VARIABLES + + with open(filename, "r") as f: + f.readline() + + for line_nr, line in tqdm(enumerate(f)): + words = line.split(";") + year = int(words[0]) + country_code = int(words[1]) + partner_code = int(words[2]) + plot_code = int(words[3]) + sampler_code = int(words[9]) + q_flag = int(words[44]) if words[44] != "" else 0 + + # 8 is the code for "do not use" + if ( + self.metadata.deposition_type[sampler_code] not in self.DEP_TYPES_TO_USE + or sampler_code == 8 + ): + continue + + sampler_type = self.metadata.deposition_type[sampler_code] + + period = int(words[6]) + start = words[4] + stop = words[5] + + quantity = words[47] + if quantity == "": # or quantity == "0": + continue + else: + quantity = float(quantity) + + try: + self.metadata.plots.plots[country_code] + self.metadata.plots.plots[country_code][plot_code] + self.metadata.plots.plots[country_code][plot_code][sampler_code] + except KeyError: + logger.warning( + f"Some metadata is missing for {country_code=}, {plot_code=}, {sampler_code=}. Skipping" + ) + continue + + try: + self.metadata.plots.plots[country_code][plot_code][sampler_code].survey_years[ + year + ] + except KeyError as e: + logger.warning( + f"Year {year} can't be found for {country_code=}, {plot_code=}, {sampler_code=}. Only years found are {self.metadata.plots.plots[country_code][plot_code][sampler_code].survey_years.keys()}" + ) + continue + days, dtime, ts_type = self._get_days_date_ts_type( + year, country_code, plot_code, sampler_code, period, start, stop + ) + + if days is None or dtime is None or ts_type is None: + continue + station_name = Station.get_station_name(country_code, plot_code, sampler_code) + + if station_name not in stations: + stations[station_name] = {} + if ts_type not in stations[station_name]: + lat, lon, alt = self.metadata.plots.get_position( + year, country_code, plot_code, sampler_code + ) + stations[station_name][ts_type] = Station( + country_code, plot_code, sampler_code, lat, lon, alt, partner_code, ts_type + ) + + for species in vars_to_retrieve: + conc = self._get_species_conc(words[self.VAR_POSITION[species]], species) + + # Sea-salt correction + # The factor self.SEASALT_CORRECTION[species] is the factor use to go from mg/L to mg S/L (for sulpher) + if species in self.SEASALT_CORRECTION: + na_factor = ( + self._get_species_conc(words[self.VAR_POSITION["wetna"]], "wetna") + * self.SEASALT_FACTORS["wetna"] + * self.SEASALT_CORRECTION[species] + ) + cl_factor = ( + self._get_species_conc(words[self.VAR_POSITION["wetcl"]], "wetcl") + * self.SEASALT_FACTORS["wetcl"] + * self.SEASALT_CORRECTION[species] + ) + seasalt_correction = min(na_factor, cl_factor) + + if seasalt_correction > conc and conc > 0: + logger.warning( + f"Seasalt correction {seasalt_correction} is larger than concentration {conc} for {species}" + ) + if conc > 0: + conc -= seasalt_correction + + # Unit correction + conc *= quantity / days + + stations[station_name][ts_type].add_measurement( + species, dtime, conc, self.UNITS[species], q_flag + ) + + station_datas = [] + for station_name in stations: + for ts_type in stations[station_name]: + station = stations[station_name][ts_type] + station_data = StationData() + station_data.var_info = BrowseDict(**station.var_info) + for species in station.data.keys(): + station_data[species] = self._clean_data_with_flags( + station.data[species], + station.dtime[species], + station.flags[species], + species, + ) + station_data.dtime = station.dtime[species] + + station_data.country = station.country + + # Needs to convert coordinates to correct type! + station_data.station_coords = { + "latitude": station.lat, + "longitude": station.lon, + "altitude": station.alt, + } + + station_data.latitude = station.lat + station_data.longitude = station.lon + station_data.altitude = station.alt + + station_data.filename = filename + station_data.ts_type = station.ts_type + station_data.ts_type_src = station.ts_type + station_data.station_name = station.station_name + station_data._append_meta_item("sampler_type", station.sampler_type) + + station_data.data_id = self.data_id + + station_datas.append(station_data) + return UngriddedData.from_station_data(station_datas, add_meta_keys="sampler_type") + + def _get_species_conc(self, conc_str: str, species: str) -> float: + conc = float(conc_str) if conc_str != "" else np.nan + if conc == -1: + logger.warning(f"Value for {species} found to be {conc}") + conc = np.nan + return conc + + def _get_days_date_ts_type( + self, + year: int, + country_code: int, + plot_code: int, + sampler_code: int, + period: int, + start: str | datetime, + stop: str | datetime, + ) -> Tuple[float | None, datetime | None, str | None]: + if start != "" and stop != "": + if isinstance(start, str): + start = datetime.strptime(start, "%Y-%m-%d") + if isinstance(stop, str): + stop = datetime.strptime(stop, "%Y-%m-%d") + + if (stop - start).days <= 0: + return None, None, None + + return (stop - start).days, start, self._get_tstype(start, stop) + + if self.metadata is None: + raise ValueError(f"Metadata is not read yet") + + try: + days = self.metadata.plots.get_days(year, country_code, plot_code, sampler_code) + except ValueError as e: + logger.warning(repr(e)) + return None, None, None + + if days == 0: + return None, None, None + + try: + dtime = self.metadata.plots.get_date( + year, country_code, plot_code, sampler_code, period + ) + except ValueError: + return None, None, None + + ts_type = self.metadata.plots.get_ts_type(year, country_code, plot_code, sampler_code) + + return days, dtime, ts_type + + def _get_tstype(self, start: datetime, stop: datetime) -> str: + days = (stop - start).days + + return SurveyYear.get_tstype(days) + + def _clean_data_with_flags( + self, + data: list[float], + time: list[datetime], + flags: list[int], + species: str, + ) -> list[float]: + data_array = np.array(data) + flags_array = np.array(flags) + years = np.array([i.year for i in time]) + + for year in range(self.MIN_YEAR, self.MAX_YEAR): + yr_flags = flags_array[np.where(years == year)] + # yr_flags can have 0 length + try: + quality = np.sum(np.where(yr_flags == 0)) / len(yr_flags) + if quality < self.QUALITY_LIMIT: + logger.warning( + f"Quailty of {quality} found for {species} in year {year}. Setting data this year to NaN" + ) + data_array[np.where(years == year)] = np.nan + + except RuntimeWarning: + # yr_flags has 0 length + logger.warning(f"No data for species {species} in year {year}.") + + return list(data_array) diff --git a/pyaerocom/plugins/mscw_ctm/reader.py b/pyaerocom/plugins/mscw_ctm/reader.py index 09791aea4..00106ecf8 100755 --- a/pyaerocom/plugins/mscw_ctm/reader.py +++ b/pyaerocom/plugins/mscw_ctm/reader.py @@ -69,6 +69,9 @@ class ReadMscwCtm: # by the original data but computed on import) AUX_REQUIRES = { "depso4": ["dryso4", "wetso4"], + "depoxs": ["dryoxs", "wetoxs"], + "depoxn": ["dryoxn", "wetoxn"], + "deprdn": ["dryrdn", "wetrdn"], "concbc": ["concbcf", "concbcc"], "concno3": ["concno3c", "concno3f"], "concoa": ["concoac", "concoaf"], @@ -108,6 +111,9 @@ class ReadMscwCtm: # not iris.cube.Cube instance AUX_FUNS = { "depso4": add_dataarrays, + "depoxs": add_dataarrays, + "depoxn": add_dataarrays, + "deprdn": add_dataarrays, "concbc": add_dataarrays, "concno3": add_dataarrays, "concoa": add_dataarrays, diff --git a/pyaerocom/ungriddeddata.py b/pyaerocom/ungriddeddata.py index f07fef615..9855ac6bb 100644 --- a/pyaerocom/ungriddeddata.py +++ b/pyaerocom/ungriddeddata.py @@ -1899,12 +1899,20 @@ def apply_filters(self, var_outlier_ranges=None, **filter_attributes): ) if set_flags_nan: if not data.has_flag_data: - raise MetaDataError( + # jgriesfeller 20230210 + # not sure if raising this exception is the right thing to do + # the fake variables (vars computed from other variables) might not have + # and do not need flags (because that has been done during the read of the + # variable they are computed from) + # disabling and logging it for now + # raise MetaDataError( + logger.info( 'Cannot apply filter "set_flags_nan" to ' "UngriddedData object, since it does not " "contain flag information" ) - data = data.set_flags_nan(inplace=True) + else: + data = data.set_flags_nan(inplace=True) if region_id: data = data.filter_region(region_id) return data diff --git a/pyaerocom/units_helpers.py b/pyaerocom/units_helpers.py index b17be3f03..f62f73a07 100644 --- a/pyaerocom/units_helpers.py +++ b/pyaerocom/units_helpers.py @@ -39,6 +39,8 @@ #: factor UCONV_MUL_FACS = pd.DataFrame( [ + # ["dryso4", "mg/m2/d", "mgS m-2 d-1", M_S / M_SO4], + # ["drynh4", "mg/m2/d", "mgN m-2 d-1", M_N/ M_NH4], # ["concso4", "ug S/m3", "ug m-3", M_SO4 / M_S], # ["SO4ugSm3", "ug/m3", "ug S m-3", M_S / M_SO4], # ["concso4pm25", "ug S/m3", "ug m-3", M_SO4 / M_S], @@ -64,6 +66,7 @@ # may be used to specify alternative names for custom units defined # in UCONV_MUL_FACS + UALIASES = { # mass concentrations "ug S m-3": "ug S/m3", @@ -80,6 +83,7 @@ # deposition rates (explicit) ## sulphur species "mgS/m2/h": "mg S m-2 h-1", + "mg/m2/h": "mg m-2 h-1", "mgS/m**2/h": "mg S m-2 h-1", "mgSm-2h-1": "mg S m-2 h-1", "mgSm**-2h-1": "mg S m-2 h-1", diff --git a/pyproject.toml b/pyproject.toml index e4fcaf146..91358aa09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ ReadGAW = "pyaerocom.plugins.gaw.reader:ReadGAW" ReadGhost = "pyaerocom.plugins.ghost.reader:ReadGhost" ReadMEP = "pyaerocom.plugins.mep.reader:ReadMEP" ReadICOS = "pyaerocom.plugins.icos.reader:ReadICOS" +ReadIPCForest = "pyaerocom.plugins.ipcforests.reader:ReadIPCForest" [tool.flit.sdist] include = ["LICENSE", "README.*", "pyaerocom_env.yml", "tests"] diff --git a/tests/io/test_ebas_varinfo.py b/tests/io/test_ebas_varinfo.py index b1fe67cce..73b049837 100644 --- a/tests/io/test_ebas_varinfo.py +++ b/tests/io/test_ebas_varinfo.py @@ -168,6 +168,16 @@ ("wetoxs", None, None, None, None, ["concprcpoxs"], 1), ("wetoxn", None, None, None, None, ["concprcpoxn"], 1), ("wetrdn", None, None, None, None, ["concprcprdn"], 1), + ("wetoxsc", None, None, None, None, ["concprcpoxsc"], 1), + ("wetoxst", None, None, None, None, ["concprcpoxst"], 1), + ("wetna", None, None, None, None, ["concprcpna"], 1), + ("proxydryoxs", None, None, None, None, ["concprcpoxs"], 1), + ("proxydryoxn", None, None, None, None, ["concprcpoxn"], 1), + ("proxydryrdn", None, None, None, None, ["concprcprdn"], 1), + ("proxydryo3", None, None, None, None, ["vmro3"], 1), + ("wetno3", None, None, None, None, ["concprcpno3"], 1), + ("wetnh4", None, None, None, None, ["concprcpnh4"], 1), + ("wetso4", None, None, None, None, ["concprcpso4"], 1), ( "prmm", ["precipitation_amount_off", "precipitation_amount"], diff --git a/tests/io/test_read_ebas.py b/tests/io/test_read_ebas.py index 3b07f0cd7..c5acbdb2b 100644 --- a/tests/io/test_read_ebas.py +++ b/tests/io/test_read_ebas.py @@ -310,6 +310,59 @@ def test_PROVIDES_VARIABLES(reader: ReadEbas): "concso4coarse", "concnh4coarse", "concno3pm25", + "vmrnh3", + "proxydryoxn", + "proxywetpm25", + "concss25", + "concprcpno3", + "concprcpso4", + "concCoc25", + "concom25", + "concprcpnh4", + "concsscoarse", + "proxydryhno3", + "proxydryhono", + "proxydryn2o5", + "proxydrynh3", + "proxydrynh4", + "proxydryno2", + "proxydryno2no2", + "proxydryno3c", + "proxydryno3f", + "proxydryo3", + "proxydryoxs", + "proxydrypm10", + "proxydrypm25", + "proxydryrdn", + "proxydryso2", + "proxydryso4", + "proxywethno3", + "proxywethono", + "proxywetn2o5", + "proxywetnh3", + "proxywetnh4", + "proxywetno2", + "proxywetno2no2", + "proxywetno3c", + "proxywetno3f", + "proxyweto3", + "proxywetoxn", + "proxywetoxs", + "proxywetpm10", + "proxywetrdn", + "proxywetso2", + "proxywetso4", + "vmrhno3", + "vmrtp", + "wetnh4", + "wetno3", + "wetso4", + "wetna", + "concprcpoxst", + "wetoxsc", + "concprcpoxsc", + "wetoxst", + "concprcpna", } assert set(reader.PROVIDES_VARIABLES) == (PROVIDES_VARIABLES) diff --git a/tests/plugins/ipcforests/__init__.py b/tests/plugins/ipcforests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/plugins/ipcforests/test_reader.py b/tests/plugins/ipcforests/test_reader.py new file mode 100644 index 000000000..2033fb05d --- /dev/null +++ b/tests/plugins/ipcforests/test_reader.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from pyaerocom import const +from pyaerocom.plugins.ipcforests.metadata import MetadataReader as ReadIPCForestMeta +from pyaerocom.plugins.ipcforests.reader import ReadIPCForest +from tests.conftest import lustre_unavail + +try: + IPC_PATH = Path(const.OBSLOCS_UNGRIDDED[const.IPCFORESTS_NAME]) +except KeyError: + pytest.mark.skip(reason=f"IPCForests path not initialised due to non existence in CI") + +# stationnames are not consistent between variables! +# wetoxn +STATION_NAMES = ("DE-604-2", "NO-7-2", "UK-718-2") +# fakedrypm10 +# STATION_NAMES = ("Birkenes II", "La Coulonche", "Jarczew") + +VARS_DEFAULT = {"wetoxn"} +VARS_PROVIDED = VARS_DEFAULT + + +@lustre_unavail +@pytest.fixture(scope="module") +def reader() -> ReadIPCForest: + return ReadIPCForest(data_dir=str(IPC_PATH)) + + +@lustre_unavail +@pytest.fixture(scope="module") +def meta_reader() -> ReadIPCForestMeta: + return ReadIPCForestMeta(str(IPC_PATH)) + + +@lustre_unavail +@pytest.fixture() +def station_files(station: str) -> list[Path]: + p = IPC_PATH.glob("dp_dem.csv") + files = [x for x in p if x.is_file()] + # assert files, f"no files for {station}" + assert files + return files + + +@lustre_unavail +def test_DATASET_NAME(reader: ReadIPCForest): + assert reader.DATA_ID == const.IPCFORESTS_NAME + + +@lustre_unavail +def test_DEFAULT_VARS(reader: ReadIPCForest): + assert set(reader.DEFAULT_VARS) >= VARS_DEFAULT + + +@lustre_unavail +def test_METADATA(meta_reader: ReadIPCForestMeta): + assert len(meta_reader.deposition_type) >= 3, "found less deposition types than expected" + + +@lustre_unavail +def test_PROVIDES_VARIABLES(reader: ReadIPCForest): + return set(reader.PROVIDES_VARIABLES) >= VARS_PROVIDED + + +@lustre_unavail +def test_read_file( + reader: ReadIPCForest, +): + data = reader.read(vars_to_retrieve=VARS_DEFAULT) + assert set(data.contains_vars) == VARS_DEFAULT + + +@lustre_unavail +def test_read_station( + reader: ReadIPCForest, +): + # IPCForest reader does not support partial read of stations at this time + # not easy to implement due to being a single file dataset + data = reader.read( + vars_to_retrieve=VARS_DEFAULT, + ) + for station in STATION_NAMES: + assert station in data.unique_station_names + + +@lustre_unavail +def test_reader_gives_correct_IPC_PATH(reader: ReadIPCForest): + assert str(IPC_PATH) == reader.data_dir diff --git a/tests/plugins/test_entry_points.py b/tests/plugins/test_entry_points.py index 390ee10c1..3e397cf0d 100644 --- a/tests/plugins/test_entry_points.py +++ b/tests/plugins/test_entry_points.py @@ -15,4 +15,4 @@ def test_gridded(): def test_ungridded(): names = {ep.name for ep in metadata.entry_points(group="pyaerocom.ungridded")} assert names, "no entry points found" - assert names == {"ReadGAW", "ReadGhost", "ReadMEP", "ReadICOS"} + assert names == {"ReadGAW", "ReadGhost", "ReadMEP", "ReadICOS", "ReadIPCForest"} diff --git a/tests/test_config.py b/tests/test_config.py index add0e1fcf..3adfba53d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -114,21 +114,23 @@ def test_Config_has_access_users_database(): @lustre_avail @pytest.mark.parametrize( - "cfg_id,basedir,init_obslocs_ungridded,init_data_search_dirs", + "cfg_id,basedir,init_obslocs_ungridded,init_data_search_dirs,data_searchdirno", [ - ("metno", None, False, False), - ("metno", None, True, False), - ("metno", None, True, True), - ("metno", f"/home/{USER}", True, True), - ("users-db", None, False, False), + ("metno", None, False, False, 0), + ("metno", None, True, False, 0), + ("metno", None, True, True, 0), + ("metno", f"/home/{USER}", True, True, 2), + ("users-db", None, False, False, 0), ], ) -def test_Config_read_config(cfg_id, basedir, init_obslocs_ungridded, init_data_search_dirs): +def test_Config_read_config( + cfg_id, basedir, init_obslocs_ungridded, init_data_search_dirs, data_searchdirno +): cfg = testmod.Config(try_infer_environment=False) cfg_file = cfg._config_files[cfg_id] assert Path(cfg_file).exists() cfg.read_config(cfg_file, basedir, init_obslocs_ungridded, init_data_search_dirs) - assert len(cfg.DATA_SEARCH_DIRS) == 0 + assert len(cfg.DATA_SEARCH_DIRS) == data_searchdirno assert len(cfg.OBSLOCS_UNGRIDDED) == 0 assert Path(cfg.OUTPUTDIR).exists() assert Path(cfg.COLOCATEDDATADIR).exists() diff --git a/tests/test_varcollection.py b/tests/test_varcollection.py index ed702aa94..fff03d390 100644 --- a/tests/test_varcollection.py +++ b/tests/test_varcollection.py @@ -81,9 +81,9 @@ def test_VarCollection_get_var_error(collection: VarCollection): "search_pattern,num", [ ("*blaaaaaaa*", 0), - ("dep*", 1), + ("dep*", 7), ("od*", 26), - ("conc*", 82), + ("conc*", 90), ], ) def test_VarCollection_find(collection: VarCollection, search_pattern: str, num: int):