From 9434d608e3db11c3c59e9e2e6dc34877a419c51f Mon Sep 17 00:00:00 2001 From: xincjin-NOAA Date: Mon, 3 Jun 2024 21:16:52 +0000 Subject: [PATCH] refactor amsua --- .../bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 49 +++----- ush/ioda/bufr2ioda/bufr2ioda_ncep_1bmusa.py | 107 ++++++++++++++++++ .../bufr2ioda/bufr2ioda_ncep_amsua_base.py | 86 ++++++++++++++ ush/ioda/bufr2ioda/bufr2ioda_ncep_esamua.py | 102 +++++++++++++++++ ush/ioda/bufr2ioda/bufr2ioda_ssmis.py | 9 -- ush/ioda/bufr2ioda/bufr2ioda_ssmis.yaml | 27 +++-- ush/ioda/bufr2ioda/combine_base.py | 37 ++---- ush/ioda/bufr2ioda/run_bufr_converter.py | 31 +++++ 8 files changed, 364 insertions(+), 84 deletions(-) create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_ncep_1bmusa.py create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_ncep_amsua_base.py create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_ncep_esamua.py create mode 100644 ush/ioda/bufr2ioda/run_bufr_converter.py diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index fcad812ef..c4e69a7ae 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -258,43 +258,29 @@ class Bufr2IodaAmusa(Bufr2IodaBase): - def __init__(self, yaml_order, *args, **kwargs): - self.yaml_order = yaml_order - super().__init__(*args, **kwargs) - self.config.update(config_json) - - if self.yaml_order: - self.yaml_config = yaml_es - else: - self.yaml_config = yaml_1b - - def get_yaml_file(self): - if self.yaml_order: - return self.config['yaml_file'][0] - else: - return self.config['yaml_file'][1] - - -class Bufr2IodaAmusaChange(Bufr2IodaAmusa): def __init__(self, *args, **kwargs): - self.yaml_order = args[0] super().__init__(*args, **kwargs) - if self.yaml_order: - self.yaml_config = yaml_1b + self.config.update(config_json) + cycle_datetime = config["PDY"] + if cycle_datetime >= AMSUA_TYPE_CHANGE_DATETIME: + yaml_order = YAML_NORMAL else: - self.yaml_config = yaml_es + yaml_order = not YAML_NORMAL + logger.info(f'yaml order is {yaml_order}') - def get_yaml_file(self): - if self.yaml_order: - return self.config['yaml_file'][1] - else: - return self.config['yaml_file'][0] + # if self.yaml_order: + # self.yaml_config = yaml_es + # else: + # self.yaml_config = yaml_1b def get_ac_dir(self): return self.config['ac_dir'] @timing_decorator def re_map_variable(self): + if yaml_order and self.data_type == ESAMUA: + return + # TODO replace this follow that in GSI # read_bufrtovs.f90 # antcorr_application.f90 @@ -341,15 +327,6 @@ def apply_corr(self, sat_id, ta, ifov): args = parser.parse_args() log_level = 'DEBUG' if args.verbose else 'INFO' logger = Logger(os.path.basename(__file__), level=log_level) - amsua_files = [] - splits = set() - - cycle_datetime = config["PDY"] - if cycle_datetime >= AMSUA_TYPE_CHANGE_DATETIME: - yaml_order = YAML_NORMAL - else: - yaml_order = not YAML_NORMAL - logger.info(f'yaml order is {yaml_order}') convert_bamua = Bufr2IodaAmusa(yaml_order, config) convert_bamua.execute() diff --git a/ush/ioda/bufr2ioda/bufr2ioda_ncep_1bmusa.py b/ush/ioda/bufr2ioda/bufr2ioda_ncep_1bmusa.py new file mode 100644 index 000000000..3fd422240 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_ncep_1bmusa.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +from combine_base import Bufr2IodaBase +from wxflow import Logger, save_as_yaml +from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr, R1000, R1000000 +from utils import timing_decorator + +logger = Logger(os.path.basename(__file__), level='INFO') + +config_json = dict(yaml_file='bufr2ioda_ncep_1bamua_ta.yaml', + data_type="1bamua_td") + +bufr = { + 'obsdatain': '/scratch2/NCEPDEV/stmp3/Xin.C.Jin/bufr2ioda/gdas.t00z.1bamua.tm00.bufr_d', + 'splits': {'satId': {'category': {'map': {'_209': 'n18', + '_223': 'n19', + '_3': 'metop-b', + '_4': 'metop-a', + '_5': 'metop-c' + }, + 'variable': 'satelliteIdentifier' + }}}, + 'variables': {'brightnessTemperature': {'query': '*/BRITCSTC/TMBR'}, + 'fieldOfViewNumber': {'query': '*/FOVN'}, + 'heightOfStation': {'query': '*/HMSL'}, + 'latitude': {'query': '*/CLAT'}, + 'longitude': {'query': '*/CLON'}, + 'satelliteIdentifier': {'query': '*/SAID'}, + 'sensorAzimuthAngle': {'query': '*/BEARAZ'}, + 'sensorChannelNumber': {'query': '*/BRITCSTC/CHNM'}, + 'sensorScanAngle': {'sensorScanAngle': {'fieldOfViewNumber': '*/FOVN', + 'scanStart': -48.333, + 'scanStep': 3.333, + 'sensor': 'amsua'}}, + 'sensorZenithAngle': {'query': '*/SAZA'}, + 'solarAzimuthAngle': {'query': '*/SOLAZI'}, + 'solarZenithAngle': {'query': '*/SOZA'}} +} + +encoder = { + 'backend': 'netcdf', + 'dimensions': [{'name': 'Channel', + 'path': '*/BRITCSTC', + 'source': 'variables/sensorChannelNumber'}], + 'globals': [{'name': 'platformCommonName', + 'type': 'string', + 'value': 'AMSU-A'}, + {'name': 'platformLongDescription', + 'type': 'string', + 'value': 'MTYP 021-023 PROC AMSU-A 1B Tb'}], + 'obsdataout': 'temporary_ta_{splits/satId}_1716389684.nc', + 'variables': [ + {'longName': 'Satellite Identifier', + 'name': 'MetaData/satelliteIdentifier', + 'source': 'variables/satelliteIdentifier'}, + {'longName': 'Altitude of Satellite', + 'name': 'MetaData/heightOfStation', + 'source': 'variables/heightOfStation', + 'units': 'm'}, + {'longName': 'Solar Zenith Angle', + 'name': 'MetaData/solarZenithAngle', + 'range': [0, 180], + 'source': 'variables/solarZenithAngle', + 'units': 'degree'}, + {'longName': 'Solar Azimuth Angle', + 'name': 'MetaData/solarAzimuthAngle', + 'range': [0, 360], + 'source': 'variables/solarAzimuthAngle', + 'units': 'degree'}, + {'longName': 'Sensor Zenith Angle', + 'name': 'MetaData/sensorZenithAngle', + 'range': [0, 90], + 'source': 'variables/sensorZenithAngle', + 'units': 'degree'}, + {'longName': 'Sensor Azimuth Angle', + 'name': 'MetaData/sensorAzimuthAngle', + 'range': [0, 360], + 'source': 'variables/sensorAzimuthAngle', + 'units': 'degree'}, + {'longName': 'Sensor View Angle', + 'name': 'MetaData/sensorViewAngle', + 'source': 'variables/sensorScanAngle', + 'units': 'degree'}, + {'longName': 'Sensor Channel Number', + 'name': 'MetaData/sensorChannelNumber', + 'source': 'variables/sensorChannelNumber'}, + {'longName': 'Field of View Number', + 'name': 'MetaData/sensorScanPosition', + 'source': 'variables/fieldOfViewNumber'}, + {'chunks': [1000, 15], + 'compressionLevel': 4, + 'longName': 'Antenna Temperature', + 'name': 'ObsValue/brightnessTemperature', + 'range': [90, 380], + 'source': 'variables/brightnessTemperature', + 'units': 'K'} + ] +} + + +class Bufr2IodaAmusa(Bufr2IodaBase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.config.update(config_json) + self.yaml_config = dict(bufr=bufr, encoder=encoder) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_ncep_amsua_base.py b/ush/ioda/bufr2ioda/bufr2ioda_ncep_amsua_base.py new file mode 100644 index 000000000..0992f45bd --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_ncep_amsua_base.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +import os +from combine_base import Bufr2IodaBase +from wxflow import Logger, save_as_yaml +from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr, R1000, R1000000 +from utils import timing_decorator + +logger = Logger(os.path.basename(__file__), level='INFO') +AMSUA_TYPE_CHANGE_DATETIME = "20231200" +BAMUA = '1BAMUA' +ESAMUA = 'ESAMUA' + +config_json = { + "subsets": ["NC005030", "NC005031", "NC005032", "NC005034", "NC005039"], + "data_description": "NC005030 NESDIS SATWIND, GOES IR(LW); NC005031 NESDIS SATWIND, GOES WV-IMG/DL; " + "NC005032 NESDIS SATWIND, GOES VIS; NC005034 NESDIS SATWIND, GOES WV-IMG/CT; " + "NC005039 NESDIS SATWIND, GOES IR(SW)", + "data_provider": "U.S. NOAA/NDESDIS", + "sensor_info": {"sensor_name": "ABI", "sensor_full_name": "Advanced Baseline Imager", "sensor_id": 617}, + "satellite_info": [ + {"satellite_name": "GOES-16", "satellite_full_name": "Geostationary Operational Satellite - 16", + "satellite_id": 270}, + {"satellite_name": "GOES-17", "satellite_full_name": "Geostationary Operational Satellite - 17", + "satellite_id": 271}, + {"satellite_name": "GOES-18", "satellite_full_name": "Geostationary Operational Satellite - 18", + "satellite_id": 272} + ], + "ac_dir": "./" +} + + +class Bufr2IodaAmusaBase(Bufr2IodaBase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.config.update(config_json) + self.cycle_datetime = self.config["PDY"] + + # if self.yaml_order: + # self.yaml_config = yaml_es + # else: + # self.yaml_config = yaml_1b + + def get_ac_dir(self): + return self.config['ac_dir'] + + @timing_decorator + def re_map_variable(self): + if self.cycle_datetime > AMSUA_TYPE_CHANGE_DATETIME and self.data_type == ESAMUA: + return + + # TODO replace this follow that in GSI + # read_bufrtovs.f90 + # antcorr_application.f90 + # search the keyword “ta2tb” for details + + for sat_id in self.sat_ids: + logger.info(f'Converting for {sat_id}, ...') + ta = self.get_container_variable('variables', 'brightnessTemperature', sat_id) + if ta.shape[0]: + ifov = self.get_container_variable('variables', 'fieldOfViewNumber', sat_id) + logger.info(f'ta before correction1: {ta[:100, :]}') + tb = self.apply_corr(sat_id, ta, ifov) + logger.info(f'tb after correction1: {tb[:100, :]}') + self.replace_container_variable('variables', 'brightnessTemperature', tb, sat_id) + + def apply_corr(self, sat_id, ta, ifov): + ac = ACCoeff(self.get_ac_dir()) # TODO add later + llll = 1 # TODO how to set this + if llll == 1: + if sat_id not in ['n15', 'n16']: + # Convert antenna temperature to brightness temperature + ifov = ifov.astype(int) - 1 + for i in range(ta.shape[1]): + logger.info(f'inside loop for allpy ta to tb: i = {i}') + x = ta[:, i] + # logger.info(f'ta before correction: {x[:100]}') + if self.yaml_order: + x = apply_ant_corr(i, ac, ifov, x) + else: + x = remove_ant_corr(i, ac, ifov, x) + # logger.info(f'ta after correction: {x[:100]}') + x[x >= R1000] = R1000000 + ta[:, i] = x + else: + pass # TODO after know how to set llll + return ta diff --git a/ush/ioda/bufr2ioda/bufr2ioda_ncep_esamua.py b/ush/ioda/bufr2ioda/bufr2ioda_ncep_esamua.py new file mode 100644 index 000000000..46cd7f9cb --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_ncep_esamua.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +from combine_base import Bufr2IodaBase +from wxflow import Logger, save_as_yaml +from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr, R1000, R1000000 +from utils import timing_decorator + +logger = Logger(os.path.basename(__file__), level='INFO') + +config_json = dict(yaml_file="bufr2ioda_ncep_esamua.yaml", + data_type="esamua",) + +bufr = {'obsdatain': '/scratch2/NCEPDEV/stmp3/Xin.C.Jin/bufr2ioda/gdas.t00z.esamua.tm00.bufr_d', + 'splits': {'satId': {'category': {'map': {'_209': 'n18', + '_223': 'n19', + '_3': 'metop-b', + '_4': 'metop-a', + '_5': 'metop-c'}, + 'variable': 'satelliteIdentifier'}}}, + 'variables': {'brightnessTemperature': {'query': '*/ATNCHV/TMBRST'}, + 'fieldOfViewNumber': {'query': '*/FOVN'}, + 'heightOfStation': {'query': '*/SELV'}, + 'latitude': {'query': '*/CLATH'}, + 'longitude': {'query': '*/CLONH'}, + 'satelliteIdentifier': {'query': '*/SAID'}, + 'sensorAzimuthAngle': {'query': '*/BEARAZ'}, + 'sensorChannelNumber': {'query': '*/ATNCHV/INCN'}, + 'sensorScanAngle': {'sensorScanAngle': {'fieldOfViewNumber': '*/FOVN', + 'scanStart': -48.333, + 'scanStep': 3.333, + 'sensor': 'amsua'}}, + 'sensorZenithAngle': {'query': '*/SAZA'}, + 'solarAzimuthAngle': {'query': '*/SOLAZI'}, + 'solarZenithAngle': {'query': '*/SOZA'}}} + +encoder = {'backend': 'netcdf', + 'dimensions': [{'name': 'Channel', + 'path': '*/ATNCHV', + 'source': 'variables/sensorChannelNumber'}], + 'globals': [{'name': 'platformCommonName', + 'type': 'string', + 'value': 'AMSUA'}, + {'name': 'platformLongDescription', + 'type': 'string', + 'value': 'MTYP 021-033 RARS(EARS,AP,SA) AMSU-A 1C Tb ' + 'DATA)'}], + 'obsdataout': 'temporary_es_{splits/satId}_1716389684.nc', + 'variables': [ + {'longName': 'Satellite Identifier', + 'name': 'MetaData/satelliteIdentifier', + 'source': 'variables/satelliteIdentifier'}, + {'longName': 'Field Of View Number', + 'name': 'MetaData/sensorScanPosition', + 'source': 'variables/fieldOfViewNumber'}, + {'longName': 'Altitude of Satellite', + 'name': 'MetaData/heightOfStation', + 'source': 'variables/heightOfStation', + 'units': 'm'}, + {'longName': 'Solar Zenith Angle', + 'name': 'MetaData/solarZenithAngle', + 'range': [0, 180], + 'source': 'variables/solarZenithAngle', + 'units': 'degree'}, + {'longName': 'Solar Azimuth Angle', + 'name': 'MetaData/solarAzimuthAngle', + 'range': [0, 360], + 'source': 'variables/solarAzimuthAngle', + 'units': 'degree'}, + {'longName': 'Sensor Zenith Angle', + 'name': 'MetaData/sensorZenithAngle', + 'range': [0, 90], + 'source': 'variables/sensorZenithAngle', + 'units': 'degree'}, + {'longName': 'Sensor Azimuth Angle', + 'name': 'MetaData/sensorAzimuthAngle', + 'range': [0, 360], + 'source': 'variables/sensorAzimuthAngle', + 'units': 'degree'}, + {'longName': 'Sensor View Angle', + 'name': 'MetaData/sensorViewAngle', + 'source': 'variables/sensorScanAngle', + 'units': 'degree'}, + {'longName': 'Sensor Channel Number', + 'name': 'MetaData/sensorChannelNumber', + 'source': 'variables/sensorChannelNumber'}, + {'chunks': [1000, 15], + 'compressionLevel': 4, + 'coordinates': 'longitude latitude Channel', + 'longName': 'Brightness Temperature', + 'name': 'ObsValue/brightnessTemperature', + 'range': [100, 500], + 'source': 'variables/brightnessTemperature', + 'units': 'K'}]} + + +class Bufr2IodaEsamusa(Bufr2IodaBase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.config.update(config_json) + self.yaml_config = dict(bufr=bufr, encoder=encoder) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_ssmis.py b/ush/ioda/bufr2ioda/bufr2ioda_ssmis.py index 79a6deffb..dd2be3ef2 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_ssmis.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_ssmis.py @@ -86,15 +86,6 @@ 'units': 'K'}]}} # This will be moved out in the future. it should be an input when call this program. -config = { - 'RUN': 123, - 'current_cycle': '2022010412', - 'DATA': 'abcd', - 'DMPDIR': 'abc', - 'COM_OBS': 'abs', - 'PDY': '20220104', - 'cyc': '12', -} class Bufr2IodaSsmis(Bufr2IodaBase): diff --git a/ush/ioda/bufr2ioda/bufr2ioda_ssmis.yaml b/ush/ioda/bufr2ioda/bufr2ioda_ssmis.yaml index 56ec6b1d4..160037d17 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_ssmis.yaml +++ b/ush/ioda/bufr2ioda/bufr2ioda_ssmis.yaml @@ -49,7 +49,6 @@ bufr: second: '*/SECO' year: '*/YEAR' encoder: - backend: netcdf dimensions: - name: Channel path: '*/SSMISCHN' @@ -63,6 +62,19 @@ encoder: value: MTYP 021-203 SSMIS ATENNA/BRIGHTNESS TEMPERATURE DATA obsdataout: temporary_{splits/satId}_1716389684.nc variables: + - longName: Satellite Identifier + name: MetaData/satelliteIdentifier + source: variables/satelliteIdentifier + - longName: Sensor Channel Number + name: MetaData/sensorChannelNumber + source: variables/sensorChannelNumber + - chunks: + - 10000 + - 22 + longName: 3-by-3 Averaged Brightness Temperature + name: ObsValue/brightnessTemperature + source: variables/remappedBT + units: K - longName: Datetime name: MetaData/dateTime source: variables/timestamp @@ -78,16 +90,3 @@ encoder: name: MetaData/longitude source: variables/longitude units: degree_east - - longName: Satellite Identifier - name: MetaData/satelliteIdentifier - source: variables/satelliteIdentifier - - longName: Sensor Channel Number - name: MetaData/sensorChannelNumber - source: variables/sensorChannelNumber - - chunks: - - 10000 - - 22 - longName: 3-by-3 Averaged Brightness Temperature - name: ObsValue/brightnessTemperature - source: variables/remappedBT - units: K diff --git a/ush/ioda/bufr2ioda/combine_base.py b/ush/ioda/bufr2ioda/combine_base.py index f5cd6298f..9133939e5 100644 --- a/ush/ioda/bufr2ioda/combine_base.py +++ b/ush/ioda/bufr2ioda/combine_base.py @@ -22,13 +22,12 @@ } bufr_base = { - 'variables': { - 'timestamp': {'datetime': {'day': '*/DAYS', + 'variables': {'timestamp': {'datetime': {'day': '*/DAYS', 'hour': '*/HOUR', 'minute': '*/MINU', 'month': '*/MNTH', 'second': '*/SECO', - 'year': '*/YEAR'}}} + 'year': '*/YEAR'}}}, } encoder_base = { @@ -46,12 +45,15 @@ 'name': 'MetaData/longitude', 'source': 'variables/longitude', 'units': 'degree_east'}, + ] } + class Bufr2IodaBase: def __init__(self, config_para): json_object = json.dumps(json_template_base, indent=4) self.config = json.loads(Jinja(json_object, config_para).render) # make it a method if the base is not unique + self.data_type = None self.yaml_config = None self.yaml_path = None self.ioda_files = None @@ -68,8 +70,12 @@ def initialization(self): def update_config(self, config_json): self.config.update(config_json) - def set_yaml(self): - save_as_yaml(ssmis_yaml, yaml_file) + def set_yaml(self, yaml_path): + + self.yaml_config['bufr']['variables'].update(bufr_base['variables']) + self.yaml_config['encoder']['variables'] += encoder_base['variables'] + + save_as_yaml(self.yaml_config, yaml_path) def get_container_variable(self, group, variable, sat_id): return self.container.get(group + '/' + variable, [sat_id, ]) @@ -80,17 +86,6 @@ def replace_container_variable(self, group, variable, var, sat_id): def get_yaml_file(self): return self.config['yaml_file'] - def get_yaml_config(self): - with open(self.yaml_path, 'r') as yaml_file: - yaml_config = yaml.load(yaml_file, Loader=yaml.FullLoader) - return yaml_config - - def make_split_files(self, sat_ids): - # sat_id a list of ids - pattern = r'{(.*?)}' - for sat_id in sat_ids: - self.split_files[sat_id] = re.sub(pattern, sat_id, self.ioda_files) - def re_map_variable(self): # Make any changes and return for your specific case in the sub-class pass @@ -108,15 +103,7 @@ def encode(self): netcdf.Encoder(self.yaml_path).encode(self.container, self.ioda_files) def execute(self): + self.set_yaml(self.get_yaml_file()) self.initialization() self.set_container() self.encode() - - def get_info(self): - if not self.split_files: - self.make_split_files(self.splits) - sat_info = {'ioda_files': self.ioda_files, - 'split': self.splits, - 'split_files': self.split_files - } - return sat_info diff --git a/ush/ioda/bufr2ioda/run_bufr_converter.py b/ush/ioda/bufr2ioda/run_bufr_converter.py new file mode 100644 index 000000000..712b5dfc3 --- /dev/null +++ b/ush/ioda/bufr2ioda/run_bufr_converter.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +from wxflow import Logger, parse_j2yaml, cast_strdict_as_dtypedict +from bufr2ioda_ssmis import Bufr2IodaSsmis + +# Initialize root logger +logger = Logger('gen_bufr2ioda_json.py', level='INFO', colored_log=True) + +bufr_classes = [ + Bufr2IodaSsmis, +] + +config = { + 'RUN': 123, + 'current_cycle': '2022010412', + 'DATA': 'abcd', + 'DMPDIR': 'abc', + 'COM_OBS': 'abs', + 'PDY': '20220104', + 'cyc': '12', +} + + +if __name__ == "__main__": + + for bufr_class in bufr_classes: + converter = bufr_class(config) + converter.execute() + logger.info('--Finished--') + +