diff --git a/docs/sta.md b/docs/sta.md index f35dd4f..fb87740 100644 --- a/docs/sta.md +++ b/docs/sta.md @@ -15,43 +15,43 @@ The header section contains metadata about the file, such as the date performed, #### Netzsch Header Here are the commonly found metadata fields in the Netzsch instrument file header: -| Original Name | Metadata Name | Description | Example | -|-----------------------|---------------|-------------|---------| -| EXPORTTYPE | exporttype | Provides the type of export that was performed from the Netzsch Proteus software. | "DATA ALL" | -| FILE | file | Lists the manufacturer data file from which this file was produced. | "DF_FILED_DES_STA_N2_10K_231028_R1.ngb-ss3" | -| FORMAT | format | Provides the export format of the file. | "NETZSCH5" | -| FTYPE | file_type | Provides the encoding type of the file. | "ANSI" | -| IDENTITY | identity | Provides a user-entered quantity during test setup. | "DF_FILED" | -| DECIMAL | decimal | Provides the decimal character. | "POINT" | -| SEPARATOR | delimiter | Provides the delimiter used in the file. | "COMMA" | -| MTYPE | measurement_type | Provides the measurement type exported. | "DSC" | -| INSTRUMENT | instrument | Specifies the name or model of the instrument used for the measurement. | "NETZSCH STA 449F3" | -| PROJECT | project | Provides user-input information about the project or experiment associated with the data. | "Oxidative Pyrolysis" | -| DATE/TIME | date_performed | Indicates the date and time when the measurement was performed. | "2/11/2024 13:12:51 (UTC-5)" | -| CORR. FILE | correction_file | Refers to the correction file used during the measurement, if any. | | -| TEMPCAL | temperature_calibration | Provides the temperature calibration file used for the measurement. | "30-01-2024 15:52" | -| SENSITIVITY | sensitivity_calibration | Provides the sensitivity calibration file used for the measurement. | "30-01-2024 15:52" | -| LABORATORY | laboratory | Indicates the laboratory or facility where the measurement took place. | "UL FSRI" | -| OPERATOR | operator | Specifies the user-input name or identifier of the operator who performed the measurement. | "Grayson" | -| REMARK | comments | Provides any additional remarks or comments related to the measurement. | "Douglas fir, Filed, Closed pan, Pt, Kinetics validation run" | -| SAMPLE | sample | User-input name of the sample being tested. | "DF_FILED" | -| SAMPLE MASS | sample_mass | Specifies the user-input mass of the sample. | "3.99" | -| MATERIAL | material | Indicates the user-input material of the sample. | "Douglas Fir" | -| REFERENCE | reference | Refers to a user-input reference material used for comparison or calibration. Often None. | | -| REFERENCE MASS | reference_mass | Specifies the user-input mass of the reference material. Often 0. | "0" | -| TYPE OF CRUCIBLE | crucible_type | Describes the user-input type or material of the crucible used. Includes volume and presence of a lid. | "PtRh20 85 µl, with lid" | -| SAMPLE CRUCIBLE MASS | sample_crucible_mass | Specifies the user-input mass of the crucible containing the sample. | "254.00" | -| REFERENCE CRUCIBLE MASS | reference_crucible_mass | Specifies the user-input mass of the crucible containing the reference material. | "254.04" | -| PURGE {X} MFC | purge_{x}_mfc | Provides information about the one of the purge gas mass flow controller. | "NITROGEN,250.0 ml/min" | -| PROTECTIVE MFC | protective_mfc | Provides information about the protective gas mass flow controller. | "NITROGEN,250.0 ml/min" | -| DSC RANGE | dsc_range | Describes the range of the differential scanning calorimetry (DSC) measurement. | "5000" | -| TG RANGE | tg_range | Specifies the range of the thermogravimetric analysis (TG) measurement. | "35000" | -| TAU-R | tau_r | Indicates if Tau-R mode was used during the measurement. | "---" | -| CORR. CODE | correction_code | Not really sure what this represents at the moment. For all of our files it is "000". | "000" | -| EXO | exothermic | Negative or positive 1 value that represents the direction corresponding to exothermic DSC phenomena. | "-1" | -| RANGE | range | Describes the range of temperatures and/or heating rates during this measurement. | "25°C....700°C/0.0....40.0K/min" | -| SEGMENT | segment | Specifies the segment number(s) presented and the total number of segments in the temperature program. | "S1-9/9" | -| SEG. {X} | segment_{x} | Additional segment information including start and end temperatures and time or heating rate between them. | "25°C/20.0(K/min)/250°C" | +| Original Name | Metadata Name | Description | Example | Output Example | +|-----------------------|---------------|-------------|---------|----------------| +| EXPORTTYPE | export_type | Provides the type of export that was performed from the Netzsch Proteus software. | "DATA ALL" | {"export_type": "DATA ALL"} | +| FILE | file | Lists the manufacturer data file from which this file was produced. | "DF_FILED_DES_STA_N2_10K_231028_R1.ngb-ss3" | {"file": "DF_FILED_DES_STA_N2_10K_231028_R1.ngb-ss3"} | +| FORMAT | format | Provides the export format of the file. | "NETZSCH5" | {"format": "NETZSCH5"} | +| FTYPE | file_type | Provides the encoding type of the file. | "ANSI" | {"file_type": "ANSI"} | +| IDENTITY | identity | Provides a user-entered quantity during test setup. | "DF_FILED" | {"identity": "DF_FILED"} | +| DECIMAL | decimal | Provides the decimal character. | "POINT" | {"decimal": "POINT"} | +| SEPARATOR | delimiter | Provides the delimiter used in the file. | "COMMA" | {"delimiter": "COMMA"} | +| MTYPE | measurement_type | Provides the measurement type exported. | "DSC" | {"measurement_type": "DSC"} | +| INSTRUMENT | instrument | Specifies the name or model of the instrument used for the measurement. | "NETZSCH STA 449F3" | {"instrument": "NETZSCH STA 449F3"} | +| PROJECT | project | Provides user-input information about the project or experiment associated with the data. | "Oxidative Pyrolysis" | {"project": "Oxidative Pyrolysis"} | +| DATE/TIME | date_performed | Indicates the date and time when the measurement was performed. | "2/11/2024 13:12:51 (UTC-5)" | {"date_performed": "2023-10-28T13:29:27+04:00"} | +| CORR. FILE | correction_file | Refers to the correction file used during the measurement, if any. | | {"correction_file": ""} | +| TEMPCAL | temperature_calibration | Provides the temperature calibration file used for the measurement. | "30-01-2024 15:52" | {"temperature_calibration": {'date': '2023-09-10T07:44:00'}} | +| SENSITIVITY | sensitivity_calibration | Provides the sensitivity calibration file used for the measurement. | "30-01-2024 15:52" | {"sensitivity_calibration": {'date': '2023-09-10T07:44:00'}} | +| LABORATORY | laboratory | Indicates the laboratory or facility where the measurement took place. | "UL FSRI" | {"laboratory": "UL FSRI"} | +| OPERATOR | operator | Specifies the user-input name or identifier of the operator who performed the measurement. | "Grayson" | {"operator": "Grayson"} | +| REMARK | comments | Provides any additional remarks or comments related to the measurement. | "Douglas fir, Filed, Closed pan, Pt, Kinetics validation run" | {"comments": "Douglas fir, Filed, Closed pan, Pt, Kinetics validation run"} | +| SAMPLE | sample | User-input name of the sample being tested. | "DF_FILED" | {"sample": "DF_FILED"} | +| SAMPLE MASS /mg | sample_mass | Specifies the user-input mass of the sample. | "3.99" | {"sample_mass": {'value': 3.99, 'unit': 'mg'}} | +| MATERIAL | material | Indicates the user-input material of the sample. | "Douglas Fir" | {"material": "Douglas Fir"} | +| REFERENCE | reference | Refers to a user-input reference material used for comparison or calibration. Often None. | | {"reference": ""} | +| REFERENCE MASS /mg | reference_mass | Specifies the user-input mass of the reference material. Often 0. | "0" | {"reference_mass": {'value': 0, 'unit': 'mg'}} | +| TYPE OF CRUCIBLE | crucible_type | Describes the user-input type or material of the crucible used. Includes volume and presence of a lid. | "PtRh20 85 µl, with lid" | {"crucible_type": {'material': 'PtRh20', 'volume': {'value': 85.0, 'unit': 'µl'}, 'extra': 'with lid'}} | +| SAMPLE CRUCIBLE MASS | sample_crucible_mass | Specifies the user-input mass of the crucible containing the sample. | "254.00" | {"sample_crucible_mass": {'value': 241.78, 'unit': 'mg'}} | +| REFERENCE CRUCIBLE MASS | reference_crucible_mass | Specifies the user-input mass of the crucible containing the reference material. | "254.04" | {"reference_crucible_mass": {'value': 254.00, 'unit': 'mg'}} | +| PURGE {X} MFC | purge_{x}_mfc | Provides information about the one of the purge gas mass flow controller. | "NITROGEN,250.0 ml/min" | {"purge_{x}_mfc": {'gas': 'NITROGEN', 'range': 250.0, 'unit': 'ml/min'}} | +| PROTECTIVE MFC | protective_mfc | Provides information about the protective gas mass flow controller. | "NITROGEN,250.0 ml/min" | {"protective_mfc": {'gas': 'NITROGEN', 'range': 250.0, 'unit': 'ml/min'}} | +| DSC RANGE /µV | dsc_range | Describes the range of the differential scanning calorimetry (DSC) measurement. | "5000" | {"dsc_range": {'value': 5000, 'unit': 'µV'}} | +| TG RANGE /mg | tg_range | Specifies the range of the thermogravimetric analysis (TG) measurement. | "35000" | {"tg_range": {'value': 35000, 'unit': 'mg'}} | +| TAU-R | tau_r | Indicates if Tau-R mode was used during the measurement. | "---" | {"tau_r": "---"} | +| CORR. CODE | correction_code | Not really sure what this represents at the moment. For all of our files it is "000". | "000" | {"correction_code": "000"} | +| EXO | exothermic | Negative or positive 1 value that represents the direction corresponding to exothermic DSC phenomena. | "-1" | {"exothermic": "-1"} | +| RANGE | range | Describes the range of temperatures and/or heating rates during this measurement. | "25°C....700°C/0.0....40.0K/min" | {"range": "25°C....700°C/0.0....40.0K/min"} | +| SEGMENT | segment | Specifies the segment number(s) presented and the total number of segments in the temperature program. | "S1-9/9" | {"segment": "S1-9/9"} | +| SEG. {X} | segment_{x} | Additional segment information including start and end temperatures and time or heating rate between them. | "25°C/20.0(K/min)/250°C" | {"segment_{x}": {'start_temperature': {'value': 25.0, 'unit': '°C'}, 'end_temperature': {'value': 250.0, 'unit': '°C'}, 'heating_rate': {'value': 20.0, 'unit': 'K/min'}}} | ### Data diff --git a/src/lab_etl/netzsch_sta_parser.py b/src/lab_etl/netzsch_sta_parser.py index aaca5a6..8ecd7f6 100644 --- a/src/lab_etl/netzsch_sta_parser.py +++ b/src/lab_etl/netzsch_sta_parser.py @@ -74,6 +74,7 @@ def get_sta_metadata( key.strip().lower().replace(" ", "_") ) # convert key to lowercase and replace spaces with underscores (snake_case) key_mapping = { + "exporttype": "export_type", "remark": "comment", "type_of_crucible": "crucible_type", "tempcal": "temperature_calibration", @@ -82,6 +83,8 @@ def get_sta_metadata( "ftype": "file_type", "mtype": "measurement_type", "corr._code": "correction_code", + "exo": "exothermic", + "separator": "delimiter", } key = key_mapping.get(key, key) value = value.strip(", \n") @@ -325,6 +328,7 @@ def set_metadata(tbl, col_meta={}, tbl_meta={}) -> pa.Table: else v.decode("utf-8") for k, v in df.schema.metadata.items() } + print(metadata) pq.write_table( df, "tests/test_files/STA/DF_FILED_DES_STA_N2_10K_231028_R1.parquet",