diff --git a/doc/main.bib b/doc/main.bib index a3f8aac92d..81b8acf99d 100644 --- a/doc/main.bib +++ b/doc/main.bib @@ -1711,3 +1711,12 @@ @article{popp_2017_SSPlanduse volume = {42}, year = {2017} } + +@article{unlu_2024_materials, + author = {Gamze Ünlü and Florian Maczek and Jihoon Min and Stefan Frank and Fridolin Glatter and Paul Natsuo Kishimoto and Jan Streeck and Nina Eisenmenger and Volker Krey and Dominik Wiedenhofer}, + doi = {https://doi.org/10.5194/egusphere-2023-3035}, + journal = {EGUsphere [preprint]}, + title = {MESSAGEix-Materials v1.0.0: Representation of Material Flows and Stocks in an Integrated Assessment Model}, + url = {https://egusphere.copernicus.org/preprints/2024/egusphere-2023-3035/}, + year = {2024} +} diff --git a/doc/material/index.rst b/doc/material/index.rst index 8c4f627cbe..0a2fc69826 100644 --- a/doc/material/index.rst +++ b/doc/material/index.rst @@ -18,12 +18,21 @@ Code reference .. automodule:: message_ix_models.data.material :members: +.. note:: + See also :pull:`130`/the archived branch `materials-migrate `_ for a distinct version of :mod:`.material`. + That earlier PR was superseded by :pull:`188`, but contains the 1.0.0 version of MESSAGEix-Materials, which was used for the first submission of :cite:`unlu_2024_materials`. The model structure is almost identical to the default model that was added by :pull:`188`. + Compared to :pull:`188` this version differs particularly in the following areas: + + - Older base year calibration of "other industries" using outdated IEA EWEB data. + - Material demands computed in R through ``rpy2``, instead of Python implementation. + - Less accurate regional allocation/aggregation of base year demands for cement and steel. + - No use of :mod:`.tools.costs`. + Data preparation ---------------- These scripts are used to prepare and read the data into the model. -They can be turned on and off individually under `DATA_FUNCTIONS` in `__init__.py`. -For example, the buildings script (`data_buildings.py`) is only used when the buildings model outputs are given explicitly without linking the CHILLED/STURM model through a soft link. +They can be turned on and off individually under ``DATA_FUNCTIONS`` in :mod:`__init__.py`. .. automodule:: message_ix_models.model.material.data_aluminum :members: @@ -40,16 +49,13 @@ For example, the buildings script (`data_buildings.py`) is only used when the bu .. automodule:: message_ix_models.model.material.data_power_sector :members: -.. automodule:: message_ix_models.model.material.data_buildings - :members: - .. automodule:: message_ix_models.model.material.data_generic :members: .. automodule:: message_ix_models.model.material.data_ammonia_new :members: -.. automodule:: message_ix_models.model.material.data_methanol_new +.. automodule:: message_ix_models.model.material.data_methanol :members: Build and Solve the model from CLI @@ -63,12 +69,12 @@ Use ``mix-models materials-ix {SSP} build`` to add the material implementation o mix-models \ --url="ixmp://ixmp_dev/MESSAGEix-GLOBIOM 1.1-R12/baseline_DEFAULT#21" \ - --local-data "./data" material-ix SSP2 build --tag test + --local-data "./data" material-ix SSP2 build --tag test --nodes R12 -The output scenario name will be baseline_DEFAULT_test. An additional tag `--tag` can be used to add an additional suffix to the new scenario name. -The mode option `--mode` has two different inputs 'by_url' (by default) or 'by_copy'. -The first one uses the provided url to add the materials implementation on top of the scenario from the url. -This is the default option. The latter is used to create a 2 degree mitigation scenario with materials by copying carbon prices to the scenario that is specified by `--scenario_name`:: +The output scenario name will be baseline_DEFAULT_test. An additional tag ``--tag`` can be used to add an additional suffix to the new scenario name. +The mode option ``--mode`` has two different inputs 'by_url' (by default) or 'by_copy'. +The first one uses the provided ``--url`` to add the materials implementation on top of the scenario from the url. +This is the default option. The latter is used to create a 2 degree mitigation scenario with materials by copying carbon prices to the scenario that is specified by ``--scenario_name``:: mix-models --url="ixmp://ixmp_dev/MESSAGEix-Materials/scenario_name" material-ix \ build --tag test --mode by_copy @@ -78,14 +84,22 @@ This command line only builds the scenario but does not solve it. To solve the s mix-models --url="ixmp://ixmp_dev/MESSAGEix-Materials/scenario_name" material-ix \ SSP2 solve --add_calibration False --add_macro False -The solve command has the `--add_calibration` option to add MACRO calibration to a baseline scenario. `--add_macro` option solves the scenario with MACRO. -Both options are False by default.To first calibrate the scenario and then solve that scenario with MACRO both options should be set to True. +The solve command has the ``--add_calibration`` option to add MACRO calibration to a baseline scenario with a valid calibration file specified with ``--macro-file``. +The ``--add_macro`` option determines whether the scenario should be solved with MESSAGE or MESSAGE-MACRO. +MESSAGEix-Materials provides one calibration file that is only compatible with scenarios with first model year 2025 and the common model structure of a MESSAGEix-GLOBIOM scenario. +To first calibrate the scenario and then solve that scenario with MACRO both options should be set to :any`True`. + +It is also possible to shift the first model year and solve a clone with shifted years with ``--shift_model_year``. +If ``--shift_model_year`` is set together with the macro options the model year will be shifted before the MACRO calibration. + +All three options are :any:`False` by default. + Reporting ========= The reporting generates specific variables related to materials, mainly Production and Final Energy. -The resulting reporting file is generated under message_ix_models\\data\\material\\reporting_output with the name “New_Reporting_MESSAGEix-Materials_scenario_name.xlsx”. +The resulting reporting file is generated under :file:`message_ix_models/data/material/reporting_output` with the name “New_Reporting_MESSAGEix-Materials_scenario_name.xlsx”. More detailed variables related to the whole energy system and emissions are not included in this reporting. Reporting is executed by the following command:: @@ -106,35 +120,46 @@ Binary/raw data files The code relies on the following input files, stored in :file:`data/material/`: +**Cement** + :file:`CEMENT.BvR2010.xlsx` Historical cement demand data -:file:`STEEL_database_2012.xlsx` - Historical steel demand data +:file:`Global_cement_MESSAGE.xlsx` + Techno-economic parametrization data for cement sector combined (R12) -:file:`Global_steel_cement_MESSAGE.xlsx` - Techno-economic parametrization data for steel and cement sector combined (R12) +:file:`demand_cement.yaml` + Base year demand data -:file:`demand_aluminum.xlsx` - Historical aluminum demand data +**Aluminum** :file:`demand_aluminum.xlsx` Historical aluminum demand data +:file:`aluminum_trade_data.csv` + Data retrieved from IAI MFA model and used for trade calibration + :file:`aluminum_techno_economic.xlsx` Techno-economic parametrization data for aluminum sector -:file:`generic_furnace_boiler_techno_economic.xlsx` - Techno-economic parametrization data for generic furnace technologies +:file:`demand_aluminum.yaml` + Base year aluminum demand data -:file:`iamc_db ENGAGE baseline GDP PPP.xlsx` - SSP GDP projection used for material demand projections +**Iron and Steel** -:file:`MESSAGEix-Materials_final_energy_industry.xlsx` - Final energy values to calibrate base year values for industries +:file:`STEEL_database_2012.xlsx` + Historical steel demand data -:file:`residual_industry_2019.xlsx` - Final energy values to calculate the residual industry demand +:file:`Global_steel_MESSAGE.xlsx` + Techno-economic parametrization data for steel sector combined (R12) + +:file:`worldsteel_steel_trade.xlsx` + Historical data from `worldsteel Association `_ + +:file:`demand_steel.yaml` + Base year steel demand data + +**Chemicals** :file:`nh3_fertilizer_demand.xlsx` Nitrogen fertilizer demand @@ -163,6 +188,29 @@ The code relies on the following input files, stored in :file:`data/material/`: :file:`steam_cracking_hist_new_cap.csv` Steam cracker historical capacities in R12 regions +:file:`ammonia/demand_NH3.yaml` + Ammonia demand in each R12 region in year 2020 + +:file:`petrochemicals/demand_HVC.yaml` + HVC demand in each R12 region in year 2020 + +:file:`methanol/demand_methanol.yaml` + Methanol demand in each R12 region in year 2020 + +:file:`/methanol/results_material_SHAPE_comm.csv` + Commercial sector data from MESSAGEix-Buidings SHAPE scenario used to get wood demands from buildings to estimate resin demands + +:file:`/methanol/results_material_SHAPE_resid.csv` + Residential sector data from MESSAGEix-Buidings SHAPE scenario used to get wood demands from buildings to estimate resin demands + +:file:`/methanol/results_material_SSP2_comm.csv` + Commercial sector data from MESSAGEix-Buidings SSP2 scenario used to get wood demands from buildings to estimate resin demands + +:file:`/methanol/results_material_SSP2_resid.csv` + Residential sector data from MESSAGEix-Buidings SSP2 scenario used to get wood demands from buildings to estimate resin demands + +**Power sector** + :file:`NTNU_LCA_coefficients.xlsx` Material intensity (and other) coefficients for power plants based on lifecycle assessment (LCA) data from the THEMIS database, compiled in the `ADVANCE project `_. @@ -175,6 +223,32 @@ The code relies on the following input files, stored in :file:`data/material/`: :file:`LCA_commodity_mapping.xlsx` Commodity mapping (for materials) of global 11-regional MESSAGEix-GLOBIOM model to commodities of THEMIS LCA dataset. + +**Buildings** + +:file:`buildings/LED_LED_report_IAMC_sensitivity_R12.csv` + Data from MESSAGEix-Buidings LED scenarios used to get steel/cement/aluminum demands from buildings + +:file:`buildings/report_IRP_SSP2_BL_comm_R12.csv` + Commerical sector data from MESSAGEix-Buidings used to get steel/cement/aluminum demands from buildings + +:file:`buildings/report_IRP_SSP2_BL_resid_R12.csv` + Residential sector data from MESSAGEix-Buidings used to get steel/cement/aluminum demands from buildings + +**Other** + +:file:`generic_furnace_boiler_techno_economic.xlsx` + Techno-economic parametrization data for generic furnace technologies + +:file:`iamc_db ENGAGE baseline GDP PPP.xlsx` + SSP GDP projection used for material demand projections + +:file:`MESSAGEix-Materials_final_energy_industry.xlsx` + Final energy values to calibrate base year values for industries + +:file:`residual_industry_2019.xlsx` + Final energy values to calculate the residual industry demand + :file:`SSP_UE_dyn_input.xlsx` Calibration file for industry end-use energy demands @@ -190,36 +264,29 @@ The code relies on the following input files, stored in :file:`data/material/`: :file:`iea_mappings/industry.csv` Mapping of MESSAGEix-GLOBIOM industry technologies to IEA EWEB products of industry sector flows not covered by MESSAGEix-Materials -:file:`ammonia/demand_NH3.yaml` - Ammonia demand in each R12 region in year 2020 - :file:`other/mer_to_ppp_default.csv` Default conversion factors for GDP MER to PPP used in MESSAGEix-GLOBIOM SSP2 scenarios. Used to create demand projections for steel/cement/aluminum/chemicals if GDP_PPP data is not in scenario -:file:`buildings/LED_LED_report_IAMC_sensitivity_R12.csv` - Data from MESSAGEix-Buidings LED scenarios used to get steel/cement/aluminum demands from buildings -:file:`buildings/report_IRP_SSP2_BL_comm_R12.csv` - Commerical sector data from MESSAGEix-Buidings used to get steel/cement/aluminum demands from buildings -:file:`buildings/report_IRP_SSP2_BL_resid_R12.csv` - Residential sector data from MESSAGEix-Buidings used to get steel/cement/aluminum demands from buildings +:file:`material/set.yaml` +---------------------------- -:file:`/methanol/results_material_SHAPE_comm.csv` - Commercial sector data from MESSAGEix-Buidings SHAPE scenario used to get wood demands from buildings to estimate resin demands +.. literalinclude:: ../../message_ix_models/data/material/set.yaml + :language: yaml -:file:`/methanol/results_material_SHAPE_resid.csv` - Residential sector data from MESSAGEix-Buidings SHAPE scenario used to get wood demands from buildings to estimate resin demands -:file:`/methanol/results_material_SSP2_comm.csv` - Commercial sector data from MESSAGEix-Buidings SSP2 scenario used to get wood demands from buildings to estimate resin demands +Release notes +============= -:file:`/methanol/results_material_SSP2_resid.csv` - Residential sector data from MESSAGEix-Buidings SSP2 scenario used to get wood demands from buildings to estimate resin demands -:file:`material/set.yaml` ----------------------------- +This is the list of changes to MESSAGEix-Materials between each release. -.. literalinclude:: ../../message_ix_models/data/material/set.yaml - :language: yaml + +**Version 1.1.0** + +.. toctree:: + :maxdepth: 2 + + v1.1.0 diff --git a/doc/material/v1.1.0.rst b/doc/material/v1.1.0.rst new file mode 100644 index 0000000000..07b1842bb9 --- /dev/null +++ b/doc/material/v1.1.0.rst @@ -0,0 +1,80 @@ +Version 1.1.0 (August 27, 2024) +------------------------------- + +This is a release of MESSAGEix-Materials and includes model additions and enhancements. +There are also a few of changes in the command line interface. + +Model changes +~~~~~~~~~~~~~ + +**Iron & steel sector** + +This release introduces the following low-emission options: + +- Direct reduction of iron with hydrogen +- Direct reduction of iron with natural gas +- Direct reduction of iron with natural gas and CCS +- Blast furnace with CCS +- Blast furnace using charcoal +- Basic oxygen furnace with top gas recirculation + +Global trade of finished steel has been calibrated according to wordsteel Association. + +Shared input data file :file:`Global_steel_cement_MESSAGE.xlsx` has been separated for steel and cement data. + +**Non metallic minerals sector** + +Shared input data file has been separated for steel and cement data. + +CCS addon reformulated to prohibit wet CCS addon on dry clinker kiln and vice versa. + +**Non ferrous metals sector** + +Global trade of aluminum has been calibrated according to IAI MFA model. + +**Power sector** + +Material intensity of hydro power plants updated (reference?) + +**Demand generator** + +GDP and population projections are read from scenarios by default by retrieving the parametrization of ``bound_activity_up`` for the technologies ``GDP``, ``GDP_PPP`` and ``Population``. +If the data cannot be retrieved from the scenario parameters, default projections from an exogenous data file are used. + +**Other** + +MESSAGEix-Materials can be built on the :ref:`model-bare` now. + +Convenience function :func:`data_util.calculate_ini_new_cap` to calculate ``initial_new_capacity_up/lo`` based on demand parameter. + +MACRO calibration is now available for non-IIASA users who did not have access to the previously required MACRO calibration file. +See :ref:`CLI Changes` + +An option has been added to shift the first model year of a scenario to a desired year before solving. +See :ref:`CLI Changes` + + +CLI changes +~~~~~~~~~~~ + +The :func:`build_scen` command uses the ``mix-models`` CLI parameter ``--nodes`` now. +Currently ``R12`` is the only supported option. + +Scenarios can now be calibrated with MACRO by setting ``--add_calibration=True`` and setting ``--macro-file`` to an existing filename located in :file:`data/material/macro/`. + +Solving a scenario with a different first model year is possible with e.g.:: + + mix-models --url="ixmp://ixmp_dev/MESSAGEix-Materials/scenario_name" material-ix \ + SSP2 solve --shift_model_year=2025 + +Deprecations +~~~~~~~~~~~~~ +The ``material-ix`` commands ``add-buildings-ts``, ``report-2`` and ``create-bare`` have been removed. + +New features +~~~~~~~~~~~~ + +**Bug Fixes** + + +**Compatibility** diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 240b292381..dac590ccd5 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -17,6 +17,9 @@ Next release - Replace solar and wind technologies with new ones (:pull:`206`). - Reorganize input files and incorporate `first_year.csv` data into `tech_map.csv` (:pull:`221`). - Reconfigure use and implementation of technology variants/modules to be more agnostic (:pull:`221`). +- Improve and extend :doc:`/material/index` (:pull:`218`). + + - Release of MESSAGEix-Materials 1.1.0 (:doc:`/material/v1.1.0`). v2024.8.6 ========= diff --git a/message_ix_models/data/level.yaml b/message_ix_models/data/level.yaml index 4257921532..3cbb21271c 100644 --- a/message_ix_models/data/level.yaml +++ b/message_ix_models/data/level.yaml @@ -1,12 +1,5 @@ -primary: - name: Primary Energy - description: >- - A form found in nature that has not been subjected to any human engineered - conversion process. - -secondary: - name: Secondary Energy - description: Forms which have been transformed from primary energy. +export: + name: Exports final: name: Final Energy @@ -17,6 +10,16 @@ final: import: name: Imports +primary: + name: Primary Energy + description: >- + A form found in nature that has not been subjected to any human engineered + conversion process. + +secondary: + name: Secondary Energy + description: Forms which have been transformed from primary energy. + useful: name: Useful Energy description: >- @@ -32,7 +35,6 @@ water_supply: # are not currently used by model.bare.create_res. # # cooling -# export # land_use_reporting # land_use # piped-gas diff --git a/message_ix_models/data/material/aluminum/aluminum_techno_economic.xlsx b/message_ix_models/data/material/aluminum/aluminum_techno_economic.xlsx index 6dd704a1dc..b6470b9033 100644 --- a/message_ix_models/data/material/aluminum/aluminum_techno_economic.xlsx +++ b/message_ix_models/data/material/aluminum/aluminum_techno_economic.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fb2bdb6289d5151adabdd80c0758bea5393955d81f698f267254278b4b60df5b -size 120465 +oid sha256:e203bd3b535e8494767e6301fa87d11eb3d8d19de2139f27108a04bcd569f10c +size 121450 diff --git a/message_ix_models/data/material/aluminum/aluminum_trade.csv b/message_ix_models/data/material/aluminum/aluminum_trade.csv new file mode 100644 index 0000000000..0a44bbc272 --- /dev/null +++ b/message_ix_models/data/material/aluminum/aluminum_trade.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b20a9561e4c08e451bf381c2733af88fa3ea0a9f776df9689b026f5434bb01d0 +size 54793 diff --git a/message_ix_models/data/material/cement/CEMENT.BvR2010.xlsx b/message_ix_models/data/material/cement/CEMENT.BvR2010.xlsx new file mode 100644 index 0000000000..cba2d5738d --- /dev/null +++ b/message_ix_models/data/material/cement/CEMENT.BvR2010.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26116c0d9a1795a10722dec4db2b873846efe9499f91f50ba0441f5149b1e73e +size 1585074 diff --git a/message_ix_models/data/material/cement/Global_cement_MESSAGE.xlsx b/message_ix_models/data/material/cement/Global_cement_MESSAGE.xlsx new file mode 100644 index 0000000000..b2ed117cf7 --- /dev/null +++ b/message_ix_models/data/material/cement/Global_cement_MESSAGE.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b691dc7be0b7e8867dfbedf9357af942fc1ca4bca2a56108908099e92ab40bd6 +size 35184 diff --git a/message_ix_models/data/material/steel_cement/demand_cement.yaml b/message_ix_models/data/material/cement/demand_cement.yaml similarity index 100% rename from message_ix_models/data/material/steel_cement/demand_cement.yaml rename to message_ix_models/data/material/cement/demand_cement.yaml diff --git a/message_ix_models/data/material/macro/macro_calibration_input_SSP2.xlsx b/message_ix_models/data/material/macro/macro_calibration_input_SSP2.xlsx new file mode 100644 index 0000000000..2823168598 --- /dev/null +++ b/message_ix_models/data/material/macro/macro_calibration_input_SSP2.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:901951bb93a404b904920ff9c615709093764e27037b765a7ed27b3b8b31da41 +size 67928 diff --git a/message_ix_models/data/material/material.tar.gz b/message_ix_models/data/material/material.tar.gz index 0827cd96d4..1041722606 100644 --- a/message_ix_models/data/material/material.tar.gz +++ b/message_ix_models/data/material/material.tar.gz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:844229a8388f7068fa77daaa4f3f906380963521729ea188d0a2cf9df381fa8f -size 21304567 +oid sha256:d4f7eb36a5154794629291060df40dd94835a612b9ab61ce1f1bac2758629daa +size 16777041 diff --git a/message_ix_models/data/material/methanol/meth_coal_additions_fs.xlsx b/message_ix_models/data/material/methanol/meth_coal_additions_fs.xlsx deleted file mode 100644 index b32b9c0a8d..0000000000 --- a/message_ix_models/data/material/methanol/meth_coal_additions_fs.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2d1b3f1949b9918a04651602299f6e015c13a9db1ed38d83cfcfa66cbb1c3a3d -size 81510 diff --git a/message_ix_models/data/material/methanol/meth_t_d_material_pars.xlsx b/message_ix_models/data/material/methanol/meth_t_d_material_pars.xlsx index de6c8362d8..14a5aa91f2 100644 --- a/message_ix_models/data/material/methanol/meth_t_d_material_pars.xlsx +++ b/message_ix_models/data/material/methanol/meth_t_d_material_pars.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ec088063f2504ef6dfcd0e7dea4e25c04639cfcc4106a8ab2f257aff2745ae79 -size 92278 +oid sha256:a46f388f94a6ab77b0168d545f1a92da7034b5a651623dc58d6f0477d2e4cc9e +size 291 diff --git a/message_ix_models/data/material/other/generic_furnace_boiler_techno_economic.xlsx b/message_ix_models/data/material/other/generic_furnace_boiler_techno_economic.xlsx index a5c494cd99..0f53894db9 100644 --- a/message_ix_models/data/material/other/generic_furnace_boiler_techno_economic.xlsx +++ b/message_ix_models/data/material/other/generic_furnace_boiler_techno_economic.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f91735eb23ce7df1372d82fc360b3d829f6e5852b026bfda26499142b2d94b35 -size 163912 +oid sha256:4c654c51123b29f4a9ce7f5bcb3ac4c5e67efc356c781839d3c472039331a9ba +size 163759 diff --git a/message_ix_models/data/material/other/residual_industry_2019.csv b/message_ix_models/data/material/other/residual_industry_2019.csv index bb6cd920b1..5cd1704da7 100644 --- a/message_ix_models/data/material/other/residual_industry_2019.csv +++ b/message_ix_models/data/material/other/residual_industry_2019.csv @@ -1,457 +1,457 @@ "MESSAGE_region","MESSAGE_sector","MESSAGE_fuel","Value","unit" "R12_AFR","cement","biogas",0,"GWa" -"R12_AFR","cement","biomass",0.0850321542364282,"GWa" -"R12_AFR","cement","coal",3.76048635312024,"GWa" +"R12_AFR","cement","biomass",0.085032154,"GWa" +"R12_AFR","cement","coal",3.760486353,"GWa" "R12_AFR","cement","crude",0,"GWa" "R12_AFR","cement","dheat",0,"GWa" -"R12_AFR","cement","electr",0.398889029173009,"GWa" +"R12_AFR","cement","electr",0.398889029,"GWa" "R12_AFR","cement","ethanol",0,"GWa" -"R12_AFR","cement","foil",0.317658407407407,"GWa" -"R12_AFR","cement","gas",0.480723997970573,"GWa" +"R12_AFR","cement","foil",0.317658407,"GWa" +"R12_AFR","cement","gas",0.480723998,"GWa" "R12_AFR","cement","geothermal",0,"GWa" "R12_AFR","cement","hydro",0,"GWa" -"R12_AFR","cement","loil",0.0220062590055809,"GWa" +"R12_AFR","cement","loil",0.022006259,"GWa" "R12_AFR","cement","nuclear",0,"GWa" "R12_AFR","cement","other",0,"GWa" "R12_AFR","cement","solar_pv",0,"GWa" "R12_AFR","cement","solar_thermal",0,"GWa" "R12_AFR","cement","tidal",0,"GWa" -"R12_AFR","cement","total",5.06479592465753,"GWa" +"R12_AFR","cement","total",5.064795925,"GWa" "R12_AFR","cement","wind",0,"GWa" "R12_AFR","industry_residual","biogas",0,"GWa" -"R12_AFR","industry_residual","biomass",19.84911122793,"GWa" -"R12_AFR","industry_residual","coal",6.97729496981228,"GWa" +"R12_AFR","industry_residual","biomass",19.84911123,"GWa" +"R12_AFR","industry_residual","coal",6.97729497,"GWa" "R12_AFR","industry_residual","crude",0,"GWa" "R12_AFR","industry_residual","dheat",0,"GWa" -"R12_AFR","industry_residual","electr",12.4824018859716,"GWa" +"R12_AFR","industry_residual","electr",12.48240189,"GWa" "R12_AFR","industry_residual","ethanol",0,"GWa" -"R12_AFR","industry_residual","foil",2.98107856303907,"GWa" -"R12_AFR","industry_residual","gas",5.2868550767377,"GWa" +"R12_AFR","industry_residual","foil",2.981078563,"GWa" +"R12_AFR","industry_residual","gas",5.286855077,"GWa" "R12_AFR","industry_residual","geothermal",0,"GWa" "R12_AFR","industry_residual","hydro",0,"GWa" -"R12_AFR","industry_residual","loil",8.66762304084221,"GWa" +"R12_AFR","industry_residual","loil",8.667623041,"GWa" "R12_AFR","industry_residual","nuclear",0,"GWa" "R12_AFR","industry_residual","other",0,"GWa" "R12_AFR","industry_residual","solar_pv",0,"GWa" "R12_AFR","industry_residual","solar_thermal",0,"GWa" "R12_AFR","industry_residual","tidal",0,"GWa" -"R12_AFR","industry_residual","total",56.24436313483,"GWa" +"R12_AFR","industry_residual","total",56.24436313,"GWa" "R12_AFR","industry_residual","wind",0,"GWa" "R12_CHN","cement","biogas",0,"GWa" "R12_CHN","cement","biomass",0,"GWa" -"R12_CHN","cement","coal",146.000992938102,"GWa" +"R12_CHN","cement","coal",146.0009929,"GWa" "R12_CHN","cement","crude",0,"GWa" -"R12_CHN","cement","dheat",0.702811319127347,"GWa" -"R12_CHN","cement","electr",37.7781187493658,"GWa" +"R12_CHN","cement","dheat",0.702811319,"GWa" +"R12_CHN","cement","electr",37.77811875,"GWa" "R12_CHN","cement","ethanol",0,"GWa" -"R12_CHN","cement","foil",1.26330045535261,"GWa" -"R12_CHN","cement","gas",15.9092360035515,"GWa" +"R12_CHN","cement","foil",1.263300455,"GWa" +"R12_CHN","cement","gas",15.909236,"GWa" "R12_CHN","cement","geothermal",0,"GWa" "R12_CHN","cement","hydro",0,"GWa" -"R12_CHN","cement","loil",4.61361366007103,"GWa" +"R12_CHN","cement","loil",4.61361366,"GWa" "R12_CHN","cement","nuclear",0,"GWa" "R12_CHN","cement","other",0,"GWa" "R12_CHN","cement","solar_pv",0,"GWa" "R12_CHN","cement","solar_thermal",0,"GWa" "R12_CHN","cement","tidal",0,"GWa" -"R12_CHN","cement","total",206.268073128361,"GWa" +"R12_CHN","cement","total",206.2680731,"GWa" "R12_CHN","cement","wind",0,"GWa" "R12_CHN","industry_residual","biogas",0,"GWa" "R12_CHN","industry_residual","biomass",0,"GWa" -"R12_CHN","industry_residual","coal",122.560451211441,"GWa" -"R12_CHN","industry_residual","crude",0.771084931506849,"GWa" -"R12_CHN","industry_residual","dheat",29.7087563368848,"GWa" -"R12_CHN","industry_residual","electr",188.322498503932,"GWa" +"R12_CHN","industry_residual","coal",122.5604512,"GWa" +"R12_CHN","industry_residual","crude",0.771084932,"GWa" +"R12_CHN","industry_residual","dheat",29.70875634,"GWa" +"R12_CHN","industry_residual","electr",188.3224985,"GWa" "R12_CHN","industry_residual","ethanol",0,"GWa" -"R12_CHN","industry_residual","foil",29.8035267643328,"GWa" -"R12_CHN","industry_residual","gas",51.7119250599949,"GWa" -"R12_CHN","industry_residual","geothermal",0.411135635464231,"GWa" +"R12_CHN","industry_residual","foil",29.80352676,"GWa" +"R12_CHN","industry_residual","gas",51.71192506,"GWa" +"R12_CHN","industry_residual","geothermal",0.411135635,"GWa" "R12_CHN","industry_residual","hydro",0,"GWa" -"R12_CHN","industry_residual","loil",16.9678174155251,"GWa" +"R12_CHN","industry_residual","loil",16.96781742,"GWa" "R12_CHN","industry_residual","nuclear",0,"GWa" "R12_CHN","industry_residual","other",0,"GWa" "R12_CHN","industry_residual","solar_pv",0,"GWa" "R12_CHN","industry_residual","solar_thermal",0,"GWa" "R12_CHN","industry_residual","tidal",0,"GWa" -"R12_CHN","industry_residual","total",440.257195868975,"GWa" +"R12_CHN","industry_residual","total",440.2571959,"GWa" "R12_CHN","industry_residual","wind",0,"GWa" -"R12_EEU","cement","biogas",0.000701801116184678,"GWa" -"R12_EEU","cement","biomass",0.339961621765601,"GWa" -"R12_EEU","cement","coal",3.50334389041096,"GWa" -"R12_EEU","cement","crude",0.0279608252156266,"GWa" -"R12_EEU","cement","dheat",0.0619840065956367,"GWa" -"R12_EEU","cement","electr",1.71202605885337,"GWa" -"R12_EEU","cement","ethanol",5.81811263318113e-05,"GWa" -"R12_EEU","cement","foil",1.24391136199899,"GWa" -"R12_EEU","cement","gas",3.27108932394724,"GWa" -"R12_EEU","cement","geothermal",0.000306914510400812,"GWa" +"R12_EEU","cement","biogas",0.000701801,"GWa" +"R12_EEU","cement","biomass",0.339961622,"GWa" +"R12_EEU","cement","coal",3.50334389,"GWa" +"R12_EEU","cement","crude",0.027960825,"GWa" +"R12_EEU","cement","dheat",0.061984007,"GWa" +"R12_EEU","cement","electr",1.712026059,"GWa" +"R12_EEU","cement","ethanol",5.82E-05,"GWa" +"R12_EEU","cement","foil",1.243911362,"GWa" +"R12_EEU","cement","gas",3.271089324,"GWa" +"R12_EEU","cement","geothermal",0.000306915,"GWa" "R12_EEU","cement","hydro",0,"GWa" -"R12_EEU","cement","loil",0.234756399543379,"GWa" +"R12_EEU","cement","loil",0.2347564,"GWa" "R12_EEU","cement","nuclear",0,"GWa" "R12_EEU","cement","other",0,"GWa" "R12_EEU","cement","solar_pv",0,"GWa" "R12_EEU","cement","solar_thermal",0,"GWa" "R12_EEU","cement","tidal",0,"GWa" -"R12_EEU","cement","total",10.3960998353628,"GWa" +"R12_EEU","cement","total",10.39609984,"GWa" "R12_EEU","cement","wind",0,"GWa" -"R12_EEU","industry_residual","biogas",0.0551720731861999,"GWa" -"R12_EEU","industry_residual","biomass",5.3665157406139,"GWa" -"R12_EEU","industry_residual","coal",2.11487620408422,"GWa" -"R12_EEU","industry_residual","crude",0.0038128398021309,"GWa" -"R12_EEU","industry_residual","dheat",1.64502646397768,"GWa" -"R12_EEU","industry_residual","electr",11.9659949361999,"GWa" -"R12_EEU","industry_residual","ethanol",7.9337899543379e-06,"GWa" -"R12_EEU","industry_residual","foil",0.423472694317605,"GWa" -"R12_EEU","industry_residual","gas",8.5059898173516,"GWa" -"R12_EEU","industry_residual","geothermal",0.00175925748351091,"GWa" +"R12_EEU","industry_residual","biogas",0.055172073,"GWa" +"R12_EEU","industry_residual","biomass",5.366515741,"GWa" +"R12_EEU","industry_residual","coal",2.114876204,"GWa" +"R12_EEU","industry_residual","crude",0.00381284,"GWa" +"R12_EEU","industry_residual","dheat",1.645026464,"GWa" +"R12_EEU","industry_residual","electr",11.96599494,"GWa" +"R12_EEU","industry_residual","ethanol",7.93E-06,"GWa" +"R12_EEU","industry_residual","foil",0.423472694,"GWa" +"R12_EEU","industry_residual","gas",8.505989817,"GWa" +"R12_EEU","industry_residual","geothermal",0.001759257,"GWa" "R12_EEU","industry_residual","hydro",0,"GWa" -"R12_EEU","industry_residual","loil",2.16385728323186,"GWa" +"R12_EEU","industry_residual","loil",2.163857283,"GWa" "R12_EEU","industry_residual","nuclear",0,"GWa" "R12_EEU","industry_residual","other",0,"GWa" "R12_EEU","industry_residual","solar_pv",0,"GWa" -"R12_EEU","industry_residual","solar_thermal",0.00216867389649924,"GWa" +"R12_EEU","industry_residual","solar_thermal",0.002168674,"GWa" "R12_EEU","industry_residual","tidal",0,"GWa" -"R12_EEU","industry_residual","total",32.2486557011669,"GWa" +"R12_EEU","industry_residual","total",32.2486557,"GWa" "R12_EEU","industry_residual","wind",0,"GWa" "R12_FSU","cement","biogas",0,"GWa" -"R12_FSU","cement","biomass",0.0210241783358701,"GWa" -"R12_FSU","cement","coal",4.03099950279046,"GWa" -"R12_FSU","cement","crude",1.17199391171994e-05,"GWa" -"R12_FSU","cement","dheat",2.68523876966007,"GWa" -"R12_FSU","cement","electr",2.50497580365297,"GWa" +"R12_FSU","cement","biomass",0.021024178,"GWa" +"R12_FSU","cement","coal",4.030999503,"GWa" +"R12_FSU","cement","crude",1.17E-05,"GWa" +"R12_FSU","cement","dheat",2.68523877,"GWa" +"R12_FSU","cement","electr",2.504975804,"GWa" "R12_FSU","cement","ethanol",0,"GWa" -"R12_FSU","cement","foil",0.307656103500761,"GWa" -"R12_FSU","cement","gas",12.2561974299848,"GWa" +"R12_FSU","cement","foil",0.307656104,"GWa" +"R12_FSU","cement","gas",12.25619743,"GWa" "R12_FSU","cement","geothermal",0,"GWa" "R12_FSU","cement","hydro",0,"GWa" -"R12_FSU","cement","loil",0.712282294013191,"GWa" +"R12_FSU","cement","loil",0.712282294,"GWa" "R12_FSU","cement","nuclear",0,"GWa" "R12_FSU","cement","other",0,"GWa" "R12_FSU","cement","solar_pv",0,"GWa" "R12_FSU","cement","solar_thermal",0,"GWa" "R12_FSU","cement","tidal",0,"GWa" -"R12_FSU","cement","total",22.5183857990868,"GWa" +"R12_FSU","cement","total",22.5183858,"GWa" "R12_FSU","cement","wind",0,"GWa" -"R12_FSU","industry_residual","biogas",0.000158517250126839,"GWa" -"R12_FSU","industry_residual","biomass",0.415253475773719,"GWa" -"R12_FSU","industry_residual","coal",3.45643630707763,"GWa" -"R12_FSU","industry_residual","crude",0.00884482496194825,"GWa" -"R12_FSU","industry_residual","dheat",25.9021065943683,"GWa" -"R12_FSU","industry_residual","electr",20.4893152510147,"GWa" +"R12_FSU","industry_residual","biogas",0.000158517,"GWa" +"R12_FSU","industry_residual","biomass",0.415253476,"GWa" +"R12_FSU","industry_residual","coal",3.456436307,"GWa" +"R12_FSU","industry_residual","crude",0.008844825,"GWa" +"R12_FSU","industry_residual","dheat",25.90210659,"GWa" +"R12_FSU","industry_residual","electr",20.48931525,"GWa" "R12_FSU","industry_residual","ethanol",0,"GWa" -"R12_FSU","industry_residual","foil",4.39048994545916,"GWa" -"R12_FSU","industry_residual","gas",21.8859038732877,"GWa" +"R12_FSU","industry_residual","foil",4.390489945,"GWa" +"R12_FSU","industry_residual","gas",21.88590387,"GWa" "R12_FSU","industry_residual","geothermal",0,"GWa" "R12_FSU","industry_residual","hydro",0,"GWa" -"R12_FSU","industry_residual","loil",10.1775712543125,"GWa" +"R12_FSU","industry_residual","loil",10.17757125,"GWa" "R12_FSU","industry_residual","nuclear",0,"GWa" "R12_FSU","industry_residual","other",0,"GWa" "R12_FSU","industry_residual","solar_pv",0,"GWa" "R12_FSU","industry_residual","solar_thermal",0,"GWa" "R12_FSU","industry_residual","tidal",0,"GWa" -"R12_FSU","industry_residual","total",86.7260803760781,"GWa" +"R12_FSU","industry_residual","total",86.72608038,"GWa" "R12_FSU","industry_residual","wind",0,"GWa" "R12_LAM","cement","biogas",0,"GWa" -"R12_LAM","cement","biomass",2.80760947792998,"GWa" -"R12_LAM","cement","coal",1.83524570928463,"GWa" -"R12_LAM","cement","crude",0.00108360882800609,"GWa" +"R12_LAM","cement","biomass",2.807609478,"GWa" +"R12_LAM","cement","coal",1.835245709,"GWa" +"R12_LAM","cement","crude",0.001083609,"GWa" "R12_LAM","cement","dheat",0,"GWa" -"R12_LAM","cement","electr",3.07436648756976,"GWa" -"R12_LAM","cement","ethanol",0.0087682864028412,"GWa" -"R12_LAM","cement","foil",8.41689883384069,"GWa" -"R12_LAM","cement","gas",4.73865035032978,"GWa" +"R12_LAM","cement","electr",3.074366488,"GWa" +"R12_LAM","cement","ethanol",0.008768286,"GWa" +"R12_LAM","cement","foil",8.416898834,"GWa" +"R12_LAM","cement","gas",4.73865035,"GWa" "R12_LAM","cement","geothermal",0,"GWa" "R12_LAM","cement","hydro",0,"GWa" -"R12_LAM","cement","loil",0.359172948756976,"GWa" +"R12_LAM","cement","loil",0.359172949,"GWa" "R12_LAM","cement","nuclear",0,"GWa" "R12_LAM","cement","other",0,"GWa" "R12_LAM","cement","solar_pv",0,"GWa" "R12_LAM","cement","solar_thermal",0,"GWa" "R12_LAM","cement","tidal",0,"GWa" -"R12_LAM","cement","total",21.2417962526636,"GWa" +"R12_LAM","cement","total",21.24179625,"GWa" "R12_LAM","cement","wind",0,"GWa" -"R12_LAM","industry_residual","biogas",0.00344675608828006,"GWa" -"R12_LAM","industry_residual","biomass",50.5883076992643,"GWa" -"R12_LAM","industry_residual","coal",5.36297038977676,"GWa" -"R12_LAM","industry_residual","crude",0.280360014332826,"GWa" +"R12_LAM","industry_residual","biogas",0.003446756,"GWa" +"R12_LAM","industry_residual","biomass",50.5883077,"GWa" +"R12_LAM","industry_residual","coal",5.36297039,"GWa" +"R12_LAM","industry_residual","crude",0.280360014,"GWa" "R12_LAM","industry_residual","dheat",0,"GWa" -"R12_LAM","industry_residual","electr",45.880443149036,"GWa" -"R12_LAM","industry_residual","ethanol",0.175781792871639,"GWa" -"R12_LAM","industry_residual","foil",8.74841008295282,"GWa" -"R12_LAM","industry_residual","gas",21.8628289209792,"GWa" +"R12_LAM","industry_residual","electr",45.88044315,"GWa" +"R12_LAM","industry_residual","ethanol",0.175781793,"GWa" +"R12_LAM","industry_residual","foil",8.748410083,"GWa" +"R12_LAM","industry_residual","gas",21.86282892,"GWa" "R12_LAM","industry_residual","geothermal",0,"GWa" "R12_LAM","industry_residual","hydro",0,"GWa" -"R12_LAM","industry_residual","loil",16.1413009667681,"GWa" +"R12_LAM","industry_residual","loil",16.14130097,"GWa" "R12_LAM","industry_residual","nuclear",0,"GWa" "R12_LAM","industry_residual","other",0,"GWa" "R12_LAM","industry_residual","solar_pv",0,"GWa" -"R12_LAM","industry_residual","solar_thermal",0.0234145104008118,"GWa" +"R12_LAM","industry_residual","solar_thermal",0.02341451,"GWa" "R12_LAM","industry_residual","tidal",0,"GWa" -"R12_LAM","industry_residual","total",149.067266095129,"GWa" +"R12_LAM","industry_residual","total",149.0672661,"GWa" "R12_LAM","industry_residual","wind",0,"GWa" "R12_MEA","cement","biogas",0,"GWa" -"R12_MEA","cement","biomass",0.0321957889396246,"GWa" -"R12_MEA","cement","coal",3.75041373515982,"GWa" +"R12_MEA","cement","biomass",0.032195789,"GWa" +"R12_MEA","cement","coal",3.750413735,"GWa" "R12_MEA","cement","crude",0,"GWa" "R12_MEA","cement","dheat",0,"GWa" -"R12_MEA","cement","electr",0.954649594114663,"GWa" +"R12_MEA","cement","electr",0.954649594,"GWa" "R12_MEA","cement","ethanol",0,"GWa" -"R12_MEA","cement","foil",1.975315456621,"GWa" -"R12_MEA","cement","gas",6.48388289218671,"GWa" +"R12_MEA","cement","foil",1.975315457,"GWa" +"R12_MEA","cement","gas",6.483882892,"GWa" "R12_MEA","cement","geothermal",0,"GWa" "R12_MEA","cement","hydro",0,"GWa" -"R12_MEA","cement","loil",0.123194100963978,"GWa" +"R12_MEA","cement","loil",0.123194101,"GWa" "R12_MEA","cement","nuclear",0,"GWa" "R12_MEA","cement","other",0,"GWa" "R12_MEA","cement","solar_pv",0,"GWa" "R12_MEA","cement","solar_thermal",0,"GWa" "R12_MEA","cement","tidal",0,"GWa" -"R12_MEA","cement","total",13.3196515679858,"GWa" +"R12_MEA","cement","total",13.31965157,"GWa" "R12_MEA","cement","wind",0,"GWa" "R12_MEA","industry_residual","biogas",0,"GWa" -"R12_MEA","industry_residual","biomass",0.956222146118721,"GWa" -"R12_MEA","industry_residual","coal",2.90625796385084,"GWa" -"R12_MEA","industry_residual","crude",0.941807191780822,"GWa" +"R12_MEA","industry_residual","biomass",0.956222146,"GWa" +"R12_MEA","industry_residual","coal",2.906257964,"GWa" +"R12_MEA","industry_residual","crude",0.941807192,"GWa" "R12_MEA","industry_residual","dheat",0,"GWa" -"R12_MEA","industry_residual","electr",31.317901366692,"GWa" +"R12_MEA","industry_residual","electr",31.31790137,"GWa" "R12_MEA","industry_residual","ethanol",0,"GWa" -"R12_MEA","industry_residual","foil",23.646795207382,"GWa" -"R12_MEA","industry_residual","gas",116.661304997844,"GWa" +"R12_MEA","industry_residual","foil",23.64679521,"GWa" +"R12_MEA","industry_residual","gas",116.661305,"GWa" "R12_MEA","industry_residual","geothermal",0,"GWa" "R12_MEA","industry_residual","hydro",0,"GWa" -"R12_MEA","industry_residual","loil",18.0635196705987,"GWa" +"R12_MEA","industry_residual","loil",18.06351967,"GWa" "R12_MEA","industry_residual","nuclear",0,"GWa" "R12_MEA","industry_residual","other",0,"GWa" "R12_MEA","industry_residual","solar_pv",0,"GWa" -"R12_MEA","industry_residual","solar_thermal",0.00347507927447996,"GWa" +"R12_MEA","industry_residual","solar_thermal",0.003475079,"GWa" "R12_MEA","industry_residual","tidal",0,"GWa" -"R12_MEA","industry_residual","total",194.497283937468,"GWa" +"R12_MEA","industry_residual","total",194.4972839,"GWa" "R12_MEA","industry_residual","wind",0,"GWa" "R12_NAM","cement","biogas",0,"GWa" -"R12_NAM","cement","biomass",0.582628031456114,"GWa" -"R12_NAM","cement","coal",5.91121964129883,"GWa" +"R12_NAM","cement","biomass",0.582628031,"GWa" +"R12_NAM","cement","coal",5.911219641,"GWa" "R12_NAM","cement","crude",0,"GWa" -"R12_NAM","cement","dheat",0.00416448503297818,"GWa" -"R12_NAM","cement","electr",3.53510640715373,"GWa" -"R12_NAM","cement","ethanol",0.0141239218670726,"GWa" -"R12_NAM","cement","foil",1.57838338102486,"GWa" -"R12_NAM","cement","gas",12.143720920345,"GWa" +"R12_NAM","cement","dheat",0.004164485,"GWa" +"R12_NAM","cement","electr",3.535106407,"GWa" +"R12_NAM","cement","ethanol",0.014123922,"GWa" +"R12_NAM","cement","foil",1.578383381,"GWa" +"R12_NAM","cement","gas",12.14372092,"GWa" "R12_NAM","cement","geothermal",0,"GWa" "R12_NAM","cement","hydro",0,"GWa" -"R12_NAM","cement","loil",0.399005327245053,"GWa" +"R12_NAM","cement","loil",0.399005327,"GWa" "R12_NAM","cement","nuclear",0,"GWa" "R12_NAM","cement","other",0,"GWa" "R12_NAM","cement","solar_pv",0,"GWa" "R12_NAM","cement","solar_thermal",0,"GWa" "R12_NAM","cement","tidal",0,"GWa" -"R12_NAM","cement","total",24.1683521182141,"GWa" +"R12_NAM","cement","total",24.16835212,"GWa" "R12_NAM","cement","wind",0,"GWa" -"R12_NAM","industry_residual","biogas",0.0878551877219685,"GWa" -"R12_NAM","industry_residual","biomass",49.5989345636733,"GWa" -"R12_NAM","industry_residual","coal",8.41278815613902,"GWa" +"R12_NAM","industry_residual","biogas",0.087855188,"GWa" +"R12_NAM","industry_residual","biomass",49.59893456,"GWa" +"R12_NAM","industry_residual","coal",8.412788156,"GWa" "R12_NAM","industry_residual","crude",0,"GWa" -"R12_NAM","industry_residual","dheat",2.45285132864028,"GWa" -"R12_NAM","industry_residual","electr",67.6546148891426,"GWa" -"R12_NAM","industry_residual","ethanol",1.07024346778285,"GWa" -"R12_NAM","industry_residual","foil",2.2673897798072,"GWa" -"R12_NAM","industry_residual","gas",88.4095425499746,"GWa" +"R12_NAM","industry_residual","dheat",2.452851329,"GWa" +"R12_NAM","industry_residual","electr",67.65461489,"GWa" +"R12_NAM","industry_residual","ethanol",1.070243468,"GWa" +"R12_NAM","industry_residual","foil",2.26738978,"GWa" +"R12_NAM","industry_residual","gas",88.40954255,"GWa" "R12_NAM","industry_residual","geothermal",0,"GWa" "R12_NAM","industry_residual","hydro",0,"GWa" -"R12_NAM","industry_residual","loil",23.8258160388128,"GWa" +"R12_NAM","industry_residual","loil",23.82581604,"GWa" "R12_NAM","industry_residual","nuclear",0,"GWa" "R12_NAM","industry_residual","other",0,"GWa" "R12_NAM","industry_residual","solar_pv",0,"GWa" "R12_NAM","industry_residual","solar_thermal",0,"GWa" "R12_NAM","industry_residual","tidal",0,"GWa" -"R12_NAM","industry_residual","total",243.780036596271,"GWa" +"R12_NAM","industry_residual","total",243.7800366,"GWa" "R12_NAM","industry_residual","wind",0,"GWa" -"R12_PAO","cement","biogas",0.00452054515474378,"GWa" -"R12_PAO","cement","biomass",0.237255472856418,"GWa" -"R12_PAO","cement","coal",5.10774114129883,"GWa" +"R12_PAO","cement","biogas",0.004520545,"GWa" +"R12_PAO","cement","biomass",0.237255473,"GWa" +"R12_PAO","cement","coal",5.107741141,"GWa" "R12_PAO","cement","crude",0,"GWa" "R12_PAO","cement","dheat",0,"GWa" -"R12_PAO","cement","electr",2.11302347995941,"GWa" +"R12_PAO","cement","electr",2.11302348,"GWa" "R12_PAO","cement","ethanol",0,"GWa" -"R12_PAO","cement","foil",1.51895156139016,"GWa" -"R12_PAO","cement","gas",2.21860799289701,"GWa" +"R12_PAO","cement","foil",1.518951561,"GWa" +"R12_PAO","cement","gas",2.218607993,"GWa" "R12_PAO","cement","geothermal",0,"GWa" "R12_PAO","cement","hydro",0,"GWa" -"R12_PAO","cement","loil",1.04602204008118,"GWa" +"R12_PAO","cement","loil",1.04602204,"GWa" "R12_PAO","cement","nuclear",0,"GWa" "R12_PAO","cement","other",0,"GWa" "R12_PAO","cement","solar_pv",0,"GWa" "R12_PAO","cement","solar_thermal",0,"GWa" "R12_PAO","cement","tidal",0,"GWa" -"R12_PAO","cement","total",12.2461219490107,"GWa" +"R12_PAO","cement","total",12.24612195,"GWa" "R12_PAO","cement","wind",0,"GWa" -"R12_PAO","industry_residual","biogas",0.0287566237950279,"GWa" -"R12_PAO","industry_residual","biomass",7.36504221549975,"GWa" -"R12_PAO","industry_residual","coal",4.59444195104008,"GWa" -"R12_PAO","industry_residual","crude",0.0198965911973617,"GWa" +"R12_PAO","industry_residual","biogas",0.028756624,"GWa" +"R12_PAO","industry_residual","biomass",7.365042215,"GWa" +"R12_PAO","industry_residual","coal",4.594441951,"GWa" +"R12_PAO","industry_residual","crude",0.019896591,"GWa" "R12_PAO","industry_residual","dheat",0,"GWa" -"R12_PAO","industry_residual","electr",26.5017408370117,"GWa" +"R12_PAO","industry_residual","electr",26.50174084,"GWa" "R12_PAO","industry_residual","ethanol",0,"GWa" -"R12_PAO","industry_residual","foil",1.33919018772197,"GWa" -"R12_PAO","industry_residual","gas",10.1017765168696,"GWa" -"R12_PAO","industry_residual","geothermal",0.152527187975647,"GWa" +"R12_PAO","industry_residual","foil",1.339190188,"GWa" +"R12_PAO","industry_residual","gas",10.10177652,"GWa" +"R12_PAO","industry_residual","geothermal",0.152527188,"GWa" "R12_PAO","industry_residual","hydro",0,"GWa" -"R12_PAO","industry_residual","loil",11.1178578463978,"GWa" +"R12_PAO","industry_residual","loil",11.11785785,"GWa" "R12_PAO","industry_residual","nuclear",0,"GWa" "R12_PAO","industry_residual","other",0,"GWa" "R12_PAO","industry_residual","solar_pv",0,"GWa" "R12_PAO","industry_residual","solar_thermal",0,"GWa" "R12_PAO","industry_residual","tidal",0,"GWa" -"R12_PAO","industry_residual","total",61.2212311997717,"GWa" +"R12_PAO","industry_residual","total",61.2212312,"GWa" "R12_PAO","industry_residual","wind",0,"GWa" "R12_PAS","cement","biogas",0,"GWa" -"R12_PAS","cement","biomass",0.00293549594114663,"GWa" -"R12_PAS","cement","coal",25.8751749705733,"GWa" +"R12_PAS","cement","biomass",0.002935496,"GWa" +"R12_PAS","cement","coal",25.87517497,"GWa" "R12_PAS","cement","crude",0,"GWa" -"R12_PAS","cement","dheat",0.003324466768138,"GWa" -"R12_PAS","cement","electr",2.94770652993404,"GWa" -"R12_PAS","cement","ethanol",0.00122222222222222,"GWa" -"R12_PAS","cement","foil",1.29404073947235,"GWa" -"R12_PAS","cement","gas",1.99021418645358,"GWa" +"R12_PAS","cement","dheat",0.003324467,"GWa" +"R12_PAS","cement","electr",2.94770653,"GWa" +"R12_PAS","cement","ethanol",0.001222222,"GWa" +"R12_PAS","cement","foil",1.294040739,"GWa" +"R12_PAS","cement","gas",1.990214186,"GWa" "R12_PAS","cement","geothermal",0,"GWa" "R12_PAS","cement","hydro",0,"GWa" -"R12_PAS","cement","loil",0.904003777524099,"GWa" +"R12_PAS","cement","loil",0.904003778,"GWa" "R12_PAS","cement","nuclear",0,"GWa" "R12_PAS","cement","other",0,"GWa" "R12_PAS","cement","solar_pv",0,"GWa" "R12_PAS","cement","solar_thermal",0,"GWa" "R12_PAS","cement","tidal",0,"GWa" -"R12_PAS","cement","total",33.0186215517504,"GWa" +"R12_PAS","cement","total",33.01862155,"GWa" "R12_PAS","cement","wind",0,"GWa" -"R12_PAS","industry_residual","biogas",0.871191885464231,"GWa" -"R12_PAS","industry_residual","biomass",23.7981536910198,"GWa" -"R12_PAS","industry_residual","coal",15.9693980285388,"GWa" +"R12_PAS","industry_residual","biogas",0.871191885,"GWa" +"R12_PAS","industry_residual","biomass",23.79815369,"GWa" +"R12_PAS","industry_residual","coal",15.96939803,"GWa" "R12_PAS","industry_residual","crude",0,"GWa" -"R12_PAS","industry_residual","dheat",1.63626997767631,"GWa" -"R12_PAS","industry_residual","electr",57.8149666816337,"GWa" -"R12_PAS","industry_residual","ethanol",0.438693816590563,"GWa" -"R12_PAS","industry_residual","foil",7.97819601851852,"GWa" -"R12_PAS","industry_residual","gas",34.2437281334348,"GWa" -"R12_PAS","industry_residual","geothermal",0.00597564370877727,"GWa" +"R12_PAS","industry_residual","dheat",1.636269978,"GWa" +"R12_PAS","industry_residual","electr",57.81496668,"GWa" +"R12_PAS","industry_residual","ethanol",0.438693817,"GWa" +"R12_PAS","industry_residual","foil",7.978196019,"GWa" +"R12_PAS","industry_residual","gas",34.24372813,"GWa" +"R12_PAS","industry_residual","geothermal",0.005975644,"GWa" "R12_PAS","industry_residual","hydro",0,"GWa" -"R12_PAS","industry_residual","loil",16.384367854896,"GWa" +"R12_PAS","industry_residual","loil",16.38436785,"GWa" "R12_PAS","industry_residual","nuclear",0,"GWa" "R12_PAS","industry_residual","other",0,"GWa" "R12_PAS","industry_residual","solar_pv",0,"GWa" -"R12_PAS","industry_residual","solar_thermal",0.00238069824961948,"GWa" +"R12_PAS","industry_residual","solar_thermal",0.002380698,"GWa" "R12_PAS","industry_residual","tidal",0,"GWa" -"R12_PAS","industry_residual","total",159.143321675038,"GWa" +"R12_PAS","industry_residual","total",159.1433217,"GWa" "R12_PAS","industry_residual","wind",0,"GWa" "R12_RCPA","cement","biogas",0,"GWa" -"R12_RCPA","cement","biomass",0.544432749873161,"GWa" -"R12_RCPA","cement","coal",8.40656611973617,"GWa" +"R12_RCPA","cement","biomass",0.54443275,"GWa" +"R12_RCPA","cement","coal",8.40656612,"GWa" "R12_RCPA","cement","crude",0,"GWa" "R12_RCPA","cement","dheat",0,"GWa" -"R12_RCPA","cement","electr",1.19181735159817,"GWa" +"R12_RCPA","cement","electr",1.191817352,"GWa" "R12_RCPA","cement","ethanol",0,"GWa" -"R12_RCPA","cement","foil",0.0046310502283105,"GWa" -"R12_RCPA","cement","gas",0.199781151699645,"GWa" +"R12_RCPA","cement","foil",0.00463105,"GWa" +"R12_RCPA","cement","gas",0.199781152,"GWa" "R12_RCPA","cement","geothermal",0,"GWa" "R12_RCPA","cement","hydro",0,"GWa" -"R12_RCPA","cement","loil",0.289453196347032,"GWa" +"R12_RCPA","cement","loil",0.289453196,"GWa" "R12_RCPA","cement","nuclear",0,"GWa" "R12_RCPA","cement","other",0,"GWa" "R12_RCPA","cement","solar_pv",0,"GWa" "R12_RCPA","cement","solar_thermal",0,"GWa" "R12_RCPA","cement","tidal",0,"GWa" -"R12_RCPA","cement","total",10.6366816194825,"GWa" +"R12_RCPA","cement","total",10.63668162,"GWa" "R12_RCPA","cement","wind",0,"GWa" "R12_RCPA","industry_residual","biogas",0,"GWa" -"R12_RCPA","industry_residual","biomass",7.26727162290715,"GWa" -"R12_RCPA","industry_residual","coal",16.885457018899,"GWa" +"R12_RCPA","industry_residual","biomass",7.267271623,"GWa" +"R12_RCPA","industry_residual","coal",16.88545702,"GWa" "R12_RCPA","industry_residual","crude",0,"GWa" -"R12_RCPA","industry_residual","dheat",0.386870243531202,"GWa" -"R12_RCPA","industry_residual","electr",10.5830636732623,"GWa" +"R12_RCPA","industry_residual","dheat",0.386870244,"GWa" +"R12_RCPA","industry_residual","electr",10.58306367,"GWa" "R12_RCPA","industry_residual","ethanol",0,"GWa" -"R12_RCPA","industry_residual","foil",0.116575681760528,"GWa" -"R12_RCPA","industry_residual","gas",0.126814802130898,"GWa" +"R12_RCPA","industry_residual","foil",0.116575682,"GWa" +"R12_RCPA","industry_residual","gas",0.126814802,"GWa" "R12_RCPA","industry_residual","geothermal",0,"GWa" "R12_RCPA","industry_residual","hydro",0,"GWa" -"R12_RCPA","industry_residual","loil",3.34785179160325,"GWa" +"R12_RCPA","industry_residual","loil",3.347851792,"GWa" "R12_RCPA","industry_residual","nuclear",0,"GWa" "R12_RCPA","industry_residual","other",0,"GWa" "R12_RCPA","industry_residual","solar_pv",0,"GWa" "R12_RCPA","industry_residual","solar_thermal",0,"GWa" "R12_RCPA","industry_residual","tidal",0,"GWa" -"R12_RCPA","industry_residual","total",38.7139045106545,"GWa" +"R12_RCPA","industry_residual","total",38.71390451,"GWa" "R12_RCPA","industry_residual","wind",0,"GWa" "R12_SAS","cement","biogas",0,"GWa" "R12_SAS","cement","biomass",0,"GWa" -"R12_SAS","cement","coal",30.1599773505835,"GWa" +"R12_SAS","cement","coal",30.15997735,"GWa" "R12_SAS","cement","crude",0,"GWa" "R12_SAS","cement","dheat",0,"GWa" -"R12_SAS","cement","electr",0.431582522070015,"GWa" +"R12_SAS","cement","electr",0.431582522,"GWa" "R12_SAS","cement","ethanol",0,"GWa" -"R12_SAS","cement","foil",8.9286847166413,"GWa" -"R12_SAS","cement","gas",0.0101583967529173,"GWa" +"R12_SAS","cement","foil",8.928684717,"GWa" +"R12_SAS","cement","gas",0.010158397,"GWa" "R12_SAS","cement","geothermal",0,"GWa" "R12_SAS","cement","hydro",0,"GWa" -"R12_SAS","cement","loil",0.344598557077626,"GWa" +"R12_SAS","cement","loil",0.344598557,"GWa" "R12_SAS","cement","nuclear",0,"GWa" "R12_SAS","cement","other",0,"GWa" "R12_SAS","cement","solar_pv",0,"GWa" "R12_SAS","cement","solar_thermal",0,"GWa" "R12_SAS","cement","tidal",0,"GWa" -"R12_SAS","cement","total",39.8750015459158,"GWa" +"R12_SAS","cement","total",39.87500155,"GWa" "R12_SAS","cement","wind",0,"GWa" -"R12_SAS","industry_residual","biogas",0.149699873160832,"GWa" -"R12_SAS","industry_residual","biomass",92.8297218163369,"GWa" -"R12_SAS","industry_residual","coal",39.0187739780568,"GWa" +"R12_SAS","industry_residual","biogas",0.149699873,"GWa" +"R12_SAS","industry_residual","biomass",92.82972182,"GWa" +"R12_SAS","industry_residual","coal",39.01877398,"GWa" "R12_SAS","industry_residual","crude",0,"GWa" "R12_SAS","industry_residual","dheat",0,"GWa" -"R12_SAS","industry_residual","electr",56.9387698878742,"GWa" +"R12_SAS","industry_residual","electr",56.93876989,"GWa" "R12_SAS","industry_residual","ethanol",0,"GWa" -"R12_SAS","industry_residual","foil",17.1332351146626,"GWa" -"R12_SAS","industry_residual","gas",20.9828885273973,"GWa" +"R12_SAS","industry_residual","foil",17.13323511,"GWa" +"R12_SAS","industry_residual","gas",20.98288853,"GWa" "R12_SAS","industry_residual","geothermal",0,"GWa" "R12_SAS","industry_residual","hydro",0,"GWa" -"R12_SAS","industry_residual","loil",9.36286420294267,"GWa" +"R12_SAS","industry_residual","loil",9.362864203,"GWa" "R12_SAS","industry_residual","nuclear",0,"GWa" "R12_SAS","industry_residual","other",0,"GWa" "R12_SAS","industry_residual","solar_pv",0,"GWa" -"R12_SAS","industry_residual","solar_thermal",0.0880319317605277,"GWa" +"R12_SAS","industry_residual","solar_thermal",0.088031932,"GWa" "R12_SAS","industry_residual","tidal",0,"GWa" -"R12_SAS","industry_residual","total",236.503984701547,"GWa" +"R12_SAS","industry_residual","total",236.5039847,"GWa" "R12_SAS","industry_residual","wind",0,"GWa" -"R12_WEU","cement","biogas",0.0266636651445967,"GWa" -"R12_WEU","cement","biomass",1.95073096727549,"GWa" -"R12_WEU","cement","coal",9.04860969406393,"GWa" +"R12_WEU","cement","biogas",0.026663665,"GWa" +"R12_WEU","cement","biomass",1.950730967,"GWa" +"R12_WEU","cement","coal",9.048609694,"GWa" "R12_WEU","cement","crude",0,"GWa" -"R12_WEU","cement","dheat",0.2662558231862,"GWa" -"R12_WEU","cement","electr",6.99998547412481,"GWa" -"R12_WEU","cement","ethanol",0.0125287321156773,"GWa" -"R12_WEU","cement","foil",8.15165693886352,"GWa" -"R12_WEU","cement","gas",14.9128074299848,"GWa" +"R12_WEU","cement","dheat",0.266255823,"GWa" +"R12_WEU","cement","electr",6.999985474,"GWa" +"R12_WEU","cement","ethanol",0.012528732,"GWa" +"R12_WEU","cement","foil",8.151656939,"GWa" +"R12_WEU","cement","gas",14.91280743,"GWa" "R12_WEU","cement","geothermal",0,"GWa" "R12_WEU","cement","hydro",0,"GWa" -"R12_WEU","cement","loil",1.041478543379,"GWa" +"R12_WEU","cement","loil",1.041478543,"GWa" "R12_WEU","cement","nuclear",0,"GWa" "R12_WEU","cement","other",0,"GWa" "R12_WEU","cement","solar_pv",0,"GWa" -"R12_WEU","cement","solar_thermal",5.24606798579401e-05,"GWa" +"R12_WEU","cement","solar_thermal",5.25E-05,"GWa" "R12_WEU","cement","tidal",0,"GWa" -"R12_WEU","cement","total",42.4107687409944,"GWa" +"R12_WEU","cement","total",42.41076874,"GWa" "R12_WEU","cement","wind",0,"GWa" -"R12_WEU","industry_residual","biogas",0.701677519533232,"GWa" -"R12_WEU","industry_residual","biomass",22.9006996295028,"GWa" -"R12_WEU","industry_residual","coal",7.80151846106038,"GWa" +"R12_WEU","industry_residual","biogas",0.70167752,"GWa" +"R12_WEU","industry_residual","biomass",22.90069963,"GWa" +"R12_WEU","industry_residual","coal",7.801518461,"GWa" "R12_WEU","industry_residual","crude",0,"GWa" -"R12_WEU","industry_residual","dheat",9.94267000443937,"GWa" -"R12_WEU","industry_residual","electr",68.6440699133689,"GWa" -"R12_WEU","industry_residual","ethanol",0.519138321410452,"GWa" -"R12_WEU","industry_residual","foil",2.88213641476408,"GWa" -"R12_WEU","industry_residual","gas",52.6392076416793,"GWa" -"R12_WEU","industry_residual","geothermal",0.0188756183409437,"GWa" +"R12_WEU","industry_residual","dheat",9.942670004,"GWa" +"R12_WEU","industry_residual","electr",68.64406991,"GWa" +"R12_WEU","industry_residual","ethanol",0.519138321,"GWa" +"R12_WEU","industry_residual","foil",2.882136415,"GWa" +"R12_WEU","industry_residual","gas",52.63920764,"GWa" +"R12_WEU","industry_residual","geothermal",0.018875618,"GWa" "R12_WEU","industry_residual","hydro",0,"GWa" -"R12_WEU","industry_residual","loil",13.3996508515982,"GWa" +"R12_WEU","industry_residual","loil",13.39965085,"GWa" "R12_WEU","industry_residual","nuclear",0,"GWa" "R12_WEU","industry_residual","other",0,"GWa" "R12_WEU","industry_residual","solar_pv",0,"GWa" -"R12_WEU","industry_residual","solar_thermal",0.408534015093861,"GWa" +"R12_WEU","industry_residual","solar_thermal",0.408534015,"GWa" "R12_WEU","industry_residual","tidal",0,"GWa" -"R12_WEU","industry_residual","total",179.85817890931,"GWa" +"R12_WEU","industry_residual","total",179.8581789,"GWa" "R12_WEU","industry_residual","wind",0,"GWa" diff --git a/message_ix_models/data/material/petrochemicals/petrochemicals_techno_economic.xlsx b/message_ix_models/data/material/petrochemicals/petrochemicals_techno_economic.xlsx index 922d021bde..ba10803b2a 100644 --- a/message_ix_models/data/material/petrochemicals/petrochemicals_techno_economic.xlsx +++ b/message_ix_models/data/material/petrochemicals/petrochemicals_techno_economic.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:31db6a9e64a18f70db38fd36ac9dd1e2ac64e377117d4cecc738bcd6f7965ef7 -size 677693 +oid sha256:a6178b6975ee5444badcf33e5ce1d78eba40478eb599918b4634e29e5f18f362 +size 677733 diff --git a/message_ix_models/data/material/set.yaml b/message_ix_models/data/material/set.yaml index a3807d401f..e5310f6c72 100644 --- a/message_ix_models/data/material/set.yaml +++ b/message_ix_models/data/material/set.yaml @@ -1,11 +1,19 @@ -# Configuration for message_data.model.material -# -# NB these values are currently for the nitrogen fertilizer materials -# accounting. Different sets can be created once there are 2+ different -# categories of materials. +# Set configuration for the MESSAGE-Materials model +# For each set in the MESSAGEix framework, the group contains: +# - 'require': elements that must be present for the model to be set up. +# - 'remove': elements to remove. +# - 'add': elements to add. common: + node: + add: + - R12_GLB + + technology: + add: + - extract__freshwater_supply + commodity: require: - coal @@ -18,10 +26,10 @@ common: - hydrogen - lh2 - d_heat - add: - biomass + add: - water - - fresh_water + - fresh_water_supply level: require: @@ -33,9 +41,11 @@ common: - export - import - type_tec: - add: - - industry +# Not used at the moment +# maybe to be used in updated legacy reporting +# type_tec: +# add: +# - industry mode: add: @@ -56,15 +66,21 @@ common: - CO2_transformation year: - add: + require: - 1980 - 1990 + - 1995 - 2000 + - 2005 - 2010 + - 2015 - type_year: + unit: add: - - 2010 + - t/kW + - Mt + - Mt/yr + - USD/kW generic: commodity: @@ -169,6 +185,7 @@ generic: - fc_h2_resins - solar_resins - dheat_resins + mode: add: - low_temp @@ -279,10 +296,17 @@ petro_chemicals: - steam_cracker steel: + mode: + add: + - M3 + - M4 + commodity: add: - steel - pig_iron + - pig_iron_dummy + - charcoal - sponge_iron - sinter_iron - pellet_iron @@ -292,6 +316,7 @@ steel: - slag_iron - co_gas - bf_gas + - off_gas level: add: @@ -310,6 +335,7 @@ steel: - dummy_emission - end_of_life - dummy_end_of_life + - dummy_ccs technology: add: @@ -317,7 +343,12 @@ steel: - sinter_steel - pellet_steel - bf_steel + - bf_ccs_steel - dri_steel + - dri_gas_steel + - dri_gas_ccs_steel + - dri_h2_steel + - sr_steel - bof_steel - eaf_steel - prep_secondary_steel_1 @@ -335,6 +366,8 @@ steel: - export_steel - other_EOL_steel - total_EOL_steel + - bf_biomass_steel + - prod_charcoal_steel relation: add: @@ -353,6 +386,32 @@ steel: - ["steel","useful_material"] - ["steel","final_material"] + addon: + add: + - bf_ccs_steel + - dri_gas_ccs_steel + - dri_gas_steel + - dri_h2_steel + + type_addon: + add: + - bf_ccs_steel_addon + - dri_gas_ccs_steel_addon + - dri_steel_addon + + map_tec_addon: + add: + - [bf_steel, bf_ccs_steel_addon] + - [dri_gas_steel, dri_gas_ccs_steel_addon] + - [dri_steel, dri_steel_addon] + + cat_addon: + add: + - [bf_ccs_steel_addon, bf_ccs_steel] + - [dri_gas_ccs_steel_addon, dri_gas_ccs_steel] + - [dri_steel_addon, dri_gas_steel] + - [dri_steel_addon, dri_h2_steel] + cement: commodity: add: @@ -406,22 +465,25 @@ cement: type_addon: add: - - ccs_cement + - wet_ccs_cement + - dry_ccs_cement map_tec_addon: add: - - [clinker_dry_cement, ccs_cement] - - [clinker_wet_cement, ccs_cement] + - [clinker_dry_cement, dry_ccs_cement] + - [clinker_wet_cement, wet_ccs_cement] cat_addon: add: - - [ccs_cement, clinker_dry_ccs_cement] - - [ccs_cement, clinker_wet_ccs_cement] + - [dry_ccs_cement, clinker_dry_ccs_cement] + - [wet_ccs_cement, clinker_wet_ccs_cement] aluminum: commodity: add: - aluminum + - bauxite + - alumina level: add: @@ -434,6 +496,7 @@ aluminum: - useful_material - product - secondary_material + - primary_material - demand - end_of_life - dummy_end_of_life_1 @@ -443,6 +506,7 @@ aluminum: technology: add: + - refining_aluminum - soderberg_aluminum - prebake_aluminum - secondary_aluminum @@ -454,7 +518,7 @@ aluminum: - scrap_recovery_aluminum_1 - scrap_recovery_aluminum_2 - scrap_recovery_aluminum_3 - - DUMMY_alumina_supply + - DUMMY_bauxite_supply - trade_aluminum - import_aluminum - export_aluminum @@ -479,7 +543,6 @@ fertilizer: commodity: require: - electr - #- freshwater_supply removed for SSP_dev developments, is added in build if missing add: - NH3 - Fertilizer Use|Nitrogen diff --git a/message_ix_models/data/material/steel/Global_steel_MESSAGE.xlsx b/message_ix_models/data/material/steel/Global_steel_MESSAGE.xlsx new file mode 100644 index 0000000000..5d1b014fce --- /dev/null +++ b/message_ix_models/data/material/steel/Global_steel_MESSAGE.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bdfbc1ea15b443308f38ed49426e435127e91f6a5b6338b6fef6d4acef96c16 +size 105283 diff --git a/message_ix_models/data/material/steel/STEEL_database_2012.xlsx b/message_ix_models/data/material/steel/STEEL_database_2012.xlsx new file mode 100644 index 0000000000..e2427ae1a2 --- /dev/null +++ b/message_ix_models/data/material/steel/STEEL_database_2012.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1151a9491e36d951ba243dffbdda2598aa3c0c43c49813daa8d626c5420e2a44 +size 568871 diff --git a/message_ix_models/data/material/steel_cement/demand_steel.yaml b/message_ix_models/data/material/steel/demand_steel.yaml similarity index 100% rename from message_ix_models/data/material/steel_cement/demand_steel.yaml rename to message_ix_models/data/material/steel/demand_steel.yaml diff --git a/message_ix_models/data/material/steel/worldsteel_steel_trade.xlsx b/message_ix_models/data/material/steel/worldsteel_steel_trade.xlsx new file mode 100644 index 0000000000..367fed36f7 --- /dev/null +++ b/message_ix_models/data/material/steel/worldsteel_steel_trade.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6cb38631f26c4ed34ddd48a0973b73f9df259640fcd58495a08d5f31b8f6086 +size 26646 diff --git a/message_ix_models/data/material/steel_cement/CEMENT.BvR2010.xlsx b/message_ix_models/data/material/steel_cement/CEMENT.BvR2010.xlsx deleted file mode 100644 index d73aa3dd3f..0000000000 --- a/message_ix_models/data/material/steel_cement/CEMENT.BvR2010.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:56e7a31bab30f55d47bf7ebdafe7cd5059d81c1744b33d92f9a4965684dca55e -size 1586591 diff --git a/message_ix_models/data/material/steel_cement/China_steel_cement_MESSAGE.xlsx b/message_ix_models/data/material/steel_cement/China_steel_cement_MESSAGE.xlsx deleted file mode 100644 index 750b43688a..0000000000 --- a/message_ix_models/data/material/steel_cement/China_steel_cement_MESSAGE.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c399584ab299ee1dff535c7f51e42d0b8697647f114dc6e2c11643230bd9b7e1 -size 54926 diff --git a/message_ix_models/data/material/steel_cement/Global_steel_cement_MESSAGE.xlsx b/message_ix_models/data/material/steel_cement/Global_steel_cement_MESSAGE.xlsx deleted file mode 100644 index b0ed94de32..0000000000 --- a/message_ix_models/data/material/steel_cement/Global_steel_cement_MESSAGE.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:46c81f9778ab244e470f1262c918ce6662ffdb4067b82c14a5e80f1d613da3ff -size 118713 diff --git a/message_ix_models/data/material/steel_cement/STEEL_database_2012.xlsx b/message_ix_models/data/material/steel_cement/STEEL_database_2012.xlsx deleted file mode 100644 index 1a580e5f38..0000000000 --- a/message_ix_models/data/material/steel_cement/STEEL_database_2012.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0e22badd6aa7f4e686f704ac1b449a306dab2f06a20e71069e32a99f5d1a3844 -size 504346 diff --git a/message_ix_models/data/material/technology.yaml b/message_ix_models/data/material/technology.yaml index 9be53e24e6..7bf229ef8c 100644 --- a/message_ix_models/data/material/technology.yaml +++ b/message_ix_models/data/material/technology.yaml @@ -1,1353 +1,737 @@ # Collection of all material/industry technologies of MESSAGEix-Materials CH2O_synth: - description: Synthesis of formaldehyde from methanol via silver-catalyst pathway # check this - input: - commodity: test + description: Synthesis of formaldehyde from methanol via silver-catalyst pathway name: Formaldehyde Synthesis - report: string CH2O_to_resin: description: Synthesis of formaldehyde resins (proxy chemical phenol-formaldehyde) - input: - commodity: test - name: empty - report: string + name: Formaldehyde resin synthesis DUMMY_alumina_supply: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology supplying alumina (aluminum oxide) + name: Alumina supply DUMMY_coal_supply: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology supplying coal for steel sector to be used in emissions reporting + name: Coal supply for steel sector DUMMY_gas_supply: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology supplying natural gas for steel sector to be used in emissions reporting + name: Gas supply for steel sector DUMMY_limestone_supply_cement: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology supplying limestone for cement + name: Limestone supply for cement sector DUMMY_limestone_supply_steel: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology supplying limestone for steel + name: Limestone supply for steel sector DUMMY_ore_supply: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology supplying iron ore + name: Iron ore supply for steel sector MTO_petro: description: Methanol-to-Olefins process unit producing ethylene and propylene from methanol - input: - commodity: test name: Methanol-to-Olefins - report: string NH3_to_N_fertil: description: Nitrogen fertilizer production from ammonia - input: - commodity: test name: Nitrogen fertilizer production - report: string agg_ref: description: Pseudo technology aggregating petroleum products to light-oil and heavy-oil - input: - commodity: test - name: empty - report: string + name: Petroleum fuel aggregation atm_distillation_ref: - description: Most important unit in a refinery separating crude oil by - input: - commodity: test + description: First step in the refining process to separate crude oil into its components based on their boiling points name: Atmospheric distillation unit - report: string bf_steel: description: Blast furnace producing iron from iron ore - input: - commodity: test name: Blast furnace - report: string biomass_NH3: description: Ammonia synthesis with hydrogen produced from biomass gasification - input: - commodity: test name: Ammonia production from biomass - report: string biomass_NH3_ccs: description: Ammonia synthesis with hydrogen produced from biomass gasification with CCS - input: - commodity: test - name: empty - report: string + name: Bioammonia production with CCS bof_steel: - description: Basic oxygen furnace steel making route - input: - commodity: test + description: Type of steelmaking furnace used to convert molten iron produced in a blast furnace into steel name: Basic oxygen furnace - report: string - -buildings: - description: '' - input: - commodity: test - name: empty - report: string catalytic_cracking_ref: description: Catalytic cracking converts gas oils with high gasoline yield - input: - commodity: test name: Catalytic cracking process unit - report: string catalytic_reforming_ref: description: Catalytic reforming used to produce hydrogen from refinery feedstock - input: - commodity: test name: Catalytic reforming process unit - report: string clinker_dry_ccs_cement: description: Addon technology for clinker_dry_cement capturing process emissions - input: - commodity: test name: Dry process clinker kiln CCS addon - report: string clinker_dry_cement: description: Clinker kiln using the more efficient dry calcining process - input: - commodity: test name: Dry process clinker kiln - report: string clinker_wet_ccs_cement: description: Addon technology for clinker_wet_cement capturing process emissions - input: - commodity: test name: Wet process clinker kiln CCS addon - report: string clinker_wet_cement: description: Clinker kiln using the less efficient wet calcining process - input: - commodity: test - name: empty - report: string + name: Wet calcining clinker production coal_NH3: description: Ammonia synthesis with hydrogen from coal gasification - input: - commodity: test name: Ammonia production from coal - report: string coal_NH3_ccs: description: Ammonia synthesis with hydrogen from coal gasification with CCS - input: - commodity: test name: Ammonia production from coal with CCS - report: string cokeoven_steel: description: Coke oven transforming coal to coke for steel production - input: - commodity: test name: Coke oven for steel production - report: string coking_ref: description: Deep conversion technology of refinery upgrading heavy residues of vacuum distillation - input: - commodity: test name: Refinery coker process unit - report: string dheat_aluminum: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat from a district heating network for aluminum sector + name: District heat for aluminum production dheat_cement: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat from a district heating network for cement sector + name: District heat for cement production dheat_petro: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat from a district heating network for petrochemical production + name: District heat for chemical production dheat_refining: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat from a district heating network for refineries + name: District heat for refineries dheat_resins: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat from a district heating network for resin production + name: District heat for resin production dheat_steel: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat from a district heating network for steel sector + name: District heat for steel production dri_steel: - description: '' - input: - commodity: test - name: empty - report: string + description: Sponge iron production through direct reduction of iron ore + name: Direct reduction iron process eaf_steel: - description: Electric arc furnace for steel making from DRI iron or steel scrap - input: - commodity: test + description: Electric arc furnace for steel making from direct reduced iron or steel scrap name: Electric arc furnace - report: string electr_NH3: description: Integrated ammonia production facility combining on-site hydrogen electrolyser with ammonia synthesis unit - input: - commodity: test name: Ammonia production from electricity - report: string ethanol_to_ethylene_petro: description: Ethylene production via ethanol dehydration - input: - commodity: test name: Ethanol dehydration - report: string export_NFert: description: Pseudo technology representing regional nitrogen fertilizer exports to global market - input: - commodity: test name: Nitrogen fertilizer export - report: string export_NH3: description: Pseudo technology representing regional ammonia exports to global market - input: - commodity: test name: NH3 export - report: string export_aluminum: description: Pseudo technology representing regional aluminum exports to global market - input: - commodity: test name: Aluminum export - report: string export_petro: description: Pseudo technology representing regional HVC exports to global market - input: - commodity: test name: HVC export - report: string export_steel: description: Pseudo technology representing regional steel exports to global market - input: - commodity: test name: Steel export - report: string #FIXME: fuel cells are currently only producing low temp heat, but no electricity fc_h2_aluminum: description: Fuel cell using hydrogen to produce heat for aluminum and electricity - input: - commodity: test name: Hydrogen fuel-cell for aluminum process heat - report: string fc_h2_cement: description: Fuel cell using hydrogen to produce heat for cement and electricity - input: - commodity: test name: Hydrogen fuel-cell for cement process heat - report: string fc_h2_petro: description: Fuel cell using hydrogen to produce heat for petrochemicals and electricity - input: - commodity: test name: Hydrogen fuel-cell for petrochemicals process heat - report: string fc_h2_refining: description: Fuel cell using hydrogen to produce heat for refineries and electricity - input: - commodity: test name: Hydrogen fuel-cell for refinery process heat - report: string fc_h2_resins: description: Fuel cell using hydrogen to produce heat for resins and electricity - input: - commodity: test name: Hydrogen fuel-cell for resin process heat - report: string fc_h2_steel: - description: Fuel cell using hydrogen to produce heat for aluminum and electricity - input: - commodity: test + description: Fuel cell using hydrogen to produce heat for steel and electricity name: Hydrogen fuel-cell for steel process heat - report: string feedstock_t/d: - description: '' - input: - commodity: test - name: empty - report: string + description: Distribution of feedstocks for high value chemicals + name: Chemical feedstock distribution finishing_aluminum: - description: '' - input: - commodity: test - name: empty - report: string + description: Finishing of aluminum + name: Aluminum finishing finishing_steel: - description: '' - input: - commodity: test - name: empty - report: string + description: Finishing of steel + name: Steel finishing fueloil_NH3: description: Ammonia synthesis with hydrogen via partial oxidation of fueloil - input: - commodity: test name: Ammonia production from fueloil - report: string fueloil_NH3_ccs: description: Ammonia synthesis with hydrogen via partial oxidation of fueloil with CCS - input: - commodity: test name: Ammonia production from fueloil with CCS - report: string furnace_biomass_aluminum: description: Generalized furnace using biomass to provide process heat for aluminum production - input: - commodity: test name: Industrial furnace from biomass for aluminum sector - report: string furnace_biomass_cement: - description: Generalized furnace using biomass to provide process heat for aluminum production - input: - commodity: test - name: Industrial furnace from biomass for aluminum sector - report: string + description: Generalized furnace using biomass to provide process heat for cement production + name: Industrial furnace from biomass for cement sector furnace_biomass_petro: description: Generalized furnace using biomass to provide process heat for petrochemical production - input: - commodity: test name: Industrial furnace from biomass for petrochemical sector - report: string furnace_biomass_refining: description: Generalized furnace using biomass to provide process heat for refineries - input: - commodity: test name: Industrial furnace from biomass for refinery sector - report: string furnace_biomass_resins: description: Generalized furnace using biomass to provide process heat for resin production - input: - commodity: test name: Industrial furnace from biomass for resin sector - report: string furnace_biomass_steel: description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test name: Industrial furnace from biomass for iron/steel sector - report: string furnace_coal_aluminum: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using coal to provide process heat for aluminum production name: Industrial furnace from coal for aluminum sector - report: string furnace_coal_cement: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using coal to provide process heat for cement production name: Industrial furnace from coal for cement sector - report: string furnace_coal_petro: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using coal to provide process heat for petrochemicals production name: Industrial furnace from coal for petrochemicals sector - report: string furnace_coal_refining: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using coal to provide process heat for refineries name: Industrial furnace from coal for refinery sector - report: string furnace_coal_resins: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using coal to provide process heat for resins production name: Industrial furnace from coal for resins sector - report: string furnace_coal_steel: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using coal to provide process heat for steel production name: Industrial furnace from coal for steel sector - report: string furnace_coke_petro: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using coke to provide process heat for petrochemical production name: Industrial furnace from coke for petrochemical sector - report: string furnace_coke_refining: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using coke to provide process heat for refineries name: Industrial furnace from coke for refinery sector - report: string furnace_elec_aluminum: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using electricity to provide process heat for aluminum production name: Industrial furnace using electricity for aluminum sector - report: string furnace_elec_cement: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using electricity to provide process heat for cement production name: Industrial furnace using electricity for cement sector - report: string furnace_elec_petro: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using electricity to provide process heat for petrochemical production name: Industrial furnace using electricity for petrochemical sector - report: string furnace_elec_refining: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using electricity to provide process heat for refineries name: Industrial furnace using electricity for refinery sector - report: string furnace_elec_resins: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using electricity to provide process heat for resin production name: Industrial furnace using electricity for resin sector - report: string furnace_elec_steel: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using electricity to provide process heat for steel production name: Industrial furnace using electricity for steel sector - report: string furnace_ethanol_aluminum: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using ethanol to provide process heat for aluminum production name: Industrial furnace using ethanol for aluminum sector - report: string furnace_ethanol_cement: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using ethanol to provide process heat for cement production name: Industrial furnace using ethanol for cement sector - report: string furnace_ethanol_petro: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using ethanol to provide process heat for petrochemicals production name: Industrial furnace using ethanol for petrochemical sector - report: string furnace_ethanol_refining: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using ethanol to provide process heat for refineries name: Industrial furnace using ethanol for refinery sector - report: string furnace_ethanol_resins: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using ethanol to provide process heat for resins production name: Industrial furnace using ethanol for resins sector - report: string furnace_ethanol_steel: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using ethanol to provide process heat for steel production name: Industrial furnace using ethanol for steel sector - report: string furnace_foil_aluminum: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using fueloil to provide process heat for aluminum production name: Industrial furnace using fueloil for aluminum sector - report: string furnace_foil_cement: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using fueloil to provide process heat for cement production name: Industrial furnace using fueloil for cement sector - report: string furnace_foil_petro: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using fueloil to provide process heat for petrochemicals production name: Industrial furnace using fueloil for petrochemical sector - report: string furnace_foil_refining: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using fueloil to provide process heat for refineries name: Industrial furnace using fueloil for refinery sector - report: string furnace_foil_resins: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using fueloil to provide process heat for resins production name: Industrial furnace using fueloil for resin sector - report: string furnace_foil_steel: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using fueloil to provide process heat for steel production name: Industrial furnace using fueloil for steel sector - report: string furnace_gas_aluminum: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using gas to provide process heat for aluminum production name: Industrial furnace using natural gas for aluminum sector - report: string furnace_gas_cement: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using gas to provide process heat for cement production name: Industrial furnace using natural gas for cement sector - report: string furnace_gas_petro: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using gas to provide process heat for petrochemicals production name: Industrial furnace using natural gas for petrochemical sector - report: string furnace_gas_refining: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using gas to provide process heat for refineries name: Industrial furnace using natural gas for refinery sector - report: string furnace_gas_resins: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using gas to provide process heat for resins production name: Industrial furnace using natural gas for resin sector - report: string furnace_gas_steel: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using gas to provide process heat for steel production name: Industrial furnace using natural gas for steel sector - report: string furnace_h2_aluminum: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using hydrogen to provide process heat for aluminum production name: Industrial furnace using hydrogen for aluminum sector - report: string furnace_h2_cement: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using hydrogen to provide process heat for cement production name: Industrial furnace using hydrogen for cement sector - report: string furnace_h2_petro: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using hydrogen to provide process heat for petrochemicals production name: Industrial furnace using hydrogen for petrochemicals sector - report: string furnace_h2_refining: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using hydrogen to provide process heat for refineries name: Industrial furnace using hydrogen for refinery sector - report: string furnace_h2_resins: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using hydrogen to provide process heat for resins production name: Industrial furnace using hydrogen for resin sector - report: string furnace_h2_steel: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using hydrogen to provide process heat for steel production name: Industrial furnace using hydrogen for steel sector - report: string furnace_loil_aluminum: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using lightoil to provide process heat for aluminum production name: Industrial furnace using lightoil for aluminum sector - report: string furnace_loil_cement: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using lightoil to provide process heat for cement production name: Industrial furnace using lightoil for cement sector - report: string furnace_loil_petro: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using lightoil to provide process heat for petrochemical production name: Industrial furnace using lightoil for petrochemical sector - report: string furnace_loil_refining: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using lightoil to provide process heat for refineries name: Industrial furnace using lightoil for refinery sector - report: string furnace_loil_resins: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using lightoil to provide process heat for resin production name: Industrial furnace using lightoil for resin sector - report: string furnace_loil_steel: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using lightoil to provide process heat for steel production name: Industrial furnace using lightoil for steel sector - report: string furnace_methanol_aluminum: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using methanol to provide process heat for aluminum production name: Industrial furnace using methanol for aluminum sector - report: string furnace_methanol_cement: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using methanol to provide process heat for cement production name: Industrial furnace using methanol for cement sector - report: string furnace_methanol_petro: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using methanol to provide process heat for petrochemical production name: Industrial furnace using methanol for petrochemical sector - report: string furnace_methanol_refining: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using methanol to provide process heat for refineries name: Industrial furnace using methanol for refinery sector - report: string furnace_methanol_resins: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using methanol to provide process heat for resins production name: Industrial furnace using methanol for resin sector - report: string furnace_methanol_steel: - description: Generalized furnace using biomass to provide process heat for steel production - input: - commodity: test + description: Generalized furnace using methanol to provide process heat for steel production name: Industrial furnace using methanol for steel sector - report: string gas_NH3: description: Ammonia synthesis with hydrogen via steam reforming of methane - input: - commodity: test name: Ammonia production from natural gas - report: string gas_NH3_ccs: description: Ammonia synthesis with hydrogen via steam reforming of methane with CCS - input: - commodity: test name: Ammonia production from natural gas with CCS - report: string gas_processing_petro: - description: '' - input: - commodity: test - name: empty - report: string + description: Processing of raw natural gas to separate gas liquids + name: Natural gas processing grinding_ballmill_cement: - description: '' - input: - commodity: test - name: empty - report: string + description: Cement grinding using a ballmill + name: Ballmill cement grinding grinding_vertmill_cement: - description: '' - input: - commodity: test - name: empty - report: string + description: Cement grinding using a vertical mill + name: Vertical cement grinding hp_elec_aluminum: description: Electric heat pump used to provide process heat for aluminum production - input: - commodity: test name: Electric heat pump for aluminum process heat - report: string hp_elec_cement: description: Electric heat pump used to provide process heat for cement production - input: - commodity: test - name: empty - report: string + name: Electric heat pump for cement process heat hp_elec_petro: description: Electric heat pump used to provide process heat for petrochemical production - input: - commodity: test - name: empty - report: string + name: Electric heat pump for petrochemical process heat hp_elec_refining: description: Electric heat pump used to provide process heat for refinery processes - input: - commodity: test - name: empty - report: string + name: Electric heat pump for refining process heat hp_elec_resins: description: Electric heat pump used to provide process heat for resin production - input: - commodity: test - name: empty - report: string + name: Electric heat pump for resins production process heat hp_elec_steel: description: Electric heat pump used to provide process heat for steel production - input: - commodity: test - name: empty - report: string + name: Electric heat pump for steel process heat hp_gas_aluminum: description: Gas powered heat pump used to provide process heat for aluminum production - input: - commodity: test - name: empty - report: string + name: Gas powered heat pump for aluminum process heat hp_gas_cement: - description: Gas powered heat pump used to provide process heat for aluminum production - input: - commodity: test - name: empty - report: string + description: Gas powered heat pump used to provide process heat for cement production + name: Gas powered heat pump for cement process heat hp_gas_petro: description: Gas powered heat pump used to provide process heat for petrochemical production - input: - commodity: test - name: empty - report: string + name: Gas powered heat pump for petrochemical process heat hp_gas_refining: description: Gas powered heat pump used to provide process heat for refineries - input: - commodity: test - name: empty - report: string + name: Gas powered heat pump for refining process heat hp_gas_resins: description: Gas powered heat pump used to provide process heat for resin production - input: - commodity: test - name: empty - report: string + name: Gas powered heat pump for resin production process heat hp_gas_steel: description: Gas powered heat pump used to provide process heat for steel production - input: - commodity: test - name: empty - report: string + name: Gas powered heat pump for steel process heat hydro_cracking_ref: description: Hydrocracking unit of a refinery converting gas oils with hydrogen to lighter petroleum products - input: - commodity: test - name: empty - report: string + name: Hydrocracking hydrotreating_ref: description: Hydrotreating unit of a refinery - input: - commodity: test name: Hydrotreating refinery unit - report: string import_NFert: description: Pseudo technology representing imports of nitrogen fertilizer from a global pool - input: - commodity: test - name: empty - report: string + name: Nitrogen fertilizer imports import_NH3: description: Pseudo technology representing imports of ammonia from a global pool - input: - commodity: test name: Ammonia import - report: string import_aluminum: description: Pseudo technology representing imports of aluminum from a global pool - input: - commodity: test - name: empty - report: string + name: Aluminum imports import_petro: description: Pseudo technology representing imports of HVCs from a global pool - input: - commodity: test - name: empty - report: string + name: High Value Chemicals imports import_steel: description: Pseudo technology representing imports of steel from a global pool - input: - commodity: test - name: empty - report: string + name: Steel imports manuf_aluminum: description: Technology representing the manufacturing steps from crude alu to alu products - input: - commodity: test - name: empty - report: string + name: Aluminum manufacturing manuf_steel: description: Technology representing the manufacturing steps from crude steel to steel products - input: - commodity: test name: Steel products manufacturing - report: string meth_bal: description: Pseudo technology moving methanol from primary(_material) to secondary(_material) level - input: - commodity: test name: Methanol balance - report: string meth_bio: description: Methanol production via biomass gasification - input: - commodity: test - name: empty - report: string + name: Biomethanol production meth_bio_ccs: description: Methanol production via biomass gasification with CCS - input: - commodity: test - name: empty - report: string + name: Biomethanol production with CCS meth_coal: description: Methanol production via coal gasification - input: - commodity: test - name: empty - report: string + name: Methanol production from coal meth_coal_ccs: description: Methanol production via coal gasification with CCS - input: - commodity: test name: Methanol production from coal with CCS - report: string meth_exp: description: Pseudo technology representing export of methanol to the global pool - input: - commodity: test name: Methanol export - report: string meth_h2: description: Methanol production from synthetic syngas (H2 and CO2) - input: - commodity: test name: Methanol production from hydrogen - report: string meth_imp: description: Pseudo technology representing imports of methanol from the global pool - input: - commodity: test - name: Methanol import - report: string + name: Methanol imports meth_ng: description: Methanol production via steam reforming of natural gas - input: - commodity: test name: Methanol production from natural gas - report: string meth_ng_ccs: description: Methanol production via steam reforming of natural gas with CCS - input: - commodity: test - name: empty - report: string + name: Methanol production from gas meth_t_d: description: Pseudo technology moving methanol from secondary to final level - input: - commodity: test - name: Methanol transmission and distribution - report: string + name: Methanol fuel transport and distribution meth_t_d_material: description: Pseudo technology moving methanol from secondary_material to final_material level - input: - commodity: test - name: empty - report: string + name: Methanol feedstock transport and distribution meth_trd: description: Pseudo technology representing methanol trade on global markets - input: - commodity: test name: Methanol trade - report: string other_EOL_aluminum: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology representing the end-of-life aluminum as a share of production + name: Aluminum end of life technology other_EOL_cement: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology representing the end-of-life cement as a share of production + name: Cement end of life technology other_EOL_steel: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology representing the end-of-life steel as a share of production + name: Steel end of life technology pellet_steel: - description: '' - input: - commodity: test - name: empty - report: string + description: Technology to convert fine-grained iron ore concentrates or other iron-bearing materials into small, round pellets + name: Iron pelletizing technology prebake_aluminum: description: Aluminum production via Hall–Héroult process with prebaked electrodes - input: - commodity: test name: Aluminum production with prebaked electrodes - report: string prep_secondary_aluminum_1: - description: '' - input: - commodity: test - name: empty - report: string + description: Aluminum recycling preperation technology with lower costs and high quality scrap + name: Aluminum Recycling Preperation with low cost prep_secondary_aluminum_2: - description: '' - input: - commodity: test - name: empty - report: string + description: Aluminum recycling preperation technology with medium costs and medium quality scrap + name: Aluminum Recycling Preperation with medium cost prep_secondary_aluminum_3: - description: '' - input: - commodity: test - name: empty - report: string + description: Aluminum recycling preperation technology with high costs and low quality scrap + name: Aluminum Recycling Preperation with high cost prep_secondary_steel_1: - description: '' - input: - commodity: test - name: empty - report: string + description: Steel recycling preperation technology with lower costs and high quality scrap + name: Steel Recycling Preperation with low cost prep_secondary_steel_2: - description: '' - input: - commodity: test - name: empty - report: string + description: Steel recycling preperation technology with medium costs and medium quality scrap + name: Steel Recycling Preperation with medium cost prep_secondary_steel_3: - description: '' - input: - commodity: test - name: empty - report: string + description: Steel recycling preperation technology with high costs and low quality scrap + name: Steel Recycling Preperation with high cost production_HVC: description: Pseudo technology aggregating all chemicals belong to HVC group to HVC commodity - input: - commodity: test name: High Value Chemicals production - report: string raw_meal_prep_cement: - description: '' - input: - commodity: test - name: empty - report: string + description: Mixing raw materials to create the homogenous raw meal mixture which serves as the feedstock for the production of cement clinker. + name: Raw meal preperation residual_NH3: description: Pseudo technology that moves ammonia from level x to level y #TODO: look up levels - input: - commodity: test name: Non-fertilizer ammonia production - report: string scrap_recovery_aluminum: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology that represents the collected amount of scrap that goes to recycling + name: Aluminum scrap collection scrap_recovery_cement: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology that represents the collected amount of cement waste that goes to recycling + name: Cement waste collection scrap_recovery_steel: - description: '' - input: - commodity: test - name: empty - report: string + description: Pseudo technology that represents the collected amount of scrap that goes to recycling + name: Steel scrap collection secondary_aluminum: - description: '' - input: - commodity: test - name: empty - report: string + description: Recycling of aluminum via melting the scrap + name: Aluminum Recycling sinter_steel: - description: '' - input: - commodity: test - name: empty - report: string + description: Recycling of steel via melting the scrap + name: Steel Recycling soderberg_aluminum: description: Aluminum production via Hall–Héroult process with Söderberg electrodes - input: - commodity: test - name: Aluminum production with Söderberg electrodes - report: string + name: Aluminum production with Söderberg electrodes solar_aluminum: - description: Solar heat collected with ? for aluminum production - input: - commodity: test - name: Solar heat for aluminum production - report: string + description: Low temperature heat form solar collector collected with ? for aluminum production + name: Solar heat for aluminum solar_cement: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat form solar collector used for cement production + name: Solar heat for cement solar_petro: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat form solar collector used for petrochemical production + name: Solar heat for chemicals solar_refining: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat form solar collector used for refining production + name: Solar heat for refining solar_resins: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat form solar collector used for resin production + name: Solar heat for resins solar_steel: - description: '' - input: - commodity: test - name: empty - report: string - -sr_steel: - description: '' - input: - commodity: test - name: empty - report: string + description: Low temperature heat form solar collector used for steel production + name: Solar heat for steel steam_cracker_petro: description: Most important conventional technology for primary petrochemicals - input: - commodity: test name: Steam cracker - report: string total_EOL_aluminum: - description: '' - input: - commodity: test + description: Pseudo technology aggregating aluminum scrap after end-of-life name: empty - report: string total_EOL_cement: - description: '' - input: - commodity: test + description: Pseudo technology collecting cement scrap after end-of-life name: empty - report: string total_EOL_steel: - description: '' - input: - commodity: test + description: Pseudo technology aggregating steel scrap after end-of-life name: empty - report: string trade_NFert: description: Pseudo technology representing nitrogen fertilizer trade on global markets - input: - commodity: test - name: empty - report: string + name: Nitrogen fertilizer trade trade_NH3: description: Pseudo technology representing NH3 trade on global markets - input: - commodity: test name: Ammonia trade - report: string trade_aluminum: description: Pseudo technology representing aluminum trade on global markets - input: - commodity: test - name: empty - report: string + name: Aluminum trade trade_petro: description: Pseudo technology representing HVC trade on global markets - input: - commodity: test - name: empty - report: string + name: High Value Chemicals trade trade_steel: description: Pseudo technology representing steel trade on global markets - input: - commodity: test name: Steel trade - report: string vacuum_distillation_ref: description: Refinery process distilling heavy residue from atmospheric distillation - input: - commodity: test name: Vacuum distillation unit - report: string visbreaker_ref: description: Visbreaker unit of a refinery - input: - commodity: test - name: empty - report: string - -tecs_steel: - name: Steel making technologies - description: group of technologies that steel - child: [] - -tecs_petro: - name: Petrochemicals technologies - description: group of technologies that produce petrochemicals - child: [] - -tecs_cement: - name: Cement making technologies - description: group of technologies that steel - child: [] - -tecs_alu: - name: Aluminum making technologies - description: group of technologies that steel - child: [] - -tecs_ref: - name: Refinery technologies - description: group of refinery process units - child: [ ] - -heat_tecs_steel: - name: Steel heat supply technologies - description: group of technologies that steel - child: [] - -heat_tecs_petro: - name: Petrochemicals heat supply technologies - description: group of technologies that produce petrochemicals - child: [] - -heat_tecs_cement: - name: Cement heat supply technologies - description: group of technologies that steel - child: [] - -heat_tecs_alu: - name: Aluminum heat supply technologies - description: group of technologies that steel - child: [] - -heat_tecs_ref: - name: Refinery heat supply technologies - description: group of refinery process units - child: [ ] \ No newline at end of file + name: Visbreaker unit diff --git a/message_ix_models/data/relation/B.yaml b/message_ix_models/data/relation/B.yaml index 8294c495a9..dec682c80c 100644 --- a/message_ix_models/data/relation/B.yaml +++ b/message_ix_models/data/relation/B.yaml @@ -30,7 +30,7 @@ - FE_feedstock - FE_final_energy - FE_gaseous -- FE_industry [266/1626] +- FE_industry - FE_liquids - FE_new_biomass - FE_old_biomass @@ -55,7 +55,7 @@ - NOx_Emission_bunkers - NOx_nonenergy - OCA_Emission -- OCA_Emission_bunkers [241/1626] +- OCA_Emission_bunkers - OilPrices - PE_BACKSTOPS - PE_coal @@ -82,7 +82,7 @@ - POPtoSolvent - RES_variable_limt - SF6_Emission -- SF6_elec_red [214/1626] +- SF6_elec_red - SF6_mag_red - SO2_Emission - SO2_Emission_bunkers @@ -109,7 +109,7 @@ - TN2O_Emission - TSF6_Emission - UE_feedstock -- UE_feedstock_coal [187/1626] +- UE_feedstock_coal - UE_feedstock_foil - UE_feedstock_gas - UE_feedstock_liquid @@ -136,7 +136,7 @@ - UE_res_comm_th_gas - UE_res_comm_th_heat - UE_res_comm_th_hp -- UE_res_comm_th_hydrogen [160/1626] +- UE_res_comm_th_hydrogen - UE_res_comm_th_liquid - UE_res_comm_th_solar - UE_res_comm_th_solids @@ -160,7 +160,7 @@ - WasteGenToSO2Link - WasteGenToVOCLink - bco2_trans_disp -- bio_extr_mpen_c [136/1626] +- bio_extr_mpen_c - bio_scrub_lim - co2_trans_disp - co_gen_limit @@ -187,7 +187,7 @@ - elec_relations2_hydro - elec_relations2_nuclear - elec_relations2_oil -- elec_relations2_renewable [109/1626] +- elec_relations2_renewable - extraction_coal - extraction_gas - extraction_oil @@ -214,7 +214,7 @@ - lim_exp_cpa_oil - lim_exp_eeu_eth - lim_exp_eeu_meth -- lim_exp_fsu_LNG [82/1626] +- lim_exp_fsu_LNG - lim_exp_fsu_coal - lim_exp_fsu_eth - lim_exp_fsu_meth @@ -241,7 +241,7 @@ - nica_limit - nuc_lc_in_el - oil_based_elec_gen -- oil_extr_mpen_c [55/1626] +- oil_extr_mpen_c - oil_imp_c - oil_prod - oil_trd @@ -269,7 +269,7 @@ - sm1_ppl_res - sm3_ppl_res - solar_csp_pot -- solar_csp_pot1 [27/1626] +- solar_csp_pot1 - solar_csp_pot2 - solar_csp_pot3 - solar_csp_pot4 diff --git a/message_ix_models/model/bare.py b/message_ix_models/model/bare.py index 7928ae104f..ff117b0976 100644 --- a/message_ix_models/model/bare.py +++ b/message_ix_models/model/bare.py @@ -150,21 +150,28 @@ def get_spec(context) -> Spec: return Spec(add=add) -def name(context): +def name(context, *, unique: bool = False) -> str: """Generate a candidate name for a model given `context`. The name has a form like:: MESSAGEix-GLOBIOM R99 YA +D + MESSAGEix-GLOBIOM R99 YA a1b2c where: - "R99" is the node list/regional aggregation. - "YA" indicates the year codelist (:doc:`/pkg-data/year`). - - "+D" appears if :attr:`.Config.res_with_dummies` is true. - + - "+D" appears if `unique` is :any:`False` and :attr:`.Config.res_with_dummies` is + :any:`True`. + - A hexidecimal hash digest like "12b2c" appears if `unique` is :any:`True`. This + value is unique for every possible combination of settings on + :class:`.model.Config`; see :meth:`.Config.hexdigest`. """ cfg = context.model - return f"MESSAGEix-GLOBIOM {cfg.regions} Y{cfg.years}" + ( - " +D" if cfg.res_with_dummies else "" - ) + result = f"MESSAGEix-GLOBIOM {cfg.regions} Y{cfg.years}" + if not unique: + result += " +D" if cfg.res_with_dummies else "" + else: + result += " " + cfg.hexdigest(5) + return result diff --git a/message_ix_models/model/config.py b/message_ix_models/model/config.py index 25165e9d41..8c47cab31c 100644 --- a/message_ix_models/model/config.py +++ b/message_ix_models/model/config.py @@ -1,11 +1,12 @@ from dataclasses import dataclass, field, fields from message_ix_models.model.structure import codelists +from message_ix_models.util.config import ConfigHelper from message_ix_models.util.context import _ALIAS @dataclass -class Config: +class Config(ConfigHelper): """Settings and valid values for :mod:`message_ix_models.model` and submodules. For backwards compatibility, it is possible to access these on a :class:`Context` diff --git a/message_ix_models/model/material/bare.py b/message_ix_models/model/material/bare.py deleted file mode 100644 index 0c2859e539..0000000000 --- a/message_ix_models/model/material/bare.py +++ /dev/null @@ -1,242 +0,0 @@ -import logging -from typing import Mapping - -import message_ix -from sdmx.model.v21 import Code - -import message_ix_models -from message_ix_models import ScenarioInfo -from message_ix_models.model.material import ( - gen_data_aluminum, - gen_data_generic, - gen_data_steel, -) - -# from message_ix_models.util get_context, set_info, add_par_data -from message_ix_models.util import add_par_data - -from .build import apply_spec -from .util import read_config - -log = logging.getLogger(__name__) - -# Settings and valid values; the default is listed first -# How do we add the historical period ? - -SETTINGS = dict( - # period_start=[2010], - period_start=[1980], - first_model_year=[2020], - period_end=[2100], - regions=["China"], - res_with_dummies=[True], - time_step=[10], -) - - -def create_res(context=None, quiet=True): - """Create a 'bare' MESSAGE-GLOBIOM reference energy system (RES). - - Parameters - ---------- - quiet : bool - Only show log messages at level ``ERROR`` and higher. If :obj:`False` (default), - show log messages at level ``DEBUG`` and higher. - context : .Context - :attr:`.Context.scenario_info` determines the model name and scenario - name of the created Scenario. - - Returns - ------- - message_ix.Scenario - A scenario as described by :func:`get_spec`, prepared using - :func:`.build.apply_spec`. - """ - mp = context.get_platform() - mp.add_unit("Mt") - - # Model and scenario name for the RES - model_name = context.scenario_info["model"] - scenario_name = context.scenario_info["scenario"] - - # Create the Scenario - scenario = message_ix.Scenario( - mp, model=model_name, scenario=scenario_name, version="new" - ) - - # TODO move to message_ix - scenario.init_par("MERtoPPP", ["node", "year"]) - - # Uncomment to add dummy sets and data - context.res_with_dummies = True - - spec = get_spec(context) - apply_spec( - scenario, - spec, - # data=partial(get_data, context=context, spec=spec), - data=add_data, - quiet=quiet, - message=f"Create using message_ix_models {message_ix_models.__version__}", - ) - - return scenario - - -DATA_FUNCTIONS = [ - gen_data_steel, - gen_data_generic, - gen_data_aluminum, - # gen_data_variable -] - - -# Try to handle multiple data input functions from different materials -def add_data(scenario, dry_run=False): - """Populate `scenario` with MESSAGE-Material data.""" - - # Information about `scenario` - info = ScenarioInfo(scenario) - - # Check for two "node" values for global data, e.g. in - # ixmp://ene-ixmp/CD_Links_SSP2_v2.1_clean/baseline - if {"World", "R11_GLB"} < set(info.set["node"]): - log.warning("Remove 'R11_GLB' from node list for data generation") - info.set["node"].remove("R11_GLB") - - for func in DATA_FUNCTIONS: - # Generate or load the data; add to the Scenario - log.info(f"from {func.__name__}()") - add_par_data(scenario, func(scenario), dry_run=dry_run) - - log.info("done") - - -def get_spec(context=None) -> Mapping[str, ScenarioInfo]: - """Return the spec for the MESSAGE-Material bare RES. - - Parameters - ---------- - context : Context, optional - If not supplied, :func:`.get_context` is used to retrieve the current - context. - - Returns - ------- - :class:`dict` of :class:`.ScenarioInfo` objects - """ - # context = context or get_context(strict=True) - context = read_config() - context.use_defaults(SETTINGS) - - # The RES is the base, so does not require/remove any elements - spec = dict(require=ScenarioInfo()) - - # JM: For China model, we need to remove the default 'World'. - # GU: Remove world is already specified in set.yaml. - remove = ScenarioInfo() - # remove.set["node"] = context["material"]["common"]["region"]["remove"] - - add = ScenarioInfo() - - # Add technologies - # JM: try to find out a way to loop over 1st/2nd level - # and to just context["material"][xx]["add"] - add.set["technology"] = ( - context["material"]["steel"]["technology"]["add"] - + context["material"]["generic"]["technology"]["add"] - + context["material"]["aluminum"]["technology"]["add"] - + context["material"]["petro_chemicals"]["technology"]["add"] - ) - - # Add regions - - # # Load configuration for the specified region mapping - # nodes = set_info(f"node/{context.regions}") - # - # # Top-level "World" node - # world = nodes[nodes.index("World")] - - # Set elements: World, followed by the direct children of World - add.set["node"] = context["material"]["common"]["region"]["require"] - - add.set["relation"] = context["material"]["steel"]["relation"]["add"] - - # Add the time horizon - add.set["year"] = list( - range(context.period_start, context.period_end + 1, context.time_step) - ) - - # JM: Leave the first time period as historical year - # add.set['cat_year'] = [('firstmodelyear', - # context.period_start + context.time_step)] - - # GU: Set 2020 as the first model year, leave the rest as historical year - add.set["cat_year"] = [("firstmodelyear", context.first_model_year)] - - # Add levels - # JM: For bare model, both 'add' & 'require' need to be added. - add.set["level"] = ( - context["material"]["steel"]["level"]["add"] - + context["material"]["common"]["level"]["require"] - + context["material"]["generic"]["level"]["add"] - + context["material"]["aluminum"]["level"]["add"] - + context["material"]["petro_chemicals"]["level"]["add"] - ) - - # Add commodities - add.set["commodity"] = ( - context["material"]["steel"]["commodity"]["add"] - + context["material"]["common"]["commodity"]["require"] - + context["material"]["generic"]["commodity"]["add"] - + context["material"]["aluminum"]["commodity"]["add"] - + context["material"]["petro_chemicals"]["commodity"]["add"] - ) - - # Add other sets - - add.set["type_tec"] = context["material"]["common"]["type_tec"]["add"] - add.set["mode"] = ( - context["material"]["common"]["mode"]["require"] - + context["material"]["generic"]["mode"]["add"] - + context["material"]["petro_chemicals"]["mode"]["add"] - ) - - add.set["emission"] = ( - context["material"]["common"]["emission"]["require"] - + context["material"]["common"]["emission"]["add"] - ) - - # Add units, associated with commodities - # JM: What is 'anno' - # for c in c_list: - # try: - # unit = c.anno['unit'] - # except KeyError: - # log.warning(f"Commodity {c} lacks defined units") - # continue - # - # try: - # # Check that the unit can be parsed by the pint.UnitRegistry - # context.units(unit) - # except Exception: - # log.warning(f"Unit {unit} for commodity {c} not pint compatible") - # else: - # add.set['unit'].append(unit) - - # Deduplicate by converting to a set and then back; not strictly necessary, - # but reduces duplicate log entries - add.set["unit"] = sorted(set(add.set["unit"])) - - # JM: Manually set the first model year - add.y0 = context.first_model_year - - if context.res_with_dummies: - # Add dummy technologies - add.set["technology"].extend([Code("dummy"), Code("dummy source")]) - # Add a dummy commodity - add.set["commodity"].append(Code("dummy")) - - spec["add"] = add - spec["remove"] = remove - return spec diff --git a/message_ix_models/model/material/build.py b/message_ix_models/model/material/build.py index 6529be95d9..0ccac30770 100644 --- a/message_ix_models/model/material/build.py +++ b/message_ix_models/model/material/build.py @@ -1,15 +1,16 @@ import logging -from typing import Mapping +from typing import Any, Dict, Mapping import message_ix import pandas as pd +from message_ix_models import Context from message_ix_models.model.build import apply_spec from message_ix_models.model.material.data_aluminum import gen_data_aluminum from message_ix_models.model.material.data_ammonia_new import gen_all_NH3_fert from message_ix_models.model.material.data_cement import gen_data_cement from message_ix_models.model.material.data_generic import gen_data_generic -from message_ix_models.model.material.data_methanol_new import gen_data_methanol_new +from message_ix_models.model.material.data_methanol import gen_data_methanol from message_ix_models.model.material.data_petro import gen_data_petro_chemicals from message_ix_models.model.material.data_power_sector import gen_data_power_sector from message_ix_models.model.material.data_steel import gen_data_steel @@ -25,8 +26,14 @@ modify_demand_and_hist_activity, modify_industry_demand, ) -from message_ix_models.model.material.util import read_config -from message_ix_models.util import add_par_data, identify_nodes, package_data_path +from message_ix_models.model.material.util import path_fallback, read_config +from message_ix_models.model.structure import generate_set_elements, get_region_codes +from message_ix_models.util import ( + add_par_data, + identify_nodes, + load_package_data, + package_data_path, +) from message_ix_models.util.compat.message_data import ( calibrate_UE_gr_to_demand, calibrate_UE_share_constraints, @@ -34,19 +41,15 @@ from message_ix_models.util.compat.message_data import ( manual_updates_ENGAGE_SSP2_v417_to_v418 as engage_updates, ) -from message_ix_models.util.scenarioinfo import ScenarioInfo +from message_ix_models.util.scenarioinfo import ScenarioInfo, Spec log = logging.getLogger(__name__) -DATA_FUNCTIONS_1 = [ - # gen_data_buildings, - gen_data_methanol_new, +DATA_FUNCTIONS = [ + gen_data_methanol, gen_all_NH3_fert, - # gen_data_ammonia, ## deprecated module! gen_data_generic, gen_data_steel, -] -DATA_FUNCTIONS_2 = [ gen_data_cement, gen_data_petro_chemicals, gen_data_power_sector, @@ -54,7 +57,7 @@ ] # add as needed/implemented -SPEC_LIST = [ +SPEC_LIST = ( "generic", "common", "steel", @@ -65,98 +68,55 @@ "power_sector", "fertilizer", "methanol", -] +) # Try to handle multiple data input functions from different materials -def add_data_1(scenario, dry_run=False): +def add_data(scenario: message_ix.Scenario, dry_run: bool = False) -> None: """Populate `scenario` with MESSAGEix-Materials data.""" # Information about `scenario` - info = ScenarioInfo(scenario) - - # Check for two "node" values for global data, e.g. in - # ixmp://ene-ixmp/CD_Links_SSP2_v2.1_clean/baseline - if {"World", "R11_GLB"} < set(info.set["node"]): - log.warning("Remove 'R11_GLB' from node list for data generation") - info.set["node"].remove("R11_GLB") - if {"World", "R12_GLB"} < set(info.set["node"]): - log.warning("Remove 'R12_GLB' from node list for data generation") - info.set["node"].remove("R12_GLB") - - for func in DATA_FUNCTIONS_1 + DATA_FUNCTIONS_2: + for func in DATA_FUNCTIONS: # Generate or load the data; add to the Scenario log.info(f"from {func.__name__}()") data = func(scenario) - # if "SSP_dev" in scenario.model: - # if "emission_factor" in list(data.keys()): - # data.pop("emission_factor") + data = {k: v for k, v in data.items() if not v.empty} add_par_data(scenario, data, dry_run=dry_run) - - log.info("done") - - -def add_data_2(scenario, dry_run=False): - """Populate `scenario` with MESSAGEix-Materials data.""" - # Information about `scenario` - info = ScenarioInfo(scenario) - - # Check for two "node" values for global data, e.g. in - # ixmp://ene-ixmp/CD_Links_SSP2_v2.1_clean/baseline - if {"World", "R11_GLB"} < set(info.set["node"]): - log.warning("Remove 'R11_GLB' from node list for data generation") - info.set["node"].remove("R11_GLB") - if {"World", "R12_GLB"} < set(info.set["node"]): - log.warning("Remove 'R12_GLB' from node list for data generation") - info.set["node"].remove("R12_GLB") - - for func in DATA_FUNCTIONS_2: - # Generate or load the data; add to the Scenario - log.info(f"from {func.__name__}()") - # TODO: remove this once emission_factors are back in SSP_dev - data = func(scenario) - # if "SSP_dev" in scenario.model: - # if "emission_factor" in list(data.keys()): - # data.pop("emission_factor") - add_par_data(scenario, data, dry_run=dry_run) - log.info("done") def build( - scenario: message_ix.Scenario, old_calib: bool, iea_data_path=None + context: Context, + scenario: message_ix.Scenario, + old_calib: bool, + modify_existing_constraints: bool = True, + iea_data_path: str | None = None, ) -> message_ix.Scenario: """Set up materials accounting on `scenario`.""" + node_suffix = context.model.regions - # Get the specification - # Apply to the base scenario - spec = get_spec() - - if "water_supply" not in list(scenario.set("level")): - scenario.check_out() - # add missing water tecs - scenario.add_set("technology", "extract__freshwater_supply") - scenario.add_set("level", "water_supply") - scenario.add_set("commodity", "freshwater_supply") - - water_dict = pd.read_excel( - package_data_path("material", "other", "water_tec_pars.xlsx"), - sheet_name=None, + if node_suffix != "R12": + raise NotImplementedError( + "MESSAGEix-Materials is currently only supporting" + " MESSAGEix-GLOBIOM R12 regions" ) - for par in water_dict.keys(): - scenario.add_par(par, water_dict[par]) - scenario.commit("add missing water tecs") - apply_spec(scenario, spec, add_data_1, fast=True) # dry_run=True - if "SSP_dev" not in scenario.model: - engage_updates._correct_balance_td_efficiencies(scenario) - engage_updates._correct_coal_ppl_u_efficiencies(scenario) - engage_updates._correct_td_co2cc_emissions(scenario) - # spec = None - # apply_spec(scenario, spec, add_data_2) - from message_ix_models import ScenarioInfo + if f"{node_suffix}_GLB" not in list(scenario.platform.regions().region): + # Required for material trade model + # TODO Include this in the spec, while not using it as a value for `node_loc` + scenario.platform.add_region(f"{node_suffix}_GLB", "region", "World") - s_info = ScenarioInfo(scenario) - nodes = s_info.N + # Get the specification and apply to the base scenario + spec = make_spec(node_suffix) + apply_spec(scenario, spec, add_data, fast=True) # dry_run=True + + water_dict = pd.read_excel( + package_data_path("material", "other", "water_tec_pars.xlsx"), + sheet_name=None, + ) + scenario.check_out() + for par in water_dict.keys(): + scenario.add_par(par, water_dict[par]) + scenario.commit("add missing water tecs") # Adjust exogenous energy demand to incorporate the endogenized sectors # Adjust the historical activity of the useful level industry technologies @@ -169,16 +129,31 @@ def build( last_hist_year = scenario.par("historical_activity")["year_act"].max() modify_industry_demand(scenario, last_hist_year, iea_data_path) add_new_ind_hist_act(scenario, [last_hist_year], iea_data_path) - add_elec_i_ini_act(scenario) add_emission_accounting(scenario) - # scenario.commit("no changes") + if modify_existing_constraints: + calibrate_existing_constraints(scenario) + + return scenario + + +def calibrate_existing_constraints(scenario: message_ix.Scenario): + if "SSP_dev" not in scenario.model: + engage_updates._correct_balance_td_efficiencies(scenario) + engage_updates._correct_coal_ppl_u_efficiencies(scenario) + engage_updates._correct_td_co2cc_emissions(scenario) + + from message_ix_models import ScenarioInfo + + s_info = ScenarioInfo(scenario) + nodes = s_info.N + add_coal_lowerbound_2020(scenario) add_cement_bounds_2020(scenario) # Market penetration adjustments # NOTE: changing demand affects the market penetration - # levels for the enduse technologies. + # levels for the end-use technologies. # FIXME: context.ssp only works for SSP1/2/3 currently missing SSP4/5 calibrate_UE_gr_to_demand( scenario, @@ -193,11 +168,6 @@ def build( if "R12_CHN" in nodes: add_elec_lowerbound_2020(scenario) - # i_feed demand is zero creating a zero division error during MACRO calibration - scenario.check_out() - scenario.remove_set("sector", "i_feed") - scenario.commit("i_feed removed from sectors.") - df = scenario.par( "bound_activity_lo", filters={"node_loc": "R12_RCPA", "technology": "sp_el_I", "year_act": 2020}, @@ -206,7 +176,7 @@ def build( scenario.remove_par("bound_activity_lo", df) scenario.commit("remove sp_el_I min bound on RCPA in 2020") - return scenario + add_elec_i_ini_act(scenario) def get_spec() -> Mapping[str, ScenarioInfo]: @@ -233,3 +203,57 @@ def get_spec() -> Mapping[str, ScenarioInfo]: remove.set[set_name].extend(config.get("remove", [])) return dict(require=require, add=add, remove=remove) + + +def make_spec(regions: str, materials: str or None = SPEC_LIST) -> Spec: + sets: Dict[str, Any] = dict() + materials = ["common"] if not materials else materials + # Overrides specific to regional versions + tmp = dict() + # technology.yaml currently not used in Materials + for fn in ["set.yaml"]: # , "technology.yaml"): + # Field name + name = fn.split(".yaml")[0] + + # Load and store the data from the YAML file: either in a subdirectory for + # context.model.regions, or the top-level data directory + path = path_fallback(regions, fn).relative_to(package_data_path()) + # tmp[name] = load_private_data(*path.parts) + tmp[name] = load_package_data(*path.parts) + + # Merge contents of technology.yaml into set.yaml + # technology.yaml currently not used in Materials + sets.update(tmp.pop("set")) + + s = Spec() + + # Convert some values to codes + for material in materials: + for set_name in sets[material]: + if not all( + [ + isinstance(item, list) + for sublist in sets[material][set_name].values() + for item in sublist + ] + ): + generate_set_elements(sets[material], set_name) + + # Elements to add, remove, and require + for action in {"add", "remove", "require"}: + s[action].set[set_name].extend(sets[material][set_name].get(action, [])) + try: + s.add.set[f"{set_name} indexers"] = sets[material][set_name]["indexers"] + except KeyError: + pass + + # The set of required nodes varies according to context.model.regions + codelist = regions + try: + s["require"].set["node"].extend(map(str, get_region_codes(codelist))) + except FileNotFoundError: + raise ValueError( + f"Cannot get spec for MESSAGEix-Materials with regions={codelist!r}" + ) from None + + return s diff --git a/message_ix_models/model/material/cli.py b/message_ix_models/model/material/cli.py index 0509156241..866888ea1e 100644 --- a/message_ix_models/model/material/cli.py +++ b/message_ix_models/model/material/cli.py @@ -38,31 +38,6 @@ def cli(ssp): """MESSAGEix-Materials variant.""" -@cli.command("create-bare") -@click.option("--regions", type=click.Choice(["China", "R11", "R14"])) -@click.option("--dry_run", "-n", is_flag=True, help="Only show what would be done.") -@click.pass_obj -def create_bare(context, regions, dry_run): - """Create the RES from scratch.""" - from message_ix_models.model.bare import create_res - - if regions: - context.regions = regions - - # to allow historical years - context.period_start = 1980 - - # Otherwise it can not find the path to read the yaml files.. - # context.metadata_path = context.metadata_path / "data" - - scen = create_res(context) - build(scen, True) - - # Solve - if not dry_run: - scen.solve() - - @cli.command("build") @click.option( "--datafile", @@ -85,6 +60,7 @@ def create_bare(context, regions, dry_run): "--update_costs", default=False, ) +@common_params("nodes") @click.pass_obj def build_scen( context, datafile, iea_data_path, tag, mode, scenario_name, old_calib, update_costs @@ -137,9 +113,12 @@ def build_scen( scenario=context.scenario_info["scenario"] + "_" + tag, keep_solution=False, ) - scenario = build(scenario, old_calib=old_calib, iea_data_path=iea_data_path) + scenario = build( + context, scenario, old_calib=old_calib, iea_data_path=iea_data_path + ) else: scenario = build( + context, context.get_scenario().clone( model="MESSAGEix-Materials", scenario=output_scenario_name + "_" + tag, @@ -213,107 +192,79 @@ def build_scen( scenario.commit(f"update cost assumption to: {update_costs}") +def validate_macrofile_path(ctx, param, value): + if value and ctx.params["macro_file"]: + if not package_data_path( + "material", "macro", ctx.params["macro_file"] + ).is_file(): + raise FileNotFoundError( + "Specified file name of MACRO calibration file does not exist. Please" + "place in data/material/macro or use other file that exists." + ) + + @cli.command("solve") @click.option("--add_macro", default=True) -@click.option("--add_calibration", default=False) +@click.option("--add_calibration", default=False, callback=validate_macrofile_path) +@click.option("--macro_file", default=None, is_eager=True) +@click.option("--shift_model_year", default=False) @click.pass_obj -def solve_scen(context, add_calibration, add_macro): +def solve_scen(context, add_calibration, add_macro, macro_file, shift_model_year): """Solve a scenario. Use the --model_name and --scenario_name option to specify the scenario to solve. """ # default scenario: MESSAGEix-Materials NoPolicy scenario = context.get_scenario() - - if scenario.has_solution(): - if not add_calibration: - scenario.remove_solution() + default_solve_opt = { + "model": "MESSAGE", + "solve_options": {"lpmethod": "4", "scaind": "-1"}, + } + if shift_model_year: + if not scenario.has_solution(): + scenario.solve(**default_solve_opt) + if scenario.timeseries(year=scenario.firstmodelyear).empty: + log.info( + "Scenario has no timeseries data in baseyear. Starting" + "reporting workflow before shifting baseyear." + ) + run_reporting(context, False, False) + # Shift base year + scenario = scenario.clone( + model=scenario.model, + scenario=scenario.scenario + f"_{shift_model_year}", + shift_first_model_year=shift_model_year, + ) if add_calibration: - # Solve with 2020 base year - log.info("Solving the scenario without MACRO") log.info( "After macro calibration a new scenario with the suffix _macro is created." + "Make sure to use this scenario to solve with MACRO iterations." ) - log.info("Make sure to use this scenario to solve with MACRO iterations.") if not scenario.has_solution(): - scenario.solve( - model="MESSAGE", solve_options={"lpmethod": "4", "scaind": "-1"} + log.info( + "Uncalibrated scenario has no solution. Solving the scenario" + "without MACRO before calibration" ) + scenario.solve(**default_solve_opt) scenario.set_as_default() - # Report - from message_ix_models.model.material.report.reporting import report - from message_ix_models.report.legacy.iamc_report_hackathon import ( - report as reporting, - ) - - # Remove existing timeseries and add material timeseries - log.info("Reporting material-specific variables") - report(context, scenario) - log.info("Reporting standard variables") - reporting( - context.get_platform(), - scenario, - "False", - scenario.model, - scenario.scenario, - merge_hist=True, - merge_ts=True, - run_config="materials_run_config.yaml", - ) - - # Shift to 2025 base year - scenario = scenario.clone( - model=scenario.model, - scenario=scenario.scenario + "_2025", - shift_first_model_year=2025, - ) - scenario.set_as_default() - scenario.solve(model="MESSAGE", solve_options={"lpmethod": "4", "scaind": "-1"}) - # update cost_ref and price_ref with new solution - update_macro_calib_file( - scenario, f"SSP_dev_{context['ssp']}-R12-5y_macro_data_v0.12_mat.xlsx" - ) + # f"SSP_dev_{context['ssp']}-R12-5y_macro_data_v0.12_mat.xlsx" + update_macro_calib_file(scenario, macro_file) # After solving, add macro calibration log.info("Scenario solved, now adding MACRO calibration") - # scenario = add_macro_COVID( - # scenario, "R12-CHN-5y_macro_data_NGFS_w_rc_ind_adj_mat.xlsx" - # ) - scenario = add_macro_COVID( - scenario, f"SSP_dev_{context['ssp']}-R12-5y_macro_data_v0.12_mat.xlsx" - ) + # f"SSP_dev_{context['ssp']}-R12-5y_macro_data_v0.12_mat.xlsx" + scenario = add_macro_COVID(scenario, macro_file) log.info("Scenario successfully calibrated.") - if add_macro: # Default True - scenario.solve( - model="MESSAGE-MACRO", solve_options={"lpmethod": "4", "scaind": "-1"} - ) - scenario.set_as_default() - - if not add_macro: - # Solve - log.info("Solving the scenario without MACRO") - scenario.solve(model="MESSAGE", solve_options={"lpmethod": "4", "scaind": "-1"}) - scenario.set_as_default() - + if add_macro: + default_solve_opt.update({"model": "MESSAGE-MACRO"}) -@cli.command("add-buildings-ts") -@click.pass_obj -def add_building_ts(context): - # FIXME This import can not be resolved - from message_ix_models.reporting.materials.add_buildings_ts import ( - add_building_timeseries, - ) - - scenario = context.get_scenario() - log.info( - f"Adding buildings specific timeseries to scenario:{scenario.model} - " - f"{scenario.scenario}" - ) - add_building_timeseries(scenario) + log.info("Start solving the scenario") + scenario.solve(**default_solve_opt) + scenario.set_as_default() @cli.command("report") @@ -324,7 +275,7 @@ def add_building_ts(context): ) @click.option("--profile", default=False) @click.pass_obj -def run_reporting(context, remove_ts, profile, prep_for_explorer): +def run_reporting(context, remove_ts, profile): """Run materials specific reporting, then legacy reporting.""" from message_ix_models.model.material.report.reporting import report from message_ix_models.report.legacy.iamc_report_hackathon import ( @@ -396,31 +347,6 @@ def exit(): ) -@cli.command("report-2") -@click.pass_obj -def run_old_reporting(context): - from message_ix_models.report.legacy.iamc_report_hackathon import ( - report as reporting, - ) - - # Retrieve the scenario given by the --url option - scenario = context.get_scenario() - mp = scenario.platform - - reporting( - mp, - scenario, - # NB(PNK) this is not an error; .iamc_report_hackathon.report() expects a - # string containing "True" or "False" instead of an actual bool. - "False", - scenario.model, - scenario.scenario, - merge_hist=True, - merge_ts=True, - run_config="materials_run_config.yaml", - ) - - @cli.command("modify-cost", hidden=True) @click.option("--ssp", default="SSP2", help="Suffix to the scenario name") @click.pass_obj diff --git a/message_ix_models/model/material/data_aluminum.py b/message_ix_models/model/material/data_aluminum.py index 570cbb09e2..2cc9110538 100644 --- a/message_ix_models/model/material/data_aluminum.py +++ b/message_ix_models/model/material/data_aluminum.py @@ -1,4 +1,5 @@ from collections import defaultdict +from typing import Dict, Iterable, List import message_ix import pandas as pd @@ -68,7 +69,7 @@ def read_data_aluminum( return data_alu, data_alu_rel, data_aluminum_ts -def gen_data_alu_ts(data: pd.DataFrame, nodes: list) -> dict[str, pd.DataFrame]: +def gen_data_alu_ts(data: pd.DataFrame, nodes: list) -> Dict[str, pd.DataFrame]: """ Generates time variable parameter data for aluminum sector Parameters @@ -223,7 +224,16 @@ def gen_data_alu_rel(data: pd.DataFrame, years: list) -> dict[str, pd.DataFrame] def assign_input_outpt( - split, param_name, regions, val, t, rg, glb_reg, common, yv_ya, nodes + split, + param_name: str, + regions: pd.DataFrame, + val, + t: str, + rg: str, + glb_reg: str, + common: dict, + yv_ya: pd.DataFrame, + nodes, ): # Assign commodity and level names # Later mod can be added @@ -314,9 +324,17 @@ def assign_input_outpt( return df -def gen_data_alu_const(data, config, glb_reg, years, yv_ya, nodes): +def gen_data_alu_const( + data: pd.DataFrame, + config: dict, + glb_reg: str, + years: Iterable, + yv_ya: pd.DataFrame, + nodes: List[str], +): results = defaultdict(list) for t in config["technology"]["add"]: + t = t.id params = data.loc[(data["technology"] == t), "parameter"].unique() # Obtain the active and vintage years av = data.loc[(data["technology"] == t), "availability"].values[0] @@ -440,17 +458,17 @@ def gen_data_aluminum( parname = "demand" demand_dict = {} - df = material_demand_calc.derive_demand( - "aluminum", scenario, old_gdp=False, ssp=ssp - ) + df = material_demand_calc.derive_demand("aluminum", scenario, ssp=ssp) demand_dict[parname] = df ts_dict = gen_data_alu_ts(data_aluminum_ts, nodes) rel_dict = gen_data_alu_rel(data_aluminum_rel, modelyears) + trade_dict = gen_data_alu_trade(scenario) results_aluminum = combine_df_dictionaries( - const_dict, ts_dict, rel_dict, demand_dict + const_dict, ts_dict, rel_dict, demand_dict, trade_dict ) + return results_aluminum @@ -524,3 +542,190 @@ def gen_mock_demand_aluminum(scenario: message_ix.Scenario) -> pd.DataFrame: ) return demand2020_al + + +def gen_data_alu_trade(scenario: message_ix.Scenario) -> Dict[str, pd.DataFrame]: + results = defaultdict(list) + + data_trade = pd.read_csv( + package_data_path("material", "aluminum", "aluminum_trade.csv") + ) + + data_trade.drop_duplicates() + + s_info = ScenarioInfo(scenario) + + yv_ya = s_info.yv_ya + year_all = yv_ya["year_vtg"].unique() + + data_trade = data_trade[data_trade["Year"].isin(year_all)] + + # Divide R12_WEU as 0.7 WEU, 0.3 EEU. + data_trade.loc[(data_trade["Region"] == "Europe"), "Value"] *= 0.7 + data_trade.loc[(data_trade["Region"] == "Europe"), "Region"] = "West Europe" + + data_trade_eeu = data_trade[data_trade["Region"] == "West Europe"] + data_trade_eeu["Value"] *= 0.3 / 0.7 + data_trade_eeu["Region"] = "East Europe" + + data_trade = pd.concat([data_trade, data_trade_eeu]) + + # Sum Japan and Oceania as PAO + + condition = (data_trade["Region"] == "Japan") | (data_trade["Region"] == "Oceania") + data_trade_pao = data_trade.loc[condition] + data_trade_pao = ( + data_trade_pao.groupby(["Variable", "Year"])["Value"].sum().reset_index() + ) + + data_trade_pao["Region"] = "Pacific OECD" + data_trade = pd.concat([data_trade, data_trade_pao]) + condition_updated = (data_trade["Region"] == "Japan") | ( + data_trade["Region"] == "Oceania" + ) + data_trade = data_trade.drop(data_trade[condition_updated].index) + + data_trade.reset_index(drop=True, inplace=True) + + # Divide Other Asia 50-50 to SAS and PAS + + data_trade.loc[(data_trade["Region"] == "Other Asia"), "Value"] *= 0.5 + data_trade.loc[(data_trade["Region"] == "Other Asia"), "Region"] = "South Asia" + + data_trade_pas = data_trade[data_trade["Region"] == "South Asia"] + data_trade_pas["Region"] = "Other Pacific Asia" + + data_trade = pd.concat([data_trade, data_trade_pas]) + + # Divide Other Producing Regions 50-50s as Africa and FSU + + data_trade.loc[(data_trade["Region"] == "Other Producers"), "Value"] *= 0.5 + data_trade.loc[(data_trade["Region"] == "Other Producers"), "Region"] = "Africa" + + data_trade_fsu = data_trade[data_trade["Region"] == "Africa"] + data_trade_fsu["Region"] = "Former Soviet Union" + + data_trade = pd.concat([data_trade, data_trade_fsu]) + + # Drop non-producers + + condition = data_trade["Region"] == "Non Producers" + data_trade = data_trade.drop(data_trade[condition].index) + + s_info = ScenarioInfo(scenario) + + if "R12_CHN" in s_info.N: + region_tag = "R12_" + china_mapping = "R12_CHN" + else: + region_tag = "R11_" + china_mapping = "R11_CPA" + + region_mapping = { + "China": china_mapping, + "West Europe": region_tag + "WEU", + "East Europe": region_tag + "EEU", + "Pacific OECD": region_tag + "PAO", + "South Asia": region_tag + "SAS", + "Other Pacific Asia": region_tag + "PAS", + "Africa": region_tag + "AFR", + "Former Soviet Union": region_tag + "FSU", + "Middle East": region_tag + "MEA", + "North America": region_tag + "NAM", + "South America": region_tag + "LAM", + } + + # Add the data as historical_activity + + data_trade = data_trade.replace(region_mapping) + data_trade.rename( + columns={"Region": "node_loc", "Year": "year_act", "Value": "value"}, + inplace=True, + ) + + # Trade is at the product level. + # For imports this corresponds to: USE|Inputs|Imports + + data_import = data_trade[data_trade["Variable"] == "USE|Inputs|Imports"] + data_import_hist = data_import[data_import["year_act"] <= 2015] + data_import_hist["technology"] = "import_aluminum" + data_import_hist["mode"] = "M1" + data_import_hist["time"] = "year" + data_import_hist["unit"] = "-" + data_import_hist.drop(["Variable"], axis=1, inplace=True) + data_import_hist.reset_index(drop=True) + + # For exports this corresponds to: MANUFACTURING|Outputs|Exports + + data_export = data_trade[data_trade["Variable"] == "MANUFACTURING|Outputs|Exports"] + data_export_hist = data_export[data_export["year_act"] <= 2015] + data_export_hist["technology"] = "export_aluminum" + data_export_hist["mode"] = "M1" + data_export_hist["time"] = "year" + data_export_hist["unit"] = "-" + data_export_hist.drop(["Variable"], axis=1, inplace=True) + data_export_hist.reset_index(drop=True) + + results["historical_activity"].append(data_export_hist) + results["historical_activity"].append(data_import_hist) + + # Add data as historical_new_capacity for export + + for r in data_export_hist["node_loc"].unique(): + df_hist_cap = data_export_hist[data_export_hist["node_loc"] == r] + df_hist_cap = df_hist_cap.sort_values(by="year_act") + df_hist_cap["value_difference"] = df_hist_cap["value"].diff() + df_hist_cap["value_difference"] = df_hist_cap["value_difference"].fillna( + df_hist_cap["value"] + ) + df_hist_cap["historical_new_capacity"] = df_hist_cap["value_difference"] / 5 + + df_hist_cap = df_hist_cap.drop( + columns=["mode", "time", "value", "value_difference"], axis=1 + ) + df_hist_cap.rename( + columns={"historical_new_capacity": "value", "year_act": "year_vtg"}, + inplace=True, + ) + + df_hist_cap["value"] = df_hist_cap["value"].apply(lambda x: 0 if x < 0 else x) + df_hist_cap["unit"] = "-" + results["historical_new_capacity"].append(df_hist_cap) + + # For China fixing 2020 and 2025 values + + import_chn = data_import[ + (data_import["year_act"] == 2020) & (data_import["node_loc"] == "R12_CHN") + ] + + export_chn = data_export[ + (data_export["year_act"] == 2020) & (data_export["node_loc"] == "R12_CHN") + ] + + # Merge the DataFrames on 'node_loc' and 'year' + merged_df = pd.merge( + import_chn, + export_chn, + on=["node_loc", "year_act"], + suffixes=("_import", "_export"), + ) + + # Subtract the 'value_import' from 'value_export' to get net export value + merged_df["value"] = merged_df["value_export"] - merged_df["value_import"] + + # Select relevant columns for the final DataFrame + bound_act_net_export_chn = merged_df[["node_loc", "year_act", "value"]] + + bound_act_net_export_chn["technology"] = "export_aluminum" + bound_act_net_export_chn["mode"] = "M1" + bound_act_net_export_chn["time"] = "year" + bound_act_net_export_chn["unit"] = "-" + + bound_act_net_export_chn_2025 = bound_act_net_export_chn.replace({2020: 2025}) + + results["bound_activity_up"].append(bound_act_net_export_chn) + results["bound_activity_lo"].append(bound_act_net_export_chn) + results["bound_activity_up"].append(bound_act_net_export_chn_2025) + results["bound_activity_lo"].append(bound_act_net_export_chn_2025) + + return {par_name: pd.concat(dfs) for par_name, dfs in results.items()} diff --git a/message_ix_models/model/material/data_ammonia_new.py b/message_ix_models/model/material/data_ammonia_new.py index 8d77995e5e..5be5b12d16 100644 --- a/message_ix_models/model/material/data_ammonia_new.py +++ b/message_ix_models/model/material/data_ammonia_new.py @@ -1,3 +1,4 @@ +import message_ix import numpy as np import pandas as pd from message_ix import make_df @@ -14,7 +15,6 @@ CONVERSION_FACTOR_NH3_N = 17 / 14 - ssp_mode_map = { "SSP1": "CTS core", "SSP2": "RTS core", @@ -32,7 +32,9 @@ } -def gen_all_NH3_fert(scenario, dry_run=False): +def gen_all_NH3_fert( + scenario: message_ix.Scenario, dry_run: bool = False +) -> dict[str, pd.DataFrame]: return { **gen_data(scenario), **gen_data_rel(scenario), @@ -43,7 +45,9 @@ def gen_all_NH3_fert(scenario, dry_run=False): } -def broadcast_years(df_new, max_lt, act_years, vtg_years): +def broadcast_years( + df_new: pd.DataFrame, max_lt: int, act_years: pd.Series, vtg_years: pd.Series +) -> pd.DataFrame: if "year_act" in df_new.columns: df_new = df_new.pipe(same_node).pipe(broadcast, year_act=act_years) @@ -63,7 +67,12 @@ def broadcast_years(df_new, max_lt, act_years, vtg_years): return df_new -def gen_data(scenario, dry_run=False, add_ccs: bool = True, lower_costs=False): +def gen_data( + scenario: message_ix.Scenario, + dry_run=False, + add_ccs: bool = True, + lower_costs: bool = False, +) -> dict[str, pd.DataFrame]: s_info = ScenarioInfo(scenario) # s_info.yv_ya nodes = nodes_ex_world(s_info.N) @@ -250,7 +259,9 @@ def same_node_if_nan(df): return par_dict -def gen_data_ts(scenario, dry_run=False, add_ccs: bool = True): +def gen_data_ts( + scenario: message_ix.Scenario, dry_run: bool = False, add_ccs: bool = True +) -> dict[str, pd.DataFrame]: s_info = ScenarioInfo(scenario) # s_info.yv_ya nodes = s_info.N @@ -305,14 +316,14 @@ def gen_data_ts(scenario, dry_run=False, add_ccs: bool = True): return par_dict -def set_exp_imp_nodes(df): +def set_exp_imp_nodes(df: pd.DataFrame) -> None: if "node_dest" in df.columns: df.loc[df["technology"].str.contains("export"), "node_dest"] = "R12_GLB" if "node_origin" in df.columns: df.loc[df["technology"].str.contains("import"), "node_origin"] = "R12_GLB" -def read_demand(): +def read_demand() -> dict[str, pd.DataFrame]: """Read and clean data from :file:`CD-Links SSP2 N-fertilizer demand.Global.xlsx`.""" # Demand scenario [Mt N/year] from GLOBIOM @@ -497,7 +508,7 @@ def read_demand(): } -def gen_demand(): +def gen_demand() -> dict[str, pd.DataFrame]: N_energy = read_demand()["N_feed"] # updated feed with imports accounted demand_fs_org = pd.read_excel( @@ -523,7 +534,7 @@ def gen_demand(): return {"demand": df} -def gen_resid_demand_NH3(scenario): +def gen_resid_demand_NH3(scenario: message_ix.Scenario) -> dict[str, pd.DataFrame]: context = read_config() ssp = context["ssp"] @@ -537,13 +548,13 @@ def gen_resid_demand_NH3(scenario): return {"demand": df_residual} -def gen_land_input(scenario): +def gen_land_input(scenario: message_ix.Scenario) -> dict[str, pd.DataFrame]: df = scenario.par("land_output", {"commodity": "Fertilizer Use|Nitrogen"}) df["level"] = "final_material" return {"land_input": df} -def experiment_lower_CPA_SAS_costs(par_dict): +def experiment_lower_CPA_SAS_costs(par_dict: dict) -> dict[str, pd.DataFrame]: cost_list = ["inv_cost", "fix_cost"] scaler = { "R12_RCPA": [0.66 * 0.91, 0.75 * 0.9], diff --git a/message_ix_models/model/material/data_buildings.py b/message_ix_models/model/material/data_buildings.py deleted file mode 100644 index 085b2166a2..0000000000 --- a/message_ix_models/model/material/data_buildings.py +++ /dev/null @@ -1,293 +0,0 @@ -from collections import defaultdict - -import numpy as np -import pandas as pd -from message_ix import make_df - -from message_ix_models import ScenarioInfo -from message_ix_models.model.material.util import read_config -from message_ix_models.util import ( - copy_column, - package_data_path, - same_node, -) - -CASE_SENS = "ref" # 'min', 'max' -INPUTFILE = "LED_LED_report_IAMC_sensitivity_R12.csv" -# INPUTFILE = 'LED_LED_report_IAMC_sensitivity_R11.csv' - - -def read_timeseries_buildings(filename, scenario, case=CASE_SENS): - # Read the file and filter the given sensitivity case - bld_input_raw = pd.read_csv(package_data_path("material", "buildings", filename)) - bld_input_raw = bld_input_raw.loc[bld_input_raw.Sensitivity == case] - - bld_input_mat = bld_input_raw[ - bld_input_raw[ - "Variable" - ].str.contains( # "Floor Space|Aluminum|Cement|Steel|Final Energy" - "Floor Space|Aluminum|Cement|Steel" - ) - ] # Final Energy - Later. Need to figure out how to carve out - bld_input_mat["Region"] = "R12_" + bld_input_mat["Region"] - print("Check the year values") - print(bld_input_mat) - - bld_input_pivot = ( - bld_input_mat.melt( - id_vars=["Region", "Variable"], - var_name="Year", - value_vars=list(map(str, range(2015, 2101, 5))), - ) - .set_index(["Region", "Year", "Variable"]) - .squeeze() - .unstack() - .reset_index() - ) - - # Divide by floor area to get energy/material intensities - bld_intensity_ene_mat = bld_input_pivot.iloc[:, 2:].div( - bld_input_pivot["Energy Service|Residential|Floor Space"], axis=0 - ) - bld_intensity_ene_mat.columns = [ - s + "|Intensity" for s in bld_intensity_ene_mat.columns - ] - bld_intensity_ene_mat = pd.concat( - [ - bld_input_pivot[["Region", "Year"]], - bld_intensity_ene_mat.reindex(bld_input_pivot.index), - ], - axis=1, - ).drop(columns=["Energy Service|Residential|Floor Space|Intensity"]) - - bld_intensity_ene_mat["Energy Service|Residential|Floor Space"] = bld_input_pivot[ - "Energy Service|Residential|Floor Space" - ] - - # Material intensities are in kg/m2 - bld_data_long = bld_intensity_ene_mat.melt( - id_vars=["Region", "Year"], var_name="Variable" - ).rename(columns={"Region": "node", "Year": "year"}) - # Both for energy and material - bld_intensity_long = bld_data_long[ - bld_data_long["Variable"].str.contains("Intensity") - ].reset_index(drop=True) - bld_area_long = bld_data_long[ - bld_data_long["Variable"] == "Energy Service|Residential|Floor Space" - ].reset_index(drop=True) - - tmp = bld_intensity_long.Variable.str.split("|", expand=True) - - bld_intensity_long["commodity"] = tmp[3].str.lower() # Material type - bld_intensity_long["type"] = tmp[0] # 'Material Demand' or 'Scrap Release' - bld_intensity_long["unit"] = "kg/m2" - - bld_intensity_long = bld_intensity_long.drop(columns="Variable") - bld_area_long = bld_area_long.drop(columns="Variable") - - bld_intensity_long = bld_intensity_long.drop( - bld_intensity_long[np.isnan(bld_intensity_long.value)].index - ) - - # Derive baseyear material demand (Mt/year in 2020) - bld_demand_long = bld_input_pivot.melt( - id_vars=["Region", "Year"], var_name="Variable" - ).rename(columns={"Region": "node", "Year": "year"}) - tmp = bld_demand_long.Variable.str.split("|", expand=True) - bld_demand_long["commodity"] = tmp[3].str.lower() # Material type - # bld_demand_long = bld_demand_long[bld_demand_long['year']=="2020"].\ - # dropna(how='any') - bld_demand_long = bld_demand_long.dropna(how="any") - bld_demand_long = bld_demand_long[ - bld_demand_long["Variable"].str.contains("Material Demand") - ].drop(columns="Variable") - - return bld_intensity_long, bld_area_long, bld_demand_long - - -def get_scen_mat_demand( - commod, scenario, year="2020", inputfile=INPUTFILE, case=CASE_SENS -): - a, b, c = read_timeseries_buildings(inputfile, scenario, case) - if not year == "all": # specific year - cc = c[(c.commodity == commod) & (c.year == year)].reset_index(drop=True) - else: # all years - cc = c[(c.commodity == commod)].reset_index(drop=True) - return cc - - -def adjust_demand_param(scen): - scen_mat_demand = scen.par( - "demand", {"level": "demand"} - ) # mat demand without buildings considered - - scen.check_out() - comms = ["steel", "cement", "aluminum"] - for c in comms: - mat_building = get_scen_mat_demand(c, scen, year="all").rename( - columns={"value": "bld_demand"} - ) # mat demand (timeseries) from buildings model (Alessio) - mat_building["year"] = mat_building["year"].astype(int) - - sub_mat_demand = scen_mat_demand.loc[scen_mat_demand.commodity == c] - # print("old", sub_mat_demand.loc[sub_mat_demand.year >=2025]) - sub_mat_demand = sub_mat_demand.join( - mat_building.set_index(["node", "year", "commodity"]), - on=["node", "year", "commodity"], - how="left", - ) - sub_mat_demand["value"] = sub_mat_demand["value"] - sub_mat_demand["bld_demand"] - sub_mat_demand = sub_mat_demand.drop(columns=["bld_demand"]).dropna(how="any") - - # Only replace for year >= 2025 - scen.add_par("demand", sub_mat_demand.loc[sub_mat_demand.year >= 2025]) - # print("new", sub_mat_demand.loc[sub_mat_demand.year >=2025]) - scen.commit("Building material demand subtracted") - - -def gen_data_buildings(scenario, dry_run=False): - """Generate data for materials representation of steel industry.""" - # Load configuration - context = read_config() - config = context["material"]["buildings"] - - # New element names for buildings integrations - lev_new = config["level"]["add"][0] - comm_new = config["commodity"]["add"][0] - tec_new = config["technology"]["add"][0] # "buildings" - - print(lev_new, comm_new, tec_new, type(tec_new)) - - # Information about scenario, e.g. node, year - s_info = ScenarioInfo(scenario) - - # Buildings raw data (from Alessio) - ( - data_buildings, - data_buildings_demand, - data_buildings_mat_demand, - ) = read_timeseries_buildings(INPUTFILE, scenario, CASE_SENS) - - # List of data frames, to be concatenated together at end - results = defaultdict(list) - - # For each technology there are different input and output combinations - # Iterate over technologies - - # allyears = s_info.set['year'] #s_info.Y is only for modeling years - modelyears = s_info.Y # s_info.Y is only for modeling years - nodes = s_info.N - # fmy = s_info.y0 - nodes.remove("World") - # nodes.remove("R11_RCPA") - - # Read field values from the buildings input data - regions = list(set(data_buildings.node)) - comms = list(set(data_buildings.commodity)) - # types = list(set(data_buildings.type)) - types = ["Material Demand", "Scrap Release"] # Order matters - - common = dict(time="year", time_origin="year", time_dest="year", mode="M1") - - # Filter only the years in the base scenario - data_buildings["year"] = data_buildings["year"].astype(int) - data_buildings_demand["year"] = data_buildings_demand["year"].astype(int) - data_buildings = data_buildings[data_buildings["year"].isin(modelyears)] - data_buildings_demand = data_buildings_demand[ - data_buildings_demand["year"].isin(modelyears) - ] - - # historical demands - - for rg in regions: - for comm in comms: - # for typ in types: - - val_mat = data_buildings.loc[ - (data_buildings["type"] == types[0]) - & (data_buildings["commodity"] == comm) - & (data_buildings["node"] == rg), - ] - val_scr = data_buildings.loc[ - (data_buildings["type"] == types[1]) - & (data_buildings["commodity"] == comm) - & (data_buildings["node"] == rg), - ] - - # Material input to buildings - df = ( - make_df( - "input", - technology=tec_new, - commodity=comm, - level="demand", - year_vtg=val_mat.year, - value=val_mat.value, - unit="t", - node_loc=rg, - **common, - ) - .pipe(same_node) - .assign(year_act=copy_column("year_vtg")) - ) - results["input"].append(df) - - # Scrap output back to industry - df = ( - make_df( - "output", - technology=tec_new, - commodity=comm, - level="end_of_life", - year_vtg=val_scr.year, - value=val_scr.value, - unit="t", - node_loc=rg, - **common, - ) - .pipe(same_node) - .assign(year_act=copy_column("year_vtg")) - ) - results["output"].append(df) - - # Service output to buildings demand - df = ( - make_df( - "output", - technology=tec_new, - commodity=comm_new, - level="demand", - year_vtg=val_mat.year, - value=1, - unit="t", - node_loc=rg, - **common, - ) - .pipe(same_node) - .assign(year_act=copy_column("year_vtg")) - ) - results["output"].append(df) - - # Create external demand param - parname = "demand" - demand = data_buildings_demand - df = make_df( - parname, - level="demand", - commodity=comm_new, - value=demand.value, - unit="t", - year=demand.year, - time="year", - node=demand.node, - ) - results[parname].append(df) - - # Concatenate to one data frame per parameter - results = {par_name: pd.concat(dfs) for par_name, dfs in results.items()} - - # TODO: check the starting model/scenario, if not ENGAGE, call adjust_demand_param - if scenario.scenario == "LEDXXXX": - adjust_demand_param(scenario) - - return results diff --git a/message_ix_models/model/material/data_cement.py b/message_ix_models/model/material/data_cement.py index d1c958176c..ce9f3e565c 100644 --- a/message_ix_models/model/material/data_cement.py +++ b/message_ix_models/model/material/data_cement.py @@ -1,10 +1,15 @@ from collections import defaultdict +import message_ix import pandas as pd from message_ix import make_df from message_ix_models import ScenarioInfo -from message_ix_models.model.material.data_util import read_sector_data, read_timeseries +from message_ix_models.model.material.data_util import ( + calculate_ini_new_cap, + read_sector_data, + read_timeseries, +) from message_ix_models.model.material.material_demand import material_demand_calc from message_ix_models.model.material.util import get_ssp_from_context, read_config from message_ix_models.util import ( @@ -15,7 +20,7 @@ ) -def gen_mock_demand_cement(scenario): +def gen_mock_demand_cement(scenario: message_ix.Scenario) -> pd.DataFrame: s_info = ScenarioInfo(scenario) nodes = s_info.N nodes.remove("World") @@ -156,7 +161,9 @@ def gen_mock_demand_cement(scenario): return demand2020_cement -def gen_data_cement(scenario, dry_run=False): +def gen_data_cement( + scenario: message_ix.Scenario, dry_run: bool = False +) -> dict[str, pd.DataFrame]: """Generate data for materials representation of cement industry.""" # Load configuration context = read_config() @@ -166,15 +173,16 @@ def gen_data_cement(scenario, dry_run=False): s_info = ScenarioInfo(scenario) context.datafile = "Global_steel_cement_MESSAGE.xlsx" - data_cement = read_sector_data(scenario, "cement") + # Techno-economic assumptions + data_cement = read_sector_data(scenario, "cement", "Global_cement_MESSAGE.xlsx") # Special treatment for time-dependent Parameters - data_cement_ts = read_timeseries(scenario, "steel_cement", context.datafile) + data_cement_ts = read_timeseries(scenario, "cement", "Global_cement_MESSAGE.xlsx") tec_ts = set(data_cement_ts.technology) # set of tecs with var_cost # List of data frames, to be concatenated together at end results = defaultdict(list) - # For each technology there are differnet input and output combinations + # For each technology there are different input and output combinations # Iterate over technologies yv_ya = s_info.yv_ya @@ -182,8 +190,8 @@ def gen_data_cement(scenario, dry_run=False): # Do not parametrize GLB region the same way nodes = nodes_ex_world(s_info.N) - # for t in s_info.set['technology']: for t in config["technology"]["add"]: + t = t.id params = data_cement.loc[(data_cement["technology"] == t), "parameter"].unique() # Special treatment for time-varying params @@ -344,18 +352,41 @@ def gen_data_cement(scenario, dry_run=False): # Create external demand param parname = "demand" - df = material_demand_calc.derive_demand("cement", scenario, old_gdp=False, ssp=ssp) - results[parname].append(df) + df_demand = material_demand_calc.derive_demand("cement", scenario, ssp=ssp) + results[parname].append(df_demand) # Add CCS as addon parname = "addon_conversion" - ccs_tec = ["clinker_wet_cement", "clinker_dry_cement"] - df = make_df( - parname, mode="M1", type_addon="ccs_cement", value=1, unit="-", **common - ).pipe(broadcast, node=nodes, technology=ccs_tec) - results[parname].append(df) + + technology_1 = ["clinker_dry_cement"] + df_1 = make_df( + parname, mode="M1", type_addon="dry_ccs_cement", value=1, unit="-", **common + ).pipe(broadcast, node=nodes, technology=technology_1) + + technology_2 = ["clinker_wet_cement"] + df_2 = make_df( + parname, mode="M1", type_addon="wet_ccs_cement", value=1, unit="-", **common + ).pipe(broadcast, node=nodes, technology=technology_2) + + results[parname].append(df_1) + results[parname].append(df_2) # Concatenate to one data frame per parameter results = {par_name: pd.concat(dfs) for par_name, dfs in results.items()} + results["initial_new_capacity_up"] = pd.concat( + [ + calculate_ini_new_cap( + df_demand=df_demand.copy(deep=True), + technology="clinker_dry_ccs_cement", + material="cement", + ), + calculate_ini_new_cap( + df_demand=df_demand.copy(deep=True), + technology="clinker_wet_ccs_cement", + material="cement", + ), + ] + ) + return results diff --git a/message_ix_models/model/material/data_generic.py b/message_ix_models/model/material/data_generic.py index a5e93457f6..74a29d6780 100644 --- a/message_ix_models/model/material/data_generic.py +++ b/message_ix_models/model/material/data_generic.py @@ -1,7 +1,7 @@ from collections import defaultdict import pandas as pd -from message_ix import make_df +from message_ix import Scenario, make_df import message_ix_models.util from message_ix_models import ScenarioInfo @@ -15,7 +15,7 @@ from .util import read_config -def read_data_generic(scenario): +def read_data_generic(scenario: Scenario) -> (pd.DataFrame, pd.DataFrame): """Read and clean data from :file:`generic_furnace_boiler_techno_economic.xlsx`.""" # Read the file @@ -40,7 +40,9 @@ def read_data_generic(scenario): return data_generic, data_generic_ts -def gen_data_generic(scenario, dry_run=False): +def gen_data_generic( + scenario: Scenario, dry_run: bool = False +) -> dict[str, pd.DataFrame]: # Load configuration config = read_config()["material"]["generic"] @@ -65,6 +67,7 @@ def gen_data_generic(scenario, dry_run=False): global_region = [i for i in s_info.N if i.endswith("_GLB")][0] for t in config["technology"]["add"]: + t = t.id # years = s_info.Y params = data_generic.loc[ (data_generic["technology"] == t), "parameter" diff --git a/message_ix_models/model/material/data_methanol.py b/message_ix_models/model/material/data_methanol.py index 300fb4e9d6..4f0f08b43b 100644 --- a/message_ix_models/model/material/data_methanol.py +++ b/message_ix_models/model/material/data_methanol.py @@ -1,13 +1,17 @@ -import copy +from ast import literal_eval +from typing import TYPE_CHECKING, Dict, List import pandas as pd +import yaml from message_ix import make_df +import message_ix_models.util from message_ix_models.model.material.material_demand import material_demand_calc -from message_ix_models.model.material.util import combine_df_dictionaries, read_config -from message_ix_models.util import broadcast, package_data_path, same_node +from message_ix_models.model.material.util import read_config +from message_ix_models.util import broadcast, same_node -context = read_config() +if TYPE_CHECKING: + from message_ix import Scenario ssp_mode_map = { "SSP1": "CTS core", @@ -26,843 +30,230 @@ } -def gen_data_methanol(scenario): - ssp = context["ssp"] - # read sensitivity file +def gen_data_methanol(scenario: "Scenario") -> Dict[str, pd.DataFrame]: + """ + Generates data for methanol industry model + + Parameters + ---------- + scenario: .Scenario + """ + context = read_config() df_pars = pd.read_excel( - package_data_path("material", "methanol", "methanol_sensitivity_pars.xlsx"), + message_ix_models.util.package_data_path( + "material", "methanol", "methanol_sensitivity_pars.xlsx" + ), sheet_name="Sheet1", dtype=object, ) pars = df_pars.set_index("par").to_dict()["value"] - - # read new meth tec parameters - meth_bio_dict = gen_data_meth_bio(scenario) - meth_bio_ccs_dict = gen_meth_bio_ccs(scenario) - meth_fs_dic = combine_df_dictionaries( - gen_data_meth_bio(scenario), gen_meth_bio_ccs(scenario) - ) - meth_fuel_dic = combine_df_dictionaries(meth_bio_dict, meth_bio_ccs_dict) - for k in meth_fs_dic.keys(): - df_fuel = meth_fuel_dic[k] - df_fs = meth_fs_dic[k] - if "mode" in df_fuel.columns: - df_fuel["mode"] = "fuel" - df_fs["mode"] = "feedstock" - - meth_h2_fs_dict = gen_data_meth_h2("fs") - meth_h2_fuel_dict = gen_data_meth_h2("fuel") - meth_fuel_dic = combine_df_dictionaries(meth_fuel_dic, meth_h2_fuel_dict) - meth_fs_dic = combine_df_dictionaries(meth_fs_dic, meth_h2_fs_dict) - - df = meth_fs_dic["output"] - df.loc[(df["commodity"] == "methanol") & (df["level"] == "primary"), "level"] = ( - "primary_material" - ) - meth_fs_dic["output"] = df - - # add meth prod fs and fuel mode - par_dict_fs = add_prod_mode(scenario, "feedstock", "primary_material") - par_dict = add_prod_mode(scenario, "fuel") - - # add meth_bal fs mode - bal_fs_dict, bal_fuel_dict = add_bal_modes(scenario) - - # add meth trade fs mode - trade_dict_fuel, trade_dict_fs = add_trd_modes(scenario) - - dict_t_d_fs = pd.read_excel( - package_data_path("material", "methanol", "meth_t_d_material_pars.xlsx"), - sheet_name=None, - ) - dict_t_d_fuel = pd.read_excel( - package_data_path("material", "methanol", "meth_t_d_fuel.xlsx"), - sheet_name=None, - ) - - df_rel = dict_t_d_fs["relation_activity"] - - df_rel_meth = df_rel.loc[df_rel["technology"] == "meth_t_d", :] - df_rel_meth["value"] = df_rel_meth.apply( - lambda x: get_embodied_emi(x, pars, "meth_plastics_share"), axis=1 - ) - df_rel_hvc = df_rel.loc[df_rel["technology"] == "steam_cracker_petro", :] - df_rel_hvc["value"] = df_rel_hvc.apply( - lambda x: get_embodied_emi(x, pars, "hvc_plastics_share"), axis=1 - ) - dict_t_d_fs["relation_activity"] = pd.concat([df_rel_meth, df_rel_hvc]) - new_dict2 = combine_df_dictionaries( - meth_fuel_dic, - meth_fs_dic, - par_dict_fs, - par_dict, - bal_fs_dict, - bal_fuel_dict, - trade_dict_fuel, - trade_dict_fs, - dict_t_d_fs, - dict_t_d_fuel, - ) - - # for i in scenario.par_list(): - # try: - # df = scenario.par(i, filters={"technology": ["meth_ng", - # "meth_coal", "meth_ng_ccs", - # "meth_coal_ccs", "meth_exp", - # "meth_imp", "meth_trd", - # "meth_bal", "meth_t_d"], - # "mode": "M1"}) - # if df.size != 0: - # scenario.remove_par(i, df) - # except: - # pass - default_gdp_elasticity_2020, default_gdp_elasticity_2030 = iea_elasticity_map[ - ssp_mode_map[ssp] - ] - df_final = material_demand_calc.gen_demand_petro( - scenario, "methanol", default_gdp_elasticity_2020, default_gdp_elasticity_2030 - ) - df_final["value"] = df_final["value"].apply( - lambda x: x * pars["methanol_resid_demand_share"] - ) - new_dict2["demand"] = df_final - - new_dict2 = combine_df_dictionaries(new_dict2, add_meth_hist_act()) - - mto_dict = gen_data_meth_chemicals(scenario, "MTO") - ch2o_dict = gen_data_meth_chemicals(scenario, "Formaldehyde") - resin_dict = gen_data_meth_chemicals(scenario, "Resins") - chemicals_dict = combine_df_dictionaries(mto_dict, ch2o_dict, resin_dict) - - # generate resin demand from STURM results - df_comm = gen_resin_demand( - scenario, - pars["resin_share"], - "comm", - "REF", - "SSP2", # pars["wood_scenario"], #pars["pathway"] - ) - df_resid = gen_resin_demand( - scenario, - pars["resin_share"], - "residential", - pars["wood_scenario"], - "SSP2", - ) - df_resin_demand = df_comm.copy(deep=True) - df_resin_demand["value"] = df_comm["value"] + df_resid["value"] - new_dict2["demand"] = pd.concat([new_dict2["demand"], df_resin_demand]) - - # modify loil_trp inputs - new_dict2 = combine_df_dictionaries( - new_dict2, add_methanol_fuel_additives(scenario) - ) - - # fix efficiency of meth_t_d - # df = scenario.par("input", filters={"technology": "meth_t_d"}) - # df["value"] = 1 - # new_dict2["input"] = pd.concat([new_dict2["input"], df]) - - # update production costs with IEA data - if pars["update_old_tecs"]: - cost_dict = update_methanol_costs(scenario) - new_dict2 = combine_df_dictionaries(new_dict2, cost_dict) - - new_dict2 = combine_df_dictionaries( - new_dict2, add_meth_trade_historic(), chemicals_dict, add_meth_tec_vintages() - ) - - for i in new_dict2.keys(): - new_dict2[i] = new_dict2[i].drop_duplicates() - - # model MTBE phase out legislation if pars["mtbe_scenario"] == "phase-out": - new_dict2 = combine_df_dictionaries(new_dict2, add_mtbe_act_bound(scenario)) - - h2_modes = ["fs", "fuel"] - for mode in h2_modes: - new_dict2 = combine_df_dictionaries( - new_dict2, - pd.read_excel( - package_data_path("material", "methanol", f"h2_elec_{mode}.xlsx"), - sheet_name=None, + pars_dict = pd.read_excel( + message_ix_models.util.package_data_path( + "material", "methanol", "methanol_techno_economic.xlsx" ), + sheet_name=None, + dtype=object, ) - - return new_dict2 - - -def get_embodied_emi(row, pars, share_par): - if row["year_act"] < pars["incin_trend_end"]: - share = pars["incin_rate"] + pars["incin_trend"] * (row["year_act"] - 2020) else: - share = 0.5 - return row["value"] * (1 - share) * pars[share_par] - - -def add_prod_mode(scenario, mode, level=None): - tec_pars = [ - x for x in scenario.par_list() if ("technology" in scenario.idx_sets(x)) - ] - par_dict = {} - for i in tec_pars: - df = scenario.par( - i, - filters={ - "technology": [ - "meth_ng", - "meth_coal", - "meth_ng_ccs", - "meth_coal_ccs", - ] - }, - ) - if df.size != 0: - par_dict[i] = df - - for i in par_dict.keys(): - if "mode" in par_dict[i].columns: - par_dict[i]["mode"] = mode - df = par_dict["output"] - if level: - df.loc[df["commodity"] == "methanol", "level"] = level - par_dict["output"] = df - return par_dict - - -def add_bal_modes(scenario): - tec_pars = [ - x for x in scenario.par_list() if ("technology" in scenario.idx_sets(x)) - ] - bal_fs_dict = {} - bal_fuel_dict = {} - for i in tec_pars: - df = scenario.par(i, filters={"technology": "meth_bal"}) - if df.size != 0: - bal_fs_dict[i] = df - bal_fuel_dict[i] = df.copy(deep=True) - for i in bal_fs_dict.keys(): - if "mode" in bal_fs_dict[i].columns: - bal_fs_dict[i]["mode"] = "feedstock" - bal_fuel_dict[i]["mode"] = "fuel" - - df = bal_fs_dict["input"] - df.loc[df["commodity"] == "methanol", "level"] = "primary_material" - bal_fs_dict["input"] = df - df = bal_fs_dict["output"] - df.loc[df["commodity"] == "methanol", "level"] = "secondary_material" - bal_fs_dict["output"] = df - return bal_fs_dict, bal_fuel_dict - - -def add_trd_modes(scenario): - trade_dict_fs = {} - trade_dict_fuel = {} - for i in scenario.par_list(): - df = scenario.par( - i, filters={"technology": ["meth_imp", "meth_exp", "meth_trd"]} - ) - if df.size != 0: - trade_dict_fs[i] = df - trade_dict_fuel[i] = df.copy(deep=True) - - for i in trade_dict_fs.keys(): - if "mode" in trade_dict_fs[i].columns: - trade_dict_fs[i]["mode"] = "feedstock" - trade_dict_fuel[i]["mode"] = "fuel" - - df = trade_dict_fs["output"] - df.loc[df["technology"] == "meth_imp", "level"] = "secondary_material" - df.loc[df["technology"] == "meth_exp", "level"] = "export_fs" - df.loc[df["technology"] == "meth_trd", "level"] = "import_fs" - trade_dict_fs["output"] = df - - df = trade_dict_fs["input"] - df.loc[df["technology"] == "meth_imp", "level"] = "import_fs" - df.loc[df["technology"] == "meth_trd", "level"] = "export_fs" - df.loc[df["technology"] == "meth_exp", "level"] = "primary_material" - trade_dict_fs["input"] = df - return trade_dict_fs, trade_dict_fuel - - -def gen_data_meth_h2(mode): - h2_par_dict = pd.read_excel( - package_data_path( - "material", "methanol", f"meth_h2_techno_economic_{mode}.xlsx" - ), - sheet_name=None, - ) - if "inv_cost" in h2_par_dict.keys(): - h2_par_dict["inv_cost"] = update_costs_with_loc_factor(h2_par_dict["inv_cost"]) - h2_par_dict["fix_cost"] = update_costs_with_loc_factor(h2_par_dict["fix_cost"]) - return h2_par_dict - - -def gen_data_meth_bio(scenario): - df_bio = pd.read_excel( - package_data_path("material", "methanol", "meth_bio_techno_economic_new.xlsx"), - sheet_name=None, - ) - coal_ratio = get_cost_ratio_2020( - scenario, "meth_coal", "fix_cost" - ) # .drop("value", axis=1) - merge = df_bio["fix_cost"].merge( - on=["node_loc", "year_vtg", "year_act"], - right=coal_ratio.drop(["technology", "unit", "value"], axis=1), - ) - df_bio["fix_cost"] = merge.assign(value=lambda x: x["value"] * x["ratio"]).drop( - "ratio", axis=1 - ) - df_bio_inv = df_bio["inv_cost"] - for y in df_bio["inv_cost"]["year_vtg"].unique(): - coal_ratio = get_cost_ratio_2020( - scenario, "meth_coal", "inv_cost", ref_reg="R12_WEU", year=y - ).drop("value", axis=1) - merge = df_bio_inv.loc[df_bio_inv["year_vtg"] == y].merge( - on=["node_loc", "year_vtg"], - right=coal_ratio.drop(["technology", "unit"], axis=1), - ) - df_bio_inv.loc[df_bio_inv["year_vtg"] == y, "value"] = ( - merge.assign(value=lambda x: x["value"] * x["ratio"]) - .drop("ratio", axis=1)["value"] - .values - ) - df_bio["inv_cost"] = df_bio_inv - return df_bio - - -def gen_meth_bio_ccs(scenario): - df_bio = pd.read_excel( - package_data_path( - "material", "methanol", "meth_bio_techno_economic_new_ccs.xlsx" - ), - sheet_name=None, - ) - coal_ratio = get_cost_ratio_2020( - scenario, "meth_coal_ccs", "fix_cost" - ) # .drop("value", axis=1) - merge = df_bio["fix_cost"].merge( - on=["node_loc", "year_vtg", "year_act"], - right=coal_ratio.drop(["technology", "unit", "value"], axis=1), - ) - df_bio["fix_cost"] = merge.assign(value=lambda x: x["value"] * x["ratio"]).drop( - "ratio", axis=1 - ) - df_bio_inv = df_bio["inv_cost"] - for y in df_bio["inv_cost"]["year_vtg"].unique(): - coal_ratio = get_cost_ratio_2020( - scenario, "meth_coal_ccs", "inv_cost", ref_reg="R12_WEU", year=y - ).drop("value", axis=1) - merge = df_bio_inv.loc[df_bio_inv["year_vtg"] == y].merge( - on=["node_loc", "year_vtg"], - right=coal_ratio.drop(["technology", "unit"], axis=1), - ) - df_bio_inv.loc[df_bio_inv["year_vtg"] == y, "value"] = ( - merge.assign(value=lambda x: x["value"] * x["ratio"]) - .drop("ratio", axis=1)["value"] - .values - ) - df_bio["inv_cost"] = df_bio_inv - return df_bio - - -def gen_data_meth_chemicals(scenario, chemical): - df = pd.read_excel( - package_data_path( - "material", "methanol", "collection files", "MTO data collection.xlsx" - ), - sheet_name=chemical, - usecols=[1, 2, 3, 4, 6, 7], - ) - # exclude emissions for now - if chemical == "MTO": - df = df.iloc[:15,] - if chemical == "Formaldehyde": - df = df.iloc[:10,] - - common = dict( - # commodity="NH3", - # level="secondary_material", - mode="M1", - time="year", - time_dest="year", - time_origin="year", - emission="CO2_industry", # confirm if correct - relation="CO2_feedstocks", - ) - - all_years = scenario.vintage_and_active_years() - all_years = all_years[all_years["year_vtg"] > 1990] - - drop_regs = ["R12_GLB", "World"] - nodes = [i for i in scenario.set("node").tolist() if i not in drop_regs] - - par_dict = {k: pd.DataFrame() for k in (df["parameter"])} - for i in df["parameter"]: - for index, row in df[df["parameter"] == i].iterrows(): - par_dict[i] = pd.concat( - [ - par_dict[i], - make_df(i, **all_years.to_dict(orient="list"), **row, **common) - .pipe(broadcast, node_loc=nodes) - .pipe(same_node), - ] - ) - - if i == "relation_activity": - par_dict[i]["year_rel"] = par_dict[i]["year_act"] - par_dict[i]["node_rel"] = par_dict[i]["node_loc"] - - if "unit" in par_dict[i].columns: - par_dict[i]["unit"] = "???" - - hist_dict = { - "node_loc": "R12_CHN", - "technology": "MTO_petro", - "mode": "M1", - "time": "year", - "unit": "???", - } - nodes_ex_chn = nodes - nodes_ex_chn.remove("R12_CHN") - - bound_dict = { - "node_loc": nodes_ex_chn, - "technology": "MTO_petro", - "mode": "M1", - "time": "year", - "unit": "???", - } - - if chemical == "MTO": - par_dict["historical_activity"] = make_df( - "historical_activity", value=4.5, year_act=2015, **hist_dict - ) - par_dict["historical_new_capacity"] = make_df( - "historical_new_capacity", value=1.2, year_vtg=2015, **hist_dict - ) - par_dict["bound_total_capacity_lo"] = make_df( - "bound_total_capacity_lo", year_act=2020, value=11, **hist_dict - ) - par_dict["bound_activity_lo"] = make_df( - "bound_activity_lo", year_act=2020, value=10, **hist_dict - ) - par_dict["bound_activity_up"] = make_df( - "bound_activity_up", year_act=[2020, 2025], value=[12, 15], **hist_dict - ) - par_dict["bound_activity_lo"] = pd.concat( - [ - par_dict["bound_activity_lo"], - make_df("bound_activity_lo", year_act=2020, value=0, **bound_dict), - ] - ) - par_dict["bound_activity_up"] = pd.concat( - [ - par_dict["bound_activity_up"], - make_df("bound_activity_up", year_act=2020, value=0, **bound_dict), - ] - ) - df = par_dict["growth_activity_lo"] - par_dict["growth_activity_lo"] = df[ - ~((df["node_loc"] == "R12_CHN") & (df["year_act"] == 2020)) - ] - df = par_dict["growth_activity_up"] - par_dict["growth_activity_up"] = df[ - ~((df["node_loc"] == "R12_CHN") & (df["year_act"] == 2020)) - ] - df = par_dict["initial_activity_up"] - par_dict["initial_activity_up"] = df[ - ~(df["node_loc"] == "R12_CHN") - ] # & (df["year_act"] == 2020))] - # par_dict.pop("growth_activity_up") - # par_dict.pop("growth_activity_lo") - par_dict["inv_cost"] = update_costs_with_loc_factor(par_dict["inv_cost"]) - par_dict["fix_cost"] = update_costs_with_loc_factor(par_dict["fix_cost"]) - - return par_dict - - -def add_methanol_fuel_additives(scenario): - par_dict_loil = {} - pars = ["output", "var_cost", "relation_activity", "input", "emission_factor"] - if "loil_trp" not in scenario.set("technology").to_list(): - print( - "It seems that the selected scenario does not contain the technology:" - "loil_trp. Methanol fuel blending could not be calculated and was skipped" - ) - return par_dict_loil - for i in pars: - df = scenario.par(i, filters={"technology": "loil_trp", "mode": "M1"}) - if df.size != 0: - par_dict_loil[i] = df.copy(deep=True) - - par_dict_loil["input"] = par_dict_loil["input"][ - par_dict_loil["input"]["commodity"] == "lightoil" - ] - - df_mtbe = pd.read_excel( - package_data_path( - "material", - "methanol", - "collection files", - "Methanol production statistics (version 1).xlsx", - ), - # usecols=[1,2,3,4,6,7], - skiprows=[i for i in range(66)], - sheet_name="MTBE calc", - ) - df_mtbe = df_mtbe.iloc[1:13,] - df_mtbe["node_loc"] = "R12_" + df_mtbe["node_loc"] - df_mtbe = df_mtbe[["node_loc", "% share on trp"]] - - df_biodiesel = pd.read_excel( - package_data_path( - "material", - "methanol", - "collection files", - "Methanol production statistics (version 1).xlsx", - ), - skiprows=[i for i in range(38)], - usecols=[1, 2], - sheet_name="Biodiesel", - ) - df_biodiesel["node_loc"] = "R12_" + df_biodiesel["node_loc"] - df_total = df_biodiesel.merge(df_mtbe) - df_total = df_total.assign( - value=lambda x: (x["methanol energy %"] + x["% share on trp"]) - ) - - def get_meth_share(df, node): - return df[df["node_loc"] == node["node_loc"]]["value"].values[0] - - df_loil_meth = par_dict_loil["input"].copy(deep=True) - df_loil_meth["value"] = df_loil_meth.apply( - lambda x: get_meth_share(df_total, x), axis=1 - ) - df_loil_meth["commodity"] = "methanol" - par_dict_loil["input"]["value"] = ( - par_dict_loil["input"]["value"] - df_loil_meth["value"] - ) - - df_loil_eth = df_loil_meth.copy(deep=True) - df_loil_eth["commodity"] = "ethanol" - df_loil_eth["mode"] = "ethanol" - df_loil_eth["value"] = ( - df_loil_eth["value"] * 1.6343 - ) # energy ratio methanol/ethanol in MTBE vs ETBE - - df_loil_eth_mode = par_dict_loil["input"].copy(deep=True) - df_loil_eth_mode["mode"] = "ethanol" - df_loil_eth_mode["value"] = df_loil_eth_mode["value"] - df_loil_eth["value"] - - par_dict_loil_eth = copy.deepcopy(par_dict_loil) - par_dict_loil_eth.pop("input") - for i in par_dict_loil_eth.keys(): - par_dict_loil_eth[i]["mode"] = "ethanol" - - df_input = pd.concat( - [par_dict_loil["input"], df_loil_meth, df_loil_eth, df_loil_eth_mode] - ) - return combine_df_dictionaries(par_dict_loil_eth, {"input": df_input}) - - -def add_meth_trade_historic(): - par_dict_trade = pd.read_excel( - package_data_path("material", "methanol", "meth_trade_techno_economic.xlsx"), - sheet_name=None, - ) - par_dict_trade_fs = pd.read_excel( - package_data_path("material", "methanol", "meth_trade_techno_economic_fs.xlsx"), - sheet_name=None, - ) - par_dict_trade = combine_df_dictionaries(par_dict_trade_fs, par_dict_trade) - df = par_dict_trade["historical_new_capacity"] - df.loc[df["value"] < 0, "value"] = 0 - par_dict_trade["historical_new_capacity"] = df[(df["technology"] != "meth_imp")] - return par_dict_trade - - -def update_methanol_costs(scenario): - df_inv = pd.concat( - [ - get_scaled_cost_from_proxy_tec( - 842, scenario, "meth_coal", "inv_cost", "meth_coal" - ), - get_scaled_cost_from_proxy_tec( - 350, scenario, "meth_ng", "inv_cost", "meth_ng" - ), - get_scaled_cost_from_proxy_tec( - 500, scenario, "meth_ng_ccs", "inv_cost", "meth_ng_ccs" - ), - get_scaled_cost_from_proxy_tec( - 1430, scenario, "meth_coal_ccs", "inv_cost", "meth_coal_ccs" - ), - ] - ) - df_fix = pd.concat( - [ - get_scaled_cost_from_proxy_tec( - 42.1, scenario, "meth_coal", "fix_cost", "meth_coal" + pars_dict = pd.read_excel( + message_ix_models.util.package_data_path( + "material", "methanol", "methanol_techno_economic_high_demand.xlsx" ), - get_scaled_cost_from_proxy_tec( - 8.75, scenario, "meth_ng", "fix_cost", "meth_ng" - ), - get_scaled_cost_from_proxy_tec( - 12.5, scenario, "meth_ng_ccs", "fix_cost", "meth_ng_ccs" - ), - get_scaled_cost_from_proxy_tec( - 67, scenario, "meth_coal_ccs", "fix_cost", "meth_coal_ccs" - ), - ] - ) - df_inv["unit"] = "USD/GWa" - df_fix["unit"] = "USD/GWa" - # get_scaled_cost_from_proxy_tec(290, scenario, "meth_ng", "fix_cost", "meth_bio")]) - return {"inv_cost": df_inv, "fix_cost": df_fix} - - -def gen_resin_demand(scenario, resin_share, sector, buildings_scen, pathway="SHAPE"): - df = pd.read_csv( - package_data_path( - "material", - "methanol", - "results_material_" + pathway + "_" + sector + ".csv", + sheet_name=None, + dtype=object, ) - ) - resin_intensity = resin_share - df = df[df["scenario"] == buildings_scen] - df = df[df["material"] == "wood"].assign( - resin_demand=df["mat_demand_Mt"] * resin_intensity - ) - df["R12"] = "R12_" + df["R12"] - - common = dict( - mode="M1", - time="year", - time_dest="year", - time_origin="year", - emission="CO2_industry", # confirm if correct, - relation="CO2_feedstocks", - commodity="fcoh_resin", - unit="???", - level="final_material", - ) - all_years = scenario.vintage_and_active_years() - all_years = all_years[all_years["year_vtg"] > 1990] - drop_regs = ["R12_GLB", "World"] - nodes = [i for i in scenario.set("node").tolist() if i not in drop_regs] - df_demand = ( - make_df("demand", year=all_years["year_act"].unique()[:-1], **common) - .pipe(broadcast, node=nodes) - .merge( - df[["R12", "year", "resin_demand"]], - left_on=["node", "year"], - right_on=["R12", "year"], + for i in pars_dict.keys(): + pars_dict[i] = unpivot_input_data(pars_dict[i], i) + # TODO: only temporary hack to ensure SSP_dev compatibility + if "SSP_dev" in scenario.model: + file_path = message_ix_models.util.package_data_path( + "material", "methanol", "missing_rels.yaml" ) - ) - df_demand["value"] = df_demand["resin_demand"] - df_demand = make_df("demand", **df_demand) - return df_demand - -def gen_meth_residual_demand(gdp_elasticity_2020, gdp_elasticity_2030): - def get_demand_t1_with_income_elasticity( - demand_t0, income_t0, income_t1, elasticity - ): - return ( - elasticity * demand_t0 * ((income_t1 - income_t0) / income_t0) - ) + demand_t0 + with open(file_path, "r") as file: + missing_rels = yaml.safe_load(file) + df = pars_dict["relation_activity"] + pars_dict["relation_activity"] = df[~df["relation"].isin(missing_rels)] - df_gdp = pd.read_excel( - package_data_path("material", "methanol", "methanol demand.xlsx"), - sheet_name="GDP_baseline", + default_gdp_elasticity_2020, default_gdp_elasticity_2030 = iea_elasticity_map[ + ssp_mode_map[context["ssp"]] + ] + df_final = material_demand_calc.gen_demand_petro( + scenario, "methanol", default_gdp_elasticity_2020, default_gdp_elasticity_2030 ) - - df = df_gdp[(~df_gdp["Region"].isna()) & (df_gdp["Region"] != "World")] - df = df.dropna(axis=1) - - df_demand_meth = pd.read_excel( - package_data_path("material", "methanol", "methanol demand.xlsx"), - sheet_name="methanol_demand", - skiprows=[12], + df_final["value"] = df_final["value"].apply( + lambda x: x * pars["methanol_resid_demand_share"] ) - df_demand_meth = df_demand_meth[ - (~df_demand_meth["Region"].isna()) & (df_demand_meth["Region"] != "World") - ] - df_demand_meth = df_demand_meth.dropna(axis=1) - - df_demand = df.copy(deep=True) - df_demand = df_demand.drop([2010, 2015, 2020], axis=1) - - years = list(df_demand_meth.columns[5:]) - dem_2020 = df_demand_meth[2020].values - df_demand[2020] = dem_2020 - for i in range(len(years) - 1): - income_year1 = years[i] - income_year2 = years[i + 1] - if income_year2 >= 2030: - dem_2020 = get_demand_t1_with_income_elasticity( - dem_2020, df[income_year1], df[income_year2], gdp_elasticity_2030 + pars_dict["demand"] = df_final + + return pars_dict + + +def broadcast_nodes( + df_bc_node: pd.DataFrame, + df_final: pd.DataFrame, + node_cols: List[str], + node_cols_codes: Dict[str, pd.Series], + i: int, +) -> pd.DataFrame: + """ + Broadcast nodes that were stored in pivoted row + + Parameters + ---------- + df_bc_node: pd.DataFrame + df_final: pd.DataFrame + node_cols: List[str] + node_cols_codes: Dict[str, pd.Series] + i: int + """ + if len(node_cols) == 1: + if "node_loc" in node_cols: + df_bc_node = df_bc_node.pipe( + broadcast, node_loc=node_cols_codes["node_loc"] ) - else: - dem_2020 = get_demand_t1_with_income_elasticity( - dem_2020, df[income_year1], df[income_year2], gdp_elasticity_2020 + if "node_vtg" in node_cols: + df_bc_node = df_bc_node.pipe( + broadcast, node_vtg=node_cols_codes["node_vtg"] ) - df_demand[income_year2] = dem_2020 - - df_melt = df_demand.melt( - id_vars=["Region"], value_vars=df_demand.columns[5:], var_name="year" - ) - return make_df( - "demand", - unit="t", - level="final_material", - value=df_melt.value, - time="year", - commodity="methanol", - year=df_melt.year, - node=("R12_" + df_melt["Region"]), - ) - - -def get_cost_ratio_2020(scenario, tec_name, cost_type, ref_reg="R12_NAM", year="all"): - df = scenario.par(cost_type, filters={"technology": tec_name}) - if year == "all": - if 2020 in df.year_vtg.unique(): - ref_year = 2020 + if "node_rel" in node_cols: + df_bc_node = df_bc_node.pipe( + broadcast, node_rel=node_cols_codes["node_rel"] + ) + if "node" in node_cols: + df_bc_node = df_bc_node.pipe(broadcast, node=node_cols_codes["node"]) + if "node_share" in node_cols: + df_bc_node = df_bc_node.pipe( + broadcast, node_share=node_cols_codes["node_share"] + ) + else: + df_bc_node = df_bc_node.pipe(broadcast, node_loc=node_cols_codes["node_loc"]) + if len(df_final.loc[i][node_cols].T.unique()) == 1: + # df_bc_node["node_rel"] = df_bc_node["node_loc"] + df_bc_node = df_bc_node.pipe( + same_node + ) # not working for node_rel in installed message_ix_models version else: - ref_year = min(df.year_vtg.unique()) - df = df[df["year_vtg"] >= ref_year] - val_nam_2020 = df.loc[ - (df["node_loc"] == ref_reg) & (df["year_vtg"] == ref_year), "value" - ].iloc[0] - df["ratio"] = df["value"] / val_nam_2020 + if "node_rel" in list(df_bc_node.columns): + df_bc_node = df_bc_node.pipe( + broadcast, node_rel=node_cols_codes["node_rel"] + ) + if "node_origin" in list(df_bc_node.columns): + df_bc_node = df_bc_node.pipe( + broadcast, node_origin=node_cols_codes["node_origin"] + ) + if "node_dest" in list(df_bc_node.columns): + df_bc_node = df_bc_node.pipe( + broadcast, node_dest=node_cols_codes["node_dest"] + ) + return df_bc_node + + +def broadcast_years( + df_bc_node: pd.DataFrame, + yr_col_out: List[str], + yr_cols_codes: Dict[str, List[str]], + col: str, +) -> pd.DataFrame: + """ + Broadcast years that were stored in pivoted row + Parameters + ---------- + df_bc_node: pd.DataFrame + yr_col_out: List[str] + yr_cols_codes: ict[str, List[str]] + col: str + """ + if len(yr_col_out) == 1: + yr_list = [i[0] for i in yr_cols_codes[col]] + # print(yr_list) + if "year_act" in yr_col_out: + df_bc_node = df_bc_node.pipe(broadcast, year_act=yr_list) + if "year_vtg" in yr_col_out: + df_bc_node = df_bc_node.pipe(broadcast, year_vtg=yr_list) + if "year_rel" in yr_col_out: + df_bc_node = df_bc_node.pipe(broadcast, year_rel=yr_list) + if "year" in yr_col_out: + df_bc_node = df_bc_node.pipe(broadcast, year=yr_list) + df_bc_node[yr_col_out] = df_bc_node[yr_col_out].astype(int) else: - df = df[df["year_vtg"] == year] - val_nam_2020 = df.loc[ - (df["node_loc"] == ref_reg) & (df["year_vtg"] == year), "value" - ].iloc[0] - df["ratio"] = df["value"] / val_nam_2020 - - return df - - -def get_scaled_cost_from_proxy_tec( - value, scenario, proxy_tec, cost_type, new_tec, year="all" -): - df = get_cost_ratio_2020(scenario, proxy_tec, cost_type, year=year) - df["value"] = value * df["ratio"] - df["technology"] = new_tec - df["unit"] = "-" - return make_df(cost_type, **df) - - -def add_meth_tec_vintages(): - par_dict_ng = pd.read_excel( - package_data_path("material", "methanol", "meth_ng_techno_economic.xlsx"), - sheet_name=None, - ) - par_dict_ng_fs = pd.read_excel( - package_data_path("material", "methanol", "meth_ng_techno_economic_fs.xlsx"), - sheet_name=None, - ) - par_dict_ng.pop("historical_activity") - par_dict_ng_fs.pop("historical_activity") - - par_dict_coal = pd.read_excel( - package_data_path("material", "methanol", "meth_coal_additions.xlsx"), - sheet_name=None, - ) - par_dict_coal_fs = pd.read_excel( - package_data_path("material", "methanol", "meth_coal_additions_fs.xlsx"), - sheet_name=None, - ) - par_dict_coal.pop("historical_activity") - par_dict_coal_fs.pop("historical_activity") - return combine_df_dictionaries( - par_dict_ng, par_dict_ng_fs, par_dict_coal, par_dict_coal_fs - ) - - -def add_meth_hist_act(): - # fix demand infeasibility - par_dict = {} - df_fs = pd.read_excel( - package_data_path("material", "methanol", "meth_coal_additions_fs.xlsx"), - sheet_name="historical_activity", - ) - df_fuel = pd.read_excel( - package_data_path("material", "methanol", "meth_coal_additions.xlsx"), - sheet_name="historical_activity", - ) - par_dict["historical_activity"] = pd.concat([df_fs, df_fuel]) - # derived from graphic in - # "Methanol production statstics.xlsx/China demand split" diagram - - # hist_cap = make_df( - # "historical_new_capacity", - # node_loc="R12_CHN", - # technology="meth_coal", - # year_vtg=2015, - # value=9.6, - # unit="GW", - # ) - # par_dict["historical_new_capacity"] = hist_cap - # fix demand infeasibility - # act = scenario.par("historical_activity") - # row = act[act["technology"].str.startswith("meth")].sort_values("value", - # ascending=False).iloc[0] - # row["value"] = 0.0 - df_ng = pd.read_excel( - package_data_path("material", "methanol", "meth_ng_techno_economic.xlsx"), - sheet_name="historical_activity", - ) - df_ng_fs = pd.read_excel( - package_data_path("material", "methanol", "meth_ng_techno_economic_fs.xlsx"), - sheet_name="historical_activity", - ) - par_dict["historical_activity"] = pd.concat( - [par_dict["historical_activity"], df_ng, df_ng_fs] - ) - return par_dict + if "year_vtg" in yr_col_out: + y_v = [str(i) for i in yr_cols_codes[col]] + df_bc_node = df_bc_node.pipe(broadcast, year_vtg=y_v) + df_bc_node["year_act"] = [ + literal_eval(i)[1] for i in df_bc_node["year_vtg"] + ] + df_bc_node["year_vtg"] = [ + literal_eval(i)[0] for i in df_bc_node["year_vtg"] + ] + if "year_rel" in yr_col_out: + if "year_act" in yr_col_out: + df_bc_node = df_bc_node.pipe( + broadcast, year_act=[i[0] for i in yr_cols_codes[col]] + ) + df_bc_node["year_rel"] = df_bc_node["year_act"] + return df_bc_node + + +def unpivot_input_data(df: pd.DataFrame, par_name: str) -> pd.DataFrame: + """ + Unpivot data that is already contains columns for respective MESSAGEix parameter + Parameters + ---------- + df: pd.DataFrame + DataFrame containing parameter data with year and node values pivoted + par_name: str + name of MESSAGEix parameter + """ + df_final = df + df_final_full = pd.DataFrame() + + for i in df_final.index: + # parse strings of node columns to dictionary + node_cols = [i for i in df_final.columns if "node" in i] + remove = ["'", "[", "]", " "] + node_cols_codes = {} + for col in node_cols: + node_cols_codes[col] = pd.Series( + "".join(x for x in df_final.loc[i][col] if x not in remove).split(",") + ) + # create dataframe with required columns + df_bc_node = make_df(par_name, **df_final.loc[i]) -def update_costs_with_loc_factor(df): - loc_fact = pd.read_excel( - package_data_path("material", "methanol", "location factor collection.xlsx"), - sheet_name="comparison", - index_col=0, - ) - cost_conv = pd.read_excel( - package_data_path("material", "methanol", "location factor collection.xlsx"), - sheet_name="cost_convergence", - index_col=0, - ) - loc_fact = loc_fact.loc["WEU normalized"] - loc_fact.index = "R12_" + loc_fact.index - df = df.merge(loc_fact, left_on="node_loc", right_index=True) - df = df.merge(cost_conv, left_on="year_vtg", right_index=True) - df["value"] = ( - df["value"] - (1 - df["WEU normalized"]) * df["value"] * df["convergence"] - ) - return df[df.columns[:-2]] + # collect year values from year columns + yr_cols_codes = {} + yr_col_inp = [i for i in df_final.columns if "year" in i] + yr_col_out = [i for i in df_bc_node.columns if "year" in i] + df_bc_node[yr_col_inp] = df_final.loc[i][yr_col_inp].values + # broadcast in node dimensions + for colname in node_cols: + df_bc_node[colname] = None + df_bc_node = broadcast_nodes( + df_bc_node, df_final, node_cols, node_cols_codes, i + ) -def add_mtbe_act_bound(scenario): - mtbe_dict = {} - year = [i for i in range(2030, 2056, 5)] + [i for i in range(2060, 2111, 10)] - par_dict = { - "technology": "loil_trp", - "mode": "M1", - "unit": "-", - "time": "year", - "year_act": year, - "value": 0, - } - drop_regs = ["R12_GLB", "World"] - nodes = [i for i in scenario.set("node").tolist() if i not in drop_regs] - mtbe_dict["bound_activity_up"] = make_df("bound_activity_up", **par_dict).pipe( - broadcast, node_loc=nodes - ) - return mtbe_dict + # brodcast in year dimensions + for col in yr_col_inp: + yr_cols_codes[col] = literal_eval(df_bc_node[col].values[0]) + df_bc_node = broadcast_years(df_bc_node, yr_col_out, yr_cols_codes, col) + df_bc_node[yr_col_out] = df_bc_node[yr_col_out].astype(int) + + df_final_full = pd.concat([df_final_full, df_bc_node]) + df_final_full = df_final_full.drop_duplicates().reset_index(drop=True) + + # special treatment for relation_activity dataframes: + # relations parametrization should only contain columns where node_rel == node_loc + # except for relations acting on a geographic global level + # (where node_rel == "R**_GLB") + if par_name == "relation_activity": + df_final_full = df_final_full.drop( + df_final_full[ + (df_final_full.node_rel.values != "R12_GLB") + & (df_final_full.node_rel.values != df_final_full.node_loc.values) + ].index + ) + return make_df(par_name, **df_final_full) diff --git a/message_ix_models/model/material/data_methanol_new.py b/message_ix_models/model/material/data_methanol_new.py deleted file mode 100644 index 613971440f..0000000000 --- a/message_ix_models/model/material/data_methanol_new.py +++ /dev/null @@ -1,259 +0,0 @@ -from ast import literal_eval -from typing import TYPE_CHECKING, Dict, List - -import pandas as pd -import yaml -from message_ix import make_df - -import message_ix_models.util -from message_ix_models.model.material.material_demand import material_demand_calc -from message_ix_models.model.material.util import read_config -from message_ix_models.util import broadcast, same_node - -if TYPE_CHECKING: - from message_ix import Scenario - -ssp_mode_map = { - "SSP1": "CTS core", - "SSP2": "RTS core", - "SSP3": "RTS high", - "SSP4": "CTS high", - "SSP5": "RTS high", - "LED": "CTS core", # TODO: move to even lower projection -} - -iea_elasticity_map = { - "CTS core": (1.2, 0.25), - "CTS high": (1.3, 0.48), - "RTS core": (1.25, 0.35), - "RTS high": (1.4, 0.54), -} - - -def gen_data_methanol_new(scenario: "Scenario") -> Dict[str, pd.DataFrame]: - """ - Generates data for methanol industry model - - Parameters - ---------- - scenario: .Scenario - """ - context = read_config() - df_pars = pd.read_excel( - message_ix_models.util.package_data_path( - "material", "methanol", "methanol_sensitivity_pars.xlsx" - ), - sheet_name="Sheet1", - dtype=object, - ) - pars = df_pars.set_index("par").to_dict()["value"] - if pars["mtbe_scenario"] == "phase-out": - pars_dict = pd.read_excel( - message_ix_models.util.package_data_path( - "material", "methanol", "methanol_techno_economic.xlsx" - ), - sheet_name=None, - dtype=object, - ) - else: - pars_dict = pd.read_excel( - message_ix_models.util.package_data_path( - "material", "methanol", "methanol_techno_economic_high_demand.xlsx" - ), - sheet_name=None, - dtype=object, - ) - - for i in pars_dict.keys(): - pars_dict[i] = unpivot_input_data(pars_dict[i], i) - # TODO: only temporary hack to ensure SSP_dev compatibility - if "SSP_dev" in scenario.model: - file_path = message_ix_models.util.package_data_path( - "material", "methanol", "missing_rels.yaml" - ) - - with open(file_path, "r") as file: - missing_rels = yaml.safe_load(file) - df = pars_dict["relation_activity"] - pars_dict["relation_activity"] = df[~df["relation"].isin(missing_rels)] - - default_gdp_elasticity_2020, default_gdp_elasticity_2030 = iea_elasticity_map[ - ssp_mode_map[context["ssp"]] - ] - df_final = material_demand_calc.gen_demand_petro( - scenario, "methanol", default_gdp_elasticity_2020, default_gdp_elasticity_2030 - ) - df_final["value"] = df_final["value"].apply( - lambda x: x * pars["methanol_resid_demand_share"] - ) - pars_dict["demand"] = df_final - - return pars_dict - - -def broadcast_nodes( - df_bc_node: pd.DataFrame, - df_final: pd.DataFrame, - node_cols: List[str], - node_cols_codes: Dict[str, pd.Series], - i: int, -) -> pd.DataFrame: - """ - Broadcast nodes that were stored in pivoted row - - Parameters - ---------- - df_bc_node: pd.DataFrame - df_final: pd.DataFrame - node_cols: List[str] - node_cols_codes: Dict[str, pd.Series] - i: int - """ - if len(node_cols) == 1: - if "node_loc" in node_cols: - df_bc_node = df_bc_node.pipe( - broadcast, node_loc=node_cols_codes["node_loc"] - ) - if "node_vtg" in node_cols: - df_bc_node = df_bc_node.pipe( - broadcast, node_vtg=node_cols_codes["node_vtg"] - ) - if "node_rel" in node_cols: - df_bc_node = df_bc_node.pipe( - broadcast, node_rel=node_cols_codes["node_rel"] - ) - if "node" in node_cols: - df_bc_node = df_bc_node.pipe(broadcast, node=node_cols_codes["node"]) - if "node_share" in node_cols: - df_bc_node = df_bc_node.pipe( - broadcast, node_share=node_cols_codes["node_share"] - ) - else: - df_bc_node = df_bc_node.pipe(broadcast, node_loc=node_cols_codes["node_loc"]) - if len(df_final.loc[i][node_cols].T.unique()) == 1: - # df_bc_node["node_rel"] = df_bc_node["node_loc"] - df_bc_node = df_bc_node.pipe( - same_node - ) # not working for node_rel in installed message_ix_models version - else: - if "node_rel" in list(df_bc_node.columns): - df_bc_node = df_bc_node.pipe( - broadcast, node_rel=node_cols_codes["node_rel"] - ) - if "node_origin" in list(df_bc_node.columns): - df_bc_node = df_bc_node.pipe( - broadcast, node_origin=node_cols_codes["node_origin"] - ) - if "node_dest" in list(df_bc_node.columns): - df_bc_node = df_bc_node.pipe( - broadcast, node_dest=node_cols_codes["node_dest"] - ) - return df_bc_node - - -def broadcast_years( - df_bc_node: pd.DataFrame, - yr_col_out: List[str], - yr_cols_codes: Dict[str, List[str]], - col: str, -) -> pd.DataFrame: - """ - Broadcast years that were stored in pivoted row - Parameters - ---------- - df_bc_node: pd.DataFrame - yr_col_out: List[str] - yr_cols_codes: ict[str, List[str]] - col: str - """ - if len(yr_col_out) == 1: - yr_list = [i[0] for i in yr_cols_codes[col]] - # print(yr_list) - if "year_act" in yr_col_out: - df_bc_node = df_bc_node.pipe(broadcast, year_act=yr_list) - if "year_vtg" in yr_col_out: - df_bc_node = df_bc_node.pipe(broadcast, year_vtg=yr_list) - if "year_rel" in yr_col_out: - df_bc_node = df_bc_node.pipe(broadcast, year_rel=yr_list) - if "year" in yr_col_out: - df_bc_node = df_bc_node.pipe(broadcast, year=yr_list) - df_bc_node[yr_col_out] = df_bc_node[yr_col_out].astype(int) - else: - if "year_vtg" in yr_col_out: - y_v = [str(i) for i in yr_cols_codes[col]] - df_bc_node = df_bc_node.pipe(broadcast, year_vtg=y_v) - df_bc_node["year_act"] = [ - literal_eval(i)[1] for i in df_bc_node["year_vtg"] - ] - df_bc_node["year_vtg"] = [ - literal_eval(i)[0] for i in df_bc_node["year_vtg"] - ] - if "year_rel" in yr_col_out: - if "year_act" in yr_col_out: - df_bc_node = df_bc_node.pipe( - broadcast, year_act=[i[0] for i in yr_cols_codes[col]] - ) - df_bc_node["year_rel"] = df_bc_node["year_act"] - return df_bc_node - - -def unpivot_input_data(df: pd.DataFrame, par_name: str): - """ - Unpivot data that is already contains columns for respective MESSAGEix parameter - Parameters - ---------- - df: pd.DataFrame - DataFrame containing parameter data with year and node values pivoted - par_name: str - name of MESSAGEix parameter - """ - df_final = df - df_final_full = pd.DataFrame() - - for i in df_final.index: - # parse strings of node columns to dictionary - node_cols = [i for i in df_final.columns if "node" in i] - remove = ["'", "[", "]", " "] - node_cols_codes = {} - for col in node_cols: - node_cols_codes[col] = pd.Series( - "".join(x for x in df_final.loc[i][col] if x not in remove).split(",") - ) - - # create dataframe with required columns - df_bc_node = make_df(par_name, **df_final.loc[i]) - - # collect year values from year columns - yr_cols_codes = {} - yr_col_inp = [i for i in df_final.columns if "year" in i] - yr_col_out = [i for i in df_bc_node.columns if "year" in i] - df_bc_node[yr_col_inp] = df_final.loc[i][yr_col_inp].values - - # broadcast in node dimensions - for colname in node_cols: - df_bc_node[colname] = None - df_bc_node = broadcast_nodes( - df_bc_node, df_final, node_cols, node_cols_codes, i - ) - - # brodcast in year dimensions - for col in yr_col_inp: - yr_cols_codes[col] = literal_eval(df_bc_node[col].values[0]) - df_bc_node = broadcast_years(df_bc_node, yr_col_out, yr_cols_codes, col) - df_bc_node[yr_col_out] = df_bc_node[yr_col_out].astype(int) - - df_final_full = pd.concat([df_final_full, df_bc_node]) - df_final_full = df_final_full.drop_duplicates().reset_index(drop=True) - - # special treatment for relation_activity dataframes: - # relations parametrization should only contain columns where node_rel == node_loc - # except for relations acting on a geographic global level - # (where node_rel == "R**_GLB") - if par_name == "relation_activity": - df_final_full = df_final_full.drop( - df_final_full[ - (df_final_full.node_rel.values != "R12_GLB") - & (df_final_full.node_rel.values != df_final_full.node_loc.values) - ].index - ) - return make_df(par_name, **df_final_full) diff --git a/message_ix_models/model/material/data_petro.py b/message_ix_models/model/material/data_petro.py index 64ef141947..e7fbe66263 100644 --- a/message_ix_models/model/material/data_petro.py +++ b/message_ix_models/model/material/data_petro.py @@ -1,5 +1,7 @@ from collections import defaultdict +from typing import List, Set +import message_ix import pandas as pd from message_ix import make_df @@ -31,7 +33,7 @@ } -def read_data_petrochemicals(scenario): +def read_data_petrochemicals(scenario: message_ix.Scenario) -> pd.DataFrame: """Read and clean data from :file:`petrochemicals_techno_economic.xlsx`.""" # Ensure config is loaded, get the context @@ -54,7 +56,11 @@ def read_data_petrochemicals(scenario): return data_petro -def gen_mock_demand_petro(scenario, gdp_elasticity_2020, gdp_elasticity_2030): +def gen_mock_demand_petro( + scenario: message_ix.Scenario, + gdp_elasticity_2020: float, + gdp_elasticity_2030: float, +) -> pd.DataFrame: s_info = ScenarioInfo(scenario) modelyears = s_info.Y fy = scenario.firstmodelyear @@ -159,7 +165,9 @@ def get_demand_t1_with_income_elasticity( ) -def gen_data_petro_ts(data_petro_ts, results, tec_ts, nodes): +def gen_data_petro_ts( + data_petro_ts: pd.DataFrame, results: dict[list], tec_ts: Set[str], nodes: List[str] +) -> None: for t in tec_ts: common = dict( time="year", @@ -223,8 +231,16 @@ def gen_data_petro_ts(data_petro_ts, results, tec_ts, nodes): def assign_input_outpt( - split, param_name, regions, val, t, rg, global_region, common, nodes -): + split: str, + param_name: str, + regions: pd.DataFrame, + val: float | int, + t: str, + rg: str, + global_region: str, + common: dict, + nodes: List[str], +) -> pd.DataFrame: com = split[1] lev = split[2] mod = split[3] @@ -279,7 +295,7 @@ def assign_input_outpt( return df -def broadcast_to_regions(df, global_region, nodes): +def broadcast_to_regions(df: pd.DataFrame, global_region: str, nodes: List[str]): if "node_loc" in df.columns: if ( len(set(df["node_loc"])) == 1 @@ -291,7 +307,9 @@ def broadcast_to_regions(df, global_region, nodes): return df -def gen_data_petro_chemicals(scenario, dry_run=False): +def gen_data_petro_chemicals( + scenario: message_ix.Scenario, dry_run: bool = False +) -> dict[str, pd.DataFrame]: # Load configuration context = read_config() config = context["material"]["petro_chemicals"] @@ -316,6 +334,7 @@ def gen_data_petro_chemicals(scenario, dry_run=False): yv_ya = s_info.yv_ya for t in config["technology"]["add"]: + t = t.id # years = s_info.Y params = data_petro.loc[(data_petro["technology"] == t), "parameter"].unique() @@ -461,23 +480,7 @@ def gen_data_petro_chemicals(scenario, dry_run=False): ) results["demand"].append(demand_hvc) - # df_e = make_df(paramname, level='final_material', commodity="ethylene", \ - # value=demand_e.value, unit='t',year=demand_e.year, time='year', \ - # node=demand_e.node)#.pipe(broadcast, node=nodes) - # results["demand"].append(df_e) - # - # df_p = make_df(paramname, level='final_material', commodity="propylene", \ - # value=demand_p.value, unit='t',year=demand_p.year, time='year', \ - # node=demand_p.node)#.pipe(broadcast, node=nodes) - # results["demand"].append(df_p) - # - # df_BTX = make_df(paramname, level='final_material', commodity="BTX", \ - # value=demand_BTX.value, unit='t',year=demand_BTX.year, time='year', \ - # node=demand_BTX.node)#.pipe(broadcast, node=nodes) - # results["demand"].append(df_BTX) - # Special treatment for time-varying params - tec_ts = set(data_petro_ts.technology) # set of tecs in timeseries sheet gen_data_petro_ts(data_petro_ts, results, tec_ts, nodes) diff --git a/message_ix_models/model/material/data_power_sector.py b/message_ix_models/model/material/data_power_sector.py index a404318e39..dab8824756 100644 --- a/message_ix_models/model/material/data_power_sector.py +++ b/message_ix_models/model/material/data_power_sector.py @@ -1,11 +1,14 @@ from collections import defaultdict +import message_ix import pandas as pd from message_ix_models.util import package_data_path -def read_material_intensities(data_path, inv_cost): +def read_material_intensities( + data_path: str, inv_cost: pd.DataFrame +) -> dict[str, pd.DataFrame]: #################################################################### # read data #################################################################### @@ -14,6 +17,69 @@ def read_material_intensities(data_path, inv_cost): data_path_lca = data_path + "/NTNU_LCA_coefficients.xlsx" data_lca = pd.read_excel(data_path_lca, sheet_name="environmentalImpacts") + # For hydropower material intensity use "medium" from Kalt et al., 2021. + # Unit: t/MW + + data_lca.loc[ + ( + (data_lca["technology"] == "Hydro") + & (data_lca["technology variant"] == "mix") + & (data_lca["phase"] == "Construction") + & (data_lca["impact"] == "Iron") + ), + 2010, + ] = 45 + + data_lca.loc[ + ( + (data_lca["technology"] == "Hydro") + & (data_lca["technology variant"] == "mix") + & (data_lca["phase"] == "Construction") + & (data_lca["impact"] == "Iron") + ), + 2030, + ] = 45 + + data_lca.loc[ + ( + (data_lca["technology"] == "Hydro") + & (data_lca["technology variant"] == "mix") + & (data_lca["phase"] == "Construction") + & (data_lca["impact"] == "Aluminium") + ), + 2010, + ] = 0.572 + + data_lca.loc[ + ( + (data_lca["technology"] == "Hydro") + & (data_lca["technology variant"] == "mix") + & (data_lca["phase"] == "Construction") + & (data_lca["impact"] == "Aluminium") + ), + 2030, + ] = 0.572 + + data_lca.loc[ + ( + (data_lca["technology"] == "Hydro") + & (data_lca["technology variant"] == "mix") + & (data_lca["phase"] == "Construction") + & (data_lca["impact"] == "Cement") + ), + 2010, + ] = 787.5 + + data_lca.loc[ + ( + (data_lca["technology"] == "Hydro") + & (data_lca["technology variant"] == "mix") + & (data_lca["phase"] == "Construction") + & (data_lca["impact"] == "Cement") + ), + 2030, + ] = 787.5 + # read technology, region and commodity mappings data_path_tec_map = data_path + "/MESSAGE_global_model_technologies.xlsx" technology_mapping = pd.read_excel(data_path_tec_map, sheet_name="technology") @@ -261,7 +327,7 @@ def read_material_intensities(data_path, inv_cost): } -def maybe_init_pars(scenario): +def maybe_init_pars(scenario: message_ix.Scenario) -> None: if not scenario.has_par("input_cap_new"): scenario.init_par( "input_cap_new", @@ -400,7 +466,9 @@ def maybe_init_pars(scenario): ) -def gen_data_power_sector(scenario, dry_run=False): +def gen_data_power_sector( + scenario: message_ix.Scenario, dry_run: bool = False +) -> dict[str, pd.DataFrame]: """Generate data for materials representation of power industry.""" # Load configuration diff --git a/message_ix_models/model/material/data_steel.py b/message_ix_models/model/material/data_steel.py index 7e7f932f6c..ac875b872c 100644 --- a/message_ix_models/model/material/data_steel.py +++ b/message_ix_models/model/material/data_steel.py @@ -1,11 +1,14 @@ from collections import defaultdict +from typing import Dict, Iterable, List +import message_ix import pandas as pd from message_ix import make_df # Get endogenous material demand from buildings interface from message_ix_models import ScenarioInfo from message_ix_models.model.material.data_util import ( + calculate_ini_new_cap, read_rel, read_sector_data, read_timeseries, @@ -25,7 +28,7 @@ ) -def gen_mock_demand_steel(scenario): +def gen_mock_demand_steel(scenario: message_ix.Scenario) -> pd.DataFrame: s_info = ScenarioInfo(scenario) nodes = s_info.N nodes.remove("World") @@ -115,7 +118,9 @@ def gen_mock_demand_steel(scenario): return demand2020_steel -def gen_data_steel_ts(data_steel_ts, results, t, nodes): +def gen_data_steel_ts( + data_steel_ts: pd.DataFrame, results: Dict[str, list], t: str, nodes: List[str] +): common = dict( time="year", time_origin="year", @@ -206,7 +211,15 @@ def gen_data_steel_ts(data_steel_ts, results, t, nodes): return -def get_data_steel_const(data_steel, results, params, t, yv_ya, nodes, global_region): +def get_data_steel_const( + data_steel: pd.DataFrame, + results: Dict[str, list], + params: Iterable, + t: str, + yv_ya: pd.DataFrame, + nodes: List[str], + global_region: str, +): for par in params: # Obtain the parameter names, commodity,level,emission split = par.split("|") @@ -414,7 +427,7 @@ def gen_data_steel_rel(data_steel_rel, results, regions, modelyears): return -def gen_data_steel(scenario, dry_run=False): +def gen_data_steel(scenario: message_ix.Scenario, dry_run: bool = False): """Generate data for materials representation of steel industry.""" # Load configuration context = read_config() @@ -426,10 +439,10 @@ def gen_data_steel(scenario, dry_run=False): # Techno-economic assumptions # TEMP: now add cement sector as well # => Need to separate those since now I have get_data_steel and cement - data_steel = read_sector_data(scenario, "steel") + data_steel = read_sector_data(scenario, "steel", "Global_steel_MESSAGE.xlsx") # Special treatment for time-dependent Parameters - data_steel_ts = read_timeseries(scenario, "steel_cement", context.datafile) - data_steel_rel = read_rel(scenario, "steel_cement", context.datafile) + data_steel_ts = read_timeseries(scenario, "steel", "Global_steel_MESSAGE.xlsx") + data_steel_rel = read_rel(scenario, "steel", "Global_steel_MESSAGE.xlsx") tec_ts = set(data_steel_ts.technology) # set of tecs with var_cost @@ -445,6 +458,7 @@ def gen_data_steel(scenario, dry_run=False): # For each technology there are differnet input and output combinations # Iterate over technologies for t in config["technology"]["add"]: + t = t.id params = data_steel.loc[(data_steel["technology"] == t), "parameter"].unique() # Special treatment for time-varying params @@ -517,12 +531,60 @@ def gen_data_steel(scenario, dry_run=False): # Create external demand param parname = "demand" - df = material_demand_calc.derive_demand("steel", scenario, old_gdp=False, ssp=ssp) + df_demand = material_demand_calc.derive_demand("steel", scenario, ssp=ssp) + results[parname].append(df_demand) + + common = dict( + year_vtg=yv_ya.year_vtg, + year_act=yv_ya.year_act, + time="year", + time_origin="year", + time_dest="year", + ) + + # Add CCS as addon + parname = "addon_conversion" + bf_tec = ["bf_steel"] + df = make_df( + parname, mode="M2", type_addon="bf_ccs_steel_addon", value=1, unit="-", **common + ).pipe(broadcast, node=nodes, technology=bf_tec) + results[parname].append(df) + + dri_gas_tec = ["dri_gas_steel"] + df = make_df( + parname, + mode="M1", + type_addon="dri_gas_ccs_steel_addon", + value=1, + unit="-", + **common, + ).pipe(broadcast, node=nodes, technology=dri_gas_tec) + results[parname].append(df) + + dri_tec = ["dri_steel"] + df = make_df( + parname, mode="M1", type_addon="dri_steel_addon", value=1, unit="-", **common + ).pipe(broadcast, node=nodes, technology=dri_tec) results[parname].append(df) # Concatenate to one data frame per parameter results = {par_name: pd.concat(dfs) for par_name, dfs in results.items()} + results["initial_new_capacity_up"] = pd.concat( + [ + calculate_ini_new_cap( + df_demand=df_demand.copy(deep=True), + technology="dri_gas_ccs_steel", + material="steel", + ), + calculate_ini_new_cap( + df_demand=df_demand.copy(deep=True), + technology="bf_ccs_steel", + material="steel", + ), + ] + ) + maybe_remove_water_tec(scenario, results) return results diff --git a/message_ix_models/model/material/data_util.py b/message_ix_models/model/material/data_util.py index 77b63cfd07..6bf5a8ad89 100644 --- a/message_ix_models/model/material/data_util.py +++ b/message_ix_models/model/material/data_util.py @@ -7,10 +7,10 @@ import numpy as np import pandas as pd from genno import Computer +from message_ix import make_df from message_ix_models import ScenarioInfo from message_ix_models.model.material.util import ( - read_config, remove_from_list_if_exists, ) from message_ix_models.model.structure import get_region_codes @@ -105,15 +105,8 @@ def add_macro_COVID( MACRO-calibrated Scenario instance """ - # Excel file for calibration data - if "SSP_dev" in scen.model: - xls_file = os.path.join( - "C:/", "Users", "maczek", "Downloads", "macro", filename - ) - else: - xls_file = os.path.join("P:", "ene.model", "MACRO", "python", filename) # Making a dictionary from the MACRO Excel file - xls = pd.ExcelFile(xls_file) + xls = pd.ExcelFile(package_data_path("material", "macro", filename)) data = {} for s in xls.sheet_names: data[s] = xls.parse(s) @@ -886,7 +879,7 @@ def calc_demand_shares(iea_db_df: pd.DataFrame, base_year: int) -> pd.DataFrame: def calc_resid_ind_demand( - scen: message_ix.Scenario, baseyear: int, iea_data_path + scen: message_ix.Scenario, baseyear: int, iea_data_path: str ) -> pd.DataFrame: comms = ["i_spec", "i_therm"] path = os.path.join(iea_data_path, "REV2022_allISO_IEA.parquet") @@ -902,7 +895,7 @@ def calc_resid_ind_demand( def modify_industry_demand( - scen: message_ix.Scenario, baseyear: int, iea_data_path + scen: message_ix.Scenario, baseyear: int, iea_data_path: str ) -> None: df_demands_new = calc_resid_ind_demand(scen, baseyear, iea_data_path) scen.check_out() @@ -1019,7 +1012,7 @@ def read_iea_tec_map(tec_map_fname: str) -> pd.DataFrame: def get_hist_act_data( - map_fname: str, years: list or None = None, iea_data_path=None + map_fname: str, years: list or None = None, iea_data_path: str | None = None ) -> pd.DataFrame: """ reads IEA DB, maps and aggregates variables to MESSAGE technologies @@ -1062,7 +1055,7 @@ def get_hist_act_data( return df_final -def add_emission_accounting(scen): +def add_emission_accounting(scen: message_ix.Scenario) -> None: """ Parameters @@ -1071,12 +1064,12 @@ def add_emission_accounting(scen): """ # (1) ******* Add non-CO2 gases to the relevant relations. ******** # This is done by multiplying the input values and emission_factor - # per year,region and technology. + # per year,region and technology for furnace technologies. tec_list_residual = scen.par("emission_factor")["technology"].unique() tec_list_input = scen.par("input")["technology"].unique() - # The technology list to retrieve the input values + # The technology list to retrieve the input values for furnaces tec_list_input = [ i for i in tec_list_input if (("furnace" in i) | ("hp_gas_" in i)) ] @@ -1423,7 +1416,7 @@ def add_emission_accounting(scen): # scen.commit("add methanol CO2_industry") -def add_elec_lowerbound_2020(scen): +def add_elec_lowerbound_2020(scen: message_ix.Scenario) -> None: # To avoid zero i_spec prices only for R12_CHN, add the below section. # read input parameters for relevant technology/commodity combinations for # converting betwen final and useful energy @@ -1497,7 +1490,7 @@ def add_elec_lowerbound_2020(scen): scen.commit("added lower bound for activity of residual electricity technologies") -def add_coal_lowerbound_2020(sc): +def add_coal_lowerbound_2020(sc: message_ix.Scenario) -> None: """Set lower bounds for coal and i_spec as a calibration for 2020""" final_resid = pd.read_csv( @@ -1633,7 +1626,7 @@ def add_coal_lowerbound_2020(sc): ) -def add_cement_bounds_2020(sc): +def add_cement_bounds_2020(sc: message_ix.Scenario) -> None: """Set lower and upper bounds for gas and oil as a calibration for 2020""" final_resid = pd.read_csv( @@ -1883,7 +1876,9 @@ def add_cement_bounds_2020(sc): sc.commit("added lower and upper bound for fuels for cement 2020.") -def read_sector_data(scenario: message_ix.Scenario, sectname: str) -> pd.DataFrame: +def read_sector_data( + scenario: message_ix.Scenario, sectname: str, file: str +) -> pd.DataFrame: """ Read sector data for industry with sectname @@ -1904,9 +1899,6 @@ def read_sector_data(scenario: message_ix.Scenario, sectname: str) -> pd.DataFra import numpy as np - # Ensure config is loaded, get the context - context = read_config() - s_info = ScenarioInfo(scenario) if "R12_CHN" in s_info.N: @@ -1916,7 +1908,7 @@ def read_sector_data(scenario: message_ix.Scenario, sectname: str) -> pd.DataFra # data_df = data_steel_china.append(data_cement_china, ignore_index=True) data_df = pd.read_excel( - package_data_path("material", "steel_cement", context.datafile), + package_data_path("material", sectname, file), sheet_name=sheet_n, ) @@ -1986,6 +1978,8 @@ def add_ccs_technologies(scen: message_ix.Scenario) -> None: "gas_NH3_ccs", "coal_NH3_ccs", "fueloil_NH3_ccs", + "bf_ccs_steel", + "dri_gas_ccs_steel", ], "emission": "CO2", }, @@ -2010,7 +2004,6 @@ def add_ccs_technologies(scen: message_ix.Scenario) -> None: # Read in time-dependent parameters -# Now only used to add fuel cost for bare model def read_timeseries( scenario: message_ix.Scenario, material: str, filename: str ) -> pd.DataFrame: @@ -2075,7 +2068,9 @@ def read_timeseries( return df -def read_rel(scenario: message_ix.Scenario, material: str, filename: str): +def read_rel( + scenario: message_ix.Scenario, material: str, filename: str +) -> pd.DataFrame: """ Read relation_* type parameter data for specific industry @@ -2163,7 +2158,9 @@ def gen_te_projections( return inv_cost, fix_cost -def get_ssp_soc_eco_data(context: "Context", model: str, measure: str, tec): +def get_ssp_soc_eco_data( + context: "Context", model: str, measure: str, tec: str +) -> pd.DataFrame: """ Function to update scenario GDP and POP timeseries to SSP 3.0 and format to MESSAGEix "bound_activity_*" DataFrame @@ -2223,3 +2220,32 @@ def add_elec_i_ini_act(scenario: message_ix.Scenario) -> None: scenario.add_par(par, df_el) scenario.commit("add initial_activity_up for elec_i") return + + +def calculate_ini_new_cap( + df_demand: pd.DataFrame, technology: str, material: str +) -> pd.DataFrame: + """ + Derive initial_new_capacity_up parametrization for CCS based on cement demand + projection + Parameters + ---------- + df_demand: pd.DataFrame + DataFrame containing "demand" MESSAGEix parametrization + technology: str + name of CCS technology to be parametrized + material: str + name of the material/industry sector + Returns + ------- + DataFrame formatted to "initial_new_capacity_up" columns + """ + + SCALER = 0.005 + + CLINKER_RATIO = 0.72 if material == "cement" else 1 + df_demand["value"] *= CLINKER_RATIO * SCALER + + df_demand = df_demand.rename(columns={"node": "node_loc", "year": "year_vtg"}) + df_demand["technology"] = technology + return make_df("initial_new_capacity_up", **df_demand) diff --git a/message_ix_models/model/material/material_demand/material_demand_calc.py b/message_ix_models/model/material/material_demand/material_demand_calc.py index 458cd8b6ac..fa007d865b 100644 --- a/message_ix_models/model/material/material_demand/material_demand_calc.py +++ b/message_ix_models/model/material/material_demand/material_demand_calc.py @@ -1,3 +1,8 @@ +import logging +from pathlib import Path +from typing import Literal + +import message_ix import numpy as np import pandas as pd import yaml @@ -5,7 +10,8 @@ from scipy.optimize import curve_fit import message_ix_models.util -from message_ix_models import ScenarioInfo +from message_ix_models import Context, ScenarioInfo +from message_ix_models.model.material.data_util import get_ssp_soc_eco_data from message_ix_models.util import package_data_path file_gdp = "/iamc_db ENGAGE baseline GDP PPP.xlsx" @@ -14,8 +20,8 @@ material_data = { "aluminum": {"dir": "aluminum", "file": "/demand_aluminum.xlsx"}, - "steel": {"dir": "steel_cement", "file": "/STEEL_database_2012.xlsx"}, - "cement": {"dir": "steel_cement", "file": "/CEMENT.BvR2010.xlsx"}, + "steel": {"dir": "steel", "file": "/STEEL_database_2012.xlsx"}, + "cement": {"dir": "cement", "file": "/CEMENT.BvR2010.xlsx"}, "HVC": {"dir": "petrochemicals"}, "NH3": {"dir": "ammonia"}, "methanol": {"dir": "methanol"}, @@ -48,13 +54,15 @@ }, } +log = logging.getLogger(__name__) + -def steel_function(x, a, b, m): +def steel_function(x: pd.DataFrame | float, a: float, b: float, m: float): gdp_pcap, del_t = x return a * np.exp(b / gdp_pcap) * (1 - m) ** del_t -def cement_function(x, a, b): +def cement_function(x: pd.DataFrame | float, a: float, b: float): gdp_pcap = x[0] return a * np.exp(b / gdp_pcap) @@ -84,13 +92,15 @@ def cement_function(x, a, b): } -def gompertz(phi, mu, y, baseyear=2020): +def gompertz(phi: float, mu: float, y: pd.DataFrame | float, baseyear: int = 2020): return 1 - np.exp(-phi * np.exp(-mu * (y - baseyear))) -def read_timer_pop(datapath, material): +def read_timer_pop( + datapath: str | Path, material: Literal["cement", "steel", "aluminum"] +): df_population = pd.read_excel( - f'{datapath}/{material_data[material]["dir"]}{material_data["cement"]["file"]}', + f'{datapath}/{material_data[material]["dir"]}{material_data[material]["file"]}', sheet_name="Timer_POP", skiprows=[0, 1, 2, 30], nrows=26, @@ -104,10 +114,12 @@ def read_timer_pop(datapath, material): return df_population -def read_timer_gdp(datapath, material): +def read_timer_gdp( + datapath: str | Path, material: Literal["cement", "steel", "aluminum"] +): # Read GDP per capita data df_gdp = pd.read_excel( - f'{datapath}/{material_data[material]["dir"]}{material_data["cement"]["file"]}', + f'{datapath}/{material_data[material]["dir"]}{material_data[material]["file"]}', sheet_name="Timer_GDPCAP", skiprows=[0, 1, 2, 30], nrows=26, @@ -121,7 +133,7 @@ def read_timer_gdp(datapath, material): return df_gdp -def project_demand(df, phi, mu): +def project_demand(df: pd.DataFrame, phi: float, mu: float): df_demand = df.groupby("region", group_keys=False).apply( lambda group: group.assign( demand_pcap_base=group["demand.tot.base"].iloc[0] @@ -153,7 +165,7 @@ def project_demand(df, phi, mu): return df_demand[["region", "year", "demand_tot"]] -def read_base_demand(filepath): +def read_base_demand(filepath: str | Path): with open(filepath, "r") as file: yaml_data = file.read() @@ -170,7 +182,7 @@ def read_base_demand(filepath): return df -def read_hist_mat_demand(material): +def read_hist_mat_demand(material: Literal["cement", "steel", "aluminum"]): datapath = message_ix_models.util.package_data_path("material") if material in ["cement", "steel"]: @@ -257,14 +269,14 @@ def read_hist_mat_demand(material): .query("cons_pcap > 0") ) else: - print( + log.error( "non-available material selected. must be one of [aluminum, steel, cement]" ) df_cons = None return df_cons -def read_pop_from_scen(scen): +def read_pop_from_scen(scen: message_ix.Scenario) -> pd.DataFrame: pop = scen.par("bound_activity_up", {"technology": "Population"}) pop = pop.loc[pop.year_act >= 2020].rename( columns={"year_act": "year", "value": "pop.mil", "node_loc": "region"} @@ -273,7 +285,7 @@ def read_pop_from_scen(scen): return pop -def read_gdp_ppp_from_scen(scen): +def read_gdp_ppp_from_scen(scen: message_ix.Scenario) -> pd.DataFrame: if len(scen.par("bound_activity_up", filters={"technology": "GDP_PPP"})): gdp = scen.par("bound_activity_up", {"technology": "GDP_PPP"}) gdp = gdp.rename({"value": "gdp_ppp", "year_act": "year"}, axis=1)[ @@ -300,15 +312,38 @@ def read_gdp_ppp_from_scen(scen): return gdp -def derive_demand(material, scen, old_gdp=False, ssp="SSP2"): +def derive_demand( + material: Literal["cement", "steel", "aluminum"], + scen: message_ix.Scenario, + ssp: Literal["SSP1", "SSP2", "SSP3", "SSP4", "SSP5"] = "SSP2", +): datapath = message_ix_models.util.package_data_path("material") # read pop projection from scenario df_pop = read_pop_from_scen(scen) + if df_pop.empty: + log.info( + "Scenario does not provide Population projections. Reading default" + "timeseries instead" + ) + ctx = Context() + ctx.update(regions="R12") + df_pop = ( + get_ssp_soc_eco_data(ctx, "IIASA", "POP", "Population") + .rename( + columns={"year_act": "year", "value": "pop.mil", "node_loc": "region"} + ) + .drop(["mode", "time", "technology", "unit"], axis=1) + ) # read gdp (PPP) projection from scenario df_gdp = read_gdp_ppp_from_scen(scen) - if old_gdp: + # if not retrievable, read default from exogenous file instead + if df_gdp.empty: + log.info( + "Scenario does not provide GDP projections. Reading default" + "timeseries instead" + ) df_gdp = pd.read_excel(f"{datapath}/other{file_gdp}", sheet_name="data_R12") df_gdp = ( df_gdp[df_gdp["Scenario"] == "baseline"] @@ -338,11 +373,11 @@ def derive_demand(material, scen, old_gdp=False, ssp="SSP2"): p0=fitting_dict[material]["initial_guess"], )[0] mode = ssp_mode_map[ssp] - print(f"adjust regression parameters according to mode: {mode}") - print(f"before adjustment: {params_opt}") + log.info(f"adjust regression parameters according to mode: {mode}") + log.info(f"before adjustment: {params_opt}") for idx, multiplier in enumerate(mode_modifiers_dict[mode][material].values()): params_opt[idx] *= multiplier - print(f"after adjustment: {params_opt}") + log.info(f"after adjustment: {params_opt}") # prepare df for applying regression model and project demand df_all = pd.merge(df_pop, df_base_demand.drop(columns=["year"]), how="left") @@ -373,7 +408,12 @@ def derive_demand(material, scen, old_gdp=False, ssp="SSP2"): return df_final -def gen_demand_petro(scenario, chemical, gdp_elasticity_2020, gdp_elasticity_2030): +def gen_demand_petro( + scenario: message_ix.Scenario, + chemical: Literal["HVC", "methanol", "NH3"], + gdp_elasticity_2020: float, + gdp_elasticity_2030: float, +): chemicals_implemented = ["HVC", "methanol", "NH3"] if chemical not in chemicals_implemented: raise ValueError( @@ -402,12 +442,11 @@ def get_demand_t1_with_income_elasticity( .set_index("node_loc") .sort_index() ) - else: + elif "GDP" in list(scenario.set("technology")): gdp_mer = scenario.par("bound_activity_up", {"technology": "GDP"}) mer_to_ppp = pd.read_csv( package_data_path("material", "other", "mer_to_ppp_default.csv") ).set_index(["node", "year"]) - # mer_to_ppp = scenario.par("MERtoPPP").set_index("node", "year") # TODO: might need to be re-activated for different SSPs gdp_mer = gdp_mer.merge( mer_to_ppp.reset_index()[["node", "year", "value"]], @@ -427,7 +466,22 @@ def get_demand_t1_with_income_elasticity( .set_index("Region") .sort_index() ) - + else: + df_gdp = pd.read_excel( + f"{message_ix_models.util.package_data_path('material')}/other{file_gdp}", + sheet_name="data_R12", + ) + df_gdp_ts = ( + df_gdp[df_gdp["Scenario"] == "baseline"] + .loc[:, ["Region", *[i for i in df_gdp.columns if isinstance(i, int)]]] + .melt(id_vars="Region", var_name="year", value_name="gdp_ppp") + .query('Region != "World"') + .assign( + year=lambda x: x["year"].astype(int), + region=lambda x: "R12_" + x["Region"], + ) + .pivot(index="region", columns="year", values="gdp_ppp") + ) df_demand_2020 = read_base_demand( package_data_path() / "material" diff --git a/message_ix_models/model/material/report/reporting.py b/message_ix_models/model/material/report/reporting.py index 5c6602b79b..f7ff983b43 100644 --- a/message_ix_models/model/material/report/reporting.py +++ b/message_ix_models/model/material/report/reporting.py @@ -35,6 +35,7 @@ from message_ix.report import Reporter from message_ix_models import ScenarioInfo +from message_ix_models.util import package_data_path matplotlib.use("Agg") @@ -179,7 +180,7 @@ def report(context, scenario): # noqa: C901 nodes.remove("R12_GLB*") # Path for materials reporting output - directory = context.get_local_path("report", "materials") + directory = package_data_path("material", "reporting_output") directory.mkdir(exist_ok=True) # Generate message_ix level reporting and dump to an excel file. diff --git a/message_ix_models/model/material/util.py b/message_ix_models/model/material/util.py index 631a7a1a51..000ba33b5b 100644 --- a/message_ix_models/model/material/util.py +++ b/message_ix_models/model/material/util.py @@ -1,5 +1,6 @@ import os from pathlib import Path +from typing import Any, Union import message_ix import openpyxl as pxl @@ -45,19 +46,8 @@ def read_config() -> Context: context[key] = load_package_data(*_parts) - # Read material.yaml - # FIXME What is this personal information doing here? - # context.metadata_path=Path("C:/Users/unlu/Documents/GitHub/message_data/data") - # context.load_config("material", "set") - # Use a shorter name context["material"] = context["material set"] - - # Merge technology.yaml with set.yaml - # context["material"]["steel"]["technology"]["add"] = ( - # context.pop("transport technology") - # ) - return context @@ -192,7 +182,7 @@ def get_all_input_data_dirs() -> list[str]: return elements -def remove_from_list_if_exists(element, _list: list) -> None: +def remove_from_list_if_exists(element: Any, _list: list) -> None: """ Utility function removing element from list if it is part of the list Parameters @@ -241,13 +231,13 @@ def price_fit(df: pd.DataFrame) -> float: estimated value for price_ref in 2020 """ - pars = curve_fit(exponential, df.year, df.lvl, maxfev=5000)[0] + pars = curve_fit(exponential, df.year, df.lvl, maxfev=10000)[0] val = exponential([2020], *pars)[0] # print(df.commodity.unique(), df.node.unique(), val) return val -def cost_fit(df) -> float: +def cost_fit(df: pd.DataFrame) -> float: """ Python implementation of cost_ref parameter estimation implemented in MESSAGEix-MACRO calibration files. @@ -272,8 +262,11 @@ def cost_fit(df) -> float: def update_macro_calib_file(scenario: message_ix.Scenario, fname: str) -> None: - """ - Function to automate manual steps in MACRO calibration + """Function to automate manual steps in MACRO calibration + + Tries to open a xlsx file with the given "fname" and + writes cost_ref and price_ref values derived from scenario + "COST_NODAL_NET" and PRICE_COMMODITY" variables to the respective xlsx sheets. Parameters ---------- @@ -282,31 +275,33 @@ def update_macro_calib_file(scenario: message_ix.Scenario, fname: str) -> None: fname : str file name of MACRO file used for calibration """ - path = "C:/Users/maczek/Downloads/macro/refactored/" - wb = pxl.load_workbook(path + fname) + # Change this according to the relevant data path + path = package_data_path("material", "macro", fname) + wb = pxl.load_workbook(path) + + fmy = scenario.firstmodelyear + nodes = [ + "R12_AFR", + "R12_CHN", + "R12_EEU", + "R12_FSU", + "R12_LAM", + "R12_MEA", + "R12_NAM", + "R12_PAO", + "R12_PAS", + "R12_RCPA", + "R12_SAS", + "R12_WEU", + ] # cost_ref - years = [i for i in range(2020, 2055, 5)] - df = scenario.var("COST_NODAL_NET", filters={"year": years}) - df["node"] = pd.Categorical( - df["node"], - [ - "R12_AFR", - "R12_CHN", - "R12_EEU", - "R12_FSU", - "R12_LAM", - "R12_MEA", - "R12_NAM", - "R12_PAO", - "R12_PAS", - "R12_RCPA", - "R12_SAS", - "R12_WEU", - ], - ) - df = df[df["year"].isin([2025, 2030, 2035])].groupby(["node"]).apply(cost_fit) + years_cost = [i for i in range(fmy, fmy + 15, 5)] + df = scenario.var("COST_NODAL_NET", filters={"year": years_cost}) + df["node"] = pd.Categorical(df["node"], nodes) + df = df[df["year"].isin(years_cost)].groupby(["node"]).apply(cost_fit) ws = wb.get_sheet_by_name("cost_ref") + # write derived values to sheet. Cell B7 (MEA region) is skipped. for i in range(2, 7): ws[f"B{i}"].value = df.values[i - 2] for i in range(8, 14): @@ -314,34 +309,17 @@ def update_macro_calib_file(scenario: message_ix.Scenario, fname: str) -> None: # price_ref comms = ["i_feed", "i_spec", "i_therm", "rc_spec", "rc_therm", "transport"] - years = [i for i in range(2025, 2055, 5)] - df = scenario.var("PRICE_COMMODITY", filters={"commodity": comms, "year": years}) - df["node"] = pd.Categorical( - df["node"], - [ - "R12_AFR", - "R12_EEU", - "R12_FSU", - "R12_LAM", - "R12_MEA", - "R12_NAM", - "R12_PAO", - "R12_PAS", - "R12_SAS", - "R12_WEU", - "R12_CHN", - "R12_RCPA", - ], - ) - df["commodity"] = pd.Categorical( - df["commodity"], - ["i_feed", "i_spec", "i_therm", "rc_spec", "rc_therm", "transport"], + years_price = [i for i in range(fmy, 2055, 5)] + df = scenario.var( + "PRICE_COMMODITY", filters={"commodity": comms, "year": years_price} ) + df["node"] = pd.Categorical(df["node"], nodes) + df["commodity"] = pd.Categorical(df["commodity"], comms) df = df.groupby(["node", "commodity"]).apply(price_fit) ws = wb.get_sheet_by_name("price_ref") for i in range(2, 62): ws[f"C{i}"].value = df.values[i - 2] - wb.save(path + fname) + wb.save(path) def get_ssp_from_context(context: Context) -> str: @@ -362,6 +340,31 @@ def get_ssp_from_context(context: Context) -> str: return ssp -def maybe_remove_water_tec(scenario, results): +def maybe_remove_water_tec(scenario: message_ix.Scenario, results: dict) -> None: if len(scenario.par("output", filters={"technology": "extract_surfacewater"})): results["input"] = results["input"].replace({"freshwater_supply": "freshwater"}) + + +def path_fallback(context_or_regions: Union[Context, str], *parts) -> Path: + """Return a :class:`.Path` constructed from `parts`. + + If ``context.model.regions`` (or a string value as the first argument) is defined + and the file exists in a subdirectory of :file:`data/transport/{regions}/`, return + its path; otherwise, return the path in :file:`data/transport/`. + """ + if isinstance(context_or_regions, str): + regions = context_or_regions + else: + # Use a value from a Context object, or a default + regions = context_or_regions.model.regions + + candidates = ( + package_data_path("material", regions, *parts), + package_data_path("material", *parts), + ) + + for c in candidates: + if c.exists(): + return c + + raise FileNotFoundError(candidates) diff --git a/message_ix_models/testing/__init__.py b/message_ix_models/testing/__init__.py index d520c422fe..216dee7fda 100644 --- a/message_ix_models/testing/__init__.py +++ b/message_ix_models/testing/__init__.py @@ -215,14 +215,16 @@ def bare_res(request, context: Context, solved: bool = False) -> message_ix.Scen """ from message_ix_models.model import bare - name = bare.name(context) + # Model name: standard "MESSAGEix-GLOBIOM R12 YB" plus a suffix + model_name = bare.name(context, unique=True) + mp = context.get_platform() try: - base = message_ix.Scenario(mp, name, "baseline") + base = message_ix.Scenario(mp, model_name, "baseline") except ValueError: - log.info(f"Create '{name}/baseline' for testing") - context.scenario_info.update(model=name, scenario="baseline") + log.info(f"Create '{model_name}/baseline' for testing") + context.scenario_info.update(model=model_name, scenario="baseline") base = bare.create_res(context) if solved and not base.has_solution(): @@ -235,7 +237,7 @@ def bare_res(request, context: Context, solved: bool = False) -> message_ix.Scen # Generate a new scenario name with a random part new_name = f"baseline {b32hexencode(randbytes(3)).decode().rstrip('=').lower()}" - log.info(f"Clone to '{name}/{new_name}'") + log.info(f"Clone to '{model_name}/{new_name}'") return base.clone(scenario=new_name, keep_solution=solved) diff --git a/message_ix_models/tests/model/material/test_build.py b/message_ix_models/tests/model/material/test_build.py new file mode 100644 index 0000000000..3eb9f113d4 --- /dev/null +++ b/message_ix_models/tests/model/material/test_build.py @@ -0,0 +1,58 @@ +import logging + +import pytest + +from message_ix_models.model.material import build +from message_ix_models.model.structure import get_codes +from message_ix_models.testing import bare_res + +log = logging.getLogger(__name__) + + +@pytest.mark.parametrize( + "regions_arg, regions_exp", + [ + # ("R11", "R11"), + ("R12", "R12"), + ], +) +@pytest.mark.parametrize("material", [None]) +def test_make_spec(regions_arg, regions_exp, material): + # The spec can be generated + spec = build.make_spec(regions_arg, material) + + # The required elements of the "node" set match the configuration + nodes = get_codes(f"node/{regions_exp}") + expected = list(map(str, nodes[nodes.index("World")].child)) + assert expected == spec["require"].set["node"] + + +@pytest.mark.parametrize( + "regions, years, relations, solve", + [ + ("R12", "B", "B", False), + pytest.param( + "R11", "B", "B", False, marks=pytest.mark.xfail(raises=NotImplementedError) + ), + ], +) +def test_build_bare_res( + request, tmp_path, test_context, regions, years, relations, solve +): + """.materials.build() works on the bare RES, and the model solves.""" + # Generate the relevant bare RES + ctx = test_context + ctx.update(regions=regions, years=years, ssp="SSP2", relations=relations) + scenario = bare_res(request, ctx) + + # Build succeeds without error + # options = {"dummy_supply": True} + build.build(ctx, scenario, modify_existing_constraints=False, old_calib=True) + + if solve: + scenario.solve(solve_options=dict(lpmethod=4, iis=1)) + + # commented: Appears to be giving a false negative + # # Use Reporting calculations to check the result + # result = report.check(scenario) + # assert result.all(), f"\n{result}" diff --git a/message_ix_models/tests/model/test_bare.py b/message_ix_models/tests/model/test_bare.py index 6a84ca3b1b..5e2c1ad430 100644 --- a/message_ix_models/tests/model/test_bare.py +++ b/message_ix_models/tests/model/test_bare.py @@ -7,7 +7,7 @@ #: Number of items in the respective YAML files. SET_SIZE = dict( commodity=18, - level=6, + level=7, node=14 + 1, # R14 is default, and 'World' exists automatically relation=20, technology=390, diff --git a/message_ix_models/tests/test_workflow.py b/message_ix_models/tests/test_workflow.py index 65164bf12d..2be4c82e64 100644 --- a/message_ix_models/tests/test_workflow.py +++ b/message_ix_models/tests/test_workflow.py @@ -159,7 +159,7 @@ def test_workflow(caplog, request, test_context, wf) -> None: start_index = 1 if caplog.messages[0].startswith("Cull") else 0 # This setting obtains the value R11 on some Windows GHA jobs, but is otherwise R14. # TODO Debug and fix. - m = f"MESSAGEix-GLOBIOM {test_context.model.regions} YB" + m = f"MESSAGEix-GLOBIOM {test_context.model.regions} YB 33029" messages = [ f"Loaded ixmp://{mp}/{m}/test_workflow#1", f"Step runs on ixmp://{mp}/{m}/test_workflow#1", diff --git a/message_ix_models/util/config.py b/message_ix_models/util/config.py index 7b784edf59..c76919385c 100644 --- a/message_ix_models/util/config.py +++ b/message_ix_models/util/config.py @@ -1,6 +1,8 @@ import logging import os -from dataclasses import dataclass, field, fields, is_dataclass, replace +import pickle +from dataclasses import asdict, dataclass, field, fields, is_dataclass, replace +from hashlib import blake2s from pathlib import Path from typing import Any, Hashable, List, Mapping, MutableMapping, Optional, Sequence, Set @@ -30,14 +32,16 @@ def _local_data_factory(): class ConfigHelper: """Mix-in for :class:`dataclass`-based configuration classes. - This provides 3 methods—:meth:`read_file`, :meth:`replace`, and :meth:`from_dict`— - that help to use :class:`dataclass` classes for handling :mod:`message_ix_models` + This provides methods :meth:`read_file`, :meth:`replace`, and :meth:`from_dict` that + help to use :class:`dataclass` classes for handling :mod:`message_ix_models` configuration. All 3 methods take advantage of name manipulations: the characters "-" and " " are replaced with underscores ("_"). This allows to write the names of attributes in legible ways—e.g. "attribute name" or “attribute-name” instead of "attribute_name"— in configuration files and/or code. + + It also add :meth:`hexdigest`. """ @classmethod @@ -131,6 +135,26 @@ def from_dict(cls, data: Mapping): """Construct an instance from `data` with name manipulation.""" return cls(**{k: v for k, v in cls._munge_dict(data, "raise", "mapping key")}) + def hexdigest(self, length: int = -1) -> str: + """Return a hex digest that is unique for distinct settings on the instance. + + Returns + ------- + str + If `length` is non-zero, a string of this length; otherwise a 32-character + string from :meth:`.blake2s.hexdigest`. + """ + # - Dump the dataclass instance to nested, sorted tuples. This is used instead + # of dataclass.astuple() which allows e.g. units to pass as a (possibly + # unsorted) dict. + # - Pickle this collection. + # - Hash. + h = blake2s( + pickle.dumps(asdict(self, dict_factory=lambda kv: tuple(sorted(kv)))) + ) + # Return the whole digest or a part + return h.hexdigest()[0 : length if length > 0 else h.digest_size] + @dataclass class Config: