Skip to content

Commit

Permalink
Change usage of materials to module when possible
Browse files Browse the repository at this point in the history
  • Loading branch information
@measrainsey
measrainsey committed Sep 2, 2024
1 parent 3dea659 commit 8662c7c
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 66 deletions.
50 changes: 24 additions & 26 deletions message_ix_models/tools/costs/decay.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from message_ix_models.util import package_data_path

from .config import Config
from .regional_differentiation import get_raw_technology_mapping, subset_materials_map
from .regional_differentiation import get_raw_technology_mapping, subset_module_map


def get_cost_reduction_data(module) -> pd.DataFrame:
Expand Down Expand Up @@ -32,16 +32,17 @@ def get_cost_reduction_data(module) -> pd.DataFrame:
# Get full list of technologies from mapping
tech_map = energy_map = get_raw_technology_mapping("energy")

if module == "materials":
materials_map = get_raw_technology_mapping("materials")
materials_sub = subset_materials_map(materials_map)
# if module is not energy, run subset_module_map
if module != "energy":
module_map = get_raw_technology_mapping(module)
module_sub = subset_module_map(module_map)

# Remove energy technologies that exist in materials mapping
# Remove energy technologies that exist in module mapping
energy_map = energy_map.query(
"message_technology not in @materials_sub.message_technology"
"message_technology not in @module_sub.message_technology"
)

tech_map = pd.concat([energy_map, materials_sub], ignore_index=True)
tech_map = pd.concat([energy_map, module_sub], ignore_index=True)

# Read in raw data
gea_file_path = package_data_path("costs", "energy", "cost_reduction.csv")
Expand All @@ -60,9 +61,9 @@ def get_cost_reduction_data(module) -> pd.DataFrame:
.reset_index(drop=1)
).reindex(["message_technology", "reduction_rate", "cost_reduction"], axis=1)

# For materials technologies with map_tech == energy, map to base technologies
# For module technologies with map_tech == energy, map to base technologies
# and use cost reduction data
materials_rates_energy = (
module_rates_energy = (
tech_map.query("reg_diff_source == 'energy'")
.drop(columns=["reg_diff_source", "base_year_reference_region_cost"])
.merge(
Expand All @@ -80,7 +81,7 @@ def get_cost_reduction_data(module) -> pd.DataFrame:

# Combine technologies that have cost reduction rates
df_reduction_techs = pd.concat(
[energy_rates, materials_rates_energy], ignore_index=True
[energy_rates, module_rates_energy], ignore_index=True
)
df_reduction_techs = df_reduction_techs.drop_duplicates().reset_index(drop=1)

Expand All @@ -93,10 +94,9 @@ def get_cost_reduction_data(module) -> pd.DataFrame:
"key": "z",
}
)

# For remaining materials technologies that are not mapped to energy technologies,
# For remaining module technologies that are not mapped to energy technologies,
# assume no cost reduction
materials_rates_noreduction = (
module_rates_noreduction = (
tech_map.query(
"message_technology not in @df_reduction_techs.message_technology"
)
Expand All @@ -105,9 +105,9 @@ def get_cost_reduction_data(module) -> pd.DataFrame:
.drop(columns=["key"])
).reindex(["message_technology", "reduction_rate", "cost_reduction"], axis=1)

# Concatenate base and materials rates
# Concatenate base and module rates
all_rates = pd.concat(
[energy_rates, materials_rates_energy, materials_rates_noreduction],
[energy_rates, module_rates_energy, module_rates_noreduction],
ignore_index=True,
).reset_index(drop=1)

Expand All @@ -127,7 +127,7 @@ def get_technology_reduction_scenarios_data(
Raw data on cost reduction scenarios are read from
:file:`data/costs/[module]/scenarios_reduction_[module].csv`.
Assumptions are made for the materials module for technologies' cost reduction
Assumptions are made for the non-energy module for technologies' cost reduction
scenarios that are not given.
Parameters
Expand All @@ -154,25 +154,23 @@ def get_technology_reduction_scenarios_data(
["message_technology", "first_year_original"]
]

if module == "materials":
materials_first_year_file = package_data_path(
"costs", "materials", "tech_map.csv"
)
materials_first_year = pd.read_csv(materials_first_year_file)[
if module != "energy":
module_first_year_file = package_data_path("costs", module, "tech_map.csv")
module_first_year = pd.read_csv(module_first_year_file)[
["message_technology", "first_year_original"]
]
df_first_year = pd.concat(
[df_first_year, materials_first_year], ignore_index=True
[df_first_year, module_first_year], ignore_index=True
).drop_duplicates()

tech_map = tech_energy = get_raw_technology_mapping("energy")

if module == "materials":
tech_materials = subset_materials_map(get_raw_technology_mapping("materials"))
if module != "energy":
tech_module = subset_module_map(get_raw_technology_mapping(module))
tech_energy = tech_energy.query(
"message_technology not in @tech_materials.message_technology"
"message_technology not in @tech_module.message_technology"
)
tech_map = pd.concat([tech_energy, tech_materials], ignore_index=True)
tech_map = pd.concat([tech_energy, tech_module], ignore_index=True)

tech_map = tech_map.reindex(
["message_technology", "reg_diff_source", "reg_diff_technology"], axis=1
Expand Down
93 changes: 53 additions & 40 deletions message_ix_models/tools/costs/regional_differentiation.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import logging
from functools import lru_cache
from itertools import product
from typing import Literal, Mapping
from typing import Mapping

import numpy as np
import pandas as pd
Expand Down Expand Up @@ -170,7 +170,7 @@ def get_intratec_data() -> pd.DataFrame:
return pd.read_csv(file, comment="#", skipinitialspace=True)


def get_raw_technology_mapping(module: Literal["energy", "materials"]) -> pd.DataFrame:
def get_raw_technology_mapping(module) -> pd.DataFrame:
"""Retrieve a technology mapping for `module`.
The data are read from a CSV file at :file:`data/{module}/tech_map.csv`.
Expand Down Expand Up @@ -198,8 +198,8 @@ def get_raw_technology_mapping(module: Literal["energy", "materials"]) -> pd.Dat
return pd.read_csv(path, comment="#")


def subset_materials_map(raw_map):
"""Subset materials mapping for only technologies that have sufficient data.
def subset_module_map(raw_map):
"""Subset non-energy module mapping for only technologies that have sufficient data.
Parameters
----------
Expand All @@ -217,7 +217,7 @@ def subset_materials_map(raw_map):
- base_year_reference_region_cost: manually specified base year cost
of the technology in the reference region (in 2005 USD)
"""
# - Remove materials technologies that are missing both a reg_diff_source and a
# - Remove module technologies that are missing both a reg_diff_source and a
# base_year_reference_region_cost
# - Round base_year_reference_region_cost to nearest integer
sub_map = (
Expand All @@ -232,7 +232,7 @@ def subset_materials_map(raw_map):
return sub_map


def adjust_technology_mapping(module: Literal["energy", "materials"]) -> pd.DataFrame:
def adjust_technology_mapping(module) -> pd.DataFrame:
"""Adjust technology mapping based on sources and assumptions.
Parameters
Expand All @@ -258,22 +258,22 @@ def adjust_technology_mapping(module: Literal["energy", "materials"]) -> pd.Data
if module == "energy":
return raw_map_energy

elif module == "materials":
raw_map_materials = get_raw_technology_mapping("materials")
sub_map_materials = subset_materials_map(raw_map_materials)
else:
raw_map_module = get_raw_technology_mapping(module)
sub_map_module = subset_module_map(raw_map_module)

# If message_technology in sub_map_materials is in raw_map_energy and
# If message_technology in sub_map_module is in raw_map_energy and
# base_year_reference_region_cost is not null/empty, then replace
# base_year_reference_region_cost in raw_map_energy with
# base_year_reference_region_cost in sub_map_materials
materials_replace = (
sub_map_materials.query(
# base_year_reference_region_cost in sub_map_module
module_replace = (
sub_map_module.query(
"message_technology in @raw_map_energy.message_technology"
)
.rename(
columns={
"message_technology": "material_message_technology",
"base_year_reference_region_cost": "material_base_cost",
"base_year_reference_region_cost": "module_base_cost",
}
)
.drop(columns=["reg_diff_source", "reg_diff_technology"])
Expand All @@ -285,8 +285,8 @@ def adjust_technology_mapping(module: Literal["energy", "materials"]) -> pd.Data
)
.assign(
base_year_reference_region_cost=lambda x: np.where(
x.material_base_cost.notnull(),
x.material_base_cost,
x.module_base_cost.notnull(),
x.module_base_cost,
x.base_year_reference_region_cost,
)
)
Expand All @@ -304,12 +304,12 @@ def adjust_technology_mapping(module: Literal["energy", "materials"]) -> pd.Data
# Subset to only rows where reg_diff_source is "energy"
# Merge with raw_map_energy on reg_diff_technology
# If the "base_year_reference_region_cost" is not
# null/empty in raw_materials_map,
# null/empty in raw_module_map,
# then use that.
# If the base_year_reference_region_cost is null/empty in raw_materials_map,
# If the base_year_reference_region_cost is null/empty in raw_module_map,
# then use the base_year_reference_region_cost from the mapped energy technology
materials_map_energy = (
sub_map_materials.query("reg_diff_source == 'energy'")
module_map_energy = (
sub_map_module.query("reg_diff_source == 'energy'")
.drop(columns=["reg_diff_source"])
.rename(
columns={
Expand Down Expand Up @@ -345,39 +345,54 @@ def adjust_technology_mapping(module: Literal["energy", "materials"]) -> pd.Data
)
)

# Get technologies that are mapped to Intratec AND have a base year cost
# Assign map_techonology as "all"
materials_map_intratec = sub_map_materials.query(
"reg_diff_source == 'intratec' and "
"base_year_reference_region_cost.notnull()"
).assign(reg_diff_technology="all")

# Get technologies that don't have a map source but do have a base year cost
# For these technologies, assume no regional differentiation
# So use the reference region base year cost as the base year cost
# across all regions
materials_map_noregdiff = sub_map_materials.query(
module_map_noregdiff = sub_map_module.query(
"reg_diff_source.isnull() and base_year_reference_region_cost.notnull()"
)

# Concatenate materials_replace and materials_map_energy
# Concatenate module_replace, module_map_energy, and module_map_noregdiff
# Drop duplicates
materials_all = (
module_all = (
pd.concat(
[
materials_replace,
materials_map_energy,
materials_map_intratec,
materials_map_noregdiff,
module_replace,
module_map_energy,
module_map_noregdiff,
]
)
.drop_duplicates()
.reset_index(drop=True)
)

# Get list of technologies in raw_map_materials that are not in materials_all
missing_tech = raw_map_materials.query(
"message_technology not in @materials_all.message_technology"
# If module == "materials", then get materials_map_intratec
# and concatenate with module_all
if module == "materials":
# Get technologies that are mapped to Intratec AND have a base year cost
# Assign map_techonology as "all"
materials_map_intratec = sub_map_module.query(
"reg_diff_source == 'intratec' and "
"base_year_reference_region_cost.notnull()"
).assign(reg_diff_technology="all")

# Concatenate materials_map_intratec and module_all
# Drop duplicates
module_all = (
pd.concat(
[
module_all,
materials_map_intratec,
]
)
.drop_duplicates()
.reset_index(drop=True)
)

# Get list of technologies in raw_map_module that are not in module_all
missing_tech = raw_map_module.query(
"message_technology not in @module_all.message_technology"
).message_technology.unique()

log.info(
Expand All @@ -386,7 +401,7 @@ def adjust_technology_mapping(module: Literal["energy", "materials"]) -> pd.Data
+ "\n".join(missing_tech)
)

return materials_all
return module_all


def get_weo_regional_differentiation(config: "Config") -> pd.DataFrame:
Expand Down Expand Up @@ -655,8 +670,6 @@ def apply_regional_differentiation(config: "Config") -> pd.DataFrame:
)
)

filt_weo.query("message_technology == 'coal_ppl'")

# Filter for reg_diff_source == "intratec"
# Then merge with output of get_intratec_regional_differentiation
# If the base_year_reference_region_cost is empty,
Expand Down

0 comments on commit 8662c7c

Please sign in to comment.