From d5cd6369586c2d9e87691e68729af2bbd0aa4328 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 4 Sep 2023 16:30:51 +0200 Subject: [PATCH 01/44] Add .util.sdmx.{read,write} --- message_ix_models/util/sdmx.py | 62 +++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/message_ix_models/util/sdmx.py b/message_ix_models/util/sdmx.py index b271ab207d..cb99a81e48 100644 --- a/message_ix_models/util/sdmx.py +++ b/message_ix_models/util/sdmx.py @@ -1,10 +1,19 @@ """Utilities for handling objects from :mod:`sdmx`.""" import logging -from typing import Dict, List, Mapping, Union +from datetime import datetime +from importlib.metadata import version +from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Union +import sdmx +import sdmx.message from iam_units import registry from sdmx.model.v21 import AnnotableArtefact, Annotation, Code, InternationalString +from .common import package_data_path + +if TYPE_CHECKING: + from os import PathLike + log = logging.getLogger(__name__) CodeLike = Union[str, Code] @@ -96,3 +105,54 @@ def eval_anno(obj: AnnotableArtefact, id: str): except Exception as e: # Something that can't be eval()'d, e.g. a plain string log.debug(f"Could not eval({value!r}): {e}") return value + + +def read(urn: str, base_dir: Optional["PathLike"] = None): + """Read SDMX object from package data given its `urn`.""" + # Identify a path that matches `urn` + base_dir = base_dir or package_data_path("sdmx") + paths = sorted( + set(base_dir.glob(f"*{urn}*.xml")) | set(base_dir.glob(f"*{urn.upper()}*.xml")) + ) + + if len(paths) > 1: + log.info( + f"Match {paths[0].relative_to(base_dir)} for {urn!r}; {len(paths) -1 } " + "other result(s)" + ) + + with open(paths[0], "rb") as f: + msg = sdmx.read_sdmx(f) + + for _, cls in msg.iter_collections(): + try: + return next(iter(msg.objects(cls).values())) + except StopIteration: + pass + + +def write(obj, base_dir: Optional["PathLike"] = None): + """Store an SDMX object as package data.""" + # Set the URN of the object + obj.urn = sdmx.urn.make(obj) + + # Wrap the object in a StructureMessage + msg = sdmx.message.StructureMessage( + header=sdmx.message.Header( + source=f"Generated by message_ix_models {version('message_ix_models')}", + prepared=datetime.now(), + ) + ) + msg.add(obj) + + # Identify a path to write the file + base_dir = base_dir or package_data_path("sdmx") + basename = obj.urn.split("=")[-1] + path = base_dir.joinpath(f"{basename}.xml") + + # Write + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "wb") as f: + f.write(sdmx.to_xml(msg, pretty_print=True)) + + log.info(f"Wrote {path}") From 19ebd92c147dcc5c588aff4335f504b31b589254 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 4 Sep 2023 16:31:59 +0200 Subject: [PATCH 02/44] Add .project.ssp.structure.generate() --- message_ix_models/project/ssp/structure.py | 191 +++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 message_ix_models/project/ssp/structure.py diff --git a/message_ix_models/project/ssp/structure.py b/message_ix_models/project/ssp/structure.py new file mode 100644 index 0000000000..64451d2a65 --- /dev/null +++ b/message_ix_models/project/ssp/structure.py @@ -0,0 +1,191 @@ +"""Manipulate data structures for working with the SSPs.""" +import logging +from textwrap import wrap +from typing import TYPE_CHECKING + +import sdmx +import sdmx.model.v30 as m +import sdmx.urn + +from message_ix_models.util.sdmx import write + +if TYPE_CHECKING: + from message_ix_models import Context + +log = logging.getLogger(__name__) + + +DESC = """ +This code list is *not* officially published by ICONICS; rather, it is a rendering into +SDMX of structural information that is provided by ICONICS participants. It may change +or be superseded at any time. + +Each code has a single digit ID like "1"; the "original-id" annotation (and the start of +the name) give a string like "SSP1". These original IDs are *only* unique if using data +enumerated by solely by one code list or the other; if mixing the two, then they will be +ambiguous. The URNs of the codes or parts thereof (for instance, "ICONICS:SSP(2017).1") +are, by construction, unique. +""" + +CL_INFO = ( + dict( + version="2017", + name_extra="2017 edition", + desc_extra="This is the original set of SSPs as described in" + " https://doi.org/10.1016/j.gloenvcha.2016.05.009.", + ), + dict( + version="2024", + name_extra="2024 edition", + desc_extra="This set of SSPs is currently under development; for details, see " + "https://depts.washington.edu/iconics/.", + ), +) + +CODE_INFO = ( + dict( + id="1", + name="Sustainability", + name_long="Sustainability – Taking the Green Road", + description="""Low challenges to mitigation and adaptation. + +The world shifts gradually, but pervasively, toward a more sustainable path, emphasizing +more inclusive development that respects perceived environmental boundaries. Management +of the global commons slowly improves, educational and health investments accelerate the +demographic transition, and the emphasis on economic growth shifts toward a broader +emphasis on human well-being. Driven by an increasing commitment to achieving +development goals, inequality is reduced both across and within countries. Consumption +is oriented toward low material growth and lower resource and energy intensity.""", + ), + dict( + id="2", + name="Middle of the Road", + name_long="Middle of the Road", + description="""Medium challenges to mitigation and adaptation. + +The world follows a path in which social, economic, and technological trends do not +shift markedly from historical patterns. Development and income growth proceeds +unevenly, with some countries making relatively good progress while others fall short of +expectations. Global and national institutions work toward but make slow progress in +achieving sustainable development goals. Environmental systems experience degradation, +although there are some improvements and overall the intensity of resource and energy +use declines. Global population growth is moderate and levels off in the second half of +the century. Income inequality persists or improves only slowly and challenges to +reducing vulnerability to societal and environmental changes remain.""", + ), + dict( + id="3", + name="Regional Rivalry", + name_long="Regional Rivalry – A Rocky Road", + description="""High challenges to mitigation and adaptation. + +A resurgent nationalism, concerns about competitiveness and security, and regional +conflicts push countries to increasingly focus on domestic or, at most, regional issues. +Policies shift over time to become increasingly oriented toward national and regional +security issues. Countries focus on achieving energy and food security goals within +their own regions at the expense of broader-based development. Investments in education +and technological development decline. Economic development is slow, consumption is +material-intensive, and inequalities persist or worsen over time. Population growth is +low in industrialized and high in developing countries. A low international priority for +addressing environmental concerns leads to strong environmental degradation in some +regions.""", + ), + dict( + id="4", + name="Inequality", + name_long="Inequality – A Road Divided", + description="""Low challenges to mitigation, high challenges to adaptation. + +Highly unequal investments in human capital, combined with increasing disparities in +economic opportunity and political power, lead to increasing inequalities and +stratification both across and within countries. Over time, a gap widens between an +internationally-connected society that contributes to knowledge- and capital-intensive +sectors of the global economy, and a fragmented collection of lower-income, poorly +educated societies that work in a labor intensive, low-tech economy. Social cohesion +degrades and conflict and unrest become increasingly common. Technology development is +high in the high-tech economy and sectors. The globally connected energy sector +diversifies, with investments in both carbon-intensive fuels like coal and +unconventional oil, but also low-carbon energy sources. Environmental policies focus on +local issues around middle and high income areas.""", + ), + dict( + id="5", + name="Fossil-fueled Development", + name_long="Fossil-fueled Development – Taking the Highway", + description="""High challenges to mitigation, low challenges to adaptation. + +This world places increasing faith in competitive markets, innovation and participatory +societies to produce rapid technological progress and development of human capital as +the path to sustainable development. Global markets are increasingly integrated. There +are also strong investments in health, education, and institutions to enhance human and +social capital. At the same time, the push for economic and social development is +coupled with the exploitation of abundant fossil fuel resources and the adoption of +resource and energy intensive lifestyles around the world. All these factors lead to +rapid growth of the global economy, while global population peaks and declines in the +21st century. Local environmental problems like air pollution are successfully managed. +There is faith in the ability to effectively manage social and ecological systems, +including by geo-engineering if necessary.""", + ), +) + + +def generate(context: "Context"): + """Generate SDMX code lists containing the SSPs.""" + # Create an AgencyScheme containing ICONICS + as_ = m.AgencyScheme( + id="AGENCIES", + description="Agencies referenced by data structures in message_ix_models", + version="0.1", + ) + + IIASA_ECE = m.Agency( + id="IIASA_ECE", name="IIASA Energy, Climate, and Environment Program" + ) + + ICONICS = m.Agency( + id="ICONICS", + name="International Committee on New Integrated Climate Change Assessment " + "Scenarios", + contact=[m.Contact(uri=["https://depts.washington.edu/iconics/"])], + ) + + as_.maintainer = IIASA_ECE + as_.append(IIASA_ECE) + as_.append(ICONICS) + + write(as_) + + for cl_info in CL_INFO: + # Create the codelist: format the name and description + cl = m.Codelist( + id="SSP", + name=f"Shared Socioeconomic Pathways ({cl_info['name_extra']})", + description="\n".join( + wrap(DESC.strip(), width=len(DESC)) + ["", cl_info["desc_extra"]] + ), + version=cl_info["version"], + maintainer=ICONICS, + ) + + # Add one Code for each SSP + for info in CODE_INFO: + # Construct the original ID + original_id = f"SSP{info['id']}" + + # Format the name, description; add an annotation + c = m.Code( + id=info["id"], + name=f"{original_id}: {info['name']}", + description="\n".join( + [f"{original_id}: {info['name_long']}", "", info["description"]] + ), + annotations=[m.Annotation(id="original-id", text=original_id)], + ) + + # Append to the code list + cl.append(c) + + # Construct a URN + c.urn = sdmx.urn.make(c, maintainable_parent=cl) + + write(cl) From 1615f7b0921da85178ca2a36cf844593c7c8c288 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 4 Sep 2023 16:32:29 +0200 Subject: [PATCH 03/44] Add basic CLI for .project.ssp --- message_ix_models/cli.py | 1 + message_ix_models/project/ssp/__init__.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 message_ix_models/project/ssp/__init__.py diff --git a/message_ix_models/cli.py b/message_ix_models/cli.py index 33e0ddaebb..9d5233cf78 100644 --- a/message_ix_models/cli.py +++ b/message_ix_models/cli.py @@ -114,6 +114,7 @@ def debug(ctx): "message_ix_models.model.snapshot", "message_ix_models.model.structure", "message_ix_models.model.water.cli", + "message_ix_models.project.ssp", "message_ix_models.report.cli", ] diff --git a/message_ix_models/project/ssp/__init__.py b/message_ix_models/project/ssp/__init__.py new file mode 100644 index 0000000000..65fcced043 --- /dev/null +++ b/message_ix_models/project/ssp/__init__.py @@ -0,0 +1,15 @@ +import click + +from .structure import generate + + +@click.group("ssp") +def cli(): + pass + + +@cli.command("gen-structures") +@click.pass_obj +def gen_structures(context): + """(Re)Generate the SSP data structures in SDMX.""" + generate(context) From 6ee501c200c5a92b5b14edc3a575ee7e235119b5 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 4 Sep 2023 16:32:50 +0200 Subject: [PATCH 04/44] Test .project.ssp.structure.generate() --- message_ix_models/tests/project/__init__.py | 0 message_ix_models/tests/project/test_ssp.py | 9 +++++++++ 2 files changed, 9 insertions(+) create mode 100644 message_ix_models/tests/project/__init__.py create mode 100644 message_ix_models/tests/project/test_ssp.py diff --git a/message_ix_models/tests/project/__init__.py b/message_ix_models/tests/project/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/message_ix_models/tests/project/test_ssp.py b/message_ix_models/tests/project/test_ssp.py new file mode 100644 index 0000000000..468698592f --- /dev/null +++ b/message_ix_models/tests/project/test_ssp.py @@ -0,0 +1,9 @@ +from message_ix_models.project.ssp import generate + + +def test_generate(test_context): + generate(test_context) + + +def test_cli(mix_models_cli): + mix_models_cli.assert_exit_0(["ssp", "gen-structures"]) From fd9b1d983816618d10800a8dbbfa0b1d241cae59 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 4 Sep 2023 16:33:38 +0200 Subject: [PATCH 05/44] Add initial outputs from .ssp.structure.generate() --- .../data/sdmx/ICONICS:SSP(2017).xml | 125 ++++++++++++++++++ .../data/sdmx/ICONICS:SSP(2024).xml | 125 ++++++++++++++++++ .../data/sdmx/IIASA_ECE:AGENCIES(0.1).xml | 23 ++++ 3 files changed, 273 insertions(+) create mode 100644 message_ix_models/data/sdmx/ICONICS:SSP(2017).xml create mode 100644 message_ix_models/data/sdmx/ICONICS:SSP(2024).xml create mode 100644 message_ix_models/data/sdmx/IIASA_ECE:AGENCIES(0.1).xml diff --git a/message_ix_models/data/sdmx/ICONICS:SSP(2017).xml b/message_ix_models/data/sdmx/ICONICS:SSP(2017).xml new file mode 100644 index 0000000000..f2a7eb6b47 --- /dev/null +++ b/message_ix_models/data/sdmx/ICONICS:SSP(2017).xml @@ -0,0 +1,125 @@ + + + false + 2023-09-04T16:31:44.701872 + Generated by message_ix_models 2023.5.32.dev20+g8d51636 + + + + + Shared Socioeconomic Pathways (2017 edition) + This code list is *not* officially published by ICONICS; rather, it is a rendering into SDMX of structural information that is provided by ICONICS participants. It may change or be superseded at any time. Each code has a single digit ID like "1"; the "original-id" annotation (and the start of the name) give a string like "SSP1". These original IDs are *only* unique if using data enumerated by solely by one code list or the other; if mixing the two, then they will be ambiguous. The URNs of the codes or parts thereof (for instance, "ICONICS:SSP(2017).1") are, by construction, unique. + +This is the original set of SSPs as described in https://doi.org/10.1016/j.gloenvcha.2016.05.009. + + + + SSP1 + + + SSP1: Sustainability + SSP1: Sustainability – Taking the Green Road + +Low challenges to mitigation and adaptation. + +The world shifts gradually, but pervasively, toward a more sustainable path, emphasizing +more inclusive development that respects perceived environmental boundaries. Management +of the global commons slowly improves, educational and health investments accelerate the +demographic transition, and the emphasis on economic growth shifts toward a broader +emphasis on human well-being. Driven by an increasing commitment to achieving +development goals, inequality is reduced both across and within countries. Consumption +is oriented toward low material growth and lower resource and energy intensity. + + + + + SSP2 + + + SSP2: Middle of the Road + SSP2: Middle of the Road + +Medium challenges to mitigation and adaptation. + +The world follows a path in which social, economic, and technological trends do not +shift markedly from historical patterns. Development and income growth proceeds +unevenly, with some countries making relatively good progress while others fall short of +expectations. Global and national institutions work toward but make slow progress in +achieving sustainable development goals. Environmental systems experience degradation, +although there are some improvements and overall the intensity of resource and energy +use declines. Global population growth is moderate and levels off in the second half of +the century. Income inequality persists or improves only slowly and challenges to +reducing vulnerability to societal and environmental changes remain. + + + + + SSP3 + + + SSP3: Regional Rivalry + SSP3: Regional Rivalry – A Rocky Road + +High challenges to mitigation and adaptation. + +A resurgent nationalism, concerns about competitiveness and security, and regional +conflicts push countries to increasingly focus on domestic or, at most, regional issues. +Policies shift over time to become increasingly oriented toward national and regional +security issues. Countries focus on achieving energy and food security goals within +their own regions at the expense of broader-based development. Investments in education +and technological development decline. Economic development is slow, consumption is +material-intensive, and inequalities persist or worsen over time. Population growth is +low in industrialized and high in developing countries. A low international priority for +addressing environmental concerns leads to strong environmental degradation in some +regions. + + + + + SSP4 + + + SSP4: Inequality + SSP4: Inequality – A Road Divided + +Low challenges to mitigation, high challenges to adaptation. + +Highly unequal investments in human capital, combined with increasing disparities in +economic opportunity and political power, lead to increasing inequalities and +stratification both across and within countries. Over time, a gap widens between an +internationally-connected society that contributes to knowledge- and capital-intensive +sectors of the global economy, and a fragmented collection of lower-income, poorly +educated societies that work in a labor intensive, low-tech economy. Social cohesion +degrades and conflict and unrest become increasingly common. Technology development is +high in the high-tech economy and sectors. The globally connected energy sector +diversifies, with investments in both carbon-intensive fuels like coal and +unconventional oil, but also low-carbon energy sources. Environmental policies focus on +local issues around middle and high income areas. + + + + + SSP5 + + + SSP5: Fossil-fueled Development + SSP5: Fossil-fueled Development – Taking the Highway + +High challenges to mitigation, low challenges to adaptation. + +This world places increasing faith in competitive markets, innovation and participatory +societies to produce rapid technological progress and development of human capital as +the path to sustainable development. Global markets are increasingly integrated. There +are also strong investments in health, education, and institutions to enhance human and +social capital. At the same time, the push for economic and social development is +coupled with the exploitation of abundant fossil fuel resources and the adoption of +resource and energy intensive lifestyles around the world. All these factors lead to +rapid growth of the global economy, while global population peaks and declines in the +21st century. Local environmental problems like air pollution are successfully managed. +There is faith in the ability to effectively manage social and ecological systems, +including by geo-engineering if necessary. + + + + + diff --git a/message_ix_models/data/sdmx/ICONICS:SSP(2024).xml b/message_ix_models/data/sdmx/ICONICS:SSP(2024).xml new file mode 100644 index 0000000000..d4a202c8fc --- /dev/null +++ b/message_ix_models/data/sdmx/ICONICS:SSP(2024).xml @@ -0,0 +1,125 @@ + + + false + 2023-09-04T16:31:44.703451 + Generated by message_ix_models 2023.5.32.dev20+g8d51636 + + + + + Shared Socioeconomic Pathways (2024 edition) + This code list is *not* officially published by ICONICS; rather, it is a rendering into SDMX of structural information that is provided by ICONICS participants. It may change or be superseded at any time. Each code has a single digit ID like "1"; the "original-id" annotation (and the start of the name) give a string like "SSP1". These original IDs are *only* unique if using data enumerated by solely by one code list or the other; if mixing the two, then they will be ambiguous. The URNs of the codes or parts thereof (for instance, "ICONICS:SSP(2017).1") are, by construction, unique. + +This set of SSPs is currently under development; for details, see https://depts.washington.edu/iconics/. + + + + SSP1 + + + SSP1: Sustainability + SSP1: Sustainability – Taking the Green Road + +Low challenges to mitigation and adaptation. + +The world shifts gradually, but pervasively, toward a more sustainable path, emphasizing +more inclusive development that respects perceived environmental boundaries. Management +of the global commons slowly improves, educational and health investments accelerate the +demographic transition, and the emphasis on economic growth shifts toward a broader +emphasis on human well-being. Driven by an increasing commitment to achieving +development goals, inequality is reduced both across and within countries. Consumption +is oriented toward low material growth and lower resource and energy intensity. + + + + + SSP2 + + + SSP2: Middle of the Road + SSP2: Middle of the Road + +Medium challenges to mitigation and adaptation. + +The world follows a path in which social, economic, and technological trends do not +shift markedly from historical patterns. Development and income growth proceeds +unevenly, with some countries making relatively good progress while others fall short of +expectations. Global and national institutions work toward but make slow progress in +achieving sustainable development goals. Environmental systems experience degradation, +although there are some improvements and overall the intensity of resource and energy +use declines. Global population growth is moderate and levels off in the second half of +the century. Income inequality persists or improves only slowly and challenges to +reducing vulnerability to societal and environmental changes remain. + + + + + SSP3 + + + SSP3: Regional Rivalry + SSP3: Regional Rivalry – A Rocky Road + +High challenges to mitigation and adaptation. + +A resurgent nationalism, concerns about competitiveness and security, and regional +conflicts push countries to increasingly focus on domestic or, at most, regional issues. +Policies shift over time to become increasingly oriented toward national and regional +security issues. Countries focus on achieving energy and food security goals within +their own regions at the expense of broader-based development. Investments in education +and technological development decline. Economic development is slow, consumption is +material-intensive, and inequalities persist or worsen over time. Population growth is +low in industrialized and high in developing countries. A low international priority for +addressing environmental concerns leads to strong environmental degradation in some +regions. + + + + + SSP4 + + + SSP4: Inequality + SSP4: Inequality – A Road Divided + +Low challenges to mitigation, high challenges to adaptation. + +Highly unequal investments in human capital, combined with increasing disparities in +economic opportunity and political power, lead to increasing inequalities and +stratification both across and within countries. Over time, a gap widens between an +internationally-connected society that contributes to knowledge- and capital-intensive +sectors of the global economy, and a fragmented collection of lower-income, poorly +educated societies that work in a labor intensive, low-tech economy. Social cohesion +degrades and conflict and unrest become increasingly common. Technology development is +high in the high-tech economy and sectors. The globally connected energy sector +diversifies, with investments in both carbon-intensive fuels like coal and +unconventional oil, but also low-carbon energy sources. Environmental policies focus on +local issues around middle and high income areas. + + + + + SSP5 + + + SSP5: Fossil-fueled Development + SSP5: Fossil-fueled Development – Taking the Highway + +High challenges to mitigation, low challenges to adaptation. + +This world places increasing faith in competitive markets, innovation and participatory +societies to produce rapid technological progress and development of human capital as +the path to sustainable development. Global markets are increasingly integrated. There +are also strong investments in health, education, and institutions to enhance human and +social capital. At the same time, the push for economic and social development is +coupled with the exploitation of abundant fossil fuel resources and the adoption of +resource and energy intensive lifestyles around the world. All these factors lead to +rapid growth of the global economy, while global population peaks and declines in the +21st century. Local environmental problems like air pollution are successfully managed. +There is faith in the ability to effectively manage social and ecological systems, +including by geo-engineering if necessary. + + + + + diff --git a/message_ix_models/data/sdmx/IIASA_ECE:AGENCIES(0.1).xml b/message_ix_models/data/sdmx/IIASA_ECE:AGENCIES(0.1).xml new file mode 100644 index 0000000000..2cf52ab5bb --- /dev/null +++ b/message_ix_models/data/sdmx/IIASA_ECE:AGENCIES(0.1).xml @@ -0,0 +1,23 @@ + + + false + 2023-09-04T16:31:44.700655 + Generated by message_ix_models 2023.5.32.dev20+g8d51636 + + + + + Agencies referenced by data structures in message_ix_models + + IIASA Energy, Climate, and Environment Program + + + International Committee on New Integrated Climate Change Assessment Scenarios + + + + + + + + From 025456fd56d895731d3f0e21157a860846bf3678 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 4 Sep 2023 16:33:53 +0200 Subject: [PATCH 06/44] Test .util.sdmx.{read,write} --- message_ix_models/tests/util/test_sdmx.py | 30 ++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/message_ix_models/tests/util/test_sdmx.py b/message_ix_models/tests/util/test_sdmx.py index 552f0ac777..40713dc130 100644 --- a/message_ix_models/tests/util/test_sdmx.py +++ b/message_ix_models/tests/util/test_sdmx.py @@ -1,9 +1,10 @@ import logging import re +import pytest from sdmx.model.v21 import Annotation, Code -from message_ix_models.util.sdmx import eval_anno +from message_ix_models.util.sdmx import eval_anno, read def test_eval_anno(caplog): @@ -23,3 +24,30 @@ def test_eval_anno(caplog): c.annotations.append(Annotation(id="qux", text="3 + 4")) assert 7 == eval_anno(c, id="qux") + + +@pytest.mark.parametrize( + "urn, expected", + ( + ("ICONICS:SSP(2017)", "Codelist=ICONICS:SSP(2017)"), + ("ICONICS:SSP(2024)", "Codelist=ICONICS:SSP(2024)"), + ("SSP(2017)", "Codelist=ICONICS:SSP(2017)"), + ("SSP(2024)", "Codelist=ICONICS:SSP(2024)"), + ("SSP", "Codelist=ICONICS:SSP(2017)"), + ("AGENCIES", "AgencyScheme=IIASA_ECE:AGENCIES(0.1)"), + ), +) +def test_read0(urn, expected): + obj = read(urn) + assert expected in obj.urn + + +def test_read1(): + SSPS = read("ssp") + + # Identify an SSP by matching strings in its name + code0 = next(filter(lambda c: "2" in repr(c), iter(SSPS))) + code1 = next(filter(lambda c: "SSP2" in repr(c), iter(SSPS))) + code2 = next(filter(lambda c: "middle of the road" in repr(c).lower(), iter(SSPS))) + + assert code0 is code1 is code2 From f59e3479ae36bd746f111692abb683c83311b5d8 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 4 Sep 2023 16:36:20 +0200 Subject: [PATCH 07/44] Update test_cli_help --- message_ix_models/model/snapshot.py | 2 +- message_ix_models/tests/test_cli.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/message_ix_models/model/snapshot.py b/message_ix_models/model/snapshot.py index 17b9402afa..7f219f7797 100644 --- a/message_ix_models/model/snapshot.py +++ b/message_ix_models/model/snapshot.py @@ -125,7 +125,7 @@ def load(scenario: Scenario, snapshot_id: int) -> None: read_excel(scenario, path) -@click.group("snapshot", help="__doc__") +@click.group("snapshot", help=__doc__) def cli(): # pragma: no cover pass diff --git a/message_ix_models/tests/test_cli.py b/message_ix_models/tests/test_cli.py index 461e03798f..0756cd0eb3 100644 --- a/message_ix_models/tests/test_cli.py +++ b/message_ix_models/tests/test_cli.py @@ -8,6 +8,10 @@ COMMANDS = [ tuple(), ("debug",), + ("report",), + ("snapshot",), + ("ssp", "gen-structures"), + ("techs",), ("water-ix",), ] From 2f4575e8773015e8bb0034c35393db7c113de14c Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 4 Sep 2023 16:42:18 +0200 Subject: [PATCH 08/44] Extend same_node for node_{share,loc,} --- message_ix_models/util/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/message_ix_models/util/__init__.py b/message_ix_models/util/__init__.py index a47639f060..fff94a0df1 100644 --- a/message_ix_models/util/__init__.py +++ b/message_ix_models/util/__init__.py @@ -547,10 +547,14 @@ def replace_par_data( log.info(f"{len(to_remove)} obs in {par_name!r}") -def same_node(df: pd.DataFrame) -> pd.DataFrame: - """Fill 'node_{dest,origin,rel}' in `df` from 'node_loc'.""" - cols = list(set(df.columns) & {"node_origin", "node_dest", "node_rel"}) - return df.assign(**{c: copy_column("node_loc") for c in cols}) +def same_node(df: pd.DataFrame, from_col="node_loc") -> pd.DataFrame: + """Fill 'node_{,dest,loc,origin,rel,share}' in `df` from `from_col`.""" + cols = list( + set(df.columns) + & ({"node", "node_loc", "node_origin", "node_dest", "node_rel", "node_share"}) + - {from_col} + ) + return df.assign(**{c: copy_column(from_col) for c in cols}) def same_time(df: pd.DataFrame) -> pd.DataFrame: From c03505ba996c67760ee0332a7eaf0966fda70ad9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 4 Sep 2023 16:51:57 +0200 Subject: [PATCH 09/44] Use a temp dir / --dry-run to test .ssp.structure.generate() --- message_ix_models/project/ssp/__init__.py | 5 ++++- message_ix_models/project/ssp/structure.py | 17 +++++++++++++---- message_ix_models/tests/project/test_ssp.py | 8 +++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/message_ix_models/project/ssp/__init__.py b/message_ix_models/project/ssp/__init__.py index 65fcced043..e4abaa1284 100644 --- a/message_ix_models/project/ssp/__init__.py +++ b/message_ix_models/project/ssp/__init__.py @@ -1,5 +1,7 @@ import click +from message_ix_models.util.click import common_params + from .structure import generate @@ -9,7 +11,8 @@ def cli(): @cli.command("gen-structures") +@common_params("dry_run") @click.pass_obj -def gen_structures(context): +def gen_structures(context, **kwargs): """(Re)Generate the SSP data structures in SDMX.""" generate(context) diff --git a/message_ix_models/project/ssp/structure.py b/message_ix_models/project/ssp/structure.py index 64451d2a65..2348993243 100644 --- a/message_ix_models/project/ssp/structure.py +++ b/message_ix_models/project/ssp/structure.py @@ -1,7 +1,7 @@ """Manipulate data structures for working with the SSPs.""" import logging from textwrap import wrap -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional import sdmx import sdmx.model.v30 as m @@ -10,6 +10,8 @@ from message_ix_models.util.sdmx import write if TYPE_CHECKING: + from os import PathLike + from message_ix_models import Context log = logging.getLogger(__name__) @@ -129,7 +131,7 @@ ) -def generate(context: "Context"): +def generate(context: "Context", base_dir: Optional["PathLike"] = None): """Generate SDMX code lists containing the SSPs.""" # Create an AgencyScheme containing ICONICS as_ = m.AgencyScheme( @@ -153,7 +155,10 @@ def generate(context: "Context"): as_.append(IIASA_ECE) as_.append(ICONICS) - write(as_) + if context.dry_run: + log.info(f"(dry run) Would write:\n{repr(as_)}") + else: + write(as_, base_dir) for cl_info in CL_INFO: # Create the codelist: format the name and description @@ -188,4 +193,8 @@ def generate(context: "Context"): # Construct a URN c.urn = sdmx.urn.make(c, maintainable_parent=cl) - write(cl) + if context.dry_run: + log.info(f"(dry run) Would write:\n{repr(cl)}") + continue + + write(cl, base_dir) diff --git a/message_ix_models/tests/project/test_ssp.py b/message_ix_models/tests/project/test_ssp.py index 468698592f..ba5e842656 100644 --- a/message_ix_models/tests/project/test_ssp.py +++ b/message_ix_models/tests/project/test_ssp.py @@ -1,9 +1,11 @@ from message_ix_models.project.ssp import generate -def test_generate(test_context): - generate(test_context) +def test_generate(tmp_path, test_context): + generate(test_context, base_dir=tmp_path) + + assert 3 == len(list(tmp_path.glob("*.xml"))) def test_cli(mix_models_cli): - mix_models_cli.assert_exit_0(["ssp", "gen-structures"]) + mix_models_cli.assert_exit_0(["ssp", "gen-structures", "--dry-run"]) From 64bb585c18287c61dc7fd96e2b2a5c34ae84f5ee Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 5 Sep 2023 10:50:25 +0200 Subject: [PATCH 10/44] Add .sdmx.make_enum(), tests --- message_ix_models/tests/util/test_sdmx.py | 30 ++++++++++++++++++++++- message_ix_models/util/sdmx.py | 25 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/message_ix_models/tests/util/test_sdmx.py b/message_ix_models/tests/util/test_sdmx.py index 40713dc130..f569286698 100644 --- a/message_ix_models/tests/util/test_sdmx.py +++ b/message_ix_models/tests/util/test_sdmx.py @@ -4,7 +4,7 @@ import pytest from sdmx.model.v21 import Annotation, Code -from message_ix_models.util.sdmx import eval_anno, read +from message_ix_models.util.sdmx import eval_anno, make_enum, read def test_eval_anno(caplog): @@ -26,6 +26,34 @@ def test_eval_anno(caplog): assert 7 == eval_anno(c, id="qux") +def test_make_enum(): + """:func:`.make_enum` works with :class:`Flag` and subclasses.""" + from enum import Flag, IntFlag + + E = make_enum("ICONICS:SSP(2017)", base=Flag) + + # Values are bitwise flags + assert not isinstance(E["1"], int) + + # Expected length + assert 2 ** (len(E) - 1) == list(E)[-1].value + + # Flags can be combined + flags = E["1"] | E["2"] + assert E["1"] & flags + assert E["2"] & flags + assert not (E["3"] & flags) + + # Similar, with IntFlag + E = make_enum("IIASA_ECE:AGENCIES(0.1)", base=IntFlag) + + # Values are ints + assert isinstance(E["IIASA_ECE"], int) + + # Expected length + assert 2 ** (len(E) - 1) == list(E)[-1].value + + @pytest.mark.parametrize( "urn, expected", ( diff --git a/message_ix_models/util/sdmx.py b/message_ix_models/util/sdmx.py index cb99a81e48..d09320e46d 100644 --- a/message_ix_models/util/sdmx.py +++ b/message_ix_models/util/sdmx.py @@ -1,6 +1,7 @@ """Utilities for handling objects from :mod:`sdmx`.""" import logging from datetime import datetime +from enum import Enum, Flag from importlib.metadata import version from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Union @@ -107,6 +108,30 @@ def eval_anno(obj: AnnotableArtefact, id: str): return value +def make_enum(urn, base=Enum): + """Create an :class:`Enum` (or `base`) subclass with members from codelist `urn`.""" + # Read the code list + cl = read(urn) + + names = ["NONE"] if issubclass(base, Flag) else [] + names.extend(code.id for code in cl) + + # Create the class + cls = base(urn, names) + + # Add a class method to look up the str() equivalent of any `value` + def missing(cls, value): + try: + return cls[str(value)] + except (KeyError, ValueError): + return None + + if not issubclass(base, Flag): + cls._missing_ = classmethod(missing) + + return cls + + def read(urn: str, base_dir: Optional["PathLike"] = None): """Read SDMX object from package data given its `urn`.""" # Identify a path that matches `urn` From d03f173a6bfe548e812451f726369cc8facf7851 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 5 Sep 2023 10:52:12 +0200 Subject: [PATCH 11/44] Add enums for SSP(2017) and SSP(2024) --- message_ix_models/project/ssp/__init__.py | 9 ++++++- message_ix_models/project/ssp/structure.py | 6 ++++- message_ix_models/tests/project/test_ssp.py | 29 ++++++++++++++++++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/message_ix_models/project/ssp/__init__.py b/message_ix_models/project/ssp/__init__.py index e4abaa1284..ca436d4007 100644 --- a/message_ix_models/project/ssp/__init__.py +++ b/message_ix_models/project/ssp/__init__.py @@ -2,7 +2,14 @@ from message_ix_models.util.click import common_params -from .structure import generate +from .structure import SSP, SSP_2017, SSP_2024, generate + +__all__ = [ + "SSP", + "SSP_2017", + "SSP_2024", + "generate", +] @click.group("ssp") diff --git a/message_ix_models/project/ssp/structure.py b/message_ix_models/project/ssp/structure.py index 2348993243..27dcac27ef 100644 --- a/message_ix_models/project/ssp/structure.py +++ b/message_ix_models/project/ssp/structure.py @@ -7,7 +7,7 @@ import sdmx.model.v30 as m import sdmx.urn -from message_ix_models.util.sdmx import write +from message_ix_models.util.sdmx import make_enum, write if TYPE_CHECKING: from os import PathLike @@ -198,3 +198,7 @@ def generate(context: "Context", base_dir: Optional["PathLike"] = None): continue write(cl, base_dir) + + +SSP = SSP_2017 = make_enum("ICONICS:SSP(2017)") +SSP_2024 = make_enum("ICONICS:SSP(2024)") diff --git a/message_ix_models/tests/project/test_ssp.py b/message_ix_models/tests/project/test_ssp.py index ba5e842656..ac6154e8ad 100644 --- a/message_ix_models/tests/project/test_ssp.py +++ b/message_ix_models/tests/project/test_ssp.py @@ -1,4 +1,6 @@ -from message_ix_models.project.ssp import generate +import pytest + +from message_ix_models.project.ssp import SSP, SSP_2017, SSP_2024, generate def test_generate(tmp_path, test_context): @@ -7,5 +9,30 @@ def test_generate(tmp_path, test_context): assert 3 == len(list(tmp_path.glob("*.xml"))) +def test_enum(): + # Enumerations have the expected length + assert 5 == len(SSP_2017) + assert 5 == len(SSP_2024) + + # Members can be accessed by ID + a = SSP_2017("1") + b = SSP_2017["1"] + + # _missing_ invoked using the constructor + c = SSP_2017(1) + + # …all retrieving the same value + assert a == b == c + + # __getattr__ lookup does not invoke _missing_ + with pytest.raises(KeyError): + SSP_2017[1] + + # Same SSP ID from different enums are not equivalent + assert SSP_2017(1) != SSP_2024(1) + assert SSP_2017(1) is not SSP_2024(1) + assert SSP(1) != SSP_2024(1) + + def test_cli(mix_models_cli): mix_models_cli.assert_exit_0(["ssp", "gen-structures", "--dry-run"]) From ba24b63727d39fa4dfa56631589734642a160867 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 5 Sep 2023 11:03:30 +0200 Subject: [PATCH 12/44] Use Windows-compatible paths in .sdmx.{read,write} --- .../{ICONICS:SSP(2017).xml => ICONICS_SSP(2017).xml} | 0 .../{ICONICS:SSP(2024).xml => ICONICS_SSP(2024).xml} | 0 ...E:AGENCIES(0.1).xml => IIASA_ECE_AGENCIES(0.1).xml} | 0 message_ix_models/util/sdmx.py | 10 +++++++--- 4 files changed, 7 insertions(+), 3 deletions(-) rename message_ix_models/data/sdmx/{ICONICS:SSP(2017).xml => ICONICS_SSP(2017).xml} (100%) rename message_ix_models/data/sdmx/{ICONICS:SSP(2024).xml => ICONICS_SSP(2024).xml} (100%) rename message_ix_models/data/sdmx/{IIASA_ECE:AGENCIES(0.1).xml => IIASA_ECE_AGENCIES(0.1).xml} (100%) diff --git a/message_ix_models/data/sdmx/ICONICS:SSP(2017).xml b/message_ix_models/data/sdmx/ICONICS_SSP(2017).xml similarity index 100% rename from message_ix_models/data/sdmx/ICONICS:SSP(2017).xml rename to message_ix_models/data/sdmx/ICONICS_SSP(2017).xml diff --git a/message_ix_models/data/sdmx/ICONICS:SSP(2024).xml b/message_ix_models/data/sdmx/ICONICS_SSP(2024).xml similarity index 100% rename from message_ix_models/data/sdmx/ICONICS:SSP(2024).xml rename to message_ix_models/data/sdmx/ICONICS_SSP(2024).xml diff --git a/message_ix_models/data/sdmx/IIASA_ECE:AGENCIES(0.1).xml b/message_ix_models/data/sdmx/IIASA_ECE_AGENCIES(0.1).xml similarity index 100% rename from message_ix_models/data/sdmx/IIASA_ECE:AGENCIES(0.1).xml rename to message_ix_models/data/sdmx/IIASA_ECE_AGENCIES(0.1).xml diff --git a/message_ix_models/util/sdmx.py b/message_ix_models/util/sdmx.py index d09320e46d..e74ed99f62 100644 --- a/message_ix_models/util/sdmx.py +++ b/message_ix_models/util/sdmx.py @@ -136,6 +136,7 @@ def read(urn: str, base_dir: Optional["PathLike"] = None): """Read SDMX object from package data given its `urn`.""" # Identify a path that matches `urn` base_dir = base_dir or package_data_path("sdmx") + urn = urn.replace(":", "_") # ":" invalid on Windows paths = sorted( set(base_dir.glob(f"*{urn}*.xml")) | set(base_dir.glob(f"*{urn.upper()}*.xml")) ) @@ -146,8 +147,11 @@ def read(urn: str, base_dir: Optional["PathLike"] = None): "other result(s)" ) - with open(paths[0], "rb") as f: - msg = sdmx.read_sdmx(f) + try: + with open(paths[0], "rb") as f: + msg = sdmx.read_sdmx(f) + except IndexError: + raise FileNotFoundError(f"'*{urn}*.xml', '*{urn.upper()}*.xml' or similar") for _, cls in msg.iter_collections(): try: @@ -172,7 +176,7 @@ def write(obj, base_dir: Optional["PathLike"] = None): # Identify a path to write the file base_dir = base_dir or package_data_path("sdmx") - basename = obj.urn.split("=")[-1] + basename = obj.urn.split("=")[-1].replace(":", "_") # ":" invalid on Windows path = base_dir.joinpath(f"{basename}.xml") # Write From 62fa4e07169c6e7f6b01d8b7078f8c97b77baf07 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 5 Sep 2023 11:34:47 +0200 Subject: [PATCH 13/44] Satisfy mypy in .project.ssp, .util.sdmx --- message_ix_models/project/ssp/structure.py | 2 +- message_ix_models/util/sdmx.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/message_ix_models/project/ssp/structure.py b/message_ix_models/project/ssp/structure.py index 27dcac27ef..c5c2e7945a 100644 --- a/message_ix_models/project/ssp/structure.py +++ b/message_ix_models/project/ssp/structure.py @@ -162,7 +162,7 @@ def generate(context: "Context", base_dir: Optional["PathLike"] = None): for cl_info in CL_INFO: # Create the codelist: format the name and description - cl = m.Codelist( + cl: m.Codelist = m.Codelist( id="SSP", name=f"Shared Socioeconomic Pathways ({cl_info['name_extra']})", description="\n".join( diff --git a/message_ix_models/util/sdmx.py b/message_ix_models/util/sdmx.py index e74ed99f62..130f72ab29 100644 --- a/message_ix_models/util/sdmx.py +++ b/message_ix_models/util/sdmx.py @@ -3,6 +3,7 @@ from datetime import datetime from enum import Enum, Flag from importlib.metadata import version +from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Union import sdmx @@ -135,7 +136,7 @@ def missing(cls, value): def read(urn: str, base_dir: Optional["PathLike"] = None): """Read SDMX object from package data given its `urn`.""" # Identify a path that matches `urn` - base_dir = base_dir or package_data_path("sdmx") + base_dir = Path(base_dir or package_data_path("sdmx")) urn = urn.replace(":", "_") # ":" invalid on Windows paths = sorted( set(base_dir.glob(f"*{urn}*.xml")) | set(base_dir.glob(f"*{urn.upper()}*.xml")) @@ -175,7 +176,7 @@ def write(obj, base_dir: Optional["PathLike"] = None): msg.add(obj) # Identify a path to write the file - base_dir = base_dir or package_data_path("sdmx") + base_dir = Path(base_dir or package_data_path("sdmx")) basename = obj.urn.split("=")[-1].replace(":", "_") # ":" invalid on Windows path = base_dir.joinpath(f"{basename}.xml") From 0acd9b0328cb8cd18ed88694fb2fa1e4123dc187 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 5 Sep 2023 12:59:08 +0200 Subject: [PATCH 14/44] Exclude "if TYPE_CHECKING:" blocks from coverage --- message_ix_models/model/macro.py | 2 +- pyproject.toml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/message_ix_models/model/macro.py b/message_ix_models/model/macro.py index b8f05d7b0a..fdeb8f5479 100644 --- a/message_ix_models/model/macro.py +++ b/message_ix_models/model/macro.py @@ -14,7 +14,7 @@ from message_ix_models.model.bare import get_spec from message_ix_models.util import nodes_ex_world -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: from sdmx.model.v21 import Code from message_ix_models import Context diff --git a/pyproject.toml b/pyproject.toml index f0ba798fc2..84e9735653 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,12 @@ tests = [ [project.scripts] mix-models = "message_ix_models.cli:main" +[tool.coverage.report] +exclude_also = [ + # Imports only used by type checkers + "if TYPE_CHECKING:", +] + [tool.isort] profile = "black" From 91b4f2b5a8731646db754d276f21a74ab01ba992 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 5 Sep 2023 14:22:14 +0200 Subject: [PATCH 15/44] Remove _missing_ convenience in .sdmx.make_enum --- message_ix_models/tests/project/test_ssp.py | 17 ++++++++--------- message_ix_models/tests/util/test_sdmx.py | 3 +++ message_ix_models/util/sdmx.py | 15 ++------------- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/message_ix_models/tests/project/test_ssp.py b/message_ix_models/tests/project/test_ssp.py index ac6154e8ad..2fbbfd4b0a 100644 --- a/message_ix_models/tests/project/test_ssp.py +++ b/message_ix_models/tests/project/test_ssp.py @@ -15,23 +15,22 @@ def test_enum(): assert 5 == len(SSP_2024) # Members can be accessed by ID - a = SSP_2017("1") - b = SSP_2017["1"] + a = SSP_2017["1"] - # _missing_ invoked using the constructor - c = SSP_2017(1) + # …or by value + b = SSP_2017(1) - # …all retrieving the same value - assert a == b == c + # …all retrieving the same member + assert a == b # __getattr__ lookup does not invoke _missing_ with pytest.raises(KeyError): SSP_2017[1] # Same SSP ID from different enums are not equivalent - assert SSP_2017(1) != SSP_2024(1) - assert SSP_2017(1) is not SSP_2024(1) - assert SSP(1) != SSP_2024(1) + assert SSP_2017["1"] != SSP_2024["1"] + assert SSP_2017["1"] is not SSP_2024["1"] + assert SSP["1"] != SSP_2024["1"] def test_cli(mix_models_cli): diff --git a/message_ix_models/tests/util/test_sdmx.py b/message_ix_models/tests/util/test_sdmx.py index f569286698..1e624af811 100644 --- a/message_ix_models/tests/util/test_sdmx.py +++ b/message_ix_models/tests/util/test_sdmx.py @@ -79,3 +79,6 @@ def test_read1(): code2 = next(filter(lambda c: "middle of the road" in repr(c).lower(), iter(SSPS))) assert code0 is code1 is code2 + + with pytest.raises(FileNotFoundError): + read("foo") diff --git a/message_ix_models/util/sdmx.py b/message_ix_models/util/sdmx.py index 130f72ab29..d95cca0f53 100644 --- a/message_ix_models/util/sdmx.py +++ b/message_ix_models/util/sdmx.py @@ -114,23 +114,12 @@ def make_enum(urn, base=Enum): # Read the code list cl = read(urn) + # Ensure the 0 member is NONE, not any of the codes names = ["NONE"] if issubclass(base, Flag) else [] names.extend(code.id for code in cl) # Create the class - cls = base(urn, names) - - # Add a class method to look up the str() equivalent of any `value` - def missing(cls, value): - try: - return cls[str(value)] - except (KeyError, ValueError): - return None - - if not issubclass(base, Flag): - cls._missing_ = classmethod(missing) - - return cls + return base(urn, names) def read(urn: str, base_dir: Optional["PathLike"] = None): From 5b280c4ece037939e1ceaf487d67c7ccfa2d24a8 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 5 Sep 2023 14:22:47 +0200 Subject: [PATCH 16/44] Add .ssp.parse(), field helper for dataclasses --- message_ix_models/project/ssp/__init__.py | 45 +++++++++++++++++++ message_ix_models/tests/project/test_ssp.py | 48 ++++++++++++++++++++- 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/message_ix_models/project/ssp/__init__.py b/message_ix_models/project/ssp/__init__.py index ca436d4007..5aed2aee31 100644 --- a/message_ix_models/project/ssp/__init__.py +++ b/message_ix_models/project/ssp/__init__.py @@ -1,3 +1,7 @@ +import logging +import re +from typing import Union + import click from message_ix_models.util.click import common_params @@ -9,8 +13,49 @@ "SSP_2017", "SSP_2024", "generate", + "parse", + "ssp_field", ] +log = logging.getLogger(__name__) + + +def parse(value: Union[str, SSP_2017, SSP_2024]) -> Union[SSP_2017, SSP_2024]: + """Parse `value` to a member of :data:`SSP_2017` or :data:`SSP_2024`.""" + if isinstance(value, (SSP_2017, SSP_2024)): + return value + + log.debug(f"Assume {value!r} is from {SSP_2017}") + + if isinstance(value, str): + return SSP_2017[re.sub("SSP([12345])", r"\1", value)] + else: + return SSP_2017(value) + + +class ssp_field: + """SSP field for use in data classes.""" + + def __init__(self, default: Union[SSP_2017, SSP_2024]): + self._default = default + + def __set_name__(self, owner, name): + self._name = "_" + name + + def __get__(self, obj, type) -> Union[SSP_2017, SSP_2024]: + if obj is None: + return None # type: ignore [return-value] + + try: + return obj.__dict__[self._name] + except KeyError: + return obj.__dict__.setdefault(self._name, self._default) + + def __set__(self, obj, value): + if value is None: + value = self._default + setattr(obj, self._name, parse(value)) + @click.group("ssp") def cli(): diff --git a/message_ix_models/tests/project/test_ssp.py b/message_ix_models/tests/project/test_ssp.py index 2fbbfd4b0a..8f8f230baf 100644 --- a/message_ix_models/tests/project/test_ssp.py +++ b/message_ix_models/tests/project/test_ssp.py @@ -1,6 +1,13 @@ import pytest -from message_ix_models.project.ssp import SSP, SSP_2017, SSP_2024, generate +from message_ix_models.project.ssp import ( + SSP, + SSP_2017, + SSP_2024, + generate, + parse, + ssp_field, +) def test_generate(tmp_path, test_context): @@ -33,5 +40,44 @@ def test_enum(): assert SSP["1"] != SSP_2024["1"] +@pytest.mark.parametrize( + "expected, value", + ( + (SSP_2017["1"], SSP_2017["1"]), # Literal value + (SSP_2017["1"], "1"), # String ID as appears in the codelist + (SSP_2017["1"], "SSP1"), # Prefixed by "SSP" + (SSP_2017["1"], 1), # Integer + ), +) +def test_parse(value, expected): + assert expected == parse(value) + + +def test_ssp_field(): + from dataclasses import dataclass + + @dataclass + class Foo: + bar: ssp_field = ssp_field(default=SSP_2017["1"]) + baz: ssp_field = ssp_field(default=SSP_2024["5"]) + + # Can be instantiated with no arguments + f = Foo() + assert SSP_2017["1"] is f.bar + assert SSP_2024["5"] is f.baz + + # Can be instantiated with different values + f = Foo(bar=SSP_2017["3"], baz=SSP_2024["3"]) + assert SSP_2017["3"] is f.bar + assert SSP_2024["3"] is f.baz + + # Values can be set and are passed through parse() + f.bar = "SSP2" + f.baz = "SSP4" + + assert SSP_2017["2"] is f.bar + assert SSP_2017["4"] is f.baz + + def test_cli(mix_models_cli): mix_models_cli.assert_exit_0(["ssp", "gen-structures", "--dry-run"]) From 4781245913d37627dc6b145b23242be828dc2ebf Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 6 Sep 2023 12:24:19 +0200 Subject: [PATCH 17/44] Add .tools.exo_data --- message_ix_models/report/computations.py | 49 +++- .../tests/tools/test_exo_data.py | 81 +++++++ message_ix_models/tools/exo_data.py | 218 ++++++++++++++++++ pyproject.toml | 5 + 4 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 message_ix_models/tests/tools/test_exo_data.py create mode 100644 message_ix_models/tools/exo_data.py diff --git a/message_ix_models/report/computations.py b/message_ix_models/report/computations.py index 9913d1ef4d..2d387aca16 100644 --- a/message_ix_models/report/computations.py +++ b/message_ix_models/report/computations.py @@ -1,15 +1,22 @@ """Atomic reporting operations for MESSAGEix-GLOBIOM.""" import itertools import logging -from typing import Any, List, Optional, Set, Union +from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Set, Tuple, Union import ixmp import pandas as pd from genno.computations import pow +from genno.core.operator import Operator from iam_units import convert_gwp from iam_units.emissions import SPECIES from ixmp.reporting import Quantity +from message_ix_models import Context + +if TYPE_CHECKING: + from genno import Computer, Key + from sdmx.model.v21 import Code + log = logging.getLogger(__name__) __all__ = [ @@ -23,6 +30,24 @@ ] +def codelist_to_groups( + codes: List["Code"], dim: str = "n" +) -> Mapping[str, Mapping[str, List[str]]]: + """Convert `codes` into a mapping from parent items to their children. + + The returned value is suitable for use with :func:`genno.computations.aggregate`. + + If this is a list of nodes per :func:`.get_codes`, then the mapping is from regions + to the ISO 3166-1 alpha-3 codes of the countries within each region. + """ + + groups = dict() + for code in filter(lambda c: len(c.child), codes): + groups[code.id] = list(map(str, code.child)) + + return {dim: groups} + + def compound_growth(qty: Quantity, dim: str) -> Quantity: """Compute compound growth along `dim` of `qty`.""" # Compute intervals along `dim` @@ -36,6 +61,28 @@ def compound_growth(qty: Quantity, dim: str) -> Quantity: return pow(qty, Quantity(dur)).cumprod(dim).shift({dim: 1}).fillna(1.0) +@Operator.define +def exogenous_data(): + """No action. + + This exists to connect :func:`.exo_data.prepare_computer` to + :meth:`genno.Computer.add`. + """ + pass # pragma: no cover + + +@exogenous_data.helper +def add_exogenous_data( + func, c: "Computer", *, context=None, source=None, source_kw=None +) -> Tuple["Key"]: + """Prepare `c` to compute exogenous data from `source`.""" + from message_ix_models.tools.exo_data import prepare_computer + + return prepare_computer( + context or Context.get_instance(-1), c, source=source, source_kw=source_kw + ) + + def get_ts( scenario: ixmp.Scenario, filters: Optional[dict] = None, diff --git a/message_ix_models/tests/tools/test_exo_data.py b/message_ix_models/tests/tools/test_exo_data.py new file mode 100644 index 0000000000..a0bce46ffa --- /dev/null +++ b/message_ix_models/tests/tools/test_exo_data.py @@ -0,0 +1,81 @@ +import pytest +from genno import Computer + +from message_ix_models.tools.exo_data import ( + ExoDataSource, + TestSource, + prepare_computer, + register_source, +) + + +class TestExoDataSource: + def test_abstract(self): + with pytest.raises(TypeError, match="Can't instantiate"): + ExoDataSource() + + def test_register_source(self): + with pytest.raises(ValueError, match="already registered for"): + register_source(TestSource) + + +@pytest.mark.parametrize("regions, N_n", [("R12", 12), ("R14", 14)]) +def test_prepare_computer(test_context, regions, N_n): + """:func:`.exo_data.prepare_computer` works as intended.""" + test_context.model.regions = regions + + c = Computer() + + # Function runs successfully `c` + keys = prepare_computer(test_context, c, "test s1", dict(measure="POP")) + + # print(c.describe(keys[-1])) + + # Computation of data runs successfully + result = c.get(keys[-1]) + + # Data has the expected dimensions + assert ("n", "y") == result.dims + + # Data is complete + assert N_n == len(result.coords["n"]) + assert 14 == len(result.coords["y"]) + + +def test_prepare_computer_exc(test_context): + c = Computer() + + with pytest.raises(ValueError, match="must be one of"): + prepare_computer(test_context, c, "test s1", dict(measure="FOO")) + + with pytest.raises(ValueError, match="No source found that can handle"): + prepare_computer(test_context, c, "not a source") + + +@pytest.mark.parametrize("regions, N_n", [("R12", 12), ("R14", 14)]) +def test_operator(test_context, regions, N_n): + """Exogenous data calculations can be set up through :meth:`.Computer.add`.""" + test_context.model.regions = regions + + c = Computer() + c.require_compat("message_ix_models.report.computations") + + # Function runs successfully `c` + keys = c.add( + "exogenous_data", + context=test_context, + source="test s1", + source_kw=dict(measure="POP"), + ) + + # print(c.describe(keys[-1])) + + # Computation of data runs successfully + result = c.get(keys[-1]) + + # Data has the expected dimensions + assert ("n", "y") == result.dims + + # Data is complete + assert N_n == len(result.coords["n"]) + assert 14 == len(result.coords["y"]) diff --git a/message_ix_models/tools/exo_data.py b/message_ix_models/tools/exo_data.py new file mode 100644 index 0000000000..073f0f5f6e --- /dev/null +++ b/message_ix_models/tools/exo_data.py @@ -0,0 +1,218 @@ +"""Generic tools for working with exogenous data sources.""" +from abc import ABC, abstractmethod +from operator import itemgetter +from typing import Dict, Mapping, Optional, Type + +from genno import Computer, Key, Quantity, quote +from genno.core.key import single_key + +from message_ix_models import ScenarioInfo +from message_ix_models.model.structure import get_codes + +__all__ = [ + "prepare_computer", + "ExoDataSource", +] + +#: Supported measures. +#: +#: .. todo:: Store this in a separate code list or concept scheme. +MEASURES = ("GDP", "POP") + +#: List of known sources for data. +SOURCES: Dict[str, Type["ExoDataSource"]] = {} + + +class ExoDataSource(ABC): + """Base class for sources of exogenous data.""" + + #: Identifier for this particular source. + id: str = "" + + @abstractmethod + def __init__(self, source: str, source_kw: Mapping) -> None: + """Handle `source` and `source_kw`. + + An implementation **must**: + + - Raise :class:`ValueError` if it does not recognize or cannot handle the + arguments in `source` or `source_kw`. + - Recognize and handle (if possible) a "measure" keyword in `source_kw` from + :data:`MEASURES`. + + It **may**: + + - Transform these into other values, for instance by mapping certain values to + others, applying regular expressions, or other operations. + - Store those values as instance attributes for use in :meth:`__call__`, below. + + It **should not** actually load data or perform any time- or memory-intensive + operations. + """ + raise ValueError + + @abstractmethod + def __call__(self) -> Quantity: + """Return the data. + + The Quantity returned by this method **must** have dimensions "n" and "y". If + the original/upstream/raw data has additional dimensions or different dimension + IDs, the code **must** transform these, make appropriate selections, etc. + """ + raise NotImplementedError + + +def prepare_computer( + context, c: "Computer", source="test", source_kw: Optional[Mapping] = None +): + """Prepare `c` to compute GDP, population, or other exogenous data. + + Returns a tuple of keys. The first, like ``{m}:n-y``, triggers the following + computations: + + 1. Load data by invoking a :class:`ExoDataSource`. + 2. Aggregate on the ``n`` (``node``) dimension according to :attr:`Config.regions`. + 3. Interpolate on the ``y`` (``year``) dimension according to :attr:`Config.years`. + + Additional key(s) include: + + - ``{m}:n-y:y0 indexed``: same as ``{m}:n-y``, indexed to values as of ``y0``, that + is, the first model year. + + .. todo:: Extend to also prepare to compute values indexed to a particular ``n``. + + Parameters + ---------- + source : str + Identifier of the source, possibly with other information to be handled by a + :class:`ExoDataSource`. + source_kw : dict, optional + Keyword arguments for a Source class. These can include indexers, selectors, or + other information needed by the source class to identify the data to be + returned. + + If the key "measure" is present, it **must** be one of :data:`MEASURES`. + """ + # Handle arguments + source_kw = source_kw or dict() + if measure := source_kw.get("measure"): + if measure not in MEASURES: + raise ValueError( + f"source_kw 'measure' must be one of {MEASURES}; got {measure!r}" + ) + else: + measure = "UNKNOWN" + + # Look up input data flow + source_obj = None + for cls in SOURCES.values(): + try: + # Instantiate a Source object to provide this data + source_obj = cls(source, source_kw or dict()) + except Exception: + pass # Class does not recognize the arguments + + if source_obj is None: + raise ValueError(f"No source found that can handle {source!r}") + + # Add structural information to the Computer + c.require_compat("message_ix_models.report.computations") + + # Retrieve the node codelist + c.add("n::codes", quote(get_codes(f"node/{context.model.regions}")), strict=True) + + # Convert the codelist into a nested dict for aggregate() + c.add("n::groups", "codelist_to_groups", "n::codes", strict=True) + + # Add information about the list of periods + if "y" not in c: + info = ScenarioInfo() + info.year_from_codes(get_codes(f"year/{context.model.years}")) + + c.add("y", quote(info.Y)) + + if "y0" not in c: + c.add("y0", itemgetter(0), "y") + + # Above as coords/indexers + c.add("y::coords", lambda years: dict(y=years), "y") + c.add("y0::coord", lambda year: dict(y=year), "y0") + + # Retrieve the raw data + k = Key(measure.lower(), "ny") + k_raw = k + source_obj.id # Tagged with the source ID + keys = [k] # Keys to return + + c.add(k_raw, source_obj) + + # Aggregate + c.add(k_raw + "agg", "aggregate", k_raw, "n::groups", keep=False) + + # Interpolate to the desired set of periods + kwargs = dict(fill_value="extrapolate") + c.add(k, "interpolate", k_raw + "agg", "y::coords", kwargs=kwargs) + + # Index to y0 + keys.append(single_key(c.add(k + "y0 indexed", "index_to", k, "y0::coord"))) + + # TODO also insert (1) index to a particular label on the "n" dimension (2) both + + return tuple(keys) + + +def register_source(cls: Type[ExoDataSource]) -> Type[ExoDataSource]: + """Register `cls` as a source of exogenous data.""" + if cls.id in SOURCES: + raise ValueError(f"{SOURCES[cls.id]} already registered for id {cls.id!r}") + SOURCES[cls.id] = cls + return cls + + +@register_source +class TestSource(ExoDataSource): + """Example source of exogenous population and GDP data. + + Parameters + ---------- + source : str + **Must** be like ``test s1``, where "s1" is a scenario ID from ("s0"…"s4"). + source_kw : dict + **Must** contain an element "measure", one of :data:`MEASURES`. + """ + + id = "TEST" + + def __init__(self, source, source_kw): + if not source.startswith("test "): + # Don't recognize this `source` string → can't provide data + raise ValueError + + # Select the data according to the `source`; in this case, scenario + *parts, scenario = source.partition("test ") + self.indexers = dict(s=scenario) + + # Map from the measure ID to a variable name + self.indexers.update( + v={"POP": "Population", "GDP": "GDP"}[source_kw["measure"]] + ) + + def __call__(self) -> Quantity: + from genno.computations import select + + # - Retrieve the data. + # - Apply the prepared indexers. + return self.random_data().pipe(select, self.indexers, drop=True) + + @staticmethod + def random_data(): + """Generate some random data with n, y, s, and v dimensions.""" + from genno.computations import relabel + from genno.testing import random_qty + from pycountry import countries + + return random_qty(dict(n=len(countries), y=2, s=5, v=2), units="kg").pipe( + relabel, + n={f"n{i}": c.alpha_3 for i, c in enumerate(countries)}, + v={"v0": "Population", "v1": "GDP"}, + y={"y0": 2010, "y1": 2050}, + ) diff --git a/pyproject.toml b/pyproject.toml index 84e9735653..bd98bbb7a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,8 @@ mix-models = "message_ix_models.cli:main" [tool.coverage.report] exclude_also = [ + # Don't complain about abstract methods, they aren't run + "@(abc\\.)?abstractmethod", # Imports only used by type checkers "if TYPE_CHECKING:", ] @@ -111,6 +113,9 @@ no_implicit_optional = false addopts = "-p no:faulthandler --cov=message_ix_models --cov-report=" filterwarnings = "ignore:distutils Version classes.*:DeprecationWarning" +[tool.ruff] +select = ["C9", "E", "F", "I", "W"] + [tool.setuptools.packages] find = {} From 268662864a93088a4a891ee2b61b2d089dbd4ba7 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 6 Sep 2023 12:24:50 +0200 Subject: [PATCH 18/44] Address mypy note in test_ssp.py --- message_ix_models/tests/project/test_ssp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_ix_models/tests/project/test_ssp.py b/message_ix_models/tests/project/test_ssp.py index 8f8f230baf..75b77c1377 100644 --- a/message_ix_models/tests/project/test_ssp.py +++ b/message_ix_models/tests/project/test_ssp.py @@ -53,7 +53,7 @@ def test_parse(value, expected): assert expected == parse(value) -def test_ssp_field(): +def test_ssp_field() -> None: from dataclasses import dataclass @dataclass From bcc59d36b7c870e63c98dffd14de6b9448af1c7c Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 6 Sep 2023 16:48:09 +0200 Subject: [PATCH 19/44] Comment upstream versions in "pytest" CI workflow --- .github/workflows/pytest.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index ca77b0ae16..c4db6fc177 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -64,9 +64,18 @@ jobs: license: ${{ secrets.GAMS_LICENSE }} - name: Install packages and dependencies + # By default, install: + # - ixmp, message_ix: from GitHub branches/tags per matrix.upstream-version (above) + # - other dependencies including genno: from PyPI. + # + # To test against unreleased code (on `main`, or other branches + # for open PRs), temporarily uncomment, add, or edit lines below + # as needed. DO NOT merge such changes to `main`. run: | + # pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main" pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.upstream-version }}" pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.upstream-version }}" + pip install .[tests] - name: Configure local data path From 32943e7dd28c09be1196d539af6708df401ea52e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 6 Sep 2023 16:49:17 +0200 Subject: [PATCH 20/44] Revert #96 --- .github/workflows/pytest.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index c4db6fc177..079f391ff5 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -27,10 +27,9 @@ jobs: - { os: windows-latest, python: "3.11" } # Versions of both ixmp and message_ix to use upstream-version: - # Temporarily disabled (iiasa/ixmp#477) - # - v3.4.0 # Minimum version given in setup.cfg - # - v3.5.0 - # - v3.6.0 + - v3.4.0 # Minimum version given in setup.cfg + - v3.5.0 + - v3.6.0 - v3.7.0 # Latest released version - main # Development version From 6ea678edc5a4429efda04c32d0c78d1342de2cdb Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 8 Sep 2023 19:11:25 +0200 Subject: [PATCH 21/44] Add pre-commit, ruff configuration - Add "Code quality" job in "pytest" CI workflow. - Remove "lint" CI workflow. - Remove flake8, isort configuration. --- .flake8 | 8 -------- .github/workflows/lint.yaml | 33 --------------------------------- .github/workflows/pytest.yaml | 16 ++++++++++++++++ .gitignore | 2 ++ .pre-commit-config.yaml | 28 ++++++++++++++++++++++++++++ pyproject.toml | 6 +++--- 6 files changed, 49 insertions(+), 44 deletions(-) delete mode 100644 .flake8 delete mode 100644 .github/workflows/lint.yaml create mode 100644 .pre-commit-config.yaml diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 6abec65193..0000000000 --- a/.flake8 +++ /dev/null @@ -1,8 +0,0 @@ -[flake8] -max-complexity = 15 -# See https://black.readthedocs.io/en/stable/faq.html -max-line-length = 88 -ignore = - E203 - W503 -enable-extensions = W504 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml deleted file mode 100644 index 058a9d5d72..0000000000 --- a/.github/workflows/lint.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Lint - -on: - push: - branches: [ main ] - pull_request: - branches: [ main, "migrate-*"] - -# Cancel previous runs that have not completed -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - lint: - uses: iiasa/actions/.github/workflows/lint.yaml@main - with: - # If the "Latest version testable on GitHub Actions" in pytest.yaml - # is not the latest 3.x stable version, adjust here to match: - # python-version: "3.x" - # NB pint is normally implied by iam-units, but we force an earlier - # version to work around https://github.com/hgrecco/pint/issues/1767 - type-hint-packages: >- - genno - iam-units - "mypy < 1.5" - "pint < 0.21" - pytest - sdmx1 - types-PyYAML - types-tqdm - "ixmp @ git+https://github.com/iiasa/ixmp.git@main" - "message-ix @ git+https://github.com/iiasa/message_ix.git@main" diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 079f391ff5..62ee50af2b 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -101,3 +101,19 @@ jobs: - name: Upload test coverage to Codecov.io uses: codecov/codecov-action@v3 + + pre-commit: + name: Code quality + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + + - name: Force recreation of pre-commit virtual environment for mypy + if: github.event_name == 'schedule' # Comment this line to run on a PR + run: gh cache delete $(gh cache list -L 999 | cut -f2 | grep pre-commit) + env: { GH_TOKEN: "${{ github.token }}" } + + - uses: pre-commit/action@v3.0.0 diff --git a/.gitignore b/.gitignore index 3f9533db90..1c1d0e1751 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ pip-delete-this-directory.txt htmlcov/ .tox/ .nox/ +.benchmarks .coverage .coverage.* .cache @@ -50,6 +51,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.ruff_cache # Translations *.mo diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..1ebb6bc72a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +repos: +- repo: local + hooks: + - id: mypy + name: mypy + always_run: true + require_serial: true + pass_filenames: false + + language: python + entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@" + additional_dependencies: + - "mypy <1.5" + - pytest + - sdmx1 + - types-PyYAML + - types-tqdm + - "ixmp @ git+https://github.com/iiasa/ixmp.git@main" + - "message-ix @ git+https://github.com/iiasa/message_ix.git@main" + args: ["."] +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 23.7.0 + hooks: + - id: black +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.287 + hooks: + - id: ruff diff --git a/pyproject.toml b/pyproject.toml index bd98bbb7a7..8a2370a94e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,9 +77,6 @@ exclude_also = [ "if TYPE_CHECKING:", ] -[tool.isort] -profile = "black" - [tool.mypy] exclude = ["doc/"] @@ -116,6 +113,9 @@ filterwarnings = "ignore:distutils Version classes.*:DeprecationWarning" [tool.ruff] select = ["C9", "E", "F", "I", "W"] +[tool.ruff.mccabe] +max-complexity = 14 + [tool.setuptools.packages] find = {} From 1fef8396a4ea7d170a081b079ba861cdf1437f6b Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 8 Sep 2023 22:40:29 +0200 Subject: [PATCH 22/44] Mark .report as not supporting ixmp, message_ix <= 3.5.0 --- doc/whatsnew.rst | 1 + message_ix_models/report/__init__.py | 8 ++++++++ message_ix_models/tests/test_report.py | 12 ++++++++++++ 3 files changed, 21 insertions(+) diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 0749dc9d46..2aa61f08d7 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -8,6 +8,7 @@ v2023.9.2 ========= - New module :mod:`message_ix_models.report` for reporting (:pull:`116`). + Use of this module requires ixmp and message_ix version 3.6.0 or greater. - Add documentation on :ref:`migrate-filter-repo` using :program:`git filter-repo` and helper scripts (:pull:`89`). v2023.7.26 diff --git a/message_ix_models/report/__init__.py b/message_ix_models/report/__init__.py index 3f968b25e2..d591490817 100644 --- a/message_ix_models/report/__init__.py +++ b/message_ix_models/report/__init__.py @@ -351,6 +351,14 @@ def prepare_reporter( Same as ``context.report["key"]`` if any, but in full resolution; else one of ``default`` or ``cli-output`` according to the other settings. """ + from importlib.metadata import version + + if version("message_ix") < "3.6": + raise NotImplementedError( + "Support for message_ix_models.report.prepare_reporter() with message_ix <=" + " 3.5.0. Please upgrade to message_ix 3.6 or later." + ) + log.info("Prepare reporter") if reporter: diff --git a/message_ix_models/tests/test_report.py b/message_ix_models/tests/test_report.py index b3a97b5202..c31d715067 100644 --- a/message_ix_models/tests/test_report.py +++ b/message_ix_models/tests/test_report.py @@ -1,4 +1,6 @@ """Tests for message_data.reporting.""" +from importlib.metadata import version + import pandas as pd import pandas.testing as pdt import pytest @@ -13,7 +15,16 @@ }, } +MARK = ( + pytest.mark.xfail( + condition=version("message_ix") < "3.6", + raises=NotImplementedError, + reason="Not supported with message_ix < 3.6", + ), +) + +@MARK[0] def test_report_bare_res(request, test_context): """Prepare and run the standard MESSAGE-GLOBIOM reporting on a bare RES.""" scenario = testing.bare_res(request, test_context, solved=True) @@ -92,6 +103,7 @@ def test_report_legacy(caplog, request, tmp_path, test_context): ) +@MARK[0] @pytest.mark.parametrize("regions", ["R11"]) def test_apply_units(request, test_context, regions): test_context.regions = regions From 11e9affc82e5f10523e08c859921bf16a62f45e4 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 8 Sep 2023 23:17:33 +0200 Subject: [PATCH 23/44] Force pandas < 2.0 with message_ix <= 3.6 in "pytest" CI workflow --- .github/workflows/pytest.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 62ee50af2b..d3400da183 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -26,17 +26,17 @@ jobs: - { os: ubuntu-latest, python: "3.11" } - { os: windows-latest, python: "3.11" } # Versions of both ixmp and message_ix to use - upstream-version: - - v3.4.0 # Minimum version given in setup.cfg - - v3.5.0 - - v3.6.0 - - v3.7.0 # Latest released version - - main # Development version + upstream: + - { version: v3.4.0, extra-deps: '"pandas<2.0"' } # Minimum version given in setup.cfg + - { version: v3.5.0, extra-deps: '"pandas<2.0"' } + - { version: v3.6.0, extra-deps: '"pandas<2.0"' } + - { version: v3.7.0, extra-deps: "" } # Latest released version + - { version: main, extra-deps: "" } # Development version fail-fast: false runs-on: ${{ matrix.version.os }} - name: ${{ matrix.version.os }}-py${{ matrix.version.python }}-upstream-${{ matrix.upstream-version }} + name: ${{ matrix.version.os }}-py${{ matrix.version.python }}-upstream-${{ matrix.upstream.version }} steps: - name: Cache test data @@ -72,10 +72,10 @@ jobs: # as needed. DO NOT merge such changes to `main`. run: | # pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main" - pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.upstream-version }}" - pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.upstream-version }}" + pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.upstream.version }}" + pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.upstream.version }}" - pip install .[tests] + pip install .[tests] ${{ matrix.upstream.extra-deps }} - name: Configure local data path run: | From 6802f884be7bd7d68b94eeab0acbfb97c1b694ce Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 8 Sep 2023 23:44:30 +0200 Subject: [PATCH 24/44] Mark .snapshot.load() as not supporting ixmp, message_ix <= 3.4 --- message_ix_models/model/snapshot.py | 8 ++++++++ message_ix_models/tests/model/test_snapshot.py | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/message_ix_models/model/snapshot.py b/message_ix_models/model/snapshot.py index 7f219f7797..65ad312dbb 100644 --- a/message_ix_models/model/snapshot.py +++ b/message_ix_models/model/snapshot.py @@ -111,6 +111,14 @@ def load(scenario: Scenario, snapshot_id: int) -> None: -------- SNAPSHOTS """ + from importlib.metadata import version + + if version("message_ix") < "3.5": + raise NotImplementedError( + "Support for message_ix_models.model.snaphot.load() with message_ix <= " + "3.4.0. Please upgrade to message_ix 3.5 or later." + ) + path = fetch(SNAPSHOTS[snapshot_id]) # Add units diff --git a/message_ix_models/tests/model/test_snapshot.py b/message_ix_models/tests/model/test_snapshot.py index 337f48fbe3..dcf80c158d 100644 --- a/message_ix_models/tests/model/test_snapshot.py +++ b/message_ix_models/tests/model/test_snapshot.py @@ -1,6 +1,7 @@ import logging import shutil import sys +from importlib.metadata import version import pytest from message_ix import Scenario @@ -34,6 +35,11 @@ def unpacked_snapshot_data(test_context, request): shutil.copytree(snapshot_data_path, dest, dirs_exist_ok=True) +@pytest.mark.xfail( + condition=version("message_ix") < "3.5", + raises=NotImplementedError, + reason="Not supported with message_ix < 3.5", +) @pytest.mark.skipif( condition=GHA and sys.platform in ("darwin", "win32"), reason="Slow." ) From 96792cd07f77ae62b33a099d282d6622d00f26a8 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 10:47:53 +0200 Subject: [PATCH 25/44] Rename "TestSource" to "DemoSource" Avoid pytest attempting to collect this class. --- message_ix_models/tests/tools/test_exo_data.py | 4 ++-- message_ix_models/tools/exo_data.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/message_ix_models/tests/tools/test_exo_data.py b/message_ix_models/tests/tools/test_exo_data.py index a0bce46ffa..086bd2003f 100644 --- a/message_ix_models/tests/tools/test_exo_data.py +++ b/message_ix_models/tests/tools/test_exo_data.py @@ -2,8 +2,8 @@ from genno import Computer from message_ix_models.tools.exo_data import ( + DemoSource, ExoDataSource, - TestSource, prepare_computer, register_source, ) @@ -16,7 +16,7 @@ def test_abstract(self): def test_register_source(self): with pytest.raises(ValueError, match="already registered for"): - register_source(TestSource) + register_source(DemoSource) @pytest.mark.parametrize("regions, N_n", [("R12", 12), ("R14", 14)]) diff --git a/message_ix_models/tools/exo_data.py b/message_ix_models/tools/exo_data.py index 073f0f5f6e..7267e97f4e 100644 --- a/message_ix_models/tools/exo_data.py +++ b/message_ix_models/tools/exo_data.py @@ -169,7 +169,7 @@ def register_source(cls: Type[ExoDataSource]) -> Type[ExoDataSource]: @register_source -class TestSource(ExoDataSource): +class DemoSource(ExoDataSource): """Example source of exogenous population and GDP data. Parameters @@ -180,7 +180,7 @@ class TestSource(ExoDataSource): **Must** contain an element "measure", one of :data:`MEASURES`. """ - id = "TEST" + id = "DEMO" def __init__(self, source, source_kw): if not source.startswith("test "): From abe3b68470e023bcf5476d7680ad92bd9e00add1 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 10:48:22 +0200 Subject: [PATCH 26/44] Add carbon emission factors from IPCC (1996) --- message_ix_models/data/ipcc/1996_v3_t1-2.csv | 49 ++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 message_ix_models/data/ipcc/1996_v3_t1-2.csv diff --git a/message_ix_models/data/ipcc/1996_v3_t1-2.csv b/message_ix_models/data/ipcc/1996_v3_t1-2.csv new file mode 100644 index 0000000000..fd7d4ea5b8 --- /dev/null +++ b/message_ix_models/data/ipcc/1996_v3_t1-2.csv @@ -0,0 +1,49 @@ +# Carbon emission factors +# +# Source: IPCC (1996), "Revised 1996 IPCC Guidelines for National Greenhouse Gas +# Inventories", Volume 2, "Greenhouse Gas Inventory Workbook", +# Table 1-2 on page 1.6. +# +# https://www.ipcc-nggip.iges.or.jp/public/gl/guidelin/ch1wb1.pdf +# https://www.ipcc-nggip.iges.or.jp/public/gl/invs5a.html +# https://www.ipcc-nggip.iges.or.jp/public/gl/invs1.html +# +# Units: tonne / terajoule +# +# This transcription omits headings, notes, parentheses and other information. +# See the original. +# +fuel, value +"Crude Oil", 20.0 +"Orimulsion", 22.0 +"Natural Gas Liquids", 17.2 +"Gasoline", 18.9 +"Jet Kerosene", 19.5 +"Other Kerosene", 19.6 +"Shale Oil", 20.0 +"Gas/Diesel Oil", 20.2 +"Residual Fuel Oil", 21.1 +"LPG", 17.2 +"Ethane", 16.8 +"Naphtha", 20.0 +"Bitumen", 22.0 +"Lubricants", 20.0 +"Petroleum Coke", 27.5 +"Refinery Feedstocks", 20.0 +"Refinery Gas", 18.2 +"Other Oil", 20.0 +"Anthracite", 26.8 +"Coking Coal", 25.8 +"Other Bituminous Coal", 25.8 +"Sub-bituminous Coal", 26.2 +"Lignite", 27.6 +"Oil Shale", 29.1 +"Peat", 28.9 +"BKB & Patent Fuel", 25.8 +"Coke Oven / Gas Coke", 29.5 +"Coke Oven Gas", 13.0 +"Blast Furnace Gas", 66.0 +"Natural Gas (Dry)", 15.3 +"Solid Biomass", 29.9 +"Liquid Biomass", 20.0 +"Gas Biomass", 30.6 From 90de7d1f01d0e2e5d67735f119b6634a1beb6d33 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 10:50:27 +0200 Subject: [PATCH 27/44] Annotate commodities with matching IPCC names --- message_ix_models/data/commodity.yaml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/message_ix_models/data/commodity.yaml b/message_ix_models/data/commodity.yaml index e02911e06f..0e30115fb1 100644 --- a/message_ix_models/data/commodity.yaml +++ b/message_ix_models/data/commodity.yaml @@ -1,11 +1,17 @@ biomass: units: GWa report: Solids|Biomass + ipcc-1996-name: "Solid Biomass" coal: name: Coal units: GWa report: Solids|Fossil + # NB same value as "Coking coal"; this choice is arbitrary + # NB in message_doc, this appears as "Hard coal", but this + # term usually refers to anthracite, which has a higher + # carbon emission factor. + ipcc-1996-name: "Other Bituminous Coal" crudeoil: name: Crude oil @@ -14,6 +20,7 @@ crudeoil: level: primary units: GWa report: Oil + ipcc-1996-name: "Crude Oil" d_heat: name: (?) District heat @@ -38,14 +45,16 @@ freshwater_supply: fueloil: name: Fuel oil - description: Heavy fuel oil. + description: Heavy fuel oil level: secondary units: GWa + ipcc-1996-name: "Residual Fuel Oil" gas: name: Natural Gas units: GWa report: Gases + ipcc-1996-name: "Natural Gas (Dry)" hydrogen: name: Gaseous hydrogen @@ -61,11 +70,19 @@ lightoil: # level: secondary units: GWa report: Liquids|Oil + # NB same value as "Crude Oil" and several others; this choice is arbitrary + ipcc-1996-name: "Other Oil" + +lignite: + name: Lignite + ipcc-1996-name: "Lignite" methanol: name: Methanol units: GWa report: Liquids|Coal + # This does not appear in the referenced source; value is 17.4 + # ipcc-1996-name: MISSING non-comm: name: Non-commercial biomass @@ -238,8 +255,6 @@ transport: # Land Cover|Forest|Natural Forest # Land Cover|Other Natural Land # Land Cover|Pasture -# lh2 -# lignite # LiquidTotal # LNG # LoggingResidues From 024db7934c90ee10488e5018ae326ef0fc30a3eb Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 10:56:16 +0200 Subject: [PATCH 28/44] Load and transform IPCC (1996) CEFs --- message_ix_models/model/emissions.py | 57 ++++++++++++++++++- .../tests/model/test_emissions.py | 27 ++++++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/message_ix_models/model/emissions.py b/message_ix_models/model/emissions.py index a357d3a5ea..7095ce6184 100644 --- a/message_ix_models/model/emissions.py +++ b/message_ix_models/model/emissions.py @@ -1,15 +1,62 @@ import logging -from typing import Optional +import re +from typing import Optional, Tuple import pandas as pd +from genno import Quantity +from genno import computations as g from iam_units import convert_gwp from message_ix import Scenario, make_df from message_ix_models import ScenarioInfo +from message_ix_models.util import package_data_path + +from .structure import get_codes log = logging.getLogger(__name__) +def get_emission_factors(units: Optional[str] = None): + """Return carbon emission factors.""" + # Prepare information about commodities + commodities = get_codes("commodity") + relabel = {} # Mapping from IPCC names/IDs to message_ix_models commodity ID + select = [] # Select only the commodities needed + for c in commodities: + try: + ipcc_name = str(c.get_annotation(id="ipcc-1996-name").text) + except KeyError: + continue + else: + relabel[ipcc_name] = c.id + select.append(c.id) + + # Load data from file; relabel; and select only the values needed + result = ( + g.load_file(package_data_path("ipcc", "1996_v3_t1-2.csv"), dims={"fuel": "c"}) + .pipe(g.relabel, dict(c=relabel)) + .pipe(g.select, dict(c=select)) + ) + + # Manually insert a value for methanol + result = g.concat( + result, + Quantity(pd.Series(17.4, pd.Index(["methanol"], name="c")), units=result.units), + ) + + result.attrs["species"] = "C" + + if units is not None: + # Identify a GWP factor for target `units`, if any + to_units, to_species = split_species(units) + gwp_factor = convert_gwp("AR5GWP100", (1.0, str(result.units)), "C", to_species) + else: + gwp_factor, to_units = 1.0, result.units + + # Multiply by the GWP factor; let genno/pint handle other conversion + return result.pipe(g.mul, gwp_factor).pipe(g.convert_units, to_units) + + def add_tax_emission( scen: Scenario, price: float, @@ -76,3 +123,11 @@ def add_tax_emission( with scen.transact("Added carbon price"): scen.add_par(name, data) + + +def split_species(unit_expr: str) -> Tuple[str, Optional[str]]: + """Split `unit_expr` to an expression without a unit mention, and maybe species.""" + if match := re.fullmatch("(.*)(CO2|C)(.*)", unit_expr): + return f"{match.group(1)}{match.group(3)}", match.group(2) + else: + return unit_expr, None diff --git a/message_ix_models/tests/model/test_emissions.py b/message_ix_models/tests/model/test_emissions.py index 4a1dbaaefb..82c9e6c1ad 100644 --- a/message_ix_models/tests/model/test_emissions.py +++ b/message_ix_models/tests/model/test_emissions.py @@ -1,9 +1,11 @@ import numpy as np +import pint +import pytest from message_ix import make_df from message_ix.models import MACRO from message_ix_models import testing -from message_ix_models.model.emissions import add_tax_emission +from message_ix_models.model.emissions import add_tax_emission, get_emission_factors def add_test_data(scenario): @@ -52,3 +54,26 @@ def test_add_tax_emission(request, caplog, test_context): "Using the first of multiple discount rates: drate=[0.05 0.03]" == caplog.messages[-1] ) + + +MARK = pytest.mark.xfail( + raises=pint.errors.DimensionalityError, reason="IAMConsortium/units#X" +) + + +@pytest.mark.parametrize( + "units, exp_coal", + ( + (None, 25.8), + pytest.param("tC / TJ", 25.8, marks=MARK), + pytest.param("t CO2 / TJ", 94.6, marks=MARK), + pytest.param("t C / kWa", 0.814, marks=MARK), + ), +) +def test_get_emission_factors(units, exp_coal): + # Data are loaded + result = get_emission_factors(units=units) + assert 8 == result.size + + # Expected values are obtained + assert exp_coal == result.sel(c="coal").item() From afc48ce6de9ce6ba912e8fbf86f5841d3f92f102 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 12:15:02 +0200 Subject: [PATCH 29/44] Complete get_emissions_factors() --- message_ix_models/model/emissions.py | 36 ++++++++++++++++--- .../tests/model/test_emissions.py | 15 +++----- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/message_ix_models/model/emissions.py b/message_ix_models/model/emissions.py index 7095ce6184..a42f0e5dca 100644 --- a/message_ix_models/model/emissions.py +++ b/message_ix_models/model/emissions.py @@ -16,8 +16,34 @@ log = logging.getLogger(__name__) -def get_emission_factors(units: Optional[str] = None): - """Return carbon emission factors.""" +def get_emission_factors(units: Optional[str] = None) -> Quantity: + """Return carbon emission factors. + + Values are from the file :file:`message_ix_models/data/ipcc/1996_v3_t1-2.csv`, in + turn from `IPCC `_ + (see Table 1-2 on page 1.6); these are the same that appear on the "Emissions from + energy" page of the MESSAGEix-GLOBIOM documentation. + + The fuel dimension and names in the source are mapped to a :math:`c` ("commodity") + dimension and labels from :ref:`commodity-yaml`, using the ``ipcc-1996-name`` + annotations appearing in the latter. A value for "methanol" that appears in the + MESSAGEix-GLOBIOM docs table but not in the source is appended. + + Parameters + ---------- + unit : str, optional + Expression for units of the returned quantity. Tested values include: + + - "tC / TJ", source units (default), + - "t CO2 / TJ", and + - "t C / kWa", internal units in MESSAGEix-GLOBIOM, for instance for + "relation_activity" entries for emissions relations. + + Returns + ------- + Quantity + with 1 dimension (:math:`c`). + """ # Prepare information about commodities commodities = get_codes("commodity") relabel = {} # Mapping from IPCC names/IDs to message_ix_models commodity ID @@ -49,12 +75,14 @@ def get_emission_factors(units: Optional[str] = None): if units is not None: # Identify a GWP factor for target `units`, if any to_units, to_species = split_species(units) - gwp_factor = convert_gwp("AR5GWP100", (1.0, str(result.units)), "C", to_species) + gwp_factor = convert_gwp( + "AR5GWP100", (1.0, str(result.units)), "C", to_species + ).magnitude else: gwp_factor, to_units = 1.0, result.units # Multiply by the GWP factor; let genno/pint handle other conversion - return result.pipe(g.mul, gwp_factor).pipe(g.convert_units, to_units) + return result.pipe(g.mul, Quantity(gwp_factor)).pipe(g.convert_units, to_units) def add_tax_emission( diff --git a/message_ix_models/tests/model/test_emissions.py b/message_ix_models/tests/model/test_emissions.py index 82c9e6c1ad..d4bead0e6d 100644 --- a/message_ix_models/tests/model/test_emissions.py +++ b/message_ix_models/tests/model/test_emissions.py @@ -1,5 +1,4 @@ import numpy as np -import pint import pytest from message_ix import make_df from message_ix.models import MACRO @@ -56,18 +55,14 @@ def test_add_tax_emission(request, caplog, test_context): ) -MARK = pytest.mark.xfail( - raises=pint.errors.DimensionalityError, reason="IAMConsortium/units#X" -) - - @pytest.mark.parametrize( "units, exp_coal", ( (None, 25.8), - pytest.param("tC / TJ", 25.8, marks=MARK), - pytest.param("t CO2 / TJ", 94.6, marks=MARK), - pytest.param("t C / kWa", 0.814, marks=MARK), + # Unit expressions and values appearing in the message_doc table + ("tC / TJ", 25.8), + ("t CO2 / TJ", 94.6), + ("t C / kWa", 0.8142), ), ) def test_get_emission_factors(units, exp_coal): @@ -76,4 +71,4 @@ def test_get_emission_factors(units, exp_coal): assert 8 == result.size # Expected values are obtained - assert exp_coal == result.sel(c="coal").item() + assert np.isclose(exp_coal, result.sel(c="coal").item(), rtol=1e-4) From 10113b9bc7224caf1cb70159c7e26d7b210ff679 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 13:14:50 +0200 Subject: [PATCH 30/44] Bump minimum iam-units to v2023.9.11 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8a2370a94e..59fc86bc70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ # When the minimum is greater than the minimum via message_ix; e.g. # message_ix >= 3.4.0 → ixmp >= 3.4.0 → genno >= 1.6.0", "genno >= 1.18.1", - "iam_units", + "iam_units >= 2023.9.11", "message_ix >= 3.4.0", "pooch", "pyam-iamc >= 0.6", From 2527554519226da8b50aae4101620246637847be Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 13:32:27 +0200 Subject: [PATCH 31/44] Adjust expected values from test_create_res() --- message_ix_models/tests/model/test_bare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_ix_models/tests/model/test_bare.py b/message_ix_models/tests/model/test_bare.py index 91bb02fb7a..727f5bc310 100644 --- a/message_ix_models/tests/model/test_bare.py +++ b/message_ix_models/tests/model/test_bare.py @@ -6,7 +6,7 @@ #: Number of items in the respective YAML files. SET_SIZE = dict( - commodity=17, + commodity=18, level=6, node=14 + 1, # R14 is default, and 'World' exists automatically relation=20, From c7bed11bf358ae4b4c186126d0b1637064bb0873 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:09:34 +0200 Subject: [PATCH 32/44] Extend Sphinx configuration - Set nitpick_ignore_regex to hide known warnings. - Set rst_prolog to allow shorter references to common classes. - Set napoleon_type_aliases, ditto in docstrings. - Add intersphinx configuration for pint. - Use a local copy of the message_data inventory, if available. --- .gitignore | 1 + doc/conf.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 1c1d0e1751..ed451cada3 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,7 @@ instance/ # Sphinx documentation doc/_autosummary/ doc/_build/ +doc/*.inv # PyBuilder target/ diff --git a/doc/conf.py b/doc/conf.py index f0cd7fcd5e..3e38e60d4a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -39,6 +39,25 @@ # html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +nitpick_ignore_regex = { + # These occur because there is no .. py:module:: directive for the *top-level* + # module or package in the respective documentation and inventories. + # TODO Remove once the respective docs are fixed + ("py:mod", "ixmp"), + ("py:mod", "message_ix"), + ("py:mod", "message_data"), + # iam-units has no Sphinx docs + ("py:.*", "iam_units.*"), + # These are a consequence of autosummary-class.rst + ("py:(obj|meth)", r".*\.Test.*\.test_.*"), +} + +rst_prolog = """ +.. |Code| replace:: :class:`~sdmx.model.common.Code` +.. |Platform| replace:: :class:`~ixmp.Platform` +.. |Scenario| replace:: :class:`~message_ix.Scenario` +""" + def setup(app: "sphinx.application.Sphinx") -> None: """Copied from pytest's conf.py to enable intersphinx references to these.""" @@ -74,7 +93,6 @@ def setup(app: "sphinx.application.Sphinx") -> None: autosummary_generate = True - # -- Options for sphinx.ext.extlinks --------------------------------------------------- extlinks = { @@ -82,7 +100,6 @@ def setup(app: "sphinx.application.Sphinx") -> None: "pull": ("https://github.com/iiasa/message-ix-models/pull/%s", "PR #%s"), } - # -- Options for sphinx.ext.intersphinx ------------------------------------------------ # For message-data, see: https://docs.readthedocs.io/en/stable/guides @@ -95,15 +112,25 @@ def setup(app: "sphinx.application.Sphinx") -> None: "message-ix": ("https://docs.messageix.org/en/latest/", None), "m-data": ( f"https://{_token}:@docs.messageix.org/projects/models-internal/en/latest/", - None, + # Use a local copy of objects.inv, if the user has one + (None, "message_data.inv"), ), "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), + "pint": ("https://pint.readthedocs.io/en/stable/", None), "pooch": ("https://www.fatiando.org/pooch/latest/", None), "pytest": ("https://docs.pytest.org/en/stable/", None), "python": ("https://docs.python.org/3/", None), "sdmx": ("https://sdmx1.readthedocs.io/en/stable/", None), } +# -- Options for sphinx.ext.napoleon --------------------------------------------------- + +napoleon_type_aliases = { + "Code": ":class:`~sdmx.model.common.Code`", + "Path": ":class:`~pathlib.Path`", + "PathLike": ":class:`os.PathLike`", +} + # -- Options for sphinx.ext.todo ------------------------------------------------------- todo_include_todos = True From 3070b078432d8b2a9cca31908351bf6022aa85e4 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:12:02 +0200 Subject: [PATCH 33/44] =?UTF-8?q?Reduce=20complexity=20threshold=20from=20?= =?UTF-8?q?14=20=E2=86=92=2013?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- message_ix_models/model/build.py | 3 ++- message_ix_models/util/__init__.py | 3 ++- pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/message_ix_models/model/build.py b/message_ix_models/model/build.py index fd8a021c9b..4c72d25291 100644 --- a/message_ix_models/model/build.py +++ b/message_ix_models/model/build.py @@ -26,7 +26,8 @@ def _add_unit(mp: ixmp.Platform, unit: str, comment: str) -> None: raise -def apply_spec( +# FIXME Reduce complexity from 14 to ≤13 +def apply_spec( # noqa: C901 scenario: Scenario, spec: Union[Spec, Mapping[str, ScenarioInfo]], data: Optional[Callable] = None, diff --git a/message_ix_models/util/__init__.py b/message_ix_models/util/__init__.py index fff94a0df1..a42d5a3a67 100644 --- a/message_ix_models/util/__init__.py +++ b/message_ix_models/util/__init__.py @@ -563,7 +563,8 @@ def same_time(df: pd.DataFrame) -> pd.DataFrame: return df.assign(**{c: copy_column("time") for c in cols}) -def strip_par_data( +# FIXME Reduce complexity from 14 to ≤13 +def strip_par_data( # noqa: C901 scenario: message_ix.Scenario, set_name: str, element: str, diff --git a/pyproject.toml b/pyproject.toml index 59fc86bc70..816439aa94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,7 +114,7 @@ filterwarnings = "ignore:distutils Version classes.*:DeprecationWarning" select = ["C9", "E", "F", "I", "W"] [tool.ruff.mccabe] -max-complexity = 14 +max-complexity = 13 [tool.setuptools.packages] find = {} From 1584d42ff370c25624409ac334a941adbb90a91a Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:16:45 +0200 Subject: [PATCH 34/44] Use "*optional*", not "optional", in docstrings Sphinx ignores only the former, but will nitpick about the latter. --- message_ix_models/model/bare.py | 2 +- message_ix_models/model/build.py | 2 +- message_ix_models/model/emissions.py | 2 +- message_ix_models/model/structure.py | 2 +- message_ix_models/report/__init__.py | 4 ++-- message_ix_models/report/util.py | 4 ++-- message_ix_models/testing.py | 2 +- message_ix_models/tools/advance.py | 4 ++-- message_ix_models/tools/exo_data.py | 2 +- message_ix_models/util/__init__.py | 8 ++++---- message_ix_models/util/_logging.py | 8 ++++---- message_ix_models/util/common.py | 2 +- message_ix_models/util/context.py | 4 ++-- message_ix_models/workflow.py | 8 ++++---- 14 files changed, 27 insertions(+), 27 deletions(-) diff --git a/message_ix_models/model/bare.py b/message_ix_models/model/bare.py index 2aed6f105f..e309653562 100644 --- a/message_ix_models/model/bare.py +++ b/message_ix_models/model/bare.py @@ -42,7 +42,7 @@ def create_res(context, quiet=True): - Model name generated by :func:`name`. - Scenario name "baseline". - quiet : bool, optional + quiet : bool, *optional* Passed to `quiet` argument of :func:`.build.apply_spec`. Returns diff --git a/message_ix_models/model/build.py b/message_ix_models/model/build.py index 4c72d25291..a04476e05a 100644 --- a/message_ix_models/model/build.py +++ b/message_ix_models/model/build.py @@ -39,7 +39,7 @@ def apply_spec( # noqa: C901 ---------- spec : .Spec Specification of changes to make to `scenario`. - data : callable, optional + data : callable, *optional* Function to add data to `scenario`. `data` can either manipulate the scenario directly, or return a :class:`dict` compatible with :func:`.add_par_data`. diff --git a/message_ix_models/model/emissions.py b/message_ix_models/model/emissions.py index a42f0e5dca..f50767004e 100644 --- a/message_ix_models/model/emissions.py +++ b/message_ix_models/model/emissions.py @@ -106,7 +106,7 @@ def add_tax_emission( scen : :class:`message_ix.Scenario` price : float Price in the first model year, in USD / tonne CO₂. - conversion_factor : float, optional + conversion_factor : float, *optional* Factor for converting `price` into the model's internal emissions units, currently USD / tonne carbon. Optional: a default value is retrieved from :mod:`iam_units`. diff --git a/message_ix_models/model/structure.py b/message_ix_models/model/structure.py index a603b2dac6..471de5a591 100644 --- a/message_ix_models/model/structure.py +++ b/message_ix_models/model/structure.py @@ -219,7 +219,7 @@ def process_units_anno(set_name: str, code: Code, quiet: bool = False) -> None: ---------- set_name : str Used in logged messages when `quiet` is :data:`False`. - quiet : bool, optional + quiet : bool, *optional* If :data:`False` (the default), log on level :ref:`WARNING ` if: - the annotation is missing, or diff --git a/message_ix_models/report/__init__.py b/message_ix_models/report/__init__.py index d591490817..d94e7d6ffa 100644 --- a/message_ix_models/report/__init__.py +++ b/message_ix_models/report/__init__.py @@ -335,10 +335,10 @@ def prepare_reporter( ---------- context : Context Containing settings in the ``report/*`` tree. - scenario : message_ix.Scenario, optional + scenario : Scenario, *optional* Scenario to report. If not given, :meth:`.Context.get_scenario` is used to retrieve a Scenario. - reporter : .Reporter, optional + reporter : message_ix.Reporter, *optional* Existing reporter to extend with computations. If not given, it is created using :meth:`.Reporter.from_scenario`. diff --git a/message_ix_models/report/util.py b/message_ix_models/report/util.py index 0b5c8b9275..bdf341cca5 100644 --- a/message_ix_models/report/util.py +++ b/message_ix_models/report/util.py @@ -102,7 +102,7 @@ def collapse(df: pd.DataFrame, var=[]) -> pd.DataFrame: Parameters ---------- - var : list of str, optional + var : list of str, *optional* Strings or dimensions to concatenate to the 'Variable' column. The first of these is usually a string value used to populate the column. These are joined using the pipe ('|') character. @@ -177,7 +177,7 @@ def copy_ts(rep: Reporter, other: str, filters: Optional[dict]) -> Key: ---------- other_url : str URL of the other scenario from which to copy time series data. - filters : dict, optional + filters : dict, *optional* Filters; passed via :func:`.store_ts` to :meth:`ixmp.TimeSeries.timeseries`. Returns diff --git a/message_ix_models/testing.py b/message_ix_models/testing.py index 353c90aae6..ac51974827 100644 --- a/message_ix_models/testing.py +++ b/message_ix_models/testing.py @@ -234,7 +234,7 @@ def bare_res(request, context: Context, solved: bool = False) -> message_ix.Scen name is used for the scenario name of the returned Scenario. context : .Context Passed to :func:`.testing.bare_res`. - solved : bool, optional + solved : bool, *optional* Return a solved Scenario. Returns diff --git a/message_ix_models/tools/advance.py b/message_ix_models/tools/advance.py index 3dc2adfba4..039f7a19a7 100644 --- a/message_ix_models/tools/advance.py +++ b/message_ix_models/tools/advance.py @@ -37,7 +37,7 @@ def get_advance_data(query: Optional[str] = None) -> pd.Series: Parameters ---------- - query : str, optional + query : str, *optional* Passed to :meth:`pandas.DataFrame.query` to limit the returned values. Returns @@ -58,7 +58,7 @@ def advance_data(variable: str, query: Optional[str] = None) -> Quantity: Parameters ---------- - query : str, optional + query : str, *optional* Passed to :func:`get_advance_data`. Returns diff --git a/message_ix_models/tools/exo_data.py b/message_ix_models/tools/exo_data.py index 7267e97f4e..815541b710 100644 --- a/message_ix_models/tools/exo_data.py +++ b/message_ix_models/tools/exo_data.py @@ -86,7 +86,7 @@ def prepare_computer( source : str Identifier of the source, possibly with other information to be handled by a :class:`ExoDataSource`. - source_kw : dict, optional + source_kw : dict, *optional* Keyword arguments for a Source class. These can include indexers, selectors, or other information needed by the source class to identify the data to be returned. diff --git a/message_ix_models/util/__init__.py b/message_ix_models/util/__init__.py index a42d5a3a67..d83eb6b839 100644 --- a/message_ix_models/util/__init__.py +++ b/message_ix_models/util/__init__.py @@ -75,7 +75,7 @@ def add_par_data( data Dict with keys that are parameter names, and values are pd.DataFrame or other arguments - dry_run : optional + dry_run : bool, *optional* Only show what would be done. See also @@ -272,7 +272,7 @@ def ffill( Dimension to fill along. Must be a column in `df`. values : list of str Labels along `dim` that must be present in the returned data frame. - expr : str, optional + expr : str, *optional* If provided, :meth:`.DataFrame.eval` is called. This can be used to assign one column to another. For instance, if `dim` == "year_vtg" and `expr` is "year_act = year_vtg", then forward filling is performed along the "year_vtg" dimension/ @@ -575,9 +575,9 @@ def strip_par_data( # noqa: C901 Parameters ---------- - dry_run : bool, optional + dry_run : bool, *optional* If :data:`True`, only show what would be done. - dump : dict, optional + dump : dict, *optional* If provided, stripped data are stored in this dictionary. Otherwise, they are discarded. diff --git a/message_ix_models/util/_logging.py b/message_ix_models/util/_logging.py index 580da22b06..63039b45a3 100644 --- a/message_ix_models/util/_logging.py +++ b/message_ix_models/util/_logging.py @@ -22,9 +22,9 @@ def silence_log(names=None, level=logging.ERROR): Parameters ---------- - names : str, optional + names : str, *optional* Space-separated names of loggers to quiet. - level : int, optional + level : int, *optional* Minimum level of log messages to allow. Examples @@ -199,9 +199,9 @@ def setup( Parameters ---------- - level : str, optional + level : str, *optional* Log level for :mod:`message_ix_models` and :mod:`message_data`. - console : bool, optional + console : bool, *optional* If :obj:`True`, print all messages to console using a :class:`Formatter`. """ # Copy to avoid modifying with the operations below diff --git a/message_ix_models/util/common.py b/message_ix_models/util/common.py index 141e88c509..97f6b583e4 100644 --- a/message_ix_models/util/common.py +++ b/message_ix_models/util/common.py @@ -176,7 +176,7 @@ def load_package_data(*parts: str, suffix: Optional[str] = ".yaml") -> Any: ---------- parts : iterable of str Used to construct a path under :file:`message_ix_models/data/`. - suffix : str, optional + suffix : str, *optional* File name suffix, including, the ".", e.g. :file:`.yaml`. Returns diff --git a/message_ix_models/util/context.py b/message_ix_models/util/context.py index cdda2260e2..038b476936 100644 --- a/message_ix_models/util/context.py +++ b/message_ix_models/util/context.py @@ -51,7 +51,7 @@ def get_instance(cls, index=0) -> "Context": Parameters ---------- - index : int, optional + index : int, *optional* Index of the Context instance to return, e.g. ``-1`` for the most recently created. """ @@ -188,7 +188,7 @@ def clone_to_dest(self, create=True) -> message_ix.Scenario: Parameters ---------- - create : bool, optional + create : bool, *optional* If :obj:`True` (the default) and the base scenario does not exist, a bare RES scenario is created. Otherwise, an exception is raised. diff --git a/message_ix_models/workflow.py b/message_ix_models/workflow.py index 55b6e02e17..3e7fef0c5c 100644 --- a/message_ix_models/workflow.py +++ b/message_ix_models/workflow.py @@ -23,12 +23,12 @@ class WorkflowStep: Parameters ---------- name : str - ``"model name/scenario name"`` for the :class:`.Scenario` produced by the step. - action : CallbackType, optional + ``"model name/scenario name"`` for the |Scenario| produced by the step. + action : CallbackType, *optional* Function to be executed to modify the base into the target Scenario. - clone : bool, optional + clone : bool, *optional* :obj:`True` to clone the base scenario the target. - target : str, optional + target : str, *optional* URL for the scenario produced by the workflow step. Parsed to :attr:`scenario_info` and :attr:`platform_info`. kwargs From f600e2c8d9f8cd085c72943cd3e6d7db58a00349 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:20:28 +0200 Subject: [PATCH 35/44] Document additions in #122 --- doc/api/report/index.rst | 27 ++++++++++++++++++++++----- doc/api/tools.rst | 25 +++++++++++++++++++++++-- doc/pkg-data/codelists.rst | 12 ++++++++++++ message_ix_models/model/emissions.py | 14 +++++++------- message_ix_models/tools/exo_data.py | 8 ++++++-- message_ix_models/util/sdmx.py | 2 +- 6 files changed, 71 insertions(+), 17 deletions(-) diff --git a/doc/api/report/index.rst b/doc/api/report/index.rst index 2305ea8c25..aad02d62ee 100644 --- a/doc/api/report/index.rst +++ b/doc/api/report/index.rst @@ -106,7 +106,7 @@ Operators .. automodule:: message_ix_models.report.computations :members: - :mod:`message_ix_models` provides the following: + :mod:`message_ix_models.report.computations` provides the following: .. autosummary:: @@ -118,11 +118,28 @@ Operators remove_ts share_curtailment - Other operators are provided by: + Other operators or genno-compatible functions are provided by: - - :mod:`message_ix.reporting.computations` - - :mod:`ixmp.reporting.computations` - - :mod:`genno.computations` + - Upstream packages: + + - :mod:`message_ix.reporting.computations` + - :mod:`ixmp.reporting.computations` + - :mod:`genno.computations` + + - Other submodules: + + - :mod:`.model.emissions`: :func:`.get_emission_factors`. + + Any of these can be made available for a :class:`.Computer` instance using :meth:`~.genno.Computer.require_compat`, for instance: + + .. code-block:: + + # Indicate that a certain module contains functions to + # be referenced by name + c.require_compat("message_ix_models.model.emissions") + + # Add computations to the graph by referencing functions + c.add("ef:c", "get_emission_factors", units="t C / kWa") Utilities --------- diff --git a/doc/api/tools.rst b/doc/api/tools.rst index 312a2bb45d..63da531252 100644 --- a/doc/api/tools.rst +++ b/doc/api/tools.rst @@ -17,12 +17,33 @@ On this page: .. automodule:: message_ix_models.tools :members: +.. currentmodule:: message_ix_models.tools.exo_data -ADVANCE data (:mod:`.tools.advance`) -==================================== +Exogenous data (:mod:`.tools.exo_data`) +======================================= + +.. automodule:: message_ix_models.tools.exo_data + :members: + :exclude-members: ExoDataSource + + .. autosummary:: + + MEASURES + SOURCES + DemoSource + ExoDataSource + prepare_computer + register_source + +.. autoclass:: ExoDataSource + :members: + :special-members: __init__, __call__ .. currentmodule:: message_ix_models.tools.advance +ADVANCE data (:mod:`.tools.advance`) +==================================== + .. autosummary:: get_advance_data advance_data diff --git a/doc/pkg-data/codelists.rst b/doc/pkg-data/codelists.rst index 29e5e6b40e..d15c025d33 100644 --- a/doc/pkg-data/codelists.rst +++ b/doc/pkg-data/codelists.rst @@ -53,3 +53,15 @@ Each of these codes has the following annotations: .. literalinclude:: ../../message_ix_models/data/technology.yaml :language: yaml + +Others +====== + +.. literalinclude:: ../../message_ix_models/data/sdmx/ICONICS_SSP(2017).xml + :language: xml + +.. literalinclude:: ../../message_ix_models/data/sdmx/ICONICS_SSP(2024).xml + :language: xml + +.. literalinclude:: ../../message_ix_models/data/sdmx/IIASA_ECE_AGENCIES(0.1).xml + :language: xml diff --git a/message_ix_models/model/emissions.py b/message_ix_models/model/emissions.py index f50767004e..a7287b4d73 100644 --- a/message_ix_models/model/emissions.py +++ b/message_ix_models/model/emissions.py @@ -25,18 +25,18 @@ def get_emission_factors(units: Optional[str] = None) -> Quantity: energy" page of the MESSAGEix-GLOBIOM documentation. The fuel dimension and names in the source are mapped to a :math:`c` ("commodity") - dimension and labels from :ref:`commodity-yaml`, using the ``ipcc-1996-name`` - annotations appearing in the latter. A value for "methanol" that appears in the - MESSAGEix-GLOBIOM docs table but not in the source is appended. + dimension and labels from :ref:`commodity.yaml `, using the + ``ipcc-1996-name`` annotations appearing in the latter. A value for "methanol" that + appears in the MESSAGEix-GLOBIOM docs table but not in the source is appended. Parameters ---------- - unit : str, optional + unit : str, *optional* Expression for units of the returned quantity. Tested values include: - - "tC / TJ", source units (default), - - "t CO2 / TJ", and - - "t C / kWa", internal units in MESSAGEix-GLOBIOM, for instance for + - “tC / TJ”, source units (default), + - “t CO2 / TJ”, and + - “t C / kWa”, internal units in MESSAGEix-GLOBIOM, for instance for "relation_activity" entries for emissions relations. Returns diff --git a/message_ix_models/tools/exo_data.py b/message_ix_models/tools/exo_data.py index 815541b710..edd17ab55d 100644 --- a/message_ix_models/tools/exo_data.py +++ b/message_ix_models/tools/exo_data.py @@ -10,8 +10,12 @@ from message_ix_models.model.structure import get_codes __all__ = [ - "prepare_computer", + "MEASURES", + "SOURCES", + "DemoSource", "ExoDataSource", + "prepare_computer", + "register_source", ] #: Supported measures. @@ -19,7 +23,7 @@ #: .. todo:: Store this in a separate code list or concept scheme. MEASURES = ("GDP", "POP") -#: List of known sources for data. +#: Known sources for data. Use :func:`register_source` to add to this collection. SOURCES: Dict[str, Type["ExoDataSource"]] = {} diff --git a/message_ix_models/util/sdmx.py b/message_ix_models/util/sdmx.py index d95cca0f53..d8fd276297 100644 --- a/message_ix_models/util/sdmx.py +++ b/message_ix_models/util/sdmx.py @@ -110,7 +110,7 @@ def eval_anno(obj: AnnotableArtefact, id: str): def make_enum(urn, base=Enum): - """Create an :class:`Enum` (or `base`) subclass with members from codelist `urn`.""" + """Create an :class:`.enum.Enum` (or `base`) with members from codelist `urn`.""" # Read the code list cl = read(urn) From f35d305274a286b1686863e0c60a7fd50f613436 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:21:18 +0200 Subject: [PATCH 36/44] Add #122 to doc/whatsnew --- doc/whatsnew.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 2aa61f08d7..db89f55bff 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -1,8 +1,19 @@ What's new ********** -.. Next release -.. ============ +Next release +============ + +- New module :mod:`.project.ssp` (:pull:`122`) to generate SDMX codelists for the 2017/original SSPs and the 2024 update, and provide these as :class:`~.enum.Enum` to other code. +- New module :mod:`.tools.exo_data` to retrieve exogenous data for, among others, population and GDP (:pull:`122`). + This module has a general API that can be implemented by provider classes. +- New function :func:`.model.emissions.get_emission_factors` and associated data file to provide data from `this table `__ in the MESSAGEix-GLOBIOM documentation (:pull:`122`). +- New functions in :mod:`.util.sdmx` (:pull:`122`): + + - :func:`~.util.sdmx.read`, :func:`~.util.sdmx.write` to retrieve/store package data in SDMX-ML. + - :func:`~.util.sdmx.make_enum` to make pure-Python :class:`~.enum.Enum` (or subclass) data structures based on SDMX code lists. + +- :func:`.same_node` also fills "node_shares", "node_loc", and "node", as appropriate (:pull:`122`). v2023.9.2 ========= From 6393a00a0e837bf8875fb93995313ed8f5658d42 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:22:56 +0200 Subject: [PATCH 37/44] Address Sphinx nitpicks in ReST files --- doc/api/disutility.rst | 10 +++++----- doc/api/model-bare.rst | 4 ++-- doc/api/model.rst | 2 +- doc/api/report/index.rst | 2 +- doc/api/util.rst | 11 ++++++----- doc/api/workflow.rst | 8 ++++---- doc/cli.rst | 4 ++-- doc/data.rst | 4 ++-- doc/distrib.rst | 6 +++--- doc/migrate.rst | 2 +- doc/pkg-data/node.rst | 4 ++-- doc/whatsnew.rst | 23 ++++++++++++----------- 12 files changed, 41 insertions(+), 39 deletions(-) diff --git a/doc/api/disutility.rst b/doc/api/disutility.rst index 67de6811d4..8f06370214 100644 --- a/doc/api/disutility.rst +++ b/doc/api/disutility.rst @@ -14,15 +14,15 @@ Method & usage Use this code by calling :func:`add`, which takes arguments that describe the concrete usage: Consumer groups - This is a list of :class:`.Code` objects describing the consumer groups. + This is a list of |Code| objects describing the consumer groups. The list must be 1-dimensional, but can be composed (as in :mod:`message_data.model.transport`) from multiple dimensions. Technologies - This is a list of :class:`.Code` objects describing the technologies for which the consumers in the different groups experience disutility. - Each object must be have 'input' and 'output' annotations (:attr:`.Code.anno`); each of these is a :class:`dict` with the keys 'commodity', 'input', and 'unit', describing the source or sink for the technology. + This is a list of |Code| objects describing the technologies for which the consumers in the different groups experience disutility. + Each object must be have 'input' and 'output' annotations (:attr:`~.Code.annotations`); each of these is a :class:`dict` with the keys 'commodity', 'input', and 'unit', describing the source or sink for the technology. Template - This is also a :class:`.Code` object, similar to those in ``technologies``; see below. + This is also a |Code| object, similar to those in ``technologies``; see below. The code creates a source technology for the “disutility” commodity. The code does *not* perform the following step(s) needed to completely parametrize the formulation: @@ -113,7 +113,7 @@ For example, the technology “t0” outputs to the commodity “output of t0” .. _disutility-units: (Dis)utility is generally dimensionless. -In :mod:`pint` and thus also :mod:`message_ix_models`, this should be represented by ``""``. +In :mod:`.pint` and thus also :mod:`message_ix_models`, this should be represented by ``""``. However, to work around `iiasa/ixmp#425 `__, :func:`data_conversion` and :func:`data_source` return data with ``"-"`` as units. See :issue:`45` for more information. diff --git a/doc/api/model-bare.rst b/doc/api/model-bare.rst index 5ba5903be1..da573792b0 100644 --- a/doc/api/model-bare.rst +++ b/doc/api/model-bare.rst @@ -2,7 +2,7 @@ Reproduce the RES (:mod:`.model.bare`) ************************************** In contrast to :mod:`.model.create`, this module creates the RES 'from scratch'. -:func:`.create_res` begins by creating a new, totally empty :class:`.Scenario` and adding data to it (instead of cloning and modifying an existing scenario). +:func:`.create_res` begins by creating a new, totally empty :class:`~message_ix.Scenario` and adding data to it (instead of cloning and modifying an existing scenario). .. note:: Currently, the Scenario returned by :func:`.create_res`… @@ -27,7 +27,7 @@ Code reference :members: :exclude-members: get_spec -.. automethod:: message_ix_models.model.bare.get_spec +.. autofunction:: get_spec Since the RES is the base for all variants of MESSAGEix-GLOBIOM, the 'require' and 'remove' portions of the spec are empty. diff --git a/doc/api/model.rst b/doc/api/model.rst index 74950f8742..03a3b4e926 100644 --- a/doc/api/model.rst +++ b/doc/api/model.rst @@ -102,4 +102,4 @@ Submodules described on separate pages: >>> AUT.parent - .. seealso:: :func:`.adapt_R11_R14` + .. seealso:: :obj:`.adapt_R11_R14` diff --git a/doc/api/report/index.rst b/doc/api/report/index.rst index aad02d62ee..dac83e7272 100644 --- a/doc/api/report/index.rst +++ b/doc/api/report/index.rst @@ -32,7 +32,7 @@ Any reporting specific to ``coal_ppl`` must be in :mod:`message_ix_models`, sinc The basic **design pattern** of :mod:`message_ix_models.report` is: - A ``global.yaml`` file (i.e. in `YAML `_ format) that contains a *concise* yet *explicit* description of the reporting computations needed for a MESSAGE-GLOBIOM model. -- :func:`~.reporting.prepare_reporter` reads the file and a Scenario object, and uses it to populate a new Reporter. +- :func:`~.report.prepare_reporter` reads the file and a Scenario object, and uses it to populate a new Reporter. This function mostly relies on the :doc:`configuration handlers ` built in to Genno to handle the different sections of the file. Features diff --git a/doc/api/util.rst b/doc/api/util.rst index 5e681ba775..34e904d7bc 100644 --- a/doc/api/util.rst +++ b/doc/api/util.rst @@ -25,9 +25,9 @@ Commonly used: ~context.Context ~scenarioinfo.ScenarioInfo ~scenarioinfo.Spec - adapt_R11_R12 - adapt_R11_R14 - as_codes + .adapt_R11_R12 + .adapt_R11_R14 + .as_codes broadcast cached check_support @@ -52,6 +52,7 @@ Commonly used: .. automodule:: message_ix_models.util :members: + :exclude-members: as_codes, eval_anno .. autodata:: message_ix_models.util.cache.SKIP_CACHE @@ -117,7 +118,7 @@ Commonly used: .. automethod:: clone_to_dest - To use this method, either decorate a command with :func:`common_params`: + To use this method, either decorate a command with :func:`.common_params`: .. code-block:: python @@ -163,7 +164,7 @@ Commonly used: :members: :mod:`.util.node` -================== +================= .. currentmodule:: message_ix_models.util.node diff --git a/doc/api/workflow.rst b/doc/api/workflow.rst index 2e2ee1824b..0beb3938b7 100644 --- a/doc/api/workflow.rst +++ b/doc/api/workflow.rst @@ -22,7 +22,7 @@ The generic pattern for workflows is: 1. A precursor scenario is obtained. - It may be returned by a prior workflow step, or loaded from a :class:`Platform`. + It may be returned by a prior workflow step, or loaded from a :class:`~ixmp.Platform`. 2. (Optional) The precursor scenario is cloned to a target model name and scenario name. 3. A function is called to operate on the scenario. This function may do zero or more of: @@ -54,15 +54,15 @@ These functions **must**: - Accept at least 2 arguments: - 1. A :class:`Context` instance. + 1. A :class:`.Context` instance. 2. The precursor scenario. 3. Optionally additional, keyword-only arguments. - Return either: - - a :class:`.Scenario` object, that can be the same object provided as an argument, or a different scenario, e.g. a clone or a different scenario, even from a different platform. + - a :class:`~.message_ix.Scenario` object, that can be the same object provided as an argument, or a different scenario, e.g. a clone or a different scenario, even from a different platform. - :class:`None`. - In this case, any modifications implemented by the step should be reflected in the :class:`.Scenario` given as an argument. + In this case, any modifications implemented by the step should be reflected in the Scenario given as an argument. The functions **may**: diff --git a/doc/cli.rst b/doc/cli.rst index ed7a2ffd43..7a1393a859 100644 --- a/doc/cli.rst +++ b/doc/cli.rst @@ -27,7 +27,7 @@ To add a specific database, you can use the ixmp CLI [1]_:: $ ixmp platform add [PLATFORMNAME] jdbc oracle [COMPUTER]:[PORT]/[SERVICENAME] [USERNAME] [PASSWORD] You may also want to make this the *default* platform. -Unless told otherwise, :mod:`message_ix_models` creates :class:`~.Platform` objects without any arguments (``mp = ixmp.Platform()``); this loads the default platform. +Unless told otherwise, :mod:`message_ix_models` creates :class:`~ixmp.Platform` objects without any arguments (``mp = ixmp.Platform()``); this loads the default platform. Set the default:: $ ixmp platform add default [PLATFORMNAME] @@ -150,7 +150,7 @@ To explain further: These options direct it to work with a different Platform. ``--model MODEL --scenario SCENARIO`` or ``--url`` - Many commands use an *existing* :class:`~.Scenario` as a starting point, and begin by cloning that Scenario to a new (model name, scenario name). + Many commands use an *existing* |Scenario| as a starting point, and begin by cloning that Scenario to a new (model name, scenario name). For any such command, these top-level options define the starting point/initial Scenario to clone/‘baseline’. In contrast, see ``--output-model``, below. diff --git a/doc/data.rst b/doc/data.rst index a2642a7272..5bf7b37e17 100644 --- a/doc/data.rst +++ b/doc/data.rst @@ -96,7 +96,7 @@ Prefer text formats *Do not* hard-code paths Data stored with (1) or (2) above can be retrieved with the utility funtions mentioned, instead of hard-coded paths. - For system-specific paths (3) only, get a :obj:`.Context` object and use it to get an appropriate :class:`.Path` object pointing to a file + For system-specific paths (3) only, get a :obj:`.Context` object and use it to get an appropriate :class:`~pathlib.Path` object pointing to a file .. code-block:: python @@ -253,7 +253,7 @@ Specific modules for model variants, projects, etc. **should**: - Define a single :mod:`dataclass ` to express the configuration options they understand. See for example :class:`.model.Config` (for constructing new models), and :class:`message_data.model.buildings.Config` (for the MESSAGEix-Buildings model variant / linkage). -- Store this on the :class:`Context` at a simple key. +- Store this on the :class:`.Context` at a simple key. For example :class:`.model.Config` is stored at ``context.model`` or ``context["model"]``. - Retrieve and respect configuration from existing objects, i.e. only duplicate settings with the same meaning when strictly necessary. - Communicate to other modules by setting the appropriate configuration values. diff --git a/doc/distrib.rst b/doc/distrib.rst index 9dbae15ac3..32a154e7ca 100644 --- a/doc/distrib.rst +++ b/doc/distrib.rst @@ -12,9 +12,9 @@ Overview Scenarios in the MESSAGEix-GLOBIOM global model family are characterized by: -- ca. 100 MB of data, depending on storage format (e.g. in a :class:`.JDBCBackend` local, HyperSQL database, or :ref:`ixmp:excel-data-format` in Excel files). -- :meth:`.solve` times of between 10 and 60 minutes, depending on hardware and configuration, plus similar amounts of time to run the legacy reporting in :mod:`message_data`. -- Memory usage of ~10 GB or more using :class:`.JDBCBackend`, currently the only supported backend. +- ca. 100 MB of data, depending on storage format (e.g. in a :class:`~ixmp.backend.jdbc.JDBCBackend` local, HyperSQL database, or :ref:`ixmp:excel-data-format` in Excel files). +- :meth:`~message_ix.Scenario.solve` times of between 10 and 60 minutes, depending on hardware and configuration, plus similar amounts of time to run the legacy reporting in :mod:`message_data`. +- Memory usage of ~10 GB or more using :class:`~ixmp.backend.jdbc.JDBCBackend`, currently the only supported backend. These resource needs can be a bottleneck in applications, for example: diff --git a/doc/migrate.rst b/doc/migrate.rst index 75388bb0c0..d8221ce80c 100644 --- a/doc/migrate.rst +++ b/doc/migrate.rst @@ -230,7 +230,7 @@ Read through all the steps first before starting. - Adjust data handling. - For example, usage of :func:`private_data_path` to locate data files must be modified to :func:`package_data_path` if the data files were moved during the migration. + For example, usage of :func:`.private_data_path` to locate data files must be modified to :func:`.package_data_path` if the data files were moved during the migration. Tests can help to ensure that these changes are effective. - Address CI checks. diff --git a/doc/pkg-data/node.rst b/doc/pkg-data/node.rst index feb2a7d53a..3f746ad8d8 100644 --- a/doc/pkg-data/node.rst +++ b/doc/pkg-data/node.rst @@ -8,7 +8,7 @@ The codes in these lists denote **regions** and **countries**. When loaded using :func:`.get_codes`, the :attr:`.Code.child` attribute is a list of child codes. See the function documentation for how to retrieve these. -.. seealso:: :func:`.adapt_R11_R12`, :func:`.adapt_R11_R14`, :func:`.identify_nodes`. +.. seealso:: :obj:`.adapt_R11_R12`, :obj:`.adapt_R11_R14`, :func:`.identify_nodes`. .. contents:: :local: @@ -100,4 +100,4 @@ Zambia (``ZMB``) ---------------- .. literalinclude:: ../../message_ix_models/data/node/ZMB.yaml - :language: yaml \ No newline at end of file + :language: yaml diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index db89f55bff..229aa69980 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -26,10 +26,11 @@ v2023.7.26 ========== - Add code and CLI commands to :doc:`fetch and load MESSAGEix-GLOBIOM snapshots ` (:pull:`102`). + Use of this module requires ixmp and message_ix version 3.5.0 or greater. - Add :func:`.util.pooch.fetch`, a thin wrapper for using :doc:`Pooch ` (:pull:`102`). - New module :mod:`message_ix_models.model.macro` with utilities for calibrating :mod:`message_ix.macro` (:pull:`104`). - New method :meth:`.Workflow.guess_target` (:pull:`104`). -- Change in behaviour of :meth:`.Workflow.add_step`: the method now returns the name of the newly-added workflow step, rather than the :class:`WorkflowStep` object added to carry out the step (:pull:`104`). +- Change in behaviour of :meth:`.Workflow.add_step`: the method now returns the name of the newly-added workflow step, rather than the :class:`.WorkflowStep` object added to carry out the step (:pull:`104`). The former is more frequently used in code that uses :class:`.Workflow`. - Add the :ref:`R17` node code list (:pull:`109`). - Add the :ref:`R20` node code list (:pull:`109`). @@ -49,7 +50,7 @@ v2023.5.13 - Add :doc:`/water/index` (:pull:`88`, :pull:`91`). - New utility function :func:`.replace_par_data` (:pull:`90`). -- :func:`.disutility.get_spec` preserves all :class:`Annotations <~.sdmx.model.v21.Annotation>` attached to the :class:`~.sdmx.model.v21.Code` object used as a template for usage technologies (:pull:`90`). +- :func:`.disutility.get_spec` preserves all :class:`Annotations ` attached to the :class:`~sdmx.model.common.Code` object used as a template for usage technologies (:pull:`90`). - Add ``CO2_Emission_Global_Total`` to the :ref:`“A” relation codelist ` (:pull:`90`). - :class:`.Adapter` and :class:`.MappingAdapter` can be imported from :mod:`message_ix_models.util` (:pull:`90`). - Bump :mod:`sdmx` requirement from v2.2.0 to v2.8.0 (:pull:`90`). @@ -77,9 +78,9 @@ v2023.5.13 ========= - Add the :ref:`ZMB` node code list (:pull:`83`). -- Add the utility :func:`same_time`, to copy the set time in paramenters (:pull:`83`). +- Add the utility :func:`.same_time`, to copy the set time in parameters (:pull:`83`). - New :class:`~message_ix_models.Config` and :class:`.model.Config` :py:mod:`dataclasses` for clearer description/handling of recognized settings stored on :class:`.Context` (:pull:`82`). - :class:`.ConfigHelper` for convenience/utility functionality in :mod:`message_ix_models`-based code. + :class:`.ConfigHelper` for convenience/utility functionality in :mod:`.message_ix_models`-based code. - New functions :func:`.generate_product`, :func:`.generate_set_elements`, :func:`.get_region_codes` in :mod:`.model.structure` (:pull:`82`). - Revise and improve the :doc:`Workflow API ` (:pull:`82`). - Adjust for pandas 1.5.0 (:pull:`81`). @@ -94,7 +95,7 @@ v2023.5.13 2022.7.25 ========= -- Add :func:`get_advance_data`, and related tools for data from the ADVANCE project, including the :ref:`node codelist ` for the data (:pull:`76`). +- Add :func:`.get_advance_data`, and related tools for data from the ADVANCE project, including the :ref:`node codelist ` for the data (:pull:`76`). - Add unit annotations to :ref:`commodity-yaml` (:pull:`76`). - New utility methods :meth:`.ScenarioInfo.io_units` to derive units for ``input`` and ``output`` parameters from :meth:`.units_for` commodity stocks and technology activities (:pull:`76`). - Transfer :func:`.add_tax_emission` from :mod:`message_data`, improve, and add tests (:pull:`76`). @@ -109,7 +110,7 @@ v2023.5.13 2022.5.6 ======== -- Bump minimum required version of :mod:`message_ix` to v3.4.0 from v3.2.0 (:pull:`71`). +- Bump minimum required version of :mod:`.message_ix` to v3.4.0 from v3.2.0 (:pull:`71`). - Add a documentation page on :doc:`distrib` (:pull:`59`). - Add :func:`.testing.not_ci` for marking tests not to be run on continuous integration services; improve :func:`~.testing.session_context` (:pull:`62`). - :func:`.apply_spec` also adds elements of the "node" set using :meth:`.ixmp.Platform.add_region` (:pull:`62`). @@ -119,14 +120,14 @@ v2023.5.13 2022.3.30 ========= -- Add :func:`adapt_R11_R12`, a function for adapting data from the :ref:`R11` to the :ref:`R12` node lists (:pull:`56`). +- Add :obj:`.adapt_R11_R12`, a function for adapting data from the :ref:`R11` to the :ref:`R12` node lists (:pull:`56`). - Work around `iiasa/ixmp#425 `__ in :func:`.disutility.data_conversion` (:ref:`docs `, :pull:`55`). 2022.3.3 ======== - Change the node name in R12.yaml from R12_CPA to R12_RCPA (:pull:`49`). -- Register “message local data” ixmp configuration file setting and use to set the :attr:`.Context.local_path` when provided. +- Register “message local data” ixmp configuration file setting and use to set the :attr:`.Context.local_path <.Config.local_data>` when provided. See :ref:`local-data` (:pull:`47`) 2022.1.26 @@ -135,7 +136,7 @@ v2023.5.13 - New :class:`.Spec` class for easier handling of specifications of model (or model variant) structure (:pull:`39`) - New utility function :func:`.util.local_data_path` (:pull:`39`). - :func:`.repr` of :class:`.Context` no longer prints a (potentially very long) list of all keys and settings (:pull:`39`). -- :func:`.as_codes` accepts a :class:`.dict` with :class:`.Code` values (:pull:`39`). +- :func:`.as_codes` accepts a :class:`.dict` with |Code| values (:pull:`39`). Earlier releases ================ @@ -162,8 +163,8 @@ Earlier releases 2021.7.6 -------- -- Add :func:`identify_nodes`, a function for identifying a :doc:`pkg-data/node` based on a :class:`.Scenario` (:pull:`24`). -- Add :func:`adapt_R11_R14`, a function for adapting data from the :ref:`R11` to the :ref:`R14` node lists (:pull:`24`). +- Add :func:`identify_nodes`, a function for identifying a :doc:`pkg-data/node` based on a |Scenario| (:pull:`24`). +- Add :obj:`.adapt_R11_R14`, a function for adapting data from the :ref:`R11` to the :ref:`R14` node lists (:pull:`24`). - Add :func:`.export_test_data` and :command:`mix-models export-test-data` command (:pull:`16`). See :ref:`export-test-data`. - Allow use of pytest's persistent cache across test sessions (:pull:`23`). From ee4bcf7aad7a98413a992c37ec4147506fbf5458 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:23:37 +0200 Subject: [PATCH 38/44] Add .project.ssp, .tools.sdmx to API docs --- doc/api/project.rst | 13 +++++++++++-- doc/api/util.rst | 9 +++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/doc/api/project.rst b/doc/api/project.rst index 71ce7d4874..c5ff47803e 100644 --- a/doc/api/project.rst +++ b/doc/api/project.rst @@ -1,7 +1,16 @@ +.. currentmodule:: message_ix_models.project + Specific research projects (:mod:`.project`) ******************************************** -.. currentmodule:: message_ix_models.project - .. automodule:: message_ix_models.project :members: + + +.. currentmodule:: message_ix_models.project.ssp + +Shared Socioeconomic Pathways (:mod:`.project.ssp`) +=================================================== + +.. automodule:: message_ix_models.project.ssp + :members: diff --git a/doc/api/util.rst b/doc/api/util.rst index 34e904d7bc..7629a25a81 100644 --- a/doc/api/util.rst +++ b/doc/api/util.rst @@ -15,6 +15,7 @@ Submodules: node pooch scenarioinfo + sdmx Commonly used: @@ -188,3 +189,11 @@ Commonly used: .. automodule:: message_ix_models.util.scenarioinfo :members: + +:mod:`.util.sdmx` +================= + +.. currentmodule:: message_ix_models.util.sdmx + +.. automodule:: message_ix_models.util.sdmx + :members: From 3dd79cb5e21cd90abd9d2419e56139e63df5fc06 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:24:08 +0200 Subject: [PATCH 39/44] Use hidden toctree block and add top-level in doc/index --- doc/index.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/index.rst b/doc/index.rst index 157fbd0d2e..0020c162ec 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -24,9 +24,39 @@ Among other tasks, the tools allow modelers to: distrib bibliography +API reference +============= + +.. currentmodule:: message_ix_models + +Commonly used classes may be imported directly from :mod:`message_ix_models`. + +.. automodule:: message_ix_models + + .. autosummary:: + + .Config + .Context + .ScenarioInfo + .Spec + .Workflow + +- :doc:`api/model` +- :doc:`api/model-bare` +- :doc:`api/model-build` +- :doc:`api/model-emissions` +- :doc:`api/model-snapshot` +- :doc:`api/disutility` +- :doc:`api/report/index` +- :doc:`api/tools` +- :doc:`api/util` +- :doc:`api/testing` +- :doc:`api/workflow` + .. toctree:: :maxdepth: 2 :caption: API reference + :hidden: api/model api/model-bare From f0605be0eacdbefedbf01e7fcc132d28bbfbb0ef Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:26:38 +0200 Subject: [PATCH 40/44] Address Sphinx nitpicks in Python files - Use ReST replacements where possible. --- message_ix_models/model/bare.py | 2 +- message_ix_models/model/disutility.py | 6 ++-- message_ix_models/model/macro.py | 6 ++-- message_ix_models/model/structure.py | 2 +- message_ix_models/report/__init__.py | 13 ++++---- message_ix_models/testing.py | 6 ++-- .../tests/model/test_disutility.py | 2 +- message_ix_models/tests/util/test_cache.py | 2 +- message_ix_models/tests/util/test_context.py | 2 +- message_ix_models/tests/util/test_node.py | 6 ++-- message_ix_models/tests/util/test_sdmx.py | 2 +- message_ix_models/util/__init__.py | 30 +++++++++---------- message_ix_models/util/click.py | 2 +- message_ix_models/util/common.py | 6 ++-- message_ix_models/util/config.py | 6 ++-- message_ix_models/util/context.py | 25 ++++++++-------- message_ix_models/util/scenarioinfo.py | 15 +++++----- message_ix_models/util/sdmx.py | 6 ++-- message_ix_models/workflow.py | 6 ++-- 19 files changed, 74 insertions(+), 71 deletions(-) diff --git a/message_ix_models/model/bare.py b/message_ix_models/model/bare.py index e309653562..d08fb8dae3 100644 --- a/message_ix_models/model/bare.py +++ b/message_ix_models/model/bare.py @@ -36,7 +36,7 @@ def create_res(context, quiet=True): Parameters ---------- - context : .Context + context : Context :attr:`.Context.scenario_info` determines the model name and scenario name of the created Scenario. If not provided, the defaults are: diff --git a/message_ix_models/model/disutility.py b/message_ix_models/model/disutility.py index bd4089a0f5..7ff71580f3 100644 --- a/message_ix_models/model/disutility.py +++ b/message_ix_models/model/disutility.py @@ -51,11 +51,11 @@ def get_spec( Parameters ---------- - groups : list of Code + groups : list of |Code| Identities of the consumer groups with distinct disutilities. - technologies : list of Code + technologies : list of |Code| The technologies to which the disutilities are applied. - template : .Code + template : |Code| """ s = Spec() diff --git a/message_ix_models/model/macro.py b/message_ix_models/model/macro.py index fdeb8f5479..015c3fe8c2 100644 --- a/message_ix_models/model/macro.py +++ b/message_ix_models/model/macro.py @@ -43,8 +43,8 @@ def generate( node list indicated by :attr:`.model.Config.regions`. - "year": All periods from the period *before* the first model year. - "commodity": The elements of `commodities`. - - "sector": If each entry of `commodities` is a :class:`.Code` and has an annotation - with id="macro-sector", the value of that annotation. Otherwise, the same as + - "sector": If each entry of `commodities` is a |Code| and has an annotation with + id="macro-sector", the value of that annotation. Otherwise, the same as `commodity`. `value` supplies the parameter value, which is the same for all observations. @@ -56,7 +56,7 @@ def generate( MACRO parameter for which to generate data. context Used with :func:`.bare.get_spec`. - commodities : list of str or Code + commodities : list of str or |Code| Commodities to include in the MESSAGE-MACRO linkage. value : float Parameter value. diff --git a/message_ix_models/model/structure.py b/message_ix_models/model/structure.py index 471de5a591..11bbabe3b9 100644 --- a/message_ix_models/model/structure.py +++ b/message_ix_models/model/structure.py @@ -113,7 +113,7 @@ def generate_product( Mapping from dimension IDs to lists of codes. name : str Name of the set. - template : .Code + template : Code Must have Python format strings for its its :attr:`id` and :attr:`name` attributes. """ diff --git a/message_ix_models/report/__init__.py b/message_ix_models/report/__init__.py index d94e7d6ffa..d5ef4eb1de 100644 --- a/message_ix_models/report/__init__.py +++ b/message_ix_models/report/__init__.py @@ -301,7 +301,7 @@ def prepare_reporter( scenario: Optional[Scenario] = None, reporter: Optional[Reporter] = None, ) -> Tuple[Reporter, Key]: - """Return a :class:`.Reporter` and `key` prepared to report a :class:`.Scenario`. + """Return a :class:`message_ix.Reporter` and `key` prepared to report a |Scenario|. The code responds to the following settings on `context`: @@ -323,11 +323,12 @@ def prepare_reporter( corresponding, full-resolution Key, if any, is returned. * - report/config - dict or Path-like or None - - If :class:`dict`, then this is passed to :meth:`.Reporter.configure`. If - Path-like, then this is the path to the reporting configuration file. If not - given, defaults to :file:`report/global.yaml`. + - If :class:`dict`, then this is passed to + :meth:`message_ix.Reporter.configure`. If Path-like, then this is the path to + the reporting configuration file. If not given, defaults to + :file:`report/global.yaml`. * - report/output_path - - Path-like, optional + - Path-like, *optional* - Path to write reporting outputs. If given, a computation ``cli-output`` is added to the Reporter which writes ``report/key`` to this path. @@ -340,7 +341,7 @@ def prepare_reporter( retrieve a Scenario. reporter : message_ix.Reporter, *optional* Existing reporter to extend with computations. If not given, it is created - using :meth:`.Reporter.from_scenario`. + using :meth:`message_ix.Reporter.from_scenario`. Returns ------- diff --git a/message_ix_models/testing.py b/message_ix_models/testing.py index ac51974827..4e84f96b7d 100644 --- a/message_ix_models/testing.py +++ b/message_ix_models/testing.py @@ -215,7 +215,7 @@ def cli_test_group(): def bare_res(request, context: Context, solved: bool = False) -> message_ix.Scenario: - """Return or create a Scenario containing the bare RES, for use in testing. + """Return or create a |Scenario| containing the bare RES, for use in testing. The Scenario has a model name like "MESSAGEix-GLOBIOM [regions] [start]:[duration]:[end]", e.g. "MESSAGEix-GLOBIOM R14 2020:10:2110" (see @@ -232,14 +232,14 @@ def bare_res(request, context: Context, solved: bool = False) -> message_ix.Scen request : .Request or None The pytest :fixture:`pytest:request` fixture. If provided the pytest test node name is used for the scenario name of the returned Scenario. - context : .Context + context : Context Passed to :func:`.testing.bare_res`. solved : bool, *optional* Return a solved Scenario. Returns ------- - .Scenario + Scenario The scenario is a fresh clone, so can be modified freely without disturbing other tests. """ diff --git a/message_ix_models/tests/model/test_disutility.py b/message_ix_models/tests/model/test_disutility.py index 8cfc7e13cf..f78342f86b 100644 --- a/message_ix_models/tests/model/test_disutility.py +++ b/message_ix_models/tests/model/test_disutility.py @@ -73,7 +73,7 @@ def spec(groups, techs, template): @pytest.fixture def scenario(request, test_context, techs): - """Fixture: a :class:`.Scenario` with technologies given by :func:`techs`.""" + """Fixture: a |Scenario| with technologies given by :func:`techs`.""" test_context.regions = "R14" s = testing.bare_res(request, test_context, solved=False) s.check_out() diff --git a/message_ix_models/tests/util/test_cache.py b/message_ix_models/tests/util/test_cache.py index 5e1b97a520..94317aac5b 100644 --- a/message_ix_models/tests/util/test_cache.py +++ b/message_ix_models/tests/util/test_cache.py @@ -16,7 +16,7 @@ class TestEncoder: def test_sdmx(self): - """:mod:`message_ix_models` configures :class:`.Encoder` for :class:`.Code`.""" + """:mod:`message_ix_models` configures :class:`.Encoder` for |Code|.""" codes0 = [sdmx_model.Code(id=f"FOO{i}", name="foo") for i in range(5)] codes1 = [f"FOO{i}" for i in range(5)] diff --git a/message_ix_models/tests/util/test_context.py b/message_ix_models/tests/util/test_context.py index d28c6df56b..5921cb4634 100644 --- a/message_ix_models/tests/util/test_context.py +++ b/message_ix_models/tests/util/test_context.py @@ -74,7 +74,7 @@ def test_clone_to_dest(self, caplog, test_context): assert s.model.startswith("baz") and s.scenario.startswith("baz") def test_dealias(self, caplog): - """Aliasing works with :meth:`Context.__init__`, :meth:`Context.update`.""" + """Aliasing works with :meth:`.Context.__init__`, :meth:`.Context.update`.""" c = Context() c.update(regions="R99") assert [] == caplog.messages # No log warnings for core Config, .model.Config diff --git a/message_ix_models/tests/util/test_node.py b/message_ix_models/tests/util/test_node.py index d859ed7c95..8cfaafef4b 100644 --- a/message_ix_models/tests/util/test_node.py +++ b/message_ix_models/tests/util/test_node.py @@ -39,7 +39,7 @@ def test_mapping_adapter(): @pytest.fixture(scope="function") def input(): - """Fixture: test data for :func:`.adapt_R11_R14`.""" + """Fixture: test data for :obj:`.adapt_R11_R14`.""" R11_all = get_codes("node/R11") R11_reg = R11_all[R11_all.index("World")].child df = make_df( @@ -61,7 +61,7 @@ def input(): ], ) def test_adapt_df(input, func, N, expected, target_nodes): - """:func:`.adapt_R11_R14` handles :class:`pandas.DataFrame`.""" + """:obj:`.adapt_R11_R14` handles :class:`pandas.DataFrame`.""" # Function runs output = func(input) @@ -86,7 +86,7 @@ def test_adapt_df(input, func, N, expected, target_nodes): [(adapt_R11_R12, VALUE[0], "R12_CHN"), (adapt_R11_R14, VALUE[1], "R14_CAS")], ) def test_adapt_qty(input, func, expected, node_loc): - """:func:`.adapt_R11_R14` handles :class:`genno.Quantity`.""" + """:obj:`.adapt_R11_R14` handles :class:`genno.Quantity`.""" # Convert to genno.Quantity df = input[PAR] input[PAR] = Quantity.from_series(df.set_index(df.columns[:-2].tolist())["value"]) diff --git a/message_ix_models/tests/util/test_sdmx.py b/message_ix_models/tests/util/test_sdmx.py index 1e624af811..16fb946be4 100644 --- a/message_ix_models/tests/util/test_sdmx.py +++ b/message_ix_models/tests/util/test_sdmx.py @@ -27,7 +27,7 @@ def test_eval_anno(caplog): def test_make_enum(): - """:func:`.make_enum` works with :class:`Flag` and subclasses.""" + """:func:`.make_enum` works with :class:`~enum.Flag` and subclasses.""" from enum import Flag, IntFlag E = make_enum("ICONICS:SSP(2017)", base=Flag) diff --git a/message_ix_models/util/__init__.py b/message_ix_models/util/__init__.py index d83eb6b839..1376bfcfe1 100644 --- a/message_ix_models/util/__init__.py +++ b/message_ix_models/util/__init__.py @@ -266,7 +266,7 @@ def ffill( Parameters ---------- - df : .DataFrame + df : pandas.DataFrame Data to fill forwards. dim : str Dimension to fill along. Must be a column in `df`. @@ -335,7 +335,7 @@ def make_io(src, dest, efficiency, on="input", **kwargs): If 'input', `efficiency` applies to the input, and the output, thus the activity level of the technology, is in dest[2] units. If 'output', the opposite. kwargs - Passed to :func:`make_df`. + Passed to :func:`~message_ix.make_df`. Returns ------- @@ -371,12 +371,11 @@ def make_matched_dfs( Parameters ---------- - base : pd.DataFrame, dict, etc. - Used to populate other columns of each data frame. - Duplicates—which occur when the target parameter has fewer dimensions than - `base`—are dropped. + base : pandas.DataFrame, dict, etc. + Used to populate other columns of each data frame. Duplicates—which occur when + the target parameter has fewer dimensions than `base`—are dropped. par_values : - Argument names (e.g. ‘fix_cost’) are passed to :func:`.make_df`. + Argument names (e.g. ‘fix_cost’) are passed to :func:`~.message_ix.make_df`. If the value is :class:`float`, it overwrites the "value" column; if :class:`pint.Quantity`, its magnitude overwrites "value" and its units the "units" column, as a formatted string. @@ -416,13 +415,13 @@ def make_source_tech( The technology has no inputs; its output commodity and/or level are determined by `common`; either single values, or :obj:`None` if the result will be - :meth:`~DataFrame.pipe`'d through :func:`broadcast`. + :meth:`~pandas.DataFrame.pipe`'d through :func:`broadcast`. Parameters ---------- - info : .Scenario or .ScenarioInfo + info : Scenario or ScenarioInfo common : dict - Passed to :func:`make_df`. + Passed to :func:`~message_ix.make_df`. **values Values for 'capacity_factor' (optional; default 1.0), 'output', 'var_cost', and optionally 'technical_lifetime'. @@ -463,11 +462,11 @@ def make_source_tech( def maybe_query(series: pd.Series, query: Optional[str]) -> pd.Series: - """Apply :meth:`pandas.Series.query` if the `query` arg is not :obj:`None`. + """Apply :meth:`pandas.DataFrame.query` if the `query` arg is not :obj:`None`. - :meth:`~pandas.Series.query` is not chainable (`pandas-dev/pandas#37941 + :meth:`~pandas.DataFrame.query` is not chainable (`pandas-dev/pandas#37941 `_). Use this function with - :func:`pandas.Series.pipe`, passing an argument that may be :obj:`None`, to have a + :meth:`pandas.Series.pipe`, passing an argument that may be :obj:`None`, to have a chainable query operation that can be a no-op. """ # Convert Series to DataFrame, query(), then retrieve the single column @@ -570,7 +569,7 @@ def strip_par_data( # noqa: C901 element: str, dry_run: bool = False, dump: Optional[Dict[str, pd.DataFrame]] = None, -): +) -> int: """Remove `element` from `set_name` in scenario, optionally dumping to `dump`. Parameters @@ -583,7 +582,8 @@ def strip_par_data( # noqa: C901 Returns ------- - Total number of rows removed across all parameters. + int + Total number of rows removed across all parameters. See also -------- diff --git a/message_ix_models/util/click.py b/message_ix_models/util/click.py index 3adaa99338..f95d2db7f3 100644 --- a/message_ix_models/util/click.py +++ b/message_ix_models/util/click.py @@ -70,7 +70,7 @@ def format_sys_argv() -> str: def store_context(context: Union[click.Context, Context], param, value): - """Callback that simply stores a value on the :class:`Context` object. + """Callback that simply stores a value on the :class:`.Context` object. Use this for parameters that are not used directly in a @click.command() function, but need to be carried by the Context for later use. diff --git a/message_ix_models/util/common.py b/message_ix_models/util/common.py index 97f6b583e4..ddae79ea2f 100644 --- a/message_ix_models/util/common.py +++ b/message_ix_models/util/common.py @@ -195,7 +195,7 @@ def load_package_data(*parts: str, suffix: Optional[str] = ".yaml") -> Any: def load_private_data(*parts: str) -> Mapping: # pragma: no cover (needs message_data) """Load a private data file from :mod:`message_data` and return its contents. - Analogous to :mod:`load_package_data`, but for non-public data. + Analogous to :func:`load_package_data`, but for non-public data. Parameters ---------- @@ -250,7 +250,7 @@ def package_data_path(*parts) -> Path: Parameters ---------- parts : sequence of str or Path - Joined to the base path using :meth:`.Path.joinpath`. + Joined to the base path using :meth:`~pathlib.PurePath.joinpath`. See also -------- @@ -268,7 +268,7 @@ def private_data_path(*parts) -> Path: # pragma: no cover (needs message_data) Parameters ---------- parts : sequence of str or Path - Joined to the base path using :meth:`.Path.joinpath`. + Joined to the base path using :meth:`~pathlib.PurePath.joinpath`. See also -------- diff --git a/message_ix_models/util/config.py b/message_ix_models/util/config.py index b8d9c8af40..0c48fe467c 100644 --- a/message_ix_models/util/config.py +++ b/message_ix_models/util/config.py @@ -131,16 +131,16 @@ class Config: #: :program:`--scenario` or :program:`--url` CLI options. scenario_info: MutableMapping[str, str] = field(default_factory=dict) - #: Like :attr:`platform_info`, used by e.g. :meth:`clone_to_dest`. + #: Like :attr:`platform_info`, used by e.g. :meth:`.clone_to_dest`. dest_platform: MutableMapping[str, str] = field(default_factory=dict) - #: Like :attr:`scenario_info`, used by e.g. :meth:`clone_to_dest`. + #: Like :attr:`scenario_info`, used by e.g. :meth:`.clone_to_dest`. dest_scenario: MutableMapping[str, str] = field(default_factory=dict) #: A scenario URL, e.g. as given by the :program:`--url` CLI option. url: Optional[str] = None - #: Like :attr:`url`, used by e.g. :meth:`clone_to_dest`. + #: Like :attr:`url`, used by e.g. :meth:`.clone_to_dest`. dest: Optional[str] = None #: Base path for cached data, e.g. as given by the :program:`--cache-path` CLI diff --git a/message_ix_models/util/context.py b/message_ix_models/util/context.py index 038b476936..98ea03c37d 100644 --- a/message_ix_models/util/context.py +++ b/message_ix_models/util/context.py @@ -283,18 +283,19 @@ def get_local_path(self, *parts: str, suffix=None) -> Path: Parameters ========== parts : - Path fragments, e.g. directories, passed to :meth:`Path.joinpath`. + Path fragments, for instance directories, passed to + :meth:`~.pathlib.PurePath.joinpath`. suffix : - File name suffix including a ".", e.g. ".csv", passed to - :meth:`Path.with_suffix`. + File name suffix including a "."—for instance, ".csv"—passed to + :meth:`~.pathlib.PurePath.with_suffix`. """ result = self.core.local_data.joinpath(*parts) return result.with_suffix(suffix) if suffix else result def get_platform(self, reload=False) -> ixmp.Platform: - """Return a :class:`ixmp.Platform` from :attr:`platform_info`. + """Return a |Platform| from :attr:`.Config.platform_info`. - When used through the CLI, :attr:`platform_info` is a 'base' platform as + When used through the CLI, :attr:`.Config.platform_info` is a 'base' platform as indicated by the --url or --platform options. If a Platform has previously been instantiated with :meth:`get_platform`, the @@ -319,16 +320,16 @@ def get_platform(self, reload=False) -> ixmp.Platform: return self["_mp"] def get_scenario(self) -> message_ix.Scenario: - """Return a :class:`message_ix.Scenario` from :attr:`scenario_info`. + """Return a |Scenario| from :attr:`~.Config.scenario_info`. - When used through the CLI, :attr:`scenario_info` is a ‘base’ scenario for an - operation, indicated by the ``--url`` or ``--platform/--model/--scenario`` - options. + When used through the CLI, :attr:`~.Config.scenario_info` is a ‘base’ scenario + for an operation, indicated by the ``--url`` or + ``--platform/--model/--scenario`` options. """ return message_ix.Scenario(self.get_platform(), **self.core.scenario_info) def set_scenario(self, scenario: message_ix.Scenario) -> None: - """Update :attr:`scenario_info` to match an existing `scenario`.""" + """Update :attr:`.Config.scenario_info` to match an existing `scenario`.""" self.core.scenario_info.update( model=scenario.model, scenario=scenario.scenario, version=scenario.version ) @@ -346,8 +347,8 @@ def handle_cli_args( ): """Handle command-line arguments. - May update the :attr:`data_path`, :attr:`platform_info`, :attr:`scenario_info`, - and/or :attr:`url` settings. + May update the :attr:`.Config.local_data`, :attr:`.Config.platform_info`, + :attr:`.Config.scenario_info`, and/or :attr:`.url` settings. """ self.core.verbose = verbose diff --git a/message_ix_models/util/scenarioinfo.py b/message_ix_models/util/scenarioinfo.py index 080f3b7dfc..485a44caeb 100644 --- a/message_ix_models/util/scenarioinfo.py +++ b/message_ix_models/util/scenarioinfo.py @@ -138,10 +138,11 @@ def __repr__(self): def units_for(self, set_name: str, id: str) -> pint.Unit: """Return the units associated with code `id` in MESSAGE set `set_name`. - :mod:`ixmp` (or the sole :class:`.JDBCBackend`, as of v3.5.0) does not handle - unit information for variables and equations (unlike parameter values), such as - MESSAGE decision variables ``ACT``, ``CAP``, etc. In :mod:`message_ix_models` - and :mod:`message_data`, the following conventions are (generally) followed: + :mod:`ixmp` (or the sole :class:`~ixmp.backend.base.JDBCBackend`, as of v3.5.0) + does not handle unit information for variables and equations (unlike parameter + values), such as MESSAGE decision variables ``ACT``, ``CAP``, etc. In + :mod:`message_ix_models` and :mod:`message_data`, the following conventions are + (generally) followed: - The units of ``ACT`` and others are consistent for each ``technology``. - The units of ``COMMODITY_BALANCE``, ``STOCK``, ``commodity_stock``, etc. are @@ -278,9 +279,9 @@ class Spec: A Spec collects 3 :class:`.ScenarioInfo` instances at the attributes :attr:`.add`, :attr:`.remove`, and :attr:`.require`. This is the type that is accepted by :func:`.apply_spec`; :doc:`model-build` describes how a Spec is used to modify a - :class:`Scenario`. A Spec may also be used to express information about the target - structure of data to be prepared; like :class:`.ScenarioInfo`, this can happen - before the target :class:`.Scenario` exists. + |Scenario|. A Spec may also be used to express information about the target + structure of data to be prepared; like ScenarioInfo, this can happen before the + target Scenario exists. Spec also provides: diff --git a/message_ix_models/util/sdmx.py b/message_ix_models/util/sdmx.py index d8fd276297..96424a5adc 100644 --- a/message_ix_models/util/sdmx.py +++ b/message_ix_models/util/sdmx.py @@ -22,13 +22,13 @@ def as_codes(data: Union[List[str], Dict[str, CodeLike]]) -> List[Code]: - """Convert *data* to a :class:`list` of :class:`.Code` objects. + """Convert `data` to a :class:`list` of |Code| objects. Various inputs are accepted: - :class:`list` of :class:`str`. - - :class:`dict`, in which keys are :attr:`.Code.id` and values are further - :class:`dict` with keys matching other :class:`.Code` attributes. + - :class:`dict`, in which keys are :attr:`~sdmx.model.common.Code.id` and values are + further :class:`dict` with keys matching other Code attributes. """ # Assemble results as a dictionary result: Dict[str, Code] = {} diff --git a/message_ix_models/workflow.py b/message_ix_models/workflow.py index 3e7fef0c5c..befb1fc635 100644 --- a/message_ix_models/workflow.py +++ b/message_ix_models/workflow.py @@ -136,11 +136,11 @@ def __repr__(self): class Workflow(Computer): - """Workflow for operations on multiple :class:`Scenarios <.Scenario>`. + """Workflow for operations on multiple :class:`Scenarios `. Parameters ---------- - context : .Context + context : Context Context object with settings common to the entire workflow. """ @@ -207,7 +207,7 @@ def truncate(self, name: str): """Truncate the workflow at the step `name`. The step `name` is replaced with a new :class:`WorkflowStep` that simply loads - the target :class:`Scenario` that would be produced by the original step. + the target |Scenario| that would be produced by the original step. Raises ------ From 380ab7098cc1cb3c7e9c9b1d2bedafe70d80bf29 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:28:15 +0200 Subject: [PATCH 41/44] Deprecate .sdmx.eval_anno --- doc/whatsnew.rst | 8 ++++++++ message_ix_models/util/sdmx.py | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 229aa69980..0989b35b05 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -4,6 +4,9 @@ What's new Next release ============ +All changes +----------- + - New module :mod:`.project.ssp` (:pull:`122`) to generate SDMX codelists for the 2017/original SSPs and the 2024 update, and provide these as :class:`~.enum.Enum` to other code. - New module :mod:`.tools.exo_data` to retrieve exogenous data for, among others, population and GDP (:pull:`122`). This module has a general API that can be implemented by provider classes. @@ -15,6 +18,11 @@ Next release - :func:`.same_node` also fills "node_shares", "node_loc", and "node", as appropriate (:pull:`122`). +Deprecations +------------ + +- :func:`.eval_anno` is deprecated; code should instead use :meth:`sdmx.model.common.AnnotableArtefact.eval_annotation`, which provides the same functionality. + v2023.9.2 ========= diff --git a/message_ix_models/util/sdmx.py b/message_ix_models/util/sdmx.py index 96424a5adc..fa43b9d2a1 100644 --- a/message_ix_models/util/sdmx.py +++ b/message_ix_models/util/sdmx.py @@ -92,10 +92,10 @@ def as_codes(data: Union[List[str], Dict[str, CodeLike]]) -> List[Code]: def eval_anno(obj: AnnotableArtefact, id: str): """Retrieve the annotation `id` from `obj`, run :func:`eval` on its contents. - This can be used for unpacking Python values (e.g. :class:`dict`) stored as an - annotation on a :class:`~sdmx.model.Code`. + .. deprecated:: 2023.9.12 - Returns :obj:`None` if no attribute exists with the given `id`. + Use :meth:`sdmx.model.common.AnnotableArtefact.eval_annotation`, which provides + the same functionality. """ try: value = str(obj.get_annotation(id=id).text) From 1c454d9f1819bf0fda28af6944258938b0352023 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:29:14 +0200 Subject: [PATCH 42/44] Add SPHINXOPTS=-n in "pytest" CI workflow Generate warnings about broken references. --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index d3400da183..ec4e8f8550 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -97,7 +97,7 @@ jobs: if: startsWith(matrix.os, 'ubuntu') env: RTD_TOKEN_MESSAGE_DATA: ${{ secrets.RTD_TOKEN_MESSAGE_DATA }} - run: make --directory=doc html + run: make --directory=doc SPHINXOPTS=-n html - name: Upload test coverage to Codecov.io uses: codecov/codecov-action@v3 From 3c14ff6989ae3e13326f8f26b9eff931cbfbfd83 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 21:36:07 +0200 Subject: [PATCH 43/44] Adjust condition for docs build step in "pytest" CI workflow --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index ec4e8f8550..d9ac719cf8 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -94,7 +94,7 @@ jobs: shell: bash - name: Test documentation build using Sphinx - if: startsWith(matrix.os, 'ubuntu') + if: startsWith(matrix.version.os, 'ubuntu') env: RTD_TOKEN_MESSAGE_DATA: ${{ secrets.RTD_TOKEN_MESSAGE_DATA }} run: make --directory=doc SPHINXOPTS=-n html From 4074c838ab8306171fea0dde0b590e2e1c923863 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 11 Sep 2023 23:07:36 +0200 Subject: [PATCH 44/44] Install docs requirements for CI --- .github/workflows/pytest.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index d9ac719cf8..547b1c3dfa 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -75,7 +75,7 @@ jobs: pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.upstream.version }}" pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.upstream.version }}" - pip install .[tests] ${{ matrix.upstream.extra-deps }} + pip install .[docs,tests] ${{ matrix.upstream.extra-deps }} - name: Configure local data path run: | @@ -97,7 +97,7 @@ jobs: if: startsWith(matrix.version.os, 'ubuntu') env: RTD_TOKEN_MESSAGE_DATA: ${{ secrets.RTD_TOKEN_MESSAGE_DATA }} - run: make --directory=doc SPHINXOPTS=-n html + run: make --directory=doc SPHINXOPTS="-n --color" html - name: Upload test coverage to Codecov.io uses: codecov/codecov-action@v3