Miscellaneous enhancements for 2023-W36

.github/workflows/pytest.yaml

@@ -26,18 +26,17 @@ jobs:
         - { os: ubuntu-latest, python: "3.11" }
         - { os: windows-latest, python: "3.11" }
         # Versions of both ixmp and message_ix to use
-        upstream-version:
-        # Temporarily disabled (iiasa/ixmp#477)
-        # - v3.4.0  # Minimum version given in setup.cfg
-        # - v3.5.0
-        # - v3.6.0
-        - v3.7.0  # Latest released version
-        - main    # Development version
+        upstream:
+        - { version: v3.4.0, extra-deps: '"pandas<2.0"' }  # Minimum version given in setup.cfg
+        - { version: v3.5.0, extra-deps: '"pandas<2.0"' }
+        - { version: v3.6.0, extra-deps: '"pandas<2.0"' }
+        - { version: v3.7.0, extra-deps: "" }  # Latest released version
+        - { version: main, extra-deps: "" }  # Development version
 
       fail-fast: false
 
     runs-on: ${{ matrix.version.os }}
-    name: ${{ matrix.version.os }}-py${{ matrix.version.python }}-upstream-${{ matrix.upstream-version }}
+    name: ${{ matrix.version.os }}-py${{ matrix.version.python }}-upstream-${{ matrix.upstream.version }}
 
     steps:
     - name: Cache test data
@@ -64,10 +63,19 @@ jobs:
         license: ${{ secrets.GAMS_LICENSE }}
 
     - name: Install packages and dependencies
+      # By default, install:
+      # - ixmp, message_ix: from GitHub branches/tags per matrix.upstream-version (above)
+      # - other dependencies including genno: from PyPI.
+      #
+      # To test against unreleased code (on `main`, or other branches
+      # for open PRs), temporarily uncomment, add, or edit lines below
+      # as needed. DO NOT merge such changes to `main`.
       run: |
-        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.upstream-version }}"
-        pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.upstream-version }}"
-        pip install .[tests]
+        # pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main"
+        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.upstream.version }}"
+        pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.upstream.version }}"
+
+        pip install .[docs,tests] ${{ matrix.upstream.extra-deps }}
 
     - name: Configure local data path
       run: |
@@ -86,10 +94,26 @@ jobs:
       shell: bash
 
     - name: Test documentation build using Sphinx
-      if: startsWith(matrix.os, 'ubuntu')
+      if: startsWith(matrix.version.os, 'ubuntu')
       env:
         RTD_TOKEN_MESSAGE_DATA: ${{ secrets.RTD_TOKEN_MESSAGE_DATA }}
-      run: make --directory=doc html
+      run: make --directory=doc SPHINXOPTS="-n --color" html
 
     - name: Upload test coverage to Codecov.io
       uses: codecov/codecov-action@v3
+
+  pre-commit:
+    name: Code quality
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+
+    - name: Force recreation of pre-commit virtual environment for mypy
+      if: github.event_name == 'schedule'  # Comment this line to run on a PR
+      run: gh cache delete $(gh cache list -L 999 | cut -f2 | grep pre-commit)
+      env: { GH_TOKEN: "${{ github.token }}" }
+
+    - uses: pre-commit/[email protected]

.gitignore

@@ -41,6 +41,7 @@ pip-delete-this-directory.txt
 htmlcov/
 .tox/
 .nox/
+.benchmarks
 .coverage
 .coverage.*
 .cache
@@ -50,6 +51,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+.ruff_cache
 
 # Translations
 *.mo
@@ -71,6 +73,7 @@ instance/
 # Sphinx documentation
 doc/_autosummary/
 doc/_build/
+doc/*.inv
 
 # PyBuilder
 target/

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+repos:
+- repo: local
+  hooks:
+  - id: mypy
+    name: mypy
+    always_run: true
+    require_serial: true
+    pass_filenames: false
+
+    language: python
+    entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@"
+    additional_dependencies:
+    - "mypy <1.5"
+    - pytest
+    - sdmx1
+    - types-PyYAML
+    - types-tqdm
+    - "ixmp @ git+https://github.com/iiasa/ixmp.git@main"
+    - "message-ix @ git+https://github.com/iiasa/message_ix.git@main"
+    args: ["."]
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 23.7.0
+  hooks:
+  - id: black
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.0.287
+  hooks:
+  - id: ruff

doc/api/disutility.rst

@@ -14,15 +14,15 @@ Method & usage
 Use this code by calling :func:`add`, which takes arguments that describe the concrete usage:
 
 Consumer groups
-   This is a list of :class:`.Code` objects describing the consumer groups.
+   This is a list of |Code| objects describing the consumer groups.
    The list must be 1-dimensional, but can be composed (as in :mod:`message_data.model.transport`) from multiple dimensions.
 
 Technologies
-   This is a list of :class:`.Code` objects describing the technologies for which the consumers in the different groups experience disutility.
-   Each object must be have 'input' and 'output' annotations (:attr:`.Code.anno`); each of these is a :class:`dict` with the keys 'commodity', 'input', and 'unit', describing the source or sink for the technology.
+   This is a list of |Code| objects describing the technologies for which the consumers in the different groups experience disutility.
+   Each object must be have 'input' and 'output' annotations (:attr:`~.Code.annotations`); each of these is a :class:`dict` with the keys 'commodity', 'input', and 'unit', describing the source or sink for the technology.
 
 Template
-   This is also a :class:`.Code` object, similar to those in ``technologies``; see below.
+   This is also a |Code| object, similar to those in ``technologies``; see below.
 
 The code creates a source technology for the “disutility” commodity.
 The code does *not* perform the following step(s) needed to completely parametrize the formulation:
@@ -113,7 +113,7 @@ For example, the technology “t0” outputs to the commodity “output of t0”
 .. _disutility-units:
 
 (Dis)utility is generally dimensionless.
-In :mod:`pint` and thus also :mod:`message_ix_models`, this should be represented by ``""``.
+In :mod:`.pint` and thus also :mod:`message_ix_models`, this should be represented by ``""``.
 However, to work around `iiasa/ixmp#425 <https://github.com/iiasa/ixmp/issues/425>`__, :func:`data_conversion` and :func:`data_source` return data with ``"-"`` as units.
 See :issue:`45` for more information.
 

doc/api/model-bare.rst

@@ -2,7 +2,7 @@ Reproduce the RES (:mod:`.model.bare`)
 **************************************
 
 In contrast to :mod:`.model.create`, this module creates the RES 'from scratch'.
-:func:`.create_res` begins by creating a new, totally empty :class:`.Scenario` and adding data to it (instead of cloning and modifying an existing scenario).
+:func:`.create_res` begins by creating a new, totally empty :class:`~message_ix.Scenario` and adding data to it (instead of cloning and modifying an existing scenario).
 
 .. note:: Currently, the Scenario returned by :func:`.create_res`…
 
@@ -27,7 +27,7 @@ Code reference
    :members:
    :exclude-members: get_spec
 
-.. automethod:: message_ix_models.model.bare.get_spec
+.. autofunction:: get_spec
 
    Since the RES is the base for all variants of MESSAGEix-GLOBIOM, the 'require' and 'remove' portions of the spec are empty.
 

doc/api/model.rst

@@ -102,4 +102,4 @@ Submodules described on separate pages:
       >>> AUT.parent
       <Code R11_WEU: Western Europe>
 
-   .. seealso:: :func:`.adapt_R11_R14`
+   .. seealso:: :obj:`.adapt_R11_R14`

doc/api/project.rst

@@ -1,7 +1,16 @@
+.. currentmodule:: message_ix_models.project
+
 Specific research projects (:mod:`.project`)
 ********************************************
 
-.. currentmodule:: message_ix_models.project
-
 .. automodule:: message_ix_models.project
    :members:
+
+
+.. currentmodule:: message_ix_models.project.ssp
+
+Shared Socioeconomic Pathways (:mod:`.project.ssp`)
+===================================================
+
+.. automodule:: message_ix_models.project.ssp
+   :members:

doc/api/report/index.rst

@@ -32,7 +32,7 @@ Any reporting specific to ``coal_ppl`` must be in :mod:`message_ix_models`, sinc
 The basic **design pattern** of :mod:`message_ix_models.report` is:
 
 - A ``global.yaml`` file (i.e. in `YAML <https://en.wikipedia.org/wiki/YAML#Example>`_ format) that contains a *concise* yet *explicit* description of the reporting computations needed for a MESSAGE-GLOBIOM model.
-- :func:`~.reporting.prepare_reporter` reads the file and a Scenario object, and uses it to populate a new Reporter.
+- :func:`~.report.prepare_reporter` reads the file and a Scenario object, and uses it to populate a new Reporter.
   This function mostly relies on the :doc:`configuration handlers <genno:config>` built in to Genno to handle the different sections of the file.
 
 Features
@@ -106,7 +106,7 @@ Operators
 .. automodule:: message_ix_models.report.computations
    :members:
 
-   :mod:`message_ix_models` provides the following:
+   :mod:`message_ix_models.report.computations` provides the following:
 
    .. autosummary::
 
@@ -118,11 +118,28 @@ Operators
       remove_ts
       share_curtailment
 
-   Other operators are provided by:
+   Other operators or genno-compatible functions are provided by:
 
-   - :mod:`message_ix.reporting.computations`
-   - :mod:`ixmp.reporting.computations`
-   - :mod:`genno.computations`
+   - Upstream packages:
+
+     - :mod:`message_ix.reporting.computations`
+     - :mod:`ixmp.reporting.computations`
+     - :mod:`genno.computations`
+
+   - Other submodules:
+
+     - :mod:`.model.emissions`: :func:`.get_emission_factors`.
+
+   Any of these can be made available for a :class:`.Computer` instance using :meth:`~.genno.Computer.require_compat`, for instance:
+
+   .. code-block::
+
+      # Indicate that a certain module contains functions to
+      # be referenced by name
+      c.require_compat("message_ix_models.model.emissions")
+
+      # Add computations to the graph by referencing functions
+      c.add("ef:c", "get_emission_factors", units="t C / kWa")
 
 Utilities
 ---------

doc/api/tools.rst

@@ -17,12 +17,33 @@ On this page:
 .. automodule:: message_ix_models.tools
    :members:
 
+.. currentmodule:: message_ix_models.tools.exo_data
 
-ADVANCE data (:mod:`.tools.advance`)
-====================================
+Exogenous data (:mod:`.tools.exo_data`)
+=======================================
+
+.. automodule:: message_ix_models.tools.exo_data
+   :members:
+   :exclude-members: ExoDataSource
+
+   .. autosummary::
+
+      MEASURES
+      SOURCES
+      DemoSource
+      ExoDataSource
+      prepare_computer
+      register_source
+
+.. autoclass:: ExoDataSource
+   :members:
+   :special-members: __init__, __call__
 
 .. currentmodule:: message_ix_models.tools.advance
 
+ADVANCE data (:mod:`.tools.advance`)
+====================================
+
 .. autosummary::
    get_advance_data
    advance_data

doc/api/util.rst

@@ -15,6 +15,7 @@ Submodules:
    node
    pooch
    scenarioinfo
+   sdmx
 
 Commonly used:
 
@@ -25,9 +26,9 @@ Commonly used:
    ~context.Context
    ~scenarioinfo.ScenarioInfo
    ~scenarioinfo.Spec
-   adapt_R11_R12
-   adapt_R11_R14
-   as_codes
+   .adapt_R11_R12
+   .adapt_R11_R14
+   .as_codes
    broadcast
    cached
    check_support
@@ -52,6 +53,7 @@ Commonly used:
 
 .. automodule:: message_ix_models.util
    :members:
+   :exclude-members: as_codes, eval_anno
 
 .. autodata:: message_ix_models.util.cache.SKIP_CACHE
 
@@ -117,7 +119,7 @@ Commonly used:
 
    .. automethod:: clone_to_dest
 
-      To use this method, either decorate a command with :func:`common_params`:
+      To use this method, either decorate a command with :func:`.common_params`:
 
       .. code-block:: python
 
@@ -163,7 +165,7 @@ Commonly used:
    :members:
 
 :mod:`.util.node`
-==================
+=================
 
 .. currentmodule:: message_ix_models.util.node
 
@@ -187,3 +189,11 @@ Commonly used:
 
 .. automodule:: message_ix_models.util.scenarioinfo
    :members:
+
+:mod:`.util.sdmx`
+=================
+
+.. currentmodule:: message_ix_models.util.sdmx
+
+.. automodule:: message_ix_models.util.sdmx
+   :members: