Merge branch 'main' into bugfix/model-filters

.github/workflows/flake8.yaml → .github/workflows/lint.yaml

@@ -1,10 +1,10 @@
 # copied from https://github.com/marketplace/actions/install-poetry-action
-name: flake8
+name: lint
 
 on: pull_request
 
 jobs:
-  flake8:
+  lint:
     runs-on: ubuntu-latest
     steps:
       #----------------------------------------------
@@ -16,7 +16,7 @@ jobs:
         id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       #----------------------------------------------
       #  -----  install & configure poetry  -----
       #----------------------------------------------
@@ -50,7 +50,19 @@ jobs:
       #----------------------------------------------
       #              run test suite
       #----------------------------------------------
+      - name: Run black
+        run: |
+          source .venv/bin/activate
+          black --check .
+      - name: Run isort
+        run: |
+          source .venv/bin/activate
+          isort --check-only .
       - name: Run flake8
         run: |
           source .venv/bin/activate
           flake8 .
+      - name: Run mypy
+        run: |
+          source .venv/bin/activate
+          mypy .      

.github/workflows/publish.yaml

@@ -22,7 +22,7 @@ jobs:
         id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       #----------------------------------------------
       #  -----  install & configure poetry  -----
       #----------------------------------------------

.github/workflows/pytest.yaml

@@ -5,22 +5,14 @@ on: pull_request
 
 jobs:
   test:
+    strategy:
+      matrix:
+        python-version:
+        - "3.10"  # Earliest version supported by ixmp4
+        - "3.11"  # Latest version supported by ixmp4
+
     runs-on: ubuntu-latest
     services:
-      oracle:
-        image: gvenzl/oracle-xe:latest
-        env:
-          ORACLE_RANDOM_PASSWORD: true
-          APP_USER: ixmp4_test
-          APP_USER_PASSWORD: ixmp4_test
-        ports:
-          - 1521:1521
-        options: >-
-          --health-cmd healthcheck.sh
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 10
-
       postgres:
         image: postgres
         env:
@@ -39,12 +31,12 @@ jobs:
       #       check-out repo and set-up python
       #----------------------------------------------
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python-version }}
       #----------------------------------------------
       #  -----  install & configure poetry  -----
       #----------------------------------------------
@@ -71,11 +63,6 @@ jobs:
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction --no-root --with dev,server
       #----------------------------------------------
-      # install oracle client
-      #----------------------------------------------
-      - name: Setup Oracledb client
-        uses: GoodManWEN/oracle-client-action@main
-      #----------------------------------------------
       # install your root project, if required
       #----------------------------------------------
       - name: Install library

DEVELOPING.md

@@ -207,7 +207,7 @@ alembic revision -m "<message>" --autogenerate
 ```
 
 You will have to run all migrations before being able to create new ones in the development database.
-Be sure to run `black` on newly created migrations!
+Be sure to run `black` and `isort` on newly created migrations!
 
 ## Tests
 
@@ -223,15 +223,13 @@ Or use `pytest` directly:
 py.test
 ```
 
-### Running tests with PostgreSQL and ORACLE
+### Running tests with PostgreSQL
 
-In order to run the local tests with PostgreSQL or ORACLE you'll need to have a local
-instance of this database running.
-The easiest way to do this using a docker container.
+In order to run the local tests with PostgreSQL you'll need to have a local instance
+of this database running. The easiest way to do this using a docker container.
 
-For PostgreSQL using the official [`postgres`](https://hub.docker.com/_/postgres) image
-is recommended. Get the latest version on you local machine using (having docker
-installed):
+Using the official [`postgres`](https://hub.docker.com/_/postgres) image is recommended.
+Get the latest version on you local machine using (having docker installed):
 
 ```console
 docker pull postgres
@@ -243,25 +241,20 @@ and run the container with:
 docker run -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=test -p 5432:5432 -d postgres
 ```
 
-for ORACLE you can use the the [`gvenzl/oracle-xe`](https://hub.docker.com/r/gvenzl/oracle-xe) image:
-
-```console
-docker pull gvenzl/oracle-xe
-docker run -e ORACLE_RANDOM_PASSWORD=true -e APP_USER=ixmp4_test -e APP_USER_PASSWORD=ixmp4_test -p 1521:1521 -d gvenzl/oracle-xe
-```
-
-please note that you'll have to wait for a few seconds for the databases to be up and
+Please note that you'll have to wait for a few seconds for the databases to be up and
 running.
 
-The `tests/docker-compose.yml` file might help you accomplish all of this with a single command for your convenience.
+The `tests/docker-compose.yml` file might help you accomplish all of this with a single
+command for your convenience.
 
 ```console
 docker-compose -f tests/docker-compose.yml up
 ```
 
 ### Profiling
 
-Some tests will output profiler information to the `.profiles/` directory (using the `profiled` fixture). You can analyze these using `snakeviz`. For example:
+Some tests will output profiler information to the `.profiles/` directory (using the
+`profiled` fixture). You can analyze these using `snakeviz`. For example:
 
 ```bash
 snakeviz .profiles/test_add_datapoints_full_benchmark.prof
@@ -317,8 +310,8 @@ It is overwritten on the fly by the poetry-dynamic-versioning plugin.
 
 1. Check that the GitHub action "Publish ixmp4" was executed correctly and that the
    release candidate was successfully uploaded to TestPyPI. The address will be
-   https://test.pypi.org/project/imxp4/<release version>rc<N>. E.g.:
-   <https://test.pypi.org/project/imxp4/0.2.0rc1/>
+   https://test.pypi.org/project/ixmp4/<release version>rc<N>. E.g.:
+   <https://test.pypi.org/project/ixmp4/0.2.0rc1/>
 1. Visit https://github.com/iiasa/ixmp4/releases and mark the new release by creating
    the tag and release simultaneously. The name of the tag is v<release version>
    (without the rc<N>).

README.md

@@ -3,7 +3,7 @@
 Copyright (c) 2023 IIASA - Energy, Climate, and Environment Program (ECE)
 
 [![license: MIT](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://github.com/iiasa/ixmp4/blob/main/LICENSE)
-[![python](https://img.shields.io/badge/python-3.10-blue?logo=python&logoColor=white)](https://github.com/iiasa/ixmp4)
+[![python](https://img.shields.io/badge/python-3.10_|_3.11-blue?logo=python&logoColor=white)](https://github.com/iiasa/ixmp4)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
 ## Overview
@@ -29,7 +29,7 @@ For installing the latest version directly from GitHub do the following.
 
 ### Requirements
 
-This project requires Python 3.10 and poetry (>= 1.2).
+This project requires Python 3.10 (or higher) and poetry (>= 1.2).
 
 ### Setup
 

doc/source/devs/modules.rst

@@ -38,7 +38,7 @@ Package/Folder Structure
 Architecture
 ------------
 
-ixmp4 provides a Python API, a REST API and a compatibility layer for Oracle, Postgres and SQLite Databases.
+ixmp4 provides a Python API, a REST API and a compatibility layer for Postgres and SQLite Databases.
 The Python API can interact with databases directly or use the REST API of a compatible ixmp4 server instance.
 
 ::

doc/source/devs/tests.rst

@@ -24,17 +24,15 @@ Alternatively, use ``pytest`` directly:
 
    py.test
 
-Running tests with PostgreSQL and ORACLE
-----------------------------------------
+Running tests with PostgreSQL
+-----------------------------
 
-In order to run the local tests with PostgreSQL or ORACLE you'll need to have a local
-instance of this database running. The easiest way to do this is using a docker
-container. 
+In order to run the local tests with PostgreSQL you'll need to have a local instance of
+this database running. The easiest way to do this is using a docker container.
 
 The docker container of the database needs to be started first and then the tests can be
-run normally using pytest. If everything is working correctly, the tests for ORACLE or
-PostgreSQL should not be skipped.
-
+run normally using pytest. If everything is working correctly, the tests for PostgreSQL
+should not be skipped.
 
 For PostgreSQL using the official `postgres <https://hub.docker.com/_/postgres>`_ image
 is recommended. Get the latest version on your local machine using (having docker
@@ -50,17 +48,6 @@ and run the container with:
 
    docker run -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=test -p 5432:5432 -d postgres
 
-for ORACLE you can use the `gvenzl/oracle-xe`
-<https://hub.docker.com/r/gvenzl/oracle-xe>`_ image:
-
-.. code:: bash
-
-   docker pull gvenzl/oracle-xe
-   docker run -e ORACLE_RANDOM_PASSWORD=true -e APP_USER=ixmp4_test -e APP_USER_PASSWORD=ixmp4_test -p 1521:1521 -d gvenzl/oracle-xe
-
-please note that you'll have to wait for a few seconds for the databases to be up and
-running.
-
 In case there are any error messages during the start up of the container along those lines:
 
 .. code:: bash

doc/source/index.rst

@@ -11,7 +11,7 @@ Copyright © 2023 IIASA - Energy, Climate, and Environment Program (ECE)
 .. |black| image:: https://img.shields.io/badge/code%20style-black-000000.svg
    :target: https://github.com/psf/black
 
-.. |python| image:: https://img.shields.io/badge/python-3.10-blue?logo=python&logoColor=white
+.. |python| image:: https://img.shields.io/badge/python-3.10_|_3.11-blue?logo=python&logoColor=white
    :target: https://github.com/iiasa/ixmp4
 
 Overview

doc/source/modules.rst

@@ -0,0 +1,70 @@
+Developer Documentation
+=======================
+
+.. toctree::
+   :maxdepth: 1
+
+   ixmp4.core/modules
+   ixmp4.data/modules
+   ixmp4.server/modules
+   ixmp4.cli
+   ixmp4.db
+   ixmp4.db.utils
+   ixmp4.conf
+   tests
+
+
+Package/Folder Structure
+------------------------
+
+.. code:: bash
+
+   .
+   ├── ixmp4
+   │   ├── cli                 # cli
+   │   ├── conf                # configuration module, loads settings etc.
+   │   ├── core                # contains the facade layer for the core python API
+   │   ├── data
+   │   │   ├── abstract        # ABCs for data source models and repositories
+   │   │   ├── api             # data source implementation for the web api
+   │   │   ├── backend         # data source backends
+   │   │   └── db              # data source implementation for databases (sqlalchemy)
+   │   ├── db                  # database management
+   │   ├── server              # web application server
+   │       └── rest            # REST endpoints
+   ├── run                     # runtime artifacts
+   └── tests                   # tests
+
+Architecture
+------------
+
+ixmp4 provides a Python API, a REST API and a compatibility layer for Postgres and SQLite Databases.
+The Python API can interact with databases directly or use the REST API of a compatible ixmp4 server instance.
+
+::
+
+   -> calls -> 
+                            Web or SQL
+            Platform         Backend                    Server       SQL Backend
+      │  ┌────────────┐   ┌───────────┐    ┌─    │   ┌──────────┐   ┌───────────┐  ─┐      │  ┌─┐
+    P │  │            │   │           │    │     │   │          │   │           │   │    S │  │ │
+    y │  │ ┌────────┐ │   │ ┌───────┐ │    │   R │   │ ┌──────┐ │   │ ┌───────┐ │   │    Q │  │D│
+    t │  │ │        │ │   │ │       │ │  ┌─┘   E │   │ │Endp. │ │   │ │       │ │   └─┐  L │  │a│
+    h │  │ │Facade  │ │   │ │Model  │ │  │     S │   │ └──────┘ │   │ │Model  │ │     │  A │  │t│
+    o │  │ └────────┘ │   │ ├───────┤ │  │     T │   │          │   │ ├───────┤ │     │  l │  │a│
+    n │  │            │   │ ├───────┤ │  │       │   │ ┌──────┐ │   │ ├───────┤ │     │  c │  │b│
+      │  │    ...     │   │ │       │ │  │     A │   │ │Endp. │ │   │ │       │ │     │  h │  │a│
+    A │  │            │   │ │Repo.  │ │  └─┐   P │   │ └──────┘ │   │ │Repo.  │ │   ┌─┘  e │  │s│
+    P │  │            │   │ └───────┘ │    │   I │   │          │   │ └───────┘ │   │    m │  │e│
+    I │  │            │   │    ...    │    │     │   │   ...    │   │    ...    │   │    y │  │ │
+      │  └────────────┘   └───────────┘    └─    │   └──────────┘   └───────────┘  ─┘      │  └─┘
+
+           ixmp4.core        ixmp4.data                ixmp4.server      ixmp4.data
+
+Note that a REST SDK in another programming language would have to implement only the
+components before the bracketed part of the diagram (``ixmp4.data.api`` + optionally a facade layer).
+
+Overall both the “facade” layer and the “data source” layer are split
+into “models” (representing a row in a database or a json object) and
+“repositories” (representing a database table or a collection of REST
+endpoints) which manage these models.