Merge branch 'master' into #1083

mitogen-hq · Oct 14, 2024 · 58efb5c · 58efb5c
2 parents b8e2d4b + 3b2b03b
commit 58efb5c
Show file tree

Hide file tree

Showing 24 changed files with 191 additions and 212 deletions.
diff --git a/.ci/README.md b/.ci/README.md
@@ -28,14 +28,15 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
-  name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
-* `DISTROS`: the `ansible_` tests can run against multiple targets
-  simultaneously, which speeds things up. This is a space-separated list of
-  DISTRO names, but additionally, supports:
+* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run
+  the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines
+  the Linux distribution, target Python interepreter & number of instances.
+  Only distributions with a pre-built Linux container image can be used.
     * `debian-py3`: when generating Ansible inventory file, set
       `ansible_python_interpreter` to `python3`, i.e. run a test where the
       target interpreter is Python 3.
     * `debian*16`: generate 16 Docker containers running Debian. Also works
       with -py3.
 
+* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name,
+  and hence the container registry used for test targets.
diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py
@@ -35,7 +35,7 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.container_specs(ci_lib.DISTROS)
+    containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split())
     ci_lib.start_containers(containers)
 
 

diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py
@@ -28,6 +28,10 @@
 )
 
 
+DISTRO_SPECS = os.environ.get(
+    'MITOGEN_TEST_DISTRO_SPECS',
+    'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004',
+)
 IMAGE_TEMPLATE = os.environ.get(
     'MITOGEN_TEST_IMAGE_TEMPLATE',
     'public.ecr.aws/n5z0e8q9/%(distro)s-test',
@@ -196,10 +200,6 @@ def __exit__(self, _1, _2, _3): pass
 
 
 GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-# Used only when MODE=mitogen
-DISTRO = os.environ.get('DISTRO', 'debian9')
-# Used only when MODE=ansible
-DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
 TMP = TempDir().path
 
 

diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py
@@ -3,8 +3,6 @@
 
 from __future__ import print_function
 
-import getpass
-import io
 import os
 import subprocess
 import sys
@@ -57,18 +55,6 @@
         os.chdir(IMAGE_PREP_DIR)
         ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
 
-    # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022
-    #       and os.environ['USER'] is not populated on Azure macOS runners.
-    os.chdir(HOSTS_DIR)
-    with io.open('default.hosts', 'r+', encoding='utf-8') as f:
-        user = getpass.getuser()
-        content = f.read()
-        content = content.replace("{{ lookup('pipe', 'whoami') }}", user)
-        f.seek(0)
-        f.write(content)
-        f.truncate()
-    ci_lib.dump_file('default.hosts')
-
     cmd = ';'.join([
         'from __future__ import print_function',
         'import os, sys',

diff --git a/.ci/mitogen_py24_tests.py b/.ci/mitogen_py24_tests.py
@@ -8,8 +8,6 @@
 os.environ.update({
     'NOCOVERAGE': '1',
     'UNIT2': '/usr/local/python2.4.6/bin/unit2',
-
-    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
     'MITOGEN_LOG_LEVEL': 'debug',
     'SKIP_ANSIBLE': '1',
 })

diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py
@@ -6,7 +6,6 @@
 import ci_lib
 
 os.environ.update({
-    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
     'MITOGEN_LOG_LEVEL': 'debug',
     'SKIP_ANSIBLE': '1',
 })

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -48,99 +48,33 @@ jobs:
           - name: Ans_311_5
             python_version: '3.11'
             tox_env: py311-mode_ansible-ansible5
-          - name: Ans_312_6
-            python_version: '3.12'
-            tox_env: py312-mode_ansible-ansible6
-          - name: Ans_312_7
-            python_version: '3.12'
-            tox_env: py312-mode_ansible-ansible7
-          - name: Ans_312_8
-            python_version: '3.12'
-            tox_env: py312-mode_ansible-ansible8
-          - name: Ans_312_9
-            python_version: '3.12'
-            tox_env: py312-mode_ansible-ansible9
-          - name: Ans_312_10
-            python_version: '3.12'
-            tox_env: py312-mode_ansible-ansible10
-          - name: Van_312_10
-            python_version: '3.12'
-            tox_env: py312-mode_ansible-ansible10-strategy_linear
-
-          - name: Mito_27_centos6
-            tox_env: py27-mode_mitogen-distro_centos6
-          - name: Mito_27_centos7
-            tox_env: py27-mode_mitogen-distro_centos7
-          - name: Mito_27_centos8
-            tox_env: py27-mode_mitogen-distro_centos8
-          - name: Mito_27_debian9
-            tox_env: py27-mode_mitogen-distro_debian9
-          - name: Mito_27_debian10
-            tox_env: py27-mode_mitogen-distro_debian10
-          - name: Mito_27_debian11
-            tox_env: py27-mode_mitogen-distro_debian11
-          - name: Mito_27_ubuntu1604
-            tox_env: py27-mode_mitogen-distro_ubuntu1604
-          - name: Mito_27_ubuntu1804
-            tox_env: py27-mode_mitogen-distro_ubuntu1804
-          - name: Mito_27_ubuntu2004
-            tox_env: py27-mode_mitogen-distro_ubuntu2004
-
-          - name: Mito_36_centos6
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos6
-          - name: Mito_36_centos7
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos7
-          - name: Mito_36_centos8
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos8
-          - name: Mito_36_debian9
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian9
-          - name: Mito_36_debian10
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian10
-          - name: Mito_36_debian11
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian11
-          - name: Mito_36_ubuntu1604
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu1604
-          - name: Mito_36_ubuntu1804
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu1804
-          - name: Mito_36_ubuntu2004
+          - name: Ans_313_6
+            python_version: '3.13'
+            tox_env: py313-mode_ansible-ansible6
+          - name: Ans_313_7
+            python_version: '3.13'
+            tox_env: py313-mode_ansible-ansible7
+          - name: Ans_313_8
+            python_version: '3.13'
+            tox_env: py313-mode_ansible-ansible8
+          - name: Ans_313_9
+            python_version: '3.13'
+            tox_env: py313-mode_ansible-ansible9
+          - name: Ans_313_10
+            python_version: '3.13'
+            tox_env: py313-mode_ansible-ansible10
+          - name: Van_313_10
+            python_version: '3.13'
+            tox_env: py313-mode_ansible-ansible10-strategy_linear
+
+          - name: Mito_27
+            tox_env: py27-mode_mitogen
+          - name: Mito_36
             python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu2004
-
-          - name: Mito_312_centos6
-            python_version: '3.12'
-            tox_env: py312-mode_mitogen-distro_centos6
-          - name: Mito_312_centos7
-            python_version: '3.12'
-            tox_env: py312-mode_mitogen-distro_centos7
-          - name: Mito_312_centos8
-            python_version: '3.12'
-            tox_env: py312-mode_mitogen-distro_centos8
-          - name: Mito_312_debian9
-            python_version: '3.12'
-            tox_env: py312-mode_mitogen-distro_debian9
-          - name: Mito_312_debian10
-            python_version: '3.12'
-            tox_env: py312-mode_mitogen-distro_debian10
-          - name: Mito_312_debian11
-            python_version: '3.12'
-            tox_env: py312-mode_mitogen-distro_debian11
-          - name: Mito_312_ubuntu1604
-            python_version: '3.12'
-            tox_env: py312-mode_mitogen-distro_ubuntu1604
-          - name: Mito_312_ubuntu1804
-            python_version: '3.12'
-            tox_env: py312-mode_mitogen-distro_ubuntu1804
-          - name: Mito_312_ubuntu2004
-            python_version: '3.12'
-            tox_env: py312-mode_mitogen-distro_ubuntu2004
+            tox_env: py36-mode_mitogen
+          - name: Mito_313
+            python_version: '3.13'
+            tox_env: py313-mode_mitogen
 
     steps:
       - uses: actions/checkout@v4
@@ -232,18 +166,21 @@ jobs:
         include:
           - name: Mito_27
             tox_env: py27-mode_mitogen
-          - name: Mito_312
-            tox_env: py312-mode_mitogen
+          - name: Mito_313
+            python_version: '3.13'
+            tox_env: py313-mode_mitogen
 
           - name: Loc_27_210
             tox_env: py27-mode_localhost-ansible2.10
-          - name: Loc_312_10
-            tox_env: py312-mode_localhost-ansible10
+          - name: Loc_313_10
+            python_version: '3.13'
+            tox_env: py313-mode_localhost-ansible10
 
           - name: Van_27_210
             tox_env: py27-mode_localhost-ansible2.10-strategy_linear
-          - name: Van_312_10
-            tox_env: py312-mode_localhost-ansible10-strategy_linear
+          - name: Van_313_10
+            python_version: '3.13'
+            tox_env: py313-mode_localhost-ansible10-strategy_linear
 
     steps:
       - uses: actions/checkout@v4

diff --git a/README.md b/README.md
@@ -1,5 +1,7 @@
 # Mitogen
 
+[![PyPI - Version](https://img.shields.io/pypi/v/mitogen)](https://pypi.org/project/mitogen/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mitogen)](https://pypi.org/project/mitogen/)
 [![Build Status](https://img.shields.io/github/actions/workflow/status/mitogen-hq/mitogen/tests.yml?branch=master)](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster)
 
 <a href="https://mitogen.networkgenomics.com/">Please see the documentation</a>.

diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst
@@ -132,13 +132,13 @@ Noteworthy Differences
   | 5               | 3.8 - 3.11      |
   +-----------------+-----------------+
   | 6               |                 |
-  +-----------------+ 3.8 - 3.12      |
+  +-----------------+ 3.8 - 3.13      |
   | 7               |                 |
   +-----------------+-----------------+
-  | 8               | 3.9 - 3.12      |
+  | 8               | 3.9 - 3.13      |
   +-----------------+-----------------+
   | 9               |                 |
-  +-----------------+ 3.10 - 3.12     |
+  +-----------------+ 3.10 - 3.13     |
   | 10              |                 |
   +-----------------+-----------------+
 

diff --git a/docs/changelog.rst b/docs/changelog.rst
@@ -18,18 +18,26 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
-In Progress (unreleased)
+In progress (unreleased)
 ------------------------
 
+* :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker
+  SSH tests
+* :gh:issue:`1083` fix: Jinja expressions are not evaluated in (not only) credential variables.
+
+
+v0.3.13 (2024-10-09)
+--------------------
+
 * :gh:issue:`1138` CI: Complete migration from Azure DevOps Pipelines to
   GitHub Actions
 * :gh:issue:`1116` :mod:`ansible_mitogen`: Support for templated variable
   `ansible_ssh_user`.
 * :gh:issue:`978` :mod:`ansible_mitogen`: Support templated Ansible SSH port.
-* :gh:issue:`1083` fix: Jinja expressions are not evaluated in (not only) credential variables.
+* :gh:issue:`1073` Python 3.13 support
 
 
-v0.3.11 (2024-10-07)
+v0.3.12 (2024-10-07)
 --------------------
 
 * :gh:issue:`1106` :mod:`ansible_mitogen`: Support for `ansible_ssh_password`

diff --git a/examples/the_basics.py b/examples/the_basics.py
@@ -10,7 +10,6 @@
 import hashlib
 import io
 import os
-import spwd
 
 import mitogen.core
 import mitogen.master
@@ -57,21 +56,6 @@ def streamy_download_file(context, path):
     }
 
 
-def get_password_hash(username):
-    """
-    Fetch a user's password hash.
-    """
-    try:
-        h = spwd.getspnam(username)
-    except KeyError:
-        return None
-
-    # mitogen.core.Secret() is a Unicode subclass with a repr() that hides the
-    # secret data. This keeps secret stuff out of logs. Like blobs, secrets can
-    # also be serialized.
-    return mitogen.core.Secret(h)
-
-
 def md5sum(path):
     """
     Return the MD5 checksum for a file.

diff --git a/mitogen/__init__.py b/mitogen/__init__.py
@@ -35,7 +35,7 @@
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 13, 'dev')
+__version__ = (0, 3, 14, 'dev')
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

diff --git a/setup.py b/setup.py
@@ -79,6 +79,7 @@ def long_description():
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: 3.12',
+        'Programming Language :: Python :: 3.13',
         'Programming Language :: Python :: Implementation :: CPython',
         'Topic :: System :: Distributed Computing',
         'Topic :: System :: Systems Administration',

diff --git a/tests/README.md b/tests/README.md
@@ -30,11 +30,19 @@ and run the tests there.
 1. Run ``test``
 
 
-# Selecting a target distribution
+# Selecting target distributions
 
-Docker target images exist for testing against CentOS and Debian, with the
-default being Debian. To select CentOS, specify `MITOGEN_TEST_DISTRO=centos` in
-the environment.
+Linux container images for testing are available at
+
+- https://github.com/orgs/mitogen-hq/packages
+- https://public.ecr.aws/n5z0e8q9
+
+The images used are determined by two environment variables
+
+- `MITOGEN_TEST_DISTRO_SPECS`
+- `MITOGEN_TEST_IMAGE_TEMPLATE`
+
+Defaults for these can be found in `.ci/ci_lib.py` & `tests/testlib.py`
 
 
 # User Accounts

diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts
@@ -4,8 +4,7 @@
 # When running the tests outside CI, make a single 'target' host which is the
 # local machine. The ansible_user override is necessary since some tests want a
 # fixed ansible.cfg remote_user setting to test against.
-# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022
-#       and os.environ['USER'] is not populated on Azure macOS runners.
+# os.environ['USER'] is an empty string on GitHub Actions macOS runners.
 target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
 
 [test-targets]

diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml
@@ -3,5 +3,7 @@
 - import_playbook: password.yml
 - import_playbook: timeouts.yml
 - import_playbook: templated_by_inv.yml
+- import_playbook: templated_by_play_keyword.yml
 - import_playbook: templated_by_play_taskvar.yml
+- import_playbook: templated_by_task_keyword.yml
 - import_playbook: variables.yml