From 34d441fb878447e6fa025c8c668208d7044f6429 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 7 Oct 2024 10:03:32 -0700 Subject: [PATCH 01/11] Remove get_password_hash, unused spwd is removed in Python 3.13. But fortunately, this function itself is never used. Part of: #1073 --- examples/the_basics.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/examples/the_basics.py b/examples/the_basics.py index 0dcd4049d..af9903b00 100644 --- a/examples/the_basics.py +++ b/examples/the_basics.py @@ -10,7 +10,6 @@ import hashlib import io import os -import spwd import mitogen.core import mitogen.master @@ -57,21 +56,6 @@ def streamy_download_file(context, path): } -def get_password_hash(username): - """ - Fetch a user's password hash. - """ - try: - h = spwd.getspnam(username) - except KeyError: - return None - - # mitogen.core.Secret() is a Unicode subclass with a repr() that hides the - # secret data. This keeps secret stuff out of logs. Like blobs, secrets can - # also be serialized. - return mitogen.core.Secret(h) - - def md5sum(path): """ Return the MD5 checksum for a file. From 9cdd51cf5b584bf828354301ab97d4a2e13e30c7 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 8 Oct 2024 15:25:13 +0100 Subject: [PATCH 02/11] Declare Python 3.13 support No code changes needed, that I could find. --- .github/workflows/tests.yml | 105 ++++++++++++++++++------------------ docs/ansible_detailed.rst | 6 +-- docs/changelog.rst | 1 + setup.py | 1 + tests/requirements.txt | 2 +- tox.ini | 9 ++-- 6 files changed, 65 insertions(+), 59 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2c819e7d9..f3f31d82f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,24 +48,24 @@ jobs: - name: Ans_311_5 python_version: '3.11' tox_env: py311-mode_ansible-ansible5 - - name: Ans_312_6 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible6 - - name: Ans_312_7 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible7 - - name: Ans_312_8 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible8 - - name: Ans_312_9 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible9 - - name: Ans_312_10 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible10 - - name: Van_312_10 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible10-strategy_linear + - name: Ans_313_6 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible6 + - name: Ans_313_7 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible7 + - name: Ans_313_8 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible8 + - name: Ans_313_9 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible9 + - name: Ans_313_10 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible10 + - name: Van_313_10 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible10-strategy_linear - name: Mito_27_centos6 tox_env: py27-mode_mitogen-distro_centos6 @@ -114,33 +114,33 @@ jobs: python_version: '3.6' tox_env: py36-mode_mitogen-distro_ubuntu2004 - - name: Mito_312_centos6 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_centos6 - - name: Mito_312_centos7 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_centos7 - - name: Mito_312_centos8 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_centos8 - - name: Mito_312_debian9 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_debian9 - - name: Mito_312_debian10 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_debian10 - - name: Mito_312_debian11 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_debian11 - - name: Mito_312_ubuntu1604 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_ubuntu1604 - - name: Mito_312_ubuntu1804 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_ubuntu1804 - - name: Mito_312_ubuntu2004 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_ubuntu2004 + - name: Mito_313_centos6 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_centos6 + - name: Mito_313_centos7 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_centos7 + - name: Mito_313_centos8 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_centos8 + - name: Mito_313_debian9 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_debian9 + - name: Mito_313_debian10 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_debian10 + - name: Mito_313_debian11 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_debian11 + - name: Mito_313_ubuntu1604 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_ubuntu1604 + - name: Mito_313_ubuntu1804 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_ubuntu1804 + - name: Mito_313_ubuntu2004 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_ubuntu2004 steps: - uses: actions/checkout@v4 @@ -232,18 +232,21 @@ jobs: include: - name: Mito_27 tox_env: py27-mode_mitogen - - name: Mito_312 - tox_env: py312-mode_mitogen + - name: Mito_313 + python_version: '3.13' + tox_env: py313-mode_mitogen - name: Loc_27_210 tox_env: py27-mode_localhost-ansible2.10 - - name: Loc_312_10 - tox_env: py312-mode_localhost-ansible10 + - name: Loc_313_10 + python_version: '3.13' + tox_env: py313-mode_localhost-ansible10 - name: Van_27_210 tox_env: py27-mode_localhost-ansible2.10-strategy_linear - - name: Van_312_10 - tox_env: py312-mode_localhost-ansible10-strategy_linear + - name: Van_313_10 + python_version: '3.13' + tox_env: py313-mode_localhost-ansible10-strategy_linear steps: - uses: actions/checkout@v4 diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 0d7966912..73ebba4dd 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -132,13 +132,13 @@ Noteworthy Differences | 5 | 3.8 - 3.11 | +-----------------+-----------------+ | 6 | | - +-----------------+ 3.8 - 3.12 | + +-----------------+ 3.8 - 3.13 | | 7 | | +-----------------+-----------------+ - | 8 | 3.9 - 3.12 | + | 8 | 3.9 - 3.13 | +-----------------+-----------------+ | 9 | | - +-----------------+ 3.10 - 3.12 | + +-----------------+ 3.10 - 3.13 | | 10 | | +-----------------+-----------------+ diff --git a/docs/changelog.rst b/docs/changelog.rst index 4f545615c..f00de0712 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,7 @@ In Progress (unreleased) * :gh:issue:`1116` :mod:`ansible_mitogen`: Support for templated variable `ansible_ssh_user`. * :gh:issue:`978` :mod:`ansible_mitogen`: Support templated Ansible SSH port. +* :gh:issue:`1073` Python 3.13 support v0.3.11 (2024-10-07) diff --git a/setup.py b/setup.py index b17dab9dd..9cb304272 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,7 @@ def long_description(): 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', diff --git a/tests/requirements.txt b/tests/requirements.txt index d64d8b873..c5671b37e 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,5 @@ cffi==1.15.1; python_version < '3.8' -cffi==1.16; python_version >= '3.8' +cffi==1.17.1; python_version >= '3.8' coverage==5.5; python_version == '2.7' coverage==6.2; python_version == '3.6' diff --git a/tox.ini b/tox.ini index 7677ed27d..6a0eb1806 100644 --- a/tox.ini +++ b/tox.ini @@ -55,10 +55,10 @@ envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, py{311}-mode_ansible-ansible{2.10,3,4,5}, - py{312}-mode_ansible-ansible{6,7,8,9,10}, - py{27,36,312}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,312}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{313}-mode_ansible-ansible{6,7,8,9,10}, + py{27,36,313}-mode_mitogen-distro_centos{6,7,8}, + py{27,36,313}-mode_mitogen-distro_debian{9,10,11}, + py{27,36,313}-mode_mitogen-distro_ubuntu{1604,1804,2004}, report, [testenv] @@ -73,6 +73,7 @@ basepython = py310: python3.10 py311: python3.11 py312: python3.12 + py313: python3.13 deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt From 62b75f7750569e840ed32e0a2515275bdf721dbb Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 9 Oct 2024 10:48:26 +0100 Subject: [PATCH 03/11] docs: shields.io badges for PyPI version & supported Python versions --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e29546720..c3cd9a87d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Mitogen +[![PyPI - Version](https://img.shields.io/pypi/v/mitogen)](https://pypi.org/project/mitogen/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mitogen)](https://pypi.org/project/mitogen/) [![Build Status](https://img.shields.io/github/actions/workflow/status/mitogen-hq/mitogen/tests.yml?branch=master)](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster) Please see the documentation. From b91407a7792e1e827082cbfda0076e5ef45c99f9 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 9 Oct 2024 11:13:42 +0100 Subject: [PATCH 04/11] docs: Correct v0.3.12 version in changelog fixes #1149 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f00de0712..016c485e8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,7 +29,7 @@ In Progress (unreleased) * :gh:issue:`1073` Python 3.13 support -v0.3.11 (2024-10-07) +v0.3.12 (2024-10-07) -------------------- * :gh:issue:`1106` :mod:`ansible_mitogen`: Support for `ansible_ssh_password` From 8dec03894170353a7588b1414cf88772d330d1f0 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 9 Oct 2024 11:14:23 +0100 Subject: [PATCH 05/11] Prepare v0.3.13 --- docs/changelog.rst | 4 ++-- mitogen/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 016c485e8..67753daf2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,8 +18,8 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -In Progress (unreleased) ------------------------- +v0.3.13 (2024-10-09) +-------------------- * :gh:issue:`1138` CI: Complete migration from Azure DevOps Pipelines to GitHub Actions diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 8320f5e7e..48baa7c8b 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ #: Library version as a tuple. -__version__ = (0, 3, 13, 'dev') +__version__ = (0, 3, 13) #: This is :data:`False` in slave contexts. Previously it was used to prevent From 47e25eb8c596ac917428fc7c65bb24075f47d4e4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 9 Oct 2024 11:16:41 +0100 Subject: [PATCH 06/11] Begin 0.3.14 development --- docs/changelog.rst | 5 +++++ mitogen/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 67753daf2..092fed261 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,11 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +In progress (unreleased) +------------------------ + + + v0.3.13 (2024-10-09) -------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 48baa7c8b..36c448d9c 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ #: Library version as a tuple. -__version__ = (0, 3, 13) +__version__ = (0, 3, 14, 'dev') #: This is :data:`False` in slave contexts. Previously it was used to prevent From 5d6a185242ca93ab45e27bd296ad2dcca45b74f8 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 9 Oct 2024 13:16:42 +0100 Subject: [PATCH 07/11] tests: Templated "remote_user" provided as Ansible playbook keyword The password is provided as a variable because there is no corresponding keyword. I get the impression that keywords are considered a legacy mechanism, so most (new) options are only overridable by variables. The port is proved as a variable for now, to test remote_name in isolation. --- tests/ansible/integration/ssh/all.yml | 1 + .../integration/ssh/templated_by_play_keyword.yml | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/ansible/integration/ssh/templated_by_play_keyword.yml diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml index ab4000517..ce964c3e3 100644 --- a/tests/ansible/integration/ssh/all.yml +++ b/tests/ansible/integration/ssh/all.yml @@ -3,5 +3,6 @@ - import_playbook: password.yml - import_playbook: timeouts.yml - import_playbook: templated_by_inv.yml +- import_playbook: templated_by_play_keyword.yml - import_playbook: templated_by_play_taskvar.yml - import_playbook: variables.yml diff --git a/tests/ansible/integration/ssh/templated_by_play_keyword.yml b/tests/ansible/integration/ssh/templated_by_play_keyword.yml new file mode 100644 index 000000000..e66cc5f37 --- /dev/null +++ b/tests/ansible/integration/ssh/templated_by_play_keyword.yml @@ -0,0 +1,11 @@ +- name: integration/ssh/templated_by_play_keyword.yml + hosts: tt_targets_bare + gather_facts: false + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + tasks: + - meta: reset_connection + - name: Templated variables in play keywords + ping: From 5e816be12c8ecafd1c07fa323a2ac75b05c008cc Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 9 Oct 2024 14:16:17 +0100 Subject: [PATCH 08/11] tests: Templated connection keywords with delegated_to --- tests/ansible/integration/ssh/all.yml | 1 + .../ssh/templated_by_task_keyword.yml | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/ansible/integration/ssh/templated_by_task_keyword.yml diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml index ce964c3e3..eada992a7 100644 --- a/tests/ansible/integration/ssh/all.yml +++ b/tests/ansible/integration/ssh/all.yml @@ -5,4 +5,5 @@ - import_playbook: templated_by_inv.yml - import_playbook: templated_by_play_keyword.yml - import_playbook: templated_by_play_taskvar.yml +- import_playbook: templated_by_task_keyword.yml - import_playbook: variables.yml diff --git a/tests/ansible/integration/ssh/templated_by_task_keyword.yml b/tests/ansible/integration/ssh/templated_by_task_keyword.yml new file mode 100644 index 000000000..df956af5b --- /dev/null +++ b/tests/ansible/integration/ssh/templated_by_task_keyword.yml @@ -0,0 +1,24 @@ +- name: integration/ssh/templated_by_task_keyword.yml + hosts: tt_targets_bare + gather_facts: false + # FIXME Resetting the connection shouldn't require credentials + # https://github.com/mitogen-hq/mitogen/issues/1132 + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + tasks: + - name: Reset connection to target that will be delegate_to + meta: reset_connection + +- name: Test connection template by task keywords, with delegate_to + hosts: test-targets[0] + gather_facts: false + tasks: + - name: Templated by task keywords, with delegate_to + delegate_to: "{{ groups.tt_targets_bare[0] }}" + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + ping: From 9859e44ee841fe7655d8bc9a2b74580be32a4810 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 10 Oct 2024 15:23:49 +0100 Subject: [PATCH 09/11] tests: Standardise on DockerizedSshDaemon.host & .port --- tests/ssh_test.py | 4 ++-- tests/testlib.py | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/ssh_test.py b/tests/ssh_test.py index ce7dce963..f8fad55ae 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -68,7 +68,7 @@ def test_stream_name(self): password='has_sudo_password', ) name = 'ssh.%s:%s' % ( - self.dockerized_ssh.get_host(), + self.dockerized_ssh.host, self.dockerized_ssh.port, ) self.assertEqual(name, context.name) @@ -186,7 +186,7 @@ def test_verbose_enabled(self): ssh_debug_level=3, ) name = 'ssh.%s:%s' % ( - self.dockerized_ssh.get_host(), + self.dockerized_ssh.host, self.dockerized_ssh.port, ) self.assertEqual(name, context.name) diff --git a/tests/testlib.py b/tests/testlib.py index 76743e823..0fd9c7d7a 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -574,14 +574,11 @@ def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): self.image = image_template % d self.start_container() - self.host = self.get_host() + self.host = get_docker_host() self.port = self.get_port(self.container_name) - def get_host(self): - return get_docker_host() - def wait_for_sshd(self): - wait_for_port(self.get_host(), self.port, pattern='OpenSSH') + wait_for_port(self.host, self.port, pattern='OpenSSH') def check_processes(self): # Get Accounting name (ucomm) & command line (args) of each process From 28e08ef94c5a57ee672511e06ed594d49284644f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 10 Oct 2024 18:33:44 +0100 Subject: [PATCH 10/11] ci: Reduce number of Jobs by parameterizing Mitogen Docker SSH tests This reduces the number of jobs from 48 to 24. The Mitogen part of the test suite has been parameterized on the Linux container targets to be run against. Both the Ansible tests & Mitogen tests now use the same source of truth to control which targets to use: environment variable MITOGEN_TEST_DISTRO_SPECS. This replaces the two mutually exclusive env vars DISTRO and DISTROS. I've also removed vestgial traces of an unused env var MITOGEN_TEST_DISTRO. Parameterization adapted from https://eli.thegreenplace.net/2014/04/02/dynamically-generating-python-test-cases refs #1058, #1059 --- .ci/README.md | 11 +++--- .ci/ansible_tests.py | 2 +- .ci/ci_lib.py | 8 ++-- .ci/mitogen_py24_tests.py | 2 - .ci/mitogen_tests.py | 1 - .github/workflows/tests.yml | 78 +++---------------------------------- docs/changelog.rst | 2 + tests/README.md | 16 ++++++-- tests/fakessh_test.py | 3 +- tests/ssh_test.py | 26 ++++++++++++- tests/su_test.py | 13 ++++++- tests/testlib.py | 25 ++++++------ tox.ini | 52 +++++++++---------------- 13 files changed, 101 insertions(+), 138 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 9a5e8898e..17b7d2dd8 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -28,14 +28,15 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This - name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` -* `DISTROS`: the `ansible_` tests can run against multiple targets - simultaneously, which speeds things up. This is a space-separated list of - DISTRO names, but additionally, supports: +* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run + the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines + the Linux distribution, target Python interepreter & number of instances. + Only distributions with a pre-built Linux container image can be used. * `debian-py3`: when generating Ansible inventory file, set `ansible_python_interpreter` to `python3`, i.e. run a test where the target interpreter is Python 3. * `debian*16`: generate 16 Docker containers running Debian. Also works with -py3. +* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name, + and hence the container registry used for test targets. diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 3ec48dfde..62dfa8f5b 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -35,7 +35,7 @@ def pause_if_interactive(): with ci_lib.Fold('docker_setup'): - containers = ci_lib.container_specs(ci_lib.DISTROS) + containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split()) ci_lib.start_containers(containers) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index dfe49b97f..afb62e023 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -28,6 +28,10 @@ ) +DISTRO_SPECS = os.environ.get( + 'MITOGEN_TEST_DISTRO_SPECS', + 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', +) IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', 'public.ecr.aws/n5z0e8q9/%(distro)s-test', @@ -196,10 +200,6 @@ def __exit__(self, _1, _2, _3): pass GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -# Used only when MODE=mitogen -DISTRO = os.environ.get('DISTRO', 'debian9') -# Used only when MODE=ansible -DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() TMP = TempDir().path diff --git a/.ci/mitogen_py24_tests.py b/.ci/mitogen_py24_tests.py index 228e79bdd..96b144eb3 100755 --- a/.ci/mitogen_py24_tests.py +++ b/.ci/mitogen_py24_tests.py @@ -8,8 +8,6 @@ os.environ.update({ 'NOCOVERAGE': '1', 'UNIT2': '/usr/local/python2.4.6/bin/unit2', - - 'MITOGEN_TEST_DISTRO': ci_lib.DISTRO, 'MITOGEN_LOG_LEVEL': 'debug', 'SKIP_ANSIBLE': '1', }) diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py index 4de94b4c3..47aa2444f 100755 --- a/.ci/mitogen_tests.py +++ b/.ci/mitogen_tests.py @@ -6,7 +6,6 @@ import ci_lib os.environ.update({ - 'MITOGEN_TEST_DISTRO': ci_lib.DISTRO, 'MITOGEN_LOG_LEVEL': 'debug', 'SKIP_ANSIBLE': '1', }) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f3f31d82f..cc20f04a2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,80 +67,14 @@ jobs: python_version: '3.13' tox_env: py313-mode_ansible-ansible10-strategy_linear - - name: Mito_27_centos6 - tox_env: py27-mode_mitogen-distro_centos6 - - name: Mito_27_centos7 - tox_env: py27-mode_mitogen-distro_centos7 - - name: Mito_27_centos8 - tox_env: py27-mode_mitogen-distro_centos8 - - name: Mito_27_debian9 - tox_env: py27-mode_mitogen-distro_debian9 - - name: Mito_27_debian10 - tox_env: py27-mode_mitogen-distro_debian10 - - name: Mito_27_debian11 - tox_env: py27-mode_mitogen-distro_debian11 - - name: Mito_27_ubuntu1604 - tox_env: py27-mode_mitogen-distro_ubuntu1604 - - name: Mito_27_ubuntu1804 - tox_env: py27-mode_mitogen-distro_ubuntu1804 - - name: Mito_27_ubuntu2004 - tox_env: py27-mode_mitogen-distro_ubuntu2004 - - - name: Mito_36_centos6 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_centos6 - - name: Mito_36_centos7 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_centos7 - - name: Mito_36_centos8 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_centos8 - - name: Mito_36_debian9 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_debian9 - - name: Mito_36_debian10 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_debian10 - - name: Mito_36_debian11 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_debian11 - - name: Mito_36_ubuntu1604 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_ubuntu1604 - - name: Mito_36_ubuntu1804 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_ubuntu1804 - - name: Mito_36_ubuntu2004 + - name: Mito_27 + tox_env: py27-mode_mitogen + - name: Mito_36 python_version: '3.6' - tox_env: py36-mode_mitogen-distro_ubuntu2004 - - - name: Mito_313_centos6 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_centos6 - - name: Mito_313_centos7 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_centos7 - - name: Mito_313_centos8 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_centos8 - - name: Mito_313_debian9 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_debian9 - - name: Mito_313_debian10 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_debian10 - - name: Mito_313_debian11 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_debian11 - - name: Mito_313_ubuntu1604 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_ubuntu1604 - - name: Mito_313_ubuntu1804 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_ubuntu1804 - - name: Mito_313_ubuntu2004 + tox_env: py36-mode_mitogen + - name: Mito_313 python_version: '3.13' - tox_env: py313-mode_mitogen-distro_ubuntu2004 + tox_env: py313-mode_mitogen steps: - uses: actions/checkout@v4 diff --git a/docs/changelog.rst b/docs/changelog.rst index 092fed261..1fad75a58 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file In progress (unreleased) ------------------------ +* :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker + SSH tests v0.3.13 (2024-10-09) diff --git a/tests/README.md b/tests/README.md index 35a8775c8..06bf7ad7c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,11 +30,19 @@ and run the tests there. 1. Run ``test`` -# Selecting a target distribution +# Selecting target distributions -Docker target images exist for testing against CentOS and Debian, with the -default being Debian. To select CentOS, specify `MITOGEN_TEST_DISTRO=centos` in -the environment. +Linux container images for testing are available at + +- https://github.com/orgs/mitogen-hq/packages +- https://public.ecr.aws/n5z0e8q9 + +The images used are determined by two environment variables + +- `MITOGEN_TEST_DISTRO_SPECS` +- `MITOGEN_TEST_IMAGE_TEMPLATE` + +Defaults for these can be found in `.ci/ci_lib.py` & `tests/testlib.py` # User Accounts diff --git a/tests/fakessh_test.py b/tests/fakessh_test.py index 523214957..2ad722df6 100644 --- a/tests/fakessh_test.py +++ b/tests/fakessh_test.py @@ -7,8 +7,8 @@ import testlib +@unittest.skip('broken') class RsyncTest(testlib.DockerMixin, testlib.TestCase): - @unittest.skip('broken') def test_rsync_from_master(self): context = self.docker_ssh_any() @@ -24,7 +24,6 @@ def test_rsync_from_master(self): self.assertTrue(context.call(os.path.exists, '/tmp/data')) self.assertTrue(context.call(os.path.exists, '/tmp/data/simple_pkg/a.py')) - @unittest.skip('broken') def test_rsync_between_direct_children(self): # master -> SSH -> mitogen__has_sudo_pubkey -> rsync(.ssh) -> master -> # mitogen__has_sudo -> rsync diff --git a/tests/ssh_test.py b/tests/ssh_test.py index f8fad55ae..e8e0e1ebd 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -37,7 +37,7 @@ def test_okay(self): self.assertEqual(3, context.call(plain_old_module.add, 1, 2)) -class SshTest(testlib.DockerMixin, testlib.TestCase): +class SshMixin(testlib.DockerMixin): def test_debug_decoding(self): # ensure filter_debug_logs() decodes the logged string. capture = testlib.LogCapturer() @@ -176,7 +176,18 @@ def test_accept_enforce_host_keys(self): fp.close() -class BannerTest(testlib.DockerMixin, testlib.TestCase): +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'SshTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (SshMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass + + +class BannerMixin(testlib.DockerMixin): # Verify the ability to disambiguate random spam appearing in the SSHd's # login banner from a legitimate password prompt. def test_verbose_enabled(self): @@ -193,6 +204,17 @@ def test_verbose_enabled(self): context.shutdown(wait=True) +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'BannerTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (BannerMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass + + class StubPermissionDeniedTest(StubSshMixin, testlib.TestCase): def test_classic_prompt(self): self.assertRaises(mitogen.ssh.PasswordError, diff --git a/tests/su_test.py b/tests/su_test.py index 234c509bb..3750454ce 100644 --- a/tests/su_test.py +++ b/tests/su_test.py @@ -23,7 +23,7 @@ def test_basic(self): self.assertEqual(argv[2], '-c') -class SuTest(testlib.DockerMixin, testlib.TestCase): +class SuMixin(testlib.DockerMixin): stub_su_path = testlib.data_path('stubs/stub-su.py') def test_slow_auth_failure(self): @@ -64,3 +64,14 @@ def test_password_okay(self): ) context = self.router.su(via=ssh, password='rootpassword') self.assertEqual(0, context.call(os.getuid)) + + +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'SuTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (SuMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass diff --git a/tests/testlib.py b/tests/testlib.py index 0fd9c7d7a..05779dc0f 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -51,7 +51,10 @@ LOG = logging.getLogger(__name__) -DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9') +DISTRO_SPECS = os.environ.get( + 'MITOGEN_TEST_DISTRO_SPECS', + 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', +) IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', 'public.ecr.aws/n5z0e8q9/%(distro)s-test', @@ -555,8 +558,9 @@ def start_container(self): self.image, ] subprocess.check_output(args) + self.port = self.get_port(self.container_name) - def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): + def __init__(self, distro_spec, image_template=IMAGE_TEMPLATE): # Code duplicated in ci_lib.py, both should be updated together distro_pattern = re.compile(r''' (?P(?P[a-z]+)[0-9]+) @@ -565,7 +569,10 @@ def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): ''', re.VERBOSE, ) - d = distro_pattern.match(distro).groupdict(default=None) + d = distro_pattern.match(distro_spec).groupdict(default=None) + + self.distro = d['distro'] + self.family = d['family'] if d.pop('py') == 'py3': self.python_path = '/usr/bin/python3' @@ -573,9 +580,7 @@ def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): self.python_path = '/usr/bin/python' self.image = image_template % d - self.start_container() self.host = get_docker_host() - self.port = self.get_port(self.container_name) def wait_for_sshd(self): wait_for_port(self.host, self.port, pattern='OpenSSH') @@ -648,12 +653,10 @@ def setUpClass(cls): if os.environ.get('SKIP_DOCKER_TESTS'): raise unittest.SkipTest('SKIP_DOCKER_TESTS is set') - # we want to be able to override test distro for some tests that need a different container spun up - daemon_args = {} - if hasattr(cls, 'mitogen_test_distro'): - daemon_args['mitogen_test_distro'] = cls.mitogen_test_distro - - cls.dockerized_ssh = DockerizedSshDaemon(**daemon_args) + # cls.dockerized_ssh is injected by dynamically generating TestCase + # subclasses. + # TODO Bite the bullet, switch to e.g. pytest + cls.dockerized_ssh.start_container() cls.dockerized_ssh.wait_for_sshd() @classmethod diff --git a/tox.ini b/tox.ini index 6a0eb1806..5afa7eb1c 100644 --- a/tox.ini +++ b/tox.ini @@ -56,9 +56,7 @@ envlist = py{27,36}-mode_ansible-ansible{2.10,3,4}, py{311}-mode_ansible-ansible{2.10,3,4,5}, py{313}-mode_ansible-ansible{6,7,8,9,10}, - py{27,36,313}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,313}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,313}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{27,36,313}-mode_mitogen, report, [testenv] @@ -105,39 +103,27 @@ setenv = NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 PIP_CONSTRAINT={toxinidir}/tests/constraints.txt - # Only applicable to MODE=mitogen - distro_centos5: DISTRO=centos5 - distro_centos6: DISTRO=centos6 - distro_centos7: DISTRO=centos7 - distro_centos8: DISTRO=centos8 - distro_debian9: DISTRO=debian9 - distro_debian10: DISTRO=debian10 - distro_debian11: DISTRO=debian11 - distro_ubuntu1604: DISTRO=ubuntu1604 - distro_ubuntu1804: DISTRO=ubuntu1804 - distro_ubuntu2004: DISTRO=ubuntu2004 - # Note the plural, only applicable to MODE=ansible # Ansible 6 - 8 (ansible-core 2.13 - 2.15) require Python 2.7 or >= 3.5 on targets - ansible6: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 - ansible7: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 - ansible8: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible6: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible7: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible8: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 # Ansible 9 (ansible-core 2.16) requires Python 2.7 or >= 3.6 on targets - ansible9: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 + ansible9: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 # Ansible 10 (ansible-core 2.17) requires Python >= 3.7 on targets - ansible10: DISTROS=debian10-py3 debian11-py3 ubuntu2004-py3 - distros_centos: DISTROS=centos6 centos7 centos8 - distros_centos5: DISTROS=centos5 - distros_centos6: DISTROS=centos6 - distros_centos7: DISTROS=centos7 - distros_centos8: DISTROS=centos8 - distros_debian: DISTROS=debian9 debian10 debian11 - distros_debian9: DISTROS=debian9 - distros_debian10: DISTROS=debian10 - distros_debian11: DISTROS=debian11 - distros_ubuntu: DISTROS=ubuntu1604 ubuntu1804 ubuntu2004 - distros_ubuntu1604: DISTROS=ubuntu1604 - distros_ubuntu1804: DISTROS=ubuntu1804 - distros_ubuntu2004: DISTROS=ubuntu2004 + ansible10: MITOGEN_TEST_DISTRO_SPECS=debian10-py3 debian11-py3 ubuntu2004-py3 + distros_centos: MITOGEN_TEST_DISTRO_SPECS=centos6 centos7 centos8 + distros_centos5: MITOGEN_TEST_DISTRO_SPECS=centos5 + distros_centos6: MITOGEN_TEST_DISTRO_SPECS=centos6 + distros_centos7: MITOGEN_TEST_DISTRO_SPECS=centos7 + distros_centos8: MITOGEN_TEST_DISTRO_SPECS=centos8 + distros_debian: MITOGEN_TEST_DISTRO_SPECS=debian9 debian10 debian11 + distros_debian9: MITOGEN_TEST_DISTRO_SPECS=debian9 + distros_debian10: MITOGEN_TEST_DISTRO_SPECS=debian10 + distros_debian11: MITOGEN_TEST_DISTRO_SPECS=debian11 + distros_ubuntu: MITOGEN_TEST_DISTRO_SPECS=ubuntu1604 ubuntu1804 ubuntu2004 + distros_ubuntu1604: MITOGEN_TEST_DISTRO_SPECS=ubuntu1604 + distros_ubuntu1804: MITOGEN_TEST_DISTRO_SPECS=ubuntu1804 + distros_ubuntu2004: MITOGEN_TEST_DISTRO_SPECS=ubuntu2004 mode_ansible: MODE=ansible mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive mode_ansible: ANSIBLE_CALLBACK_WHITELIST=profile_tasks From e9bddf0c039bef8c52fd0dd40f8cbde5a111e5b4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 8 Oct 2024 17:43:30 +0100 Subject: [PATCH 11/11] CI: Use templated ansible_user for localhost Ansible tests refs #1022, #1116 --- .ci/localhost_ansible_tests.py | 14 -------------- tests/ansible/hosts/default.hosts | 3 +-- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index c36bad269..502a9abca 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -3,8 +3,6 @@ from __future__ import print_function -import getpass -import io import os import subprocess import sys @@ -57,18 +55,6 @@ os.chdir(IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") - # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022 - # and os.environ['USER'] is not populated on Azure macOS runners. - os.chdir(HOSTS_DIR) - with io.open('default.hosts', 'r+', encoding='utf-8') as f: - user = getpass.getuser() - content = f.read() - content = content.replace("{{ lookup('pipe', 'whoami') }}", user) - f.seek(0) - f.write(content) - f.truncate() - ci_lib.dump_file('default.hosts') - cmd = ';'.join([ 'from __future__ import print_function', 'import os, sys', diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 609cd9f88..bebe90df9 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -4,8 +4,7 @@ # When running the tests outside CI, make a single 'target' host which is the # local machine. The ansible_user override is necessary since some tests want a # fixed ansible.cfg remote_user setting to test against. -# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022 -# and os.environ['USER'] is not populated on Azure macOS runners. +# os.environ['USER'] is an empty string on GitHub Actions macOS runners. target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" [test-targets]