From 1bfb81c51444b06bfc8e82b06c26acf941435c3d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 24 Oct 2024 15:35:15 +0100 Subject: [PATCH] WIP --- .ci/localhost_ansible_tests.py | 31 +--- .github/workflows/tests.yml | 141 ++++++++++-------- ansible_mitogen/loaders.py | 2 +- mitogen/sudo.py | 4 + tests/ansible/all.yml | 4 - tests/ansible/ansible.cfg | 14 +- tests/ansible/hosts/default.hosts | 2 +- tests/ansible/integration/all.yml | 36 +---- tests/ansible/integration/become/all.yml | 10 -- .../integration/become/sudo_password.yml | 82 ++-------- tests/image_prep/_user_accounts.yml | 16 -- tox.ini | 5 + 12 files changed, 117 insertions(+), 230 deletions(-) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 502a9abca..dbd6f9672 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -15,12 +15,6 @@ HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts') KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key') - -with ci_lib.Fold('unit_tests'): - os.environ['SKIP_MITOGEN'] = '1' - ci_lib.run('./run_tests -v') - - with ci_lib.Fold('job_setup'): os.chmod(KEY_PATH, int('0600', 8)) # NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n", @@ -55,28 +49,9 @@ os.chdir(IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") - cmd = ';'.join([ - 'from __future__ import print_function', - 'import os, sys', - 'print(sys.executable, os.path.realpath(sys.executable))', - ]) - for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']: - print(interpreter) - try: - subprocess.call([interpreter, '-c', cmd]) - except OSError as exc: - print(exc) - - print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1') - environ = os.environ.copy() - environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1' - try: - subprocess.call([interpreter, '-c', cmd], env=environ) - except OSError as exc: - print(exc) - - -with ci_lib.Fold('ansible'): +if (os.environ['TOX_ENV_NAME'] != 'py313-mode_localhost-bisect' + or 'FOO_OVERRIDE' in os.environ +) : os.chdir(TESTS_DIR) playbook = os.environ.get('PLAYBOOK', 'all.yml') ci_lib.run('./run_ansible_playbook.py %s %s', diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cc20f04a2..cd6292036 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,54 +24,6 @@ jobs: fail-fast: false matrix: include: - - name: Ans_27_210 - tox_env: py27-mode_ansible-ansible2.10 - - name: Ans_27_4 - tox_env: py27-mode_ansible-ansible4 - - - name: Ans_36_210 - python_version: '3.6' - tox_env: py36-mode_ansible-ansible2.10 - - name: Ans_36_4 - python_version: '3.6' - tox_env: py36-mode_ansible-ansible4 - - - name: Ans_311_210 - python_version: '3.11' - tox_env: py311-mode_ansible-ansible2.10 - - name: Ans_311_3 - python_version: '3.11' - tox_env: py311-mode_ansible-ansible3 - - name: Ans_311_4 - python_version: '3.11' - tox_env: py311-mode_ansible-ansible4 - - name: Ans_311_5 - python_version: '3.11' - tox_env: py311-mode_ansible-ansible5 - - name: Ans_313_6 - python_version: '3.13' - tox_env: py313-mode_ansible-ansible6 - - name: Ans_313_7 - python_version: '3.13' - tox_env: py313-mode_ansible-ansible7 - - name: Ans_313_8 - python_version: '3.13' - tox_env: py313-mode_ansible-ansible8 - - name: Ans_313_9 - python_version: '3.13' - tox_env: py313-mode_ansible-ansible9 - - name: Ans_313_10 - python_version: '3.13' - tox_env: py313-mode_ansible-ansible10 - - name: Van_313_10 - python_version: '3.13' - tox_env: py313-mode_ansible-ansible10-strategy_linear - - - name: Mito_27 - tox_env: py27-mode_mitogen - - name: Mito_36 - python_version: '3.6' - tox_env: py36-mode_mitogen - name: Mito_313 python_version: '3.13' tox_env: py313-mode_mitogen @@ -164,23 +116,15 @@ jobs: fail-fast: false matrix: include: - - name: Mito_27 - tox_env: py27-mode_mitogen - - name: Mito_313 + - name: Loc_313_old python_version: '3.13' - tox_env: py313-mode_mitogen - - - name: Loc_27_210 - tox_env: py27-mode_localhost-ansible2.10 - - name: Loc_313_10 + tox_env: py313-mode_localhost-old + - name: Loc_313_new python_version: '3.13' - tox_env: py313-mode_localhost-ansible10 - - - name: Van_27_210 - tox_env: py27-mode_localhost-ansible2.10-strategy_linear - - name: Van_313_10 + tox_env: py313-mode_localhost-new + - name: Loc_313_bisect python_version: '3.13' - tox_env: py313-mode_localhost-ansible10-strategy_linear + tox_env: py313-mode_localhost-bisect steps: - uses: actions/checkout@v4 @@ -260,8 +204,52 @@ jobs: PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" fi + export old_rev=2f647e9617067802647d2a461906c1241c5cac00 + export new_rev=a1a6550daf305ec9815a7b12db42c68b63426878 + + if [[ "${{ matrix.tox_env }}" == "py313-mode_localhost-bisect" ]]; then + cd .. + git clone https://github.com/ansible/ansible + cd ansible + git checkout $old_rev + cd ../mitogen + fi + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" + if [[ "${{ matrix.tox_env }}" != "py313-mode_localhost-bisect" ]]; then + exit 0 + fi + + cd ../ansible + + cat << EOF > bisect.sh + #!/usr/bin/env bash + set -o errexit -o nounset -o pipefail + + sudo -K + rm -rf build dist + + cd ../mitogen + rm -rf .tox + + export FOO_OVERRIDE=1 + "$PYTHON" -m tox -r -e "${{ matrix.tox_env }}" && rc=\$? || rc=\$? + + if [[ \$rc -ne 0 ]]; then + exit 0 + else + exit \$rc + fi + EOF + + chmod +x bisect.sh + + git bisect start + git bisect good $old_rev + git bisect bad $new_rev + git bisect run ./bisect.sh + # https://github.com/marketplace/actions/alls-green check: if: always() @@ -273,3 +261,32 @@ jobs: - uses: re-actors/alls-green@release/v1 with: jobs: ${{ toJSON(needs) }} + + +# alex@uv2204:~/src/ansible$ git show --summary 2f647e9617067802647d2a461906c1241c5cac00 +# commit 2f647e9617067802647d2a461906c1241c5cac00 +# Author: Felix Fontein +# Date: Fri Mar 31 23:14:35 2023 +0200 + +# Implement semantic markup support for Ansible documentation in validate-modules. (#80243) + +# create mode 100644 changelogs/fragments/80243-validate-modules.yml +# create mode 100644 test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/semantic_markup.py +# alex@uv2204:~/src/ansible$ git bisect good a1a6550daf305ec9815a7b12db42c68b63426878 +# You need to start by "git bisect start" + +# Do you want me to do it for you [Y/n]? ^C +# alex@uv2204:~/src/ansible$ git show --summary a1a6550daf305ec9815a7b12db42c68b63426878 +# commit a1a6550daf305ec9815a7b12db42c68b63426878 +# Author: Abhijeet Kasurde +# Date: Mon Sep 18 07:50:50 2023 -0700 + +# inventory_ini: Handle SyntaxWarning in ini parsing (#81707) + +# * handle SyntaxWarning ini inventory parsing + +# Fixes: #81328 + +# Signed-off-by: Abhijeet Kasurde + +# create mode 100644 changelogs/fragments/inventory_ini.yml diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 9d9876ac6..0883dba0f 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -87,7 +87,7 @@ def assert_supported_release(): # this is the first file our strategy plugins import, so we need to check this here # in prior Ansible versions, connection_loader.get_with_context didn't exist, so if a user # is trying to load an old Ansible version, we'll fail and error gracefully -assert_supported_release() +#assert_supported_release() from ansible.plugins.loader import action_loader diff --git a/mitogen/sudo.py b/mitogen/sudo.py index a1a7b8af7..e8ae56097 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -219,20 +219,24 @@ def _on_password_prompt(self, line, match): self.stream.name, line.decode('utf-8', 'replace')) if self.stream.conn.options.password is None: + LOG.error('%s: password prompted, not available', self.stream.name) self.stream.conn._fail_connection( PasswordError(password_required_msg) ) return if self.password_sent: + LOG.error('%s: password already sent', self.stream.name) self.stream.conn._fail_connection( PasswordError(password_incorrect_msg) ) return + LOG.info('%s: providing password', self.stream.name) self.stream.transmit_side.write( (self.stream.conn.options.password + '\n').encode('utf-8') ) + LOG.debug('%s: provided password', self.stream.name) self.password_sent = True PARTIAL_PATTERNS = [ diff --git a/tests/ansible/all.yml b/tests/ansible/all.yml index 7a3e70001..9a0c076a1 100644 --- a/tests/ansible/all.yml +++ b/tests/ansible/all.yml @@ -1,6 +1,2 @@ -- import_playbook: setup/all.yml - tags: setup -- import_playbook: regression/all.yml - tags: regression - import_playbook: integration/all.yml tags: integration diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 37edfd94a..f0d47cedb 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -3,20 +3,20 @@ any_errors_fatal = true # callbacks_enabled was added in Ansible 4 (ansible-core 2.11). # profile_tasks: Displays timing for each task and summary table of top N tasks # timer: Displays "Playbook run took 0 days, 0 hours, ..." -callbacks_enabled = - profile_tasks, - timer +#callbacks_enabled = +# profile_tasks, +# timer # callback_whitelist was deprecated in Ansible >= 8 (ansible-core >= 2.15). -callback_whitelist = - profile_tasks, - timer +#callback_whitelist = +# profile_tasks, +# timer inventory = hosts gathering = explicit strategy_plugins = ../../ansible_mitogen/plugins/strategy inventory_plugins = lib/inventory action_plugins = lib/action callback_plugins = lib/callback -stdout_callback = yaml +#stdout_callback = yaml vars_plugins = lib/vars library = lib/modules filter_plugins = lib/filters diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 2edb9d0e3..8279ad275 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -5,7 +5,7 @@ # local machine. The ansible_user override is necessary since some tests want a # fixed ansible.cfg remote_user setting to test against. # os.environ['USER'] is an empty string on GitHub Actions macOS runners. -target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" +target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" ansible_python_interpreter=python3 [test-targets] target diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index ac196584e..f0aabec64 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -3,39 +3,5 @@ # This playbook imports all tests that are known to work at present. # -- import_playbook: action/all.yml - tags: action -- import_playbook: async/all.yml - tags: async - import_playbook: become/all.yml - tags: become -- import_playbook: connection/all.yml - tags: connection -- import_playbook: connection_delegation/all.yml - tags: connection_delegation -- import_playbook: connection_loader/all.yml - tags: connection_loader -- import_playbook: context_service/all.yml - tags: context_service -- import_playbook: glibc_caches/all.yml - tags: glibc_caches -- import_playbook: interpreter_discovery/all.yml - tags: interpreter_discovery -- import_playbook: local/all.yml - tags: local -- import_playbook: module_utils/all.yml - tags: module_utils -- import_playbook: playbook_semantics/all.yml - tags: playbook_semantics -- import_playbook: process/all.yml - tags: process -- import_playbook: runner/all.yml - tags: runner -- import_playbook: ssh/all.yml - tags: ssh -- import_playbook: strategy/all.yml - tags: strategy -- import_playbook: stub_connections/all.yml - tags: stub_connections -- import_playbook: transport_config/all.yml - tags: transport_config + tags: become \ No newline at end of file diff --git a/tests/ansible/integration/become/all.yml b/tests/ansible/integration/become/all.yml index 1b507e166..3d0b05d26 100644 --- a/tests/ansible/integration/become/all.yml +++ b/tests/ansible/integration/become/all.yml @@ -1,11 +1 @@ - -- import_playbook: su_password.yml -- import_playbook: sudo_flags_failure.yml -- import_playbook: sudo_nonexistent.yml -- import_playbook: sudo_nopassword.yml - import_playbook: sudo_password.yml -- import_playbook: sudo_requiretty.yml -- import_playbook: templated_by_inv.yml -- import_playbook: templated_by_play_keywords.yml -- import_playbook: templated_by_play_vars.yml -- import_playbook: templated_by_task_keywords.yml diff --git a/tests/ansible/integration/become/sudo_password.yml b/tests/ansible/integration/become/sudo_password.yml index 070346359..6f92b8c36 100644 --- a/tests/ansible/integration/become/sudo_password.yml +++ b/tests/ansible/integration/become/sudo_password.yml @@ -3,74 +3,24 @@ - name: integration/become/sudo_password.yml hosts: test-targets tasks: + - block: + - meta: reset_connection - - name: Ensure sudo password absent but required. - shell: whoami - become: true - become_user: mitogen__pw_required - register: out - ignore_errors: true - when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + - name: Ensure password sudo succeeds + become: true + become_user: mitogen__pw_required + vars: + ansible_become_pass: pw_required_password + command: + cmd: whoami + changed_when: false + register: become_sudo_password_success_whoami - - assert: - that: | - out.failed and ( - ('password is required' in out.msg) or - ('Missing sudo password' in out.msg) or - ('password is required' in out.module_stderr) - ) - fail_msg: | - out={{ out }} - when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen - - - name: Ensure password sudo incorrect. - shell: whoami - become: true - become_user: mitogen__pw_required - register: out - vars: - ansible_become_pass: nopes - ignore_errors: true - when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen - - - assert: - that: | - out.failed and ( - ('Incorrect sudo password' in out.msg) or - ('sudo password is incorrect' in out.msg) - ) - fail_msg: | - out={{ out }} - when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen - - # TODO: https://github.com/dw/mitogen/issues/692 - # - name: Ensure password sudo succeeds. - # shell: whoami - # become: true - # become_user: mitogen__pw_required - # register: out - # vars: - # ansible_become_pass: pw_required_password - - # - assert: - # that: - # - out.stdout == 'mitogen__pw_required' + - assert: + that: + - become_sudo_password_success_whoami.stdout == 'mitogen__pw_required' + fail_msg: | + become_sudo_password_success_whoami={{ become_sudo_password_success_whoami }} tags: - sudo - sudo_password diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 0b6d5e610..9e3a808ac 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -82,16 +82,6 @@ with_items: "{{all_users}}" when: ansible_system == 'Darwin' - - name: Hide users from login window (Darwin). - when: ansible_system == 'Darwin' - with_items: "{{all_users}}" - osx_defaults: - array_add: true - domain: /Library/Preferences/com.apple.loginwindow - type: array - key: HiddenUsersList - value: ['mitogen_{{item}}'] - - name: Check if AccountsService is used stat: path: /var/lib/AccountsService/users @@ -107,12 +97,6 @@ [User] SystemAccount=true - - name: Restart AccountsService (Linux). - when: ansible_system == 'Linux' and out.stat.exists - service: - name: accounts-daemon - state: restarted - - name: Readonly homedir for one account shell: "chown -R root: ~mitogen__readonly_homedir" diff --git a/tox.ini b/tox.ini index 5afa7eb1c..56b69f080 100644 --- a/tox.ini +++ b/tox.ini @@ -84,6 +84,10 @@ deps = ansible8: ansible~=8.0 ansible9: ansible~=9.0 ansible10: ansible~=10.0 + bisect: -e../ansible + old: git+https://github.com/ansible/ansible.git@2f647e9617067802647d2a461906c1241c5cac00 + new: git+https://github.com/ansible/ansible.git@a1a6550daf305ec9815a7b12db42c68b63426878 + install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = @@ -95,6 +99,7 @@ commands = mode_mitogen: {toxinidir}/.ci/mitogen_tests.py passenv = ANSIBLE_* + FOO_OVERRIDE HOME MITOGEN_* setenv =