Merge pull request #1123 from moreati/release-0.3.10

Release 0.3.10

.ci/ansible_tests.py

@@ -6,11 +6,13 @@
 import os
 import signal
 import sys
-import textwrap
+
+import jinja2
 
 import ci_lib
 
 
+TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates')
 TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
 HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
 
@@ -52,37 +54,19 @@ def pause_if_interactive():
         distros[container['distro']].append(container['name'])
         families[container['family']].append(container['name'])
 
+    jinja_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR),
+        lstrip_blocks=True,  # Remove spaces and tabs from before a block
+        trim_blocks=True,  # Remove first newline after a block
+    )
+    inventory_template = jinja_env.get_template('test-targets.j2')
     inventory_path = os.path.join(HOSTS_DIR, 'target')
+
     with open(inventory_path, 'w') as fp:
-        fp.write('[test-targets]\n')
-        fp.writelines(
-            "%(name)s "
-            "ansible_host=%(hostname)s "
-            "ansible_port=%(port)s "
-            "ansible_python_interpreter=%(python_path)s "
-            "ansible_user=mitogen__has_sudo_nopw "
-            "ansible_password=has_sudo_nopw_password"
-            "\n"
-            % container
-            for container in containers
-        )
-
-        for distro, hostnames in sorted(distros.items(), key=lambda t: t[0]):
-            fp.write('\n[%s]\n' % distro)
-            fp.writelines('%s\n' % name for name in hostnames)
-
-        for family, hostnames in sorted(families.items(), key=lambda t: t[0]):
-            fp.write('\n[%s]\n' % family)
-            fp.writelines('%s\n' % name for name in hostnames)
-
-        fp.write(textwrap.dedent(
-            '''
-            [linux:children]
-            test-targets
-
-            [linux_containers:children]
-            test-targets
-            '''
+        fp.write(inventory_template.render(
+            containers=containers,
+            distros=distros,
+            families=families,
         ))
 
     ci_lib.dump_file(inventory_path)

.ci/azure-pipelines.yml

@@ -28,10 +28,6 @@ jobs:
     matrix:
       Mito_312:
         tox.env: py312-mode_mitogen
-      Loc_312_9:
-        tox.env: py312-mode_localhost-ansible9
-      Van_312_9:
-        tox.env: py312-mode_localhost-ansible9-strategy_linear
       Loc_312_10:
         tox.env: py312-mode_localhost-ansible10
       Van_312_10:

ansible_mitogen/connection.py

@@ -1129,6 +1129,6 @@ def put_file(self, in_path, out_path):
         self.get_chain().call(
             ansible_mitogen.target.transfer_file,
             context=self.binding.get_child_service_context(),
-            in_path=in_path,
-            out_path=out_path
+            in_path=ansible_mitogen.utils.unsafe.cast(in_path),
+            out_path=ansible_mitogen.utils.unsafe.cast(out_path)
         )

ansible_mitogen/mixins.py

@@ -280,7 +280,9 @@ def _remote_chmod(self, paths, mode, sudoable=False):
                   paths, mode, sudoable)
         return self.fake_shell(lambda: mitogen.select.Select.all(
             self._connection.get_chain().call_async(
-                ansible_mitogen.target.set_file_mode, path, mode
+                ansible_mitogen.target.set_file_mode,
+                ansible_mitogen.utils.unsafe.cast(path),
+                mode,
             )
             for path in paths
         ))

ansible_mitogen/transport_config.py

@@ -498,12 +498,13 @@ def ansible_ssh_timeout(self):
         )
 
     def ssh_args(self):
+        local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {})
         return [
             mitogen.core.to_text(term)
             for s in (
-                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
-                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
-                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
+                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
+                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
+                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars)
             )
             for term in ansible.utils.shlex.shlex_split(s or '')
         ]
@@ -738,12 +739,13 @@ def ansible_ssh_timeout(self):
         )
 
     def ssh_args(self):
+        local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {})
         return [
             mitogen.core.to_text(term)
             for s in (
-                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
-                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
-                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
+                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
+                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
+                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars)
             )
             for term in ansible.utils.shlex.shlex_split(s)
             if s

docs/changelog.rst

@@ -18,6 +18,22 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
+v0.3.10 (2024-09-20)
+--------------------
+
+* :gh:issue:`950` Fix Solaris/Illumos/SmartOS compatibility with become
+* :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template
+  module is called with a ``dest:`` filename that has an extension
+* :gh:issue:`1110` Fix :exc:`mitogen.core.StreamError` when Ansible copy
+  module is called with a file larger than 124 kibibytes
+  (:data:`ansible_mitogen.connection.Connection.SMALL_FILE_LIMIT`)
+* :gh:issue:`905` Initial support for templated ``ansible_ssh_args``,
+  ``ansible_ssh_common_args``, and ``ansible_ssh_extra_args`` variables.
+  NB: play or task scoped variables will probably still fail.
+* :gh:issue:`694` CI: Fixed a race condition and some resource leaks causing
+  some of intermittent failures when running the test suite.
+
+
 v0.3.9 (2024-08-13)
 -------------------
 

docs/contributors.rst

@@ -116,6 +116,7 @@ sponsorship and outstanding future-thinking of its early adopters.
 
     <ul>
     <li>Alex Willmer</li>
+    <li><a href="https://github.com/momiji">Christian Bourgeois </a></li>
     <li><a href="https://underwhelm.net/">Dan Dorman</a> &mdash; - <em>When I truly understand my enemy … then in that very moment I also love him.</em></li>
     <li>Daniel Foerster</li>
     <li><a href="https://www.deps.co/">Deps</a> &mdash; <em>Private Maven Repository Hosting for Java, Scala, Groovy, Clojure</em></li>
@@ -125,6 +126,7 @@ sponsorship and outstanding future-thinking of its early adopters.
     <li><a href="https://www.channable.com">rkrzr</a></li>
     <li>jgadling</li>
     <li>John F Wall &mdash; <em>Making Ansible Great with Massive Parallelism</em></li>
+    <li><a href="https://github.com/jrosser">Jonathan Rosser</a></li>
     <li>KennethC</li>
     <li><a href="https://github.com/lberruti">Luca Berruti</li>
     <li>Lewis Bellwood &mdash; <em>Happy to be apart of a great project.</em></li>

mitogen/__init__.py

@@ -35,7 +35,7 @@
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 9)
+__version__ = (0, 3, 10)
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

mitogen/parent.py

@@ -147,6 +147,8 @@ def _ioctl_cast(n):
 LINUX_TIOCSPTLCK = _ioctl_cast(1074025521)
 
 IS_LINUX = os.uname()[0] == 'Linux'
+IS_SOLARIS = os.uname()[0] == 'SunOS'
+
 
 SIGNAL_BY_NUM = dict(
     (getattr(signal, name), name)
@@ -411,7 +413,7 @@ def _acquire_controlling_tty():
         # On Linux, the controlling tty becomes the first tty opened by a
         # process lacking any prior tty.
         os.close(os.open(os.ttyname(2), os.O_RDWR))
-    if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL:
+    if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL and not IS_SOLARIS:
         # #550: prehistoric WSL does not like TIOCSCTTY.
         # On BSD an explicit ioctl is required. For some inexplicable reason,
         # Python 2.6 on Travis also requires it.
@@ -479,7 +481,8 @@ def openpty():
 
     master_fp = os.fdopen(master_fd, 'r+b', 0)
     slave_fp = os.fdopen(slave_fd, 'r+b', 0)
-    disable_echo(master_fd)
+    if not IS_SOLARIS:
+        disable_echo(master_fd)
     disable_echo(slave_fd)
     mitogen.core.set_block(slave_fd)
     return master_fp, slave_fp
@@ -2542,7 +2545,7 @@ def _signal_child(self, signum):
         # because it is setuid, so this is best-effort only.
         LOG.debug('%r: sending %s', self.proc, SIGNAL_BY_NUM[signum])
         try:
-            os.kill(self.proc.pid, signum)
+            self.proc.send_signal(signum)
         except OSError:
             e = sys.exc_info()[1]
             if e.args[0] != errno.EPERM:
@@ -2662,6 +2665,17 @@ def poll(self):
         """
         raise NotImplementedError()
 
+    def send_signal(self, sig):
+        os.kill(self.pid, sig)
+
+    def terminate(self):
+        "Ask the process to gracefully shutdown."
+        self.send_signal(signal.SIGTERM)
+
+    def kill(self):
+        "Ask the operating system to forcefully destroy the process."
+        self.send_signal(signal.SIGKILL)
+
 
 class PopenProcess(Process):
     """
@@ -2678,6 +2692,9 @@ def __init__(self, proc, stdin, stdout, stderr=None):
     def poll(self):
         return self.proc.poll()
 
+    def send_signal(self, sig):
+        self.proc.send_signal(sig)
+
 
 class ModuleForwarder(object):
     """

mitogen/unix.py

@@ -143,19 +143,23 @@ def on_shutdown(self, broker):
     def on_accept_client(self, sock):
         sock.setblocking(True)
         try:
-            pid, = struct.unpack('>L', sock.recv(4))
+            data = sock.recv(4)
+            pid, = struct.unpack('>L', data)
         except (struct.error, socket.error):
-            LOG.error('listener: failed to read remote identity: %s',
-                      sys.exc_info()[1])
+            LOG.error('listener: failed to read remote identity, got %d bytes: %s',
+                      len(data), sys.exc_info()[1])
+            sock.close()
             return
 
         context_id = self._router.id_allocator.allocate()
         try:
+            # FIXME #1109 send() returns number of bytes sent, check it
             sock.send(struct.pack('>LLL', context_id, mitogen.context_id,
                                   os.getpid()))
         except socket.error:
             LOG.error('listener: failed to assign identity to PID %d: %s',
                       pid, sys.exc_info()[1])
+            sock.close()
             return
 
         context = mitogen.parent.Context(self._router, context_id)

tests/ansible/hosts/default.hosts

@@ -12,3 +12,10 @@ target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
 target
 
 [linux_containers]
+
+[issue905]
+ssh-common-args ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
+
+[issue905:vars]
+ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ ssh_args_canary_file }}"
+ssh_args_canary_file=/tmp/ssh_args_{{ inventory_hostname }}