parallel-ssh · zephyrj · May 23, 2024 · May 23, 2024 · May 23, 2024 · May 23, 2024
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -32,7 +32,7 @@ environment:
     PYTHON_DEF: "C:\\Python38-x64"
     PYTHON_VERSION: "3.8"
     # Python versions to build wheels for
-    PYTHONVERS: C:\Python37-x64 C:\Python38-x64 C:\Python39-x64 C:\Python310-x64 C:\Python311-x64
+    PYTHONVERS: C:\Python37-x64 C:\Python38-x64 C:\Python39-x64 C:\Python310-x64 C:\Python311-x64 C:\Python312-x64
     PYTHON_ARCH: "64"
 
 install:

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -144,6 +144,7 @@ workflows:
                 - "3.7"
                 - "3.9"
                 - "3.11"
+                - "3.12"
           filters:
             tags:
               ignore: /.*/

diff --git a/ci/cibuildwheel/README.md b/ci/cibuildwheel/README.md
@@ -0,0 +1,28 @@
+# Building a manylinux wheel locally
+This directory contains files to allow the building of a local wheel on a linux using [cibuildwheel](https://github.com/pypa/cibuildwheel).
+It is configured to only build cpython wheels for python 3.10 - 3.12 on either x86_64 or aarch64 architectures but could be expanded.
+
+## Prequisites
+You must have docker or podman-docker installed on your machine along with the cibuildwheel python package.
+### Debian based OSes
+```
+sudo apt get install podman-docker
+pip install cibuildwheel
+```
+
+### Fedora based Oses
+```
+sudo dnf install podman-docker
+pip install cibuildwheel
+```
+
+## Building wheels
+All commands should be run from the root level of the repo.
+```
+./ci/cibuildwheel/build-image.sh
+cibuildwheel --config-file ci/cibuildwheel/config.toml
+```
+> Note: If you cannot build/run [rootless containers](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md) you will need to 
+> run these commands prefixed with sudo
+
+You will find the build wheels present in the `wheelhouse` directory. These can be installed into the current python environment with `pip install <wheel>`
diff --git a/ci/cibuildwheel/build-image.sh b/ci/cibuildwheel/build-image.sh
@@ -0,0 +1,11 @@
+#!/bin/bash -xe
+
+arch=$(uname -m)
+image_name="manylinux-ssh-python-${arch}"
+if [[ ${arch} == "aarch64" ]]; then
+  dockerfile="Dockerfile.aarch64"
+else
+  dockerfile="Dockerfile.2014_x86_64"
+fi
+
+docker build -t "${image_name}" -f "ci/docker/manylinux/${dockerfile}" ci/docker/manylinux
diff --git a/ci/cibuildwheel/config.toml b/ci/cibuildwheel/config.toml
@@ -0,0 +1,28 @@
+[tool.cibuildwheel]
+build = "cp310-manylinux* cp311-manylinux* cp312-manylinux*"
+skip = ""
+test-skip = ""
+archs = ["auto64"]
+build-frontend = "default"
+config-settings = {}
+dependency-versions = "pinned"
+environment = {}
+environment-pass = []
+build-verbosity = 0
+
+before-all = ""
+before-build = "pip install -U cython"
+repair-wheel-command = ""
+
+test-command = ""
+before-test = ""
+test-requires = []
+test-extras = []
+
+container-engine = "docker"
+
+manylinux-x86_64-image = "localhost/manylinux-ssh-python-x86_64"
+manylinux-aarch64-image = "localhost/manylinux-ssh-python-aarch64"
+
+[tool.cibuildwheel.linux]
+repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"
diff --git a/ci/docker/manylinux/Dockerfile.2014_x86_64 b/ci/docker/manylinux/Dockerfile.2014_x86_64
@@ -1,35 +1,40 @@
 FROM quay.io/pypa/manylinux2014_x86_64
 
-ENV OPENSSL openssl-3.1.1
-ENV LIBSSH 0.10.5
-ENV KRB 1.21.1
+ENV OPENSSL openssl-3.3.0
+ENV LIBSSH_BRANCH stable-0.10
+ENV KRB_MAJOR_MINOR 1.21
+ENV KRB_PATCH 2
 ENV SYSTEM_LIBSSH 1
 ENV CFLAGS "-g0 -s"
 
 RUN yum install -y zlib-devel cmake3 perl-core
 
-ADD libssh-${LIBSSH}.tar.xz libssh.tar.xz
+# Use a branch of 0.10 because no released version includes all fixes as of 23/05/2024.
+# See https://gitlab.com/libssh/libssh-mirror/-/issues/227
+COPY libssh-mirror-${LIBSSH_BRANCH}.tar.gz .
 ADD https://www.openssl.org/source/${OPENSSL}.tar.gz ${OPENSSL}.tar.gz
-ADD krb5-${KRB}.tar.xz krb5-${KRB}.tar.xz
+ADD https://kerberos.org/dist/krb5/${KRB_MAJOR_MINOR}/krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
 
-RUN tar -xzf ${OPENSSL}.tar.gz
 # Openssl
+RUN tar -xzf ${OPENSSL}.tar.gz
 RUN cd ${OPENSSL} && \
     ./config --prefix=/usr --openssldir=/usr/openssl threads shared && \
     make -j6 && make install_sw
 
 # Kerberos
-RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
+RUN tar -xzf krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
+RUN cd krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}/src && \
     ./configure && \
     make -j6 && \
     make install
 
 # Libssh
+RUN tar -xzf libssh-mirror-${LIBSSH_BRANCH}.tar.gz
 RUN mkdir -p build_libssh && cd build_libssh && \
-    cmake3 ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
+    cmake3 ../libssh-mirror-${LIBSSH_BRANCH} -DCMAKE_BUILD_TYPE=Release \
           -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
     make -j6 install/strip
 
-RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz
+RUN rm -rf ${OPENSSL}* libssh-mirror-${LIBSSH_BRANCH}* build_libssh krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}*
 
 VOLUME /var/cache
diff --git a/ci/docker/manylinux/Dockerfile.aarch64 b/ci/docker/manylinux/Dockerfile.aarch64
@@ -1,35 +1,40 @@
 FROM quay.io/pypa/manylinux2014_aarch64
 
-ENV OPENSSL openssl-3.1.1
-ENV LIBSSH 0.10.5
-ENV KRB 1.21.1
+ENV OPENSSL openssl-3.3.0
+ENV LIBSSH_BRANCH stable-0.10
+ENV KRB_MAJOR_MINOR 1.21
+ENV KRB_PATCH 2
 ENV SYSTEM_LIBSSH 1
 ENV CFLAGS "-g0 -s"
 
 RUN yum install epel-release -y && yum install zlib-devel cmake3 perl-core -y
 
-ADD libssh-${LIBSSH}.tar.xz libssh.tar.xz
+# Use a branch of 0.10 because no released version includes all fixes as of 23/05/2024.
+# See https://gitlab.com/libssh/libssh-mirror/-/issues/227
+COPY libssh-mirror-${LIBSSH_BRANCH}.tar.gz .
 ADD https://www.openssl.org/source/${OPENSSL}.tar.gz ${OPENSSL}.tar.gz
-ADD krb5-${KRB}.tar.xz krb5-${KRB}.tar.xz
+ADD https://kerberos.org/dist/krb5/${KRB_MAJOR_MINOR}/krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
 
-RUN tar -xzf ${OPENSSL}.tar.gz
 # Openssl
+RUN tar -xzf ${OPENSSL}.tar.gz
 RUN cd ${OPENSSL} && \
     ./config --prefix=/usr --openssldir=/usr/openssl threads shared && \
     make -j6 && make install_sw
 
 # Kerberos
-RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
+RUN tar -xzf krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
+RUN cd krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}/src && \
     ./configure && \
     make -j6 && \
     make install
 
 # Libssh
+RUN tar -xzf libssh-mirror-${LIBSSH_BRANCH}.tar.gz
 RUN mkdir -p build_libssh && cd build_libssh && \
-    cmake3 ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
+    cmake3 ../libssh-mirror-${LIBSSH_BRANCH} -DCMAKE_BUILD_TYPE=Release \
           -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
     make -j6 install/strip
 
-RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz
+RUN rm -rf ${OPENSSL}* libssh-mirror-${LIBSSH_BRANCH}* build_libssh krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}*
 
 VOLUME /var/cache
diff --git a/ci/docker/manylinux/Dockerfile.aarch64_2_24 b/ci/docker/manylinux/Dockerfile.aarch64_2_24
@@ -1,36 +1,41 @@
 FROM quay.io/pypa/manylinux_2_24_aarch64
 
-ENV OPENSSL openssl-3.1.1
-ENV LIBSSH 0.10.5
-ENV KRB 1.21.1
+ENV OPENSSL openssl-3.3.0
+ENV LIBSSH_BRANCH stable-0.10
+ENV KRB_MAJOR_MINOR 1.21
+ENV KRB_PATCH 2
 ENV SYSTEM_LIBSSH 1
 ENV CFLAGS "-g0 -s"
 ENV OPENSSL_ROOT_DIR /usr/lib
 
 RUN apt-get update -y && apt-get install zlib1g-dev cmake perl-core -y
 
-ADD libssh-${LIBSSH}.tar.xz libssh.tar.xz
+# Use a branch of 0.10 because no released version includes all fixes as of 23/05/2024.
+# See https://gitlab.com/libssh/libssh-mirror/-/issues/227
+COPY libssh-mirror-${LIBSSH_BRANCH}.tar.gz .
 ADD https://www.openssl.org/source/${OPENSSL}.tar.gz ${OPENSSL}.tar.gz
-ADD krb5-${KRB}.tar.xz krb5-${KRB}.tar.xz
+ADD https://kerberos.org/dist/krb5/${KRB_MAJOR_MINOR}/krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
 
-RUN tar -xzf ${OPENSSL}.tar.gz
 # Openssl
+RUN tar -xzf ${OPENSSL}.tar.gz
 RUN cd ${OPENSSL} && \
     ./config --prefix=/usr --openssldir=/usr/openssl threads shared && \
     make -j6 && make install_sw
 
 # Kerberos
-RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
+RUN tar -xzf krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
+RUN cd krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}/src && \
     ./configure && \
     make -j6 && \
     make install
 
 # Libssh
+RUN tar -xzf libssh-mirror-${LIBSSH_BRANCH}.tar.gz
 RUN mkdir -p build_libssh && cd build_libssh && \
-    cmake ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
+    cmake ../libssh-mirror-${LIBSSH_BRANCH} -DCMAKE_BUILD_TYPE=Release \
           -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
     make -j6 install/strip
 
-RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz
+RUN rm -rf ${OPENSSL}* libssh-mirror-${LIBSSH_BRANCH}* build_libssh krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}*
 
 VOLUME /var/cache
diff --git a/ci/docker/manylinux/Dockerfile.aarch64_2_28 b/ci/docker/manylinux/Dockerfile.aarch64_2_28
@@ -1,36 +1,42 @@
 FROM quay.io/pypa/manylinux_2_28_aarch64
 
-ENV OPENSSL openssl-3.1.1
-ENV LIBSSH 0.10.5
-ENV KRB 1.21.1
+ENV OPENSSL openssl-3.3.0
+ENV LIBSSH_BRANCH stable-0.10
+ENV KRB_MAJOR_MINOR 1.21
+ENV KRB_PATCH 2
 ENV SYSTEM_LIBSSH 1
 ENV CFLAGS "-g0 -s"
 ENV OPENSSL_ROOT_DIR /usr/lib
 
 RUN yum install zlib-devel cmake3 perl-core -y
 
-ADD libssh-${LIBSSH}.tar.xz libssh.tar.xz
+# Use a branch of 0.10 because no released version includes all fixes as of 23/05/2024.
+# See https://gitlab.com/libssh/libssh-mirror/-/issues/227
+COPY libssh-mirror-${LIBSSH_BRANCH}.tar.gz .
 ADD https://www.openssl.org/source/${OPENSSL}.tar.gz ${OPENSSL}.tar.gz
-ADD krb5-${KRB}.tar.xz krb5-${KRB}.tar.xz
+ADD https://kerberos.org/dist/krb5/${KRB_MAJOR_MINOR}/krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
+
 
-RUN tar -xzf ${OPENSSL}.tar.gz
 # Openssl
+RUN tar -xzf ${OPENSSL}.tar.gz
 RUN cd ${OPENSSL} && \
     ./config --prefix=/usr --openssldir=/usr/openssl threads shared && \
     make -j6 && make install_sw
 
 # Kerberos
-RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
+RUN tar -xzf krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
+RUN cd krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}/src && \
     ./configure && \
     make -j6 && \
     make install
 
 # Libssh
+RUN tar -xzf libssh-mirror-${LIBSSH_BRANCH}.tar.gz
 RUN mkdir -p build_libssh && cd build_libssh && \
-    cmake ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
+    cmake ../libssh-mirror-${LIBSSH_BRANCH} -DCMAKE_BUILD_TYPE=Release \
           -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
     make -j6 install/strip
 
-RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz
+RUN rm -rf ${OPENSSL}* libssh-mirror-${LIBSSH_BRANCH}* build_libssh krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}*
 
 VOLUME /var/cache
diff --git a/ci/docker/manylinux/Dockerfile.manylinux_2_24_x86_64 b/ci/docker/manylinux/Dockerfile.manylinux_2_24_x86_64
@@ -1,36 +1,41 @@
 FROM quay.io/pypa/manylinux_2_24_x86_64
 
-ENV OPENSSL openssl-3.1.1
-ENV LIBSSH 0.10.5
-ENV KRB 1.21.1
+ENV OPENSSL openssl-3.3.0
+ENV LIBSSH_BRANCH stable-0.10
+ENV KRB_MAJOR_MINOR 1.21
+ENV KRB_PATCH 2
 ENV SYSTEM_LIBSSH 1
 ENV CFLAGS "-g0 -s"
 ENV OPENSSL_ROOT_DIR /usr/lib
 
 RUN apt-get update -y && apt-get install zlib1g-dev cmake perl-core -y
 
-ADD libssh-${LIBSSH}.tar.xz libssh.tar.xz
+# Use a branch of 0.10 because no released version includes all fixes as of 23/05/2024.
+# See https://gitlab.com/libssh/libssh-mirror/-/issues/227
+COPY libssh-mirror-${LIBSSH_BRANCH}.tar.gz .
 ADD https://www.openssl.org/source/${OPENSSL}.tar.gz ${OPENSSL}.tar.gz
-ADD krb5-${KRB}.tar.xz krb5-${KRB}.tar.xz
+ADD https://kerberos.org/dist/krb5/${KRB_MAJOR_MINOR}/krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
 
-RUN tar -xzf ${OPENSSL}.tar.gz
 # Openssl
+RUN tar -xzf ${OPENSSL}.tar.gz
 RUN cd ${OPENSSL} && \
     ./config --prefix=/usr --openssldir=/usr/openssl threads shared && \
     make -j6 && make install_sw
 
 # Kerberos
-RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
+RUN tar -xzf krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
+RUN cd krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}/src && \
     ./configure && \
     make -j6 && \
     make install
 
 # Libssh
+RUN tar -xzf libssh-mirror-${LIBSSH_BRANCH}.tar.gz
 RUN mkdir -p build_libssh && cd build_libssh && \
-    cmake ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
+    cmake ../libssh-mirror-${LIBSSH_BRANCH} -DCMAKE_BUILD_TYPE=Release \
           -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
     make -j6 install/strip
 
-RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz
+RUN rm -rf ${OPENSSL}* libssh-mirror-${LIBSSH_BRANCH}* build_libssh krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}*
 
 VOLUME /var/cache
diff --git a/ci/docker/manylinux/Dockerfile.manylinux_2_28_x86_64 b/ci/docker/manylinux/Dockerfile.manylinux_2_28_x86_64
@@ -1,36 +1,42 @@
 FROM quay.io/pypa/manylinux_2_28_x86_64
 
-ENV OPENSSL openssl-3.1.1
-ENV LIBSSH 0.10.5
-ENV KRB 1.21.1
+ENV OPENSSL openssl-3.3.0
+ENV LIBSSH_BRANCH stable-0.10
+ENV KRB_MAJOR_MINOR 1.21
+ENV KRB_PATCH 2
 ENV SYSTEM_LIBSSH 1
 ENV CFLAGS "-g0 -s"
 ENV OPENSSL_ROOT_DIR /usr/lib
 
 RUN yum install -y zlib-devel cmake3 perl-core
 
-ADD libssh-${LIBSSH}.tar.xz libssh.tar.xz
+# Use a branch of 0.10 because no released version includes all fixes as of 23/05/2024.
+# See https://gitlab.com/libssh/libssh-mirror/-/issues/227
+COPY libssh-mirror-${LIBSSH_BRANCH}.tar.gz .
 ADD https://www.openssl.org/source/${OPENSSL}.tar.gz ${OPENSSL}.tar.gz
-ADD krb5-${KRB}.tar.xz krb5-${KRB}.tar.xz
+ADD https://kerberos.org/dist/krb5/${KRB_MAJOR_MINOR}/krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
+
 
-RUN tar -xzf ${OPENSSL}.tar.gz
 # Openssl
+RUN tar -xzf ${OPENSSL}.tar.gz
 RUN cd ${OPENSSL} && \
     ./config --prefix=/usr --openssldir=/usr/openssl threads shared && \
     make -j6 && make install_sw
 
 # Kerberos
-RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
+RUN tar -xzf krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}.tar.gz
+RUN cd krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}/src && \
     ./configure && \
     make -j6 && \
     make install
 
 # Libssh
+RUN tar -xzf libssh-mirror-${LIBSSH_BRANCH}.tar.gz
 RUN mkdir -p build_libssh && cd build_libssh && \
-    cmake ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
+    cmake ../libssh-mirror-${LIBSSH_BRANCH} -DCMAKE_BUILD_TYPE=Release \
           -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
     make -j6 install/strip
 
-RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz
+RUN rm -rf ${OPENSSL}* libssh-mirror-${LIBSSH_BRANCH}* build_libssh krb5-${KRB_MAJOR_MINOR}.${KRB_PATCH}*
 
 VOLUME /var/cache
diff --git a/ci/docker/manylinux/krb5-1.21.1.tar.xz b/ci/docker/manylinux/krb5-1.21.1.tar.xz