Skip to content

Add experimental igzip (#182) #669

Add experimental igzip (#182)

Add experimental igzip (#182) #669

Workflow file for this run

name: CI
on:
push:
branches:
- master
- 'releases/**'
pull_request:
release:
types:
- released
- prereleased
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
jobs:
build-test:
runs-on: ${{ matrix.conf.os }}
name: ${{ matrix.conf.os }}-${{ matrix.python-version }}-${{ matrix.conf.target-triple }}-${{ matrix.conf.target }}
strategy:
fail-fast: ${{ !( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }}
matrix:
python-version:
- '3.8'
- '3.9'
- '3.10'
- '3.11'
- '3.12'
- '3.13'
conf:
- { os: ubuntu-latest, target: x86_64, target-triple: x86_64-unknown-linux-gnu, manylinux: auto }
- { os: ubuntu-latest, target: x86_64, target-triple: x86_64-unknown-linux-musl, manylinux: musllinux_1_1 }
- { os: ubuntu-latest, target: i686, target-triple: i686-unknown-linux-gnu, manylinux: auto }
- { os: ubuntu-latest, target: i686, target-triple: i686-unknown-linux-musl, manylinux: musllinux_1_1 }
- { os: ubuntu-latest, target: aarch64, target-triple: aarch64-unknown-linux-gnu, manylinux: auto }
- { os: ubuntu-latest, target: aarch64, target-triple: aarch64-unknown-linux-musl, manylinux: musllinux_1_1 }
- { os: ubuntu-latest, target: armv7, target-triple: armv7-unknown-linux-gnueabihf, manylinux: auto }
- { os: ubuntu-latest, target: armv7, target-triple: armv7-unknown-linux-musleabihf, manylinux: musllinux_1_1 }
- { os: ubuntu-latest, target: s390x, target-triple: s390x-unknown-linux-gnu, manylinux: auto }
# - { os: ubuntu-latest, target: s390x, target-triple: s390x-unknown-linux-musl, manylinux: musllinux_1_1 } # no target musl for s390x
- { os: ubuntu-latest, target: ppc64le, target-triple: powerpc64le-unknown-linux-gnu, manylinux: auto }
# - { os: ubuntu-latest, target: ppc64le, target-triple: powerpc64le-unknown-linux-musleabihf, manylinux: musllinux_1_1 } # no target musl for ppc64le
- { os: macos-13, target: x86_64, target-triple: x86_64-apple-darwin }
- { os: macos-13, target: aarch64, target-triple: aarch64-apple-darwin }
- { os: macos-13, target: universal2, target-triple: x86_64-apple-darwin }
- { os: windows-latest, target: x86_64, target-triple: x86_64-pc-windows-msvc, python-architecture: x64 }
- { os: windows-latest, target: i686, target-triple: i686-pc-windows-msvc, python-architecture: x86 }
include:
# Windows x86_64 pypy
- conf: { os: windows-latest, target: x86_64, target-triple: x86_64-pc-windows-msvc }
python-version: pypy3.10
# Linux x86_64 pypy
- conf: { os: ubuntu-latest, target: x86_64, target-triple: x86_64-unknown-linux-gnu, manylinux: auto }
python-version: pypy3.10
# Linux arm pypy
- conf: { os: ubuntu-latest, target: aarch64, target-triple: aarch64-unknown-linux-gnu, manylinux: auto }
python-version: pypy3.10
# OSX x86_64 pypy
- conf: { os: macos-13, target: x86_64, target-triple: x86_64-apple-darwin }
python-version: pypy3.10
# OSX universal2 pypy
- conf: { os: macos-13, target: universal2, target-triple: x86_64-apple-darwin }
python-version: pypy3.10
# OSX arm pypy
- conf: { os: macos-13, target: aarch64, target-triple: aarch64-apple-darwin }
python-version: pypy3.10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
architecture: ${{ matrix.conf.python-architecture }}
allow-prereleases: true
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
target: ${{ matrix.conf.target-triple }}
- name: Install nasm (Windows / Linux)
if: runner.os != 'macOS'
uses: ilammy/setup-nasm@v1
- name: Set MSVC developer prompt
if: runner.os == 'Windows'
uses: ilammy/msvc-dev-cmd@v1
- name: Setup (Mac)
if: runner.os == 'macOS'
run: |
brew install ninja automake autoconf coreutils libtool nasm
echo "MACOSX_DEPLOYMENT_TARGET=10.12" >> $GITHUB_ENV
- name: Setup (Windows)
if: runner.os == 'Windows'
run: |
choco install ninja cmake
- name: Setup (Linux)
if: runner.os == 'Linux'
env:
TARGET_TRIPLE: ${{ matrix.conf.target-triple }}
WORKSPACE: ${{ github.workspace }}
BLOSC2_INSTALL_PREFIX: ${{ github.workspace }}/blosc2
ISAL_INSTALL_PREFIX: ${{ github.workspace }}/isal
run: |
sudo apt update
sudo apt install ninja-build -y
echo "BLOSC2_INSTALL_PREFIX=$BLOSC2_INSTALL_PREFIX" >> $GITHUB_ENV
echo "ISAL_INSTALL_PREFIX=$ISAL_INSTALL_PREFIX" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BLOSC2_INSTALL_PREFIX/lib:$BLOSC2_INSTALL_PREFIX/lib64:$ISAL_INSTALL_PREFIX/lib:$ISAL_INSTALL_PREFIX/lib64" >> $GITHUB_ENV
# so we'll just use 'cross' to build and pass it in for the action for all archs
# nothing special for the revision pin, just deterministic install
cargo install cross --git https://github.com/cross-rs/cross --rev 6d097fb
# Build blosc2
cross build --release --target $TARGET_TRIPLE --package blosc2-sys --target-dir build
blosc2_sys_dir=$(ls build/$TARGET_TRIPLE/release/build/ | grep blosc2-sys)
mv $WORKSPACE/build/$TARGET_TRIPLE/release/build/$blosc2_sys_dir/out $BLOSC2_INSTALL_PREFIX
tree -L 2 $BLOSC2_INSTALL_PREFIX
# Build isal only on 64-bit systems
# At the time of this writing, it technically builds for all unix 32-bit systems
# but ISA-L has explicitly stated they're dropping support.
if [[ "$TARGET_TRIPLE" == armv7* || "$TARGET_TRIPLE" == i686* ]]; then
echo "Not building ISA-L on 32 bit target"
mkdir -p $ISAL_INSTALL_PREFIX
else
cross build --release --target $TARGET_TRIPLE --package isal-sys --target-dir build
isal_sys_dir=$(ls build/$TARGET_TRIPLE/release/build/ | grep isal-sys)
mv $WORKSPACE/build/$TARGET_TRIPLE/release/build/$isal_sys_dir/out $ISAL_INSTALL_PREFIX
tree -L 2 $ISAL_INSTALL_PREFIX
fi
- name: Rust Tests
if: matrix.conf.target == 'x86_64' && !startsWith(matrix.python-version, 'pypy') && matrix.python-version == '3.12'
run: cargo test
- name: Build wheel (Linux)
if: runner.os == 'Linux'
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.conf.target }}
manylinux: ${{ matrix.conf.manylinux }}
docker-options: |
-e BLOSC2_INSTALL_PREFIX=${{ github.workspace }}/blosc2
-e ISAL_INSTALL_PREFIX=${{ github.workspace }}/isal
-e LD_LIBRARY_PATH=${{ github.workspace }}/blosc2/lib:${{ github.workspace }}/blosc2/lib64:${{ github.workspace }}/isal/lib:${{ github.workspace }}/isal/lib64
args: -i ${{ matrix.python-version }} --release --out dist --features use-system-blosc2-static --features use-system-isal-static
before-script-linux: |
ls -l $BLOSC2_INSTALL_PREFIX
ls -l $ISAL_INSTALL_PREFIX
- name: Build wheel (Windows)
if: runner.os == 'Windows'
run: |
python -m pip install maturin delvewheel
maturin build -i python --release --out wheels --target ${{ matrix.conf.target-triple }}
$file = Get-ChildItem -Path "wheels" | Select-Object -First 1
delvewheel repair -v "wheels\$($file.Name)" -w "dist"
- name: Build wheel (MacOS)
if: runner.os == 'macOS'
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.conf.target-triple }}
args: -i python --release --out dist
- name: Fix wheel (MacOS)
if: runner.os == 'macOS'
run: |
python -m pip install delocate
delocate-wheel -v dist/*.whl
- name: Install built wheel and Test (Native)
# TODO: I'm not sure but the actual collection of tests on windows using pypy3.10 takes forever and/or fails
if: |
!startsWith(matrix.conf.manylinux, 'musl') &&
!( matrix.python-version == 'pypy3.10' && runner.os == 'Windows' ) &&
( matrix.conf.target == 'x86_64' || matrix.conf.target == 'universal2' )
run: |
# Second install guarantees it's going to install from local dir w/ --no-index
# use first to get in dev dependencies
python -m pip install cramjam[dev] --pre --find-links dist --force-reinstall
python -m pip install cramjam --pre --no-index --find-links dist --force-reinstall
python -m pytest -vs --benchmark-skip
# Could use 'distro: alpine_latest' in 'run-on-arch-action' but seems difficult to install a specific version of python
# so we'll just use existing python alpine images to test import and cli use w/o testing archs other than x86_64
- name: Install built wheel and Test (musllinux)
# TODO: python:3.13-alpine doesn't exist yet
if: startsWith(matrix.conf.manylinux, 'musl') && matrix.conf.target == 'x86_64' && matrix.python-version != '3.13'
run: |
docker run \
-v $(pwd)/dist:/wheels \
--rm python:${{ matrix.python-version }}-alpine sh \
-c "pip install cramjam --no-index --find-links /wheels && python -c 'import cramjam'"
- name: Install built wheel and Test (Cross)
if: |
!startsWith(matrix.conf.manylinux, 'musl') &&
runner.os == 'Linux' &&
!startsWith(matrix.python-version, 'pypy') &&
contains(fromJson('["armv6", "armv7", "aarch64", "riscv64", "s390x", "ppc64le"]'), matrix.conf.target)
uses: uraimo/[email protected]
with:
arch: ${{ matrix.conf.target }}
distro: ubuntu20.04
githubToken: ${{ github.token }}
# Mount the dist directory as /artifacts in the container
dockerRunArgs: |
--volume "${PWD}/dist:/artifacts"
install: |
apt-get update
apt-get install -y --no-install-recommends python3 python3-venv software-properties-common
add-apt-repository ppa:deadsnakes/ppa
apt-get update
apt-get install -y curl python${{ matrix.python-version }}-venv
run: |
ls -lrth /artifacts
PYTHON=python${{ matrix.python-version }}
$PYTHON -m venv venv
venv/bin/pip install -U pip
venv/bin/pip install cramjam --pre --no-index --find-links /artifacts --force-reinstall
venv/bin/python -c 'import cramjam'
- name: Upload wheels
uses: actions/upload-artifact@v4
if: ${{ ( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }}
with:
name: ${{ matrix.conf.os }}-${{ matrix.python-version }}-${{ matrix.conf.target-triple }}-${{ matrix.conf.target }}
path: dist
build-wasm32-emscripten-pyodide:
runs-on: ubuntu-latest
strategy:
matrix:
python:
- "3.12"
steps:
- uses: actions/checkout@v4
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: nightly
target: wasm32-unknown-emscripten
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Install Emscripten
uses: mymindstorm/setup-emsdk@v14
with:
# This needs to match the exact expected version pyodide expects...seems a bit brittle TBH, maybe I'm missing something.
# Discover by updating pyodide in package.json and re-running 'npm run test'; it'll spit out the error of
# the expected vs actual versions.
version: '3.1.58'
- name: Build
run: |
pip install maturin
maturin build --release -i python${{ matrix.python }} --features wasm32-compat --target wasm32-unknown-emscripten -o ./dist
- uses: actions/setup-node@v3
with:
node-version: '18'
- run: npm install
- run: npm run test
- name: Upload wheels
uses: actions/upload-artifact@v4
if: ${{ ( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }}
with:
name: wasm32-unknown-emscripten-python${{ matrix.python }}
path: dist
build-sdist:
name: Build sdists
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Build sdist cramjam
run: |
python -m pip install build
python -m build --sdist -o ./dist
- name: Upload sdists
uses: actions/upload-artifact@v4
with:
name: sdist
path: dist
pypi-publish:
name: Upload release to PyPI
if: startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
needs: [build-test, build-sdist]
environment:
name: pypi
url: https://pypi.org/p/cramjam
permissions:
id-token: write
steps:
- uses: actions/download-artifact@v4
with:
path: .
merge-multiple: true
- name: List artifacts
run: ls -lhs
- name: Remove wasm32 wheels # TODO: https://discuss.python.org/t/support-wasm-wheels-on-pypi/21924
run: rm ./*wasm32.whl
- name: Publish package distributions to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
skip-existing: true
packages-dir: ./
gh-publish:
name: Publish artifacts to GH
if: startsWith(github.ref, 'refs/tags/')
permissions:
contents: write
needs: [build-test, build-sdist]
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
path: .
merge-multiple: true
- name: List artifacts
run: ls -lhs
- name: Upload to GitHub
uses: softprops/action-gh-release@v2
with:
files: ./*