Draft: Add initial build wheels CI/CD pipeline

.github/workflows/build-wheels.yml

@@ -0,0 +1,161 @@
+name: Build and test wheels
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Version to use are release version
+        required: true
+        type: string
+      test_pypi:
+        description: Upload packages to test.pypi.org
+        required: false
+        type: boolean
+        default: false
+  push:
+    tags:
+      - "*"
+
+env:
+  SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TILEDB: ${{ inputs.version }}
+
+jobs:
+  build_wheels:
+    name: Wheel ${{ matrix.buildplat[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.python }}
+    runs-on: ${{ matrix.buildplat[0] }}
+    strategy:
+      matrix:
+        buildplat:
+          - [ubuntu-22.04, manylinux_x86_64]
+          - [macos-13, macosx_x86_64]
+          - [macos-14, macosx_arm64]
+          - [windows-2022, win_amd64]
+        python: ["cp38", "cp39", "cp310", "cp311", "cp312"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Brew setup on macOS" # x-ref c8e49ba8f8b9ce
+        if: ${{ startsWith(matrix.buildplat[0], 'macos-') == true }}
+        run: |
+          set -e pipefail
+          brew update
+          brew install automake pkg-config ninja llvm bzip2 xz autoconf zlib libdeflate bcftools
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD_VERBOSITY: 3
+          CIBW_ENVIRONMENT_MACOS: >
+            CC=clang
+            CXX=clang++
+          CIBW_ARCHS: all
+          CIBW_PRERELEASE_PYTHONS: True
+          CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }}
+          CIBW_BEFORE_ALL_LINUX: yum install -y bzip2-devel xz-devel bcftools
+          CIBW_TEST_COMMAND: pytest {package}/apis/python
+          CIBW_TEST_EXTRAS: test
+          MACOSX_DEPLOYMENT_TARGET: "13.0"
+        with:
+          output-dir: wheelhouse
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.buildplat[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.python }}
+          path: "./wheelhouse/*.whl"
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    outputs:
+      sdist_name: ${{ steps.get_sdist_name.outputs.sdist_name }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - name: Get sdist package name
+        id: get_sdist_name
+        run: |
+          echo "sdist_name=$(ls dist/ | head -n 1)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+
+  test_sdist:
+    name: Test source distribution package
+    needs: [build_sdist]
+    strategy:
+      matrix:
+        os:
+          - macos-13
+          - macos-14
+          # - windows-2022 Currently we do not know how to install bcftools on windows
+          - ubuntu-22.04
+        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install bcftools MacOS
+        if: ${{ startsWith(matrix.os, 'macos-') == true }}
+        run: |
+          set -e pipefail
+          brew update
+          brew install automake pkg-config ninja llvm bzip2 xz autoconf zlib libdeflate bcftools
+
+      - name: Install bcftools Ubuntu
+        if: ${{ startsWith(matrix.os, 'ubuntu-') == true }}
+        run: sudo apt install bcftools
+
+      - name: Download sdist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+      - name: Install sdist artifact
+        run: pip install --verbose dist/${{ needs.build_sdist.outputs.sdist_name }}[test]
+
+      - uses: actions/checkout@v4
+
+      - name: Run tests
+        shell: bash
+        run: |
+          PROJECT_CWD=$PWD
+          cd ..
+          pytest -vv --showlocals $PROJECT_CWD/apis/python
+
+  upload_pypi:
+    needs: [build_wheels, test_sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    outputs:
+      package_version: ${{ steps.get_package_version.outputs.package_version }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+
+      - id: get_package_version
+        run: |
+          echo "package_version=$(ls dist/ | head -n 1 | cut -d - -f 2)" >> "$GITHUB_OUTPUT"
+
+      - name: Upload to test-pypi
+        if: inputs.test_pypi
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+      # - name: Upload to pypi
+      #   if: startsWith(github.ref, 'refs/tags/')
+      #   uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/macos.yml

@@ -29,7 +29,8 @@ jobs:
             -D CMAKE_BUILD_TYPE=Debug \
             -D CMAKE_INSTALL_PREFIX=$(pwd)/dist \
             -D OVERRIDE_INSTALL_PREFIX=OFF \
-            -D DOWNLOAD_TILEDB_PREBUILT=ON
+            -D DOWNLOAD_TILEDB_PREBUILT=ON \
+            -D TILEDBVCF_ENABLE_TESTS=ON
       - name: Build libtiledbvcf
         run: cmake --build $(pwd)/libtiledbvcf/build -j 2 --config Debug
       - name: Upload coredump as artifact if build failed
@@ -72,11 +73,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # fetch everything for python setuptools_scm
-      - name: Download libtiledbvcf artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: libtiledbvcf
-          path: dist
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -88,14 +84,16 @@ jobs:
           create-args: tiledb
       - name: Install dependencies
         run: python -m pip install --prefer-binary pyarrow==10.0.1
+      - name: Intall build dependencies
+        run: |
+          set -e pipefail
+          brew update
+          brew install automake pkg-config ninja llvm bzip2 xz autoconf zlib libdeflate bcftools
       - name: Build tiledbvcf-py
         env:
           LIBTILEDBVCF_PATH: "${{ github.workspace }}/dist"
           TileDB_DIR: "/Users/runner/micromamba/envs/ci/lib/cmake/TileDB"
         run: |
-          echo $DYLD_LIBRARY_PATH
-          echo $TileDB_DIR
-          cd apis/python
           python -m pip install -v .[test]
       - name: Confirm linking
         run: otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.cpython-*-darwin.so
@@ -166,13 +164,13 @@ jobs:
       - name: Install dependencies
         run: python -m pip install --prefer-binary pyarrow==10.0.1
       - name: Build tiledbvcf-py
-        run: cd apis/python && python -m pip install -v .[test]
+        run: python -m pip install -v .[test]
       - name: Confirm linking
         run: |
           otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.cpython-*-darwin.so
-          otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.dylib
-          otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledb.dylib
-          otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libhts.*.dylib
+          otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/lib/libtiledbvcf.dylib
+          otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/lib/libtiledb.dylib
+          otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/lib/libhts.*.dylib
       - name: Version
         run: python -c "import tiledbvcf; print(tiledbvcf.version)"
       - name: Install bcftools (for tests)