ci.yml

name: ci

on:
  push:
    branches:
      - main
    tags:
      - '**'
  pull_request: {}

env:
  COLUMNS: 150

jobs:
  coverage:
    runs-on: ubuntu-20.04

    steps:
      - uses: actions/checkout@v3

      - name: install rust nightly
        uses: dtolnay/rust-toolchain@nightly

      - id: cache-rust
        name: cache rust
        uses: Swatinem/rust-cache@v2
        with:
          key: coverage-v2

      - run: cargo install rustfilt coverage-prepare
        if: steps.cache-rust.outputs.cache-hit != 'true'

      - run: rustup component add llvm-tools-preview

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - run: pip install -r tests/requirements.txt

      - run: rustc --version --verbose

      - run: pip install -e .
        env:
          RUST_BACKTRACE: 1
          RUSTFLAGS: '-C instrument-coverage'

      - run: pip freeze

      - run: coverage run -m pytest

      - run: ls -lha
      - run: coverage xml

      - run: coverage-prepare lcov python/pydantic_core/*.so

      - uses: codecov/codecov-action@v3

  # See https://github.com/PyO3/pyo3/discussions/2781
  # tests intermittently segfault with pypy and cpython 3.7 when using `coverage run ...`, hence separate job
  test-python:
    name: test ${{ matrix.python-version }}
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - '3.7'
          - '3.8'
          - '3.9'
          - '3.10'
          - '3.11'
          - '3.12-dev'
          - 'pypy3.7'
          - 'pypy3.8'
          - 'pypy3.9'

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: install rust stable
        uses: dtolnay/rust-toolchain@stable

      - name: cache rust
        uses: Swatinem/rust-cache@v2
        with:
          key: test-v3

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - run: pip install -r tests/requirements.txt

      - run: pip install -e .
        env:
          RUST_BACKTRACE: 1

      - run: pip freeze

      - run: pytest
        env:
          HYPOTHESIS_PROFILE: slow

  test-os:
    name: test on ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos, windows]

    runs-on: ${{ matrix.os }}-latest
    steps:
      - uses: actions/checkout@v3

      - name: install rust stable
        uses: dtolnay/rust-toolchain@stable

      - name: cache rust
        uses: Swatinem/rust-cache@v2
        with:
          key: ${{ matrix.os }}-v1

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - run: pip install -r tests/requirements.txt

      - run: pip install -e .
        env:
          RUST_BACKTRACE: 1

      - run: pip freeze

      - run: pytest

      - run: cargo test

  # test with a debug build as it picks up errors which optimised release builds do not
  test-debug:
    name: test-debug ${{ matrix.python-version }}
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        python-version:
          - '3.11'
          - 'pypy3.9'

    steps:
      - uses: actions/checkout@v3
      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: install rust stable
        uses: dtolnay/rust-toolchain@stable

      - name: cache rust
        uses: Swatinem/rust-cache@v2
        with:
          key: test-debug

      - run: pip install -r tests/requirements.txt
      - run: make build-dev

      - run: pip freeze
      - run: pytest

  test-pydantic-integration:
    runs-on: ubuntu-latest
    continue-on-error: true

    steps:
      - uses: actions/checkout@v3
        with:
          repository: pydantic/pydantic

      - uses: actions/checkout@v3
        with:
          path: pydantic-core

      - name: install rust stable
        uses: dtolnay/rust-toolchain@stable

      - name: cache rust
        uses: Swatinem/rust-cache@v2
        with:
          key: test-pydantic-integration

      - uses: pdm-project/setup-pdm@v3
        with:
          cache: true

      - name: install deps
        run: |
          pdm venv create --with-pip 3.11
          pdm install -G testing
          pdm run pip install maturin
          cd pydantic-core && pdm run make build-dev

      - run: pdm info && pdm list
      - run: pdm run pytest

  lint:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: install rust stable
        uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt, clippy

      - name: cache rust
        uses: Swatinem/rust-cache@v2

      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      # used to lint js code
      - uses: actions/setup-node@v3
        with:
          node-version: '18'

      - uses: actions/cache@v3
        id: cache-py
        name: cache python
        with:
          path: ${{ env.pythonLocation }}
          key: >
            py
            ${{ env.pythonLocation }}
            ${{ hashFiles('tests/requirements-linting.txt') }}
            ${{ hashFiles('pyproject.toml') }}

      - run: pip install -r tests/requirements-linting.txt
        if: steps.cache-py.outputs.cache-hit != 'true'

      - run: make build-dev

      - run: pip freeze

      - run: make lint
      - run: make pyright

      - run: npm install
      - run: npm run lint

  bench:
    name: rust benchmarks
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: install rust nightly
        uses: dtolnay/rust-toolchain@nightly

      - name: cache rust
        uses: Swatinem/rust-cache@v2

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - run: pip install typing_extensions

      - run: cargo bench

  build-wasm-emscripten:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: set up python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: install rust nightly
        uses: dtolnay/rust-toolchain@nightly
        with:
          components: rust-src
          targets: wasm32-unknown-emscripten

      - name: cache rust
        uses: Swatinem/rust-cache@v2

      - uses: mymindstorm/setup-emsdk@v11
        with:
          # NOTE!: as per https://github.com/pydantic/pydantic-core/pull/149 this version needs to match the version
          # in node_modules/pyodide/repodata.json, to get the version, run:
          # `cat node_modules/pyodide/repodata.json | python -m json.tool | rg platform`
          version: '3.1.32'
          actions-cache-folder: emsdk-cache

      - run: pip install 'maturin>=1,<2' 'black>=22.3.0,<23' typing_extensions

      - name: build wheels
        run: make build-wasm

      - uses: actions/setup-node@v3
        with:
          node-version: '18'

      - run: npm install

      - run: npm run test

      - run: |
         ls -lh dist/
         ls -l dist/

      - uses: actions/upload-artifact@v3
        with:
          name: wasm_wheels
          path: dist

  # https://github.com/marketplace/actions/alls-green#why used for branch protection checks
  check:
    if: always()
    needs: [coverage, test-python, test-os, test-debug, lint, bench, build-wasm-emscripten]
    runs-on: ubuntu-latest
    steps:
      - name: Decide whether the needed jobs succeeded or failed
        uses: re-actors/alls-green@release/v1
        with:
          jobs: ${{ toJSON(needs) }}
          allowed-failures: coverage

  build-sdist:
    name: build sdist
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: PyO3/maturin-action@v1
        with:
          command: sdist
          args: --out dist
          rust-toolchain: stable
      - uses: actions/upload-artifact@v3
        with:
          name: pypi_files
          path: dist

  build:
    name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
    # only run on push to main and on release
    if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos, windows]
        target: [x86_64, aarch64]
        manylinux: [auto]
        include:
          - os: ubuntu
            platform: linux
          - os: windows
            ls: dir
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9 pypy3.10
          - os: windows
            ls: dir
            target: i686
            python-architecture: x86
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
          - os: macos
            target: aarch64
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
          - os: ubuntu
            platform: linux
            target: i686
          # GCC 4.8.5 in manylinux2014 container doesn't support c11 atomic
          # we use manylinux_2_24 container for aarch64 and armv7 targets instead,
          - os: ubuntu
            platform: linux
            target: aarch64
            container: messense/manylinux_2_24-cross:aarch64
          - os: ubuntu
            platform: linux
            target: armv7
            container: messense/manylinux_2_24-cross:armv7
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
          # musllinux
          - os: ubuntu
            platform: linux
            target: x86_64
            manylinux: musllinux_1_1
          - os: ubuntu
            platform: linux
            target: aarch64
            manylinux: musllinux_1_1
          - os: ubuntu
            platform: linux
            target: ppc64le
            container: messense/manylinux_2_24-cross:ppc64le
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
          - os: ubuntu
            platform: linux
            target: s390x
            container: messense/manylinux_2_24-cross:s390x
            interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
        exclude:
          # Optimized PGO builds for x86_64 manylinux and windows follow a different matrix,
          # maybe in future maturin-action can support this automatically
          - os: ubuntu
            target: x86_64
            manylinux: auto
          - os: windows
            target: x86_64
          # Windows on arm64 only supports Python 3.11+
          - os: windows
            target: aarch64

    runs-on: ${{ matrix.os }}-latest
    steps:
      - uses: actions/checkout@v3

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          architecture: ${{ matrix.python-architecture || 'x64' }}

      - run: pip install -U twine 'black>=22.3.0,<23' typing_extensions

      # generate self-schema now, so we don't have to do so inside docker in maturin build
      - run: python generate_self_schema.py

      - name: build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          manylinux: ${{ matrix.manylinux || 'auto' }}
          container: ${{ matrix.container }}
          args: --release --out dist --interpreter ${{ matrix.interpreter || '3.7 3.8 3.9 3.10 3.11 3.12 pypy3.7 pypy3.8 pypy3.9 pypy3.10' }}
          rust-toolchain: stable
          docker-options: -e CI

      - run: ${{ matrix.ls || 'ls -lh' }} dist/

      - run: twine check --strict dist/*

      - uses: actions/upload-artifact@v3
        with:
          name: pypi_files
          path: dist

  build-pgo:
    name: build pgo-optimized on ${{ matrix.platform || matrix.os }} (${{ matrix.interpreter}} - ${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
    # only run on push to main and on release
    if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, windows]
        target: [x86_64]
        manylinux: [auto]
        interpreter: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12-dev", "pypy3.7", "pypy3.8", "pypy3.9", "pypy3.10"]
        include:
          - os: ubuntu
            platform: linux
          - os: windows
            ls: dir
          - interpreter: 3.12-dev
            maturin-interpreter: "3.12"

    runs-on: ${{ matrix.os }}-latest
    steps:
      - uses: actions/checkout@v3

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.interpreter }}
          architecture: ${{ matrix.python-architecture || 'x64' }}

      - name: install rust stable
        id: rust-toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          components: llvm-tools

      - run: pip install -U 'black>=22.3.0,<23' typing_extensions

      # generate self-schema now, so we don't have to do so inside docker in maturin build
      - run: python generate_self_schema.py

      - name: build initial wheel
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          manylinux: ${{ matrix.manylinux || 'auto' }}
          args: >
            --release
            --out pgo-wheel
            --interpreter ${{ matrix.maturin-interpreter || matrix.interpreter }}
          rust-toolchain: stable
          docker-options: -e CI
        env:
          RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"

      - name: detect rust host
        run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV"
        shell: bash

      - name: generate pgo data
        run: |
          pip install -U pip
          pip install -r tests/requirements.txt
          pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall
          pytest tests/benchmarks
          rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"'

      - name: merge pgo data
        run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata

      - name: build pgo-optimized wheel
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          manylinux: ${{ matrix.manylinux || 'auto' }}
          args: >
            --release
            --out dist
            --interpreter ${{ matrix.maturin-interpreter || matrix.interpreter }}
          rust-toolchain: stable
          docker-options: -e CI
        env:
          RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"

      - run: ${{ matrix.ls || 'ls -lh' }} dist/

      - uses: actions/upload-artifact@v3
        with:
          name: pypi_files
          path: dist

  inspect-pypi-assets:
    needs: [build, build-sdist, build-pgo]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: get dist artifacts
        uses: actions/download-artifact@v3
        with:
          name: pypi_files
          path: dist

      - name: list dist files
        run: |
         ls -lh dist/
         ls -l dist/
         echo "`ls dist | wc -l` files"

      - name: extract and list sdist file
        run: |
         mkdir sdist-files
         tar -xvf dist/*.tar.gz -C sdist-files
         tree -a sdist-files

      - name: extract and list wheel file
        run: |
         ls dist/*cp310-manylinux*x86_64.whl | head -n 1
         python -m zipfile --list `ls dist/*cp310-manylinux*x86_64.whl | head -n 1`

  test-builds-arch:
    name: test build on ${{ matrix.target }}-${{ matrix.distro }}
    needs: [build]
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        target: [aarch64, armv7, s390x, ppc64le]
        distro: ['ubuntu22.04']
        include:
          - target: aarch64
            distro: alpine_latest

    steps:
      - uses: actions/checkout@v3

      - name: get dist artifacts
        uses: actions/download-artifact@v3
        with:
          name: pypi_files
          path: dist

      - uses: uraimo/run-on-arch-action@v2.5.0
        name: install & test
        with:
          arch: ${{ matrix.target }}
          distro: ${{ matrix.distro }}
          githubToken: ${{ github.token }}
          install: |
            set -x
            if command -v apt-get &> /dev/null; then
              echo "installing python & pip with apt-get..."
              apt-get update
              apt-get install -y --no-install-recommends python3 python3-pip python3-venv
            else
              echo "installing python & pip with apk..."
              apk update
              apk add python3 py3-pip
            fi
          run: |
            set -x
            # typing-extensions isn't automatically installed because of `--no-index --no-deps`
            python3 -m venv venv
            source venv/bin/activate
            python3 -m pip install -U pip typing-extensions
            python3 -m pip install -r tests/requirements.txt
            python3 -m pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall
            python3 -m pytest --ignore=tests/test_docstrings.py
            python3 -c 'import pydantic_core._pydantic_core; print(pydantic_core._pydantic_core.__version__)'

  test-builds-os:
    name: test build on ${{ matrix.os }}
    needs: [build, build-pgo]

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos, windows]

    runs-on: ${{ matrix.os }}-latest
    steps:
      - uses: actions/checkout@v3

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: get dist artifacts
        uses: actions/download-artifact@v3
        with:
          name: pypi_files
          path: dist

      - run: pip install typing-extensions
      - run: pip install -r tests/requirements.txt
      - run: pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall
      - run: pytest --ignore=tests/test_docstrings.py

  release:
    needs: [test-builds-arch, test-builds-os, build-sdist, check]
    if: success() && startsWith(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - run: pip install -U twine

      - name: check package version
        run: python .github/check_version.py

      - name: get dist artifacts
        uses: actions/download-artifact@v3
        with:
          name: pypi_files
          path: dist

      - run: twine check --strict dist/*

      - name: upload to pypi
        run: twine upload dist/*
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.pypi_token }}

      - name: get wasm dist artifacts
        uses: actions/download-artifact@v3
        with:
          name: wasm_wheels
          path: wasm

      - name: upload to github release
        uses: softprops/action-gh-release@v1
        with:
          files: |
            wasm/*.whl
          prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') }}