diff --git a/.github/workflows/build-wheels-defined.yml b/.github/workflows/build-wheels-defined.yml index 3a2a7c9..c4ed624 100644 --- a/.github/workflows/build-wheels-defined.yml +++ b/.github/workflows/build-wheels-defined.yml @@ -47,6 +47,7 @@ jobs: if: ${{ inputs.os_ubuntu_latest }} runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: @@ -56,7 +57,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python_version}} + python-version: ${{ matrix.python-version}} - name: Get Python version run: | @@ -85,6 +86,7 @@ jobs: if: ${{ inputs.os_windows_latest }} runs-on: windows-latest strategy: + fail-fast: false matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: @@ -94,7 +96,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python_version}} + python-version: ${{ matrix.python-version}} - name: Get Python version run: | @@ -120,6 +122,7 @@ jobs: if: ${{ inputs.os_macos_latest }} runs-on: macos-latest strategy: + fail-fast: false matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: @@ -129,7 +132,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python_version}} + python-version: ${{ matrix.python-version}} - name: Get Python version run: | @@ -158,18 +161,27 @@ jobs: if: ${{ inputs.os_macos_arm64 }} runs-on: macos-latest-xlarge # MacOS M1 GitHub beta runner - paid $0.16 strategy: + fail-fast: false matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Setup Python + # Temporary solution until Python version for build will be >= 3.10 (GitHub action support) + if: matrix.python-version == '3.12' + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Setup Python - MacOS M1 # Temporary solution until Python version for build will be >= 3.10 (GitHub action support) + if: matrix.python-version != '3.12' run: | - brew install python@${{ matrix.python_version }} + brew install python@${{ matrix.python-version }} # change python symlink called with default command 'python' - ln -s -f /opt/homebrew/bin/python${{ matrix.python_version }} /usr/local/bin/python + ln -s -f /opt/homebrew/bin/python${{ matrix.python-version }} /usr/local/bin/python - name: Get Python version run: | @@ -198,17 +210,25 @@ jobs: if: ${{ inputs.os_linux_armv7 }} runs-on: linux-armv7-self-hosted strategy: + fail-fast: false matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + include: + - python-version: '3.8' + CONTAINER: 'python:3.8-bullseye' + - python-version: '3.9' + CONTAINER: 'python:3.9-bullseye' + - python-version: '3.10' + CONTAINER: 'python:3.10-bullseye' + - python-version: '3.11' + CONTAINER: 'python:3.11-bullseye' + - python-version: '3.12' + CONTAINER: 'python:3.12-bullseye' + container: ${{ matrix.CONTAINER }} steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python_version}} - - name: Get Python version run: | python --version @@ -239,17 +259,25 @@ jobs: if: ${{ inputs.os_linux_arm64 }} runs-on: linux-arm64-self-hosted strategy: + fail-fast: false matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + include: + - python-version: '3.8' + CONTAINER: 'python:3.8-bullseye' + - python-version: '3.9' + CONTAINER: 'python:3.9-bullseye' + - python-version: '3.10' + CONTAINER: 'python:3.10-bullseye' + - python-version: '3.11' + CONTAINER: 'python:3.11-bullseye' + - python-version: '3.12' + CONTAINER: 'python:3.12-bullseye' + container: ${{ matrix.CONTAINER }} steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python_version}} - - name: Get Python version run: | python --version @@ -273,6 +301,8 @@ jobs: upload-python-wheels: + if: ${{ always() }} needs: [ubuntu-latest, windows-latest, macos-latest, macos-m1, linux-armv7, linux-arm64] name: Upload Python wheels uses: espressif/idf-python-wheels/.github/workflows/upload-python-wheels.yml@main + secrets: inherit diff --git a/.github/workflows/build-wheels-linux-arm64-self-hosted.yml b/.github/workflows/build-wheels-linux-arm64-self-hosted.yml deleted file mode 100644 index 1c2d678..0000000 --- a/.github/workflows/build-wheels-linux-arm64-self-hosted.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: linux-arm64-dispatch - -on: - # schedule: - # - cron: '0 0 * * 0,3' - workflow_dispatch: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - workflow_call: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - -jobs: - build-python-wheels: - name: Build Python Wheels for linux-arm64 - runs-on: [self-hosted, linux, ARM64] - strategy: - fail-fast: false - matrix: - python-version: ['3.8.17', '3.9.17', '3.10.12', '3.11.4'] - container: - image: ghcr.io/espressif/github-esp-dockerfiles/pyenv_rust_powershell:v2 - options: --privileged - steps: - - name: Prepare package list - if: ${{ inputs.packages }} - run: | - echo "input packages: ${{ inputs.packages }}" - PACKAGES=$(echo "${{ inputs.packages }}" | - sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/\([^[:space:]]\{1,\}\)/"\1"/g;s/[[:space:]]\{1,\}/,/g') - echo "packages=$PACKAGES" >> $GITHUB_ENV - echo "output packages: $PACKAGES" - - name: Checkout repository - uses: actions/checkout@v2 - - name: Prepare download folder - run: mkdir download - - name: Install build dependencies - run: | - pyenv global ${{ matrix.python-version }} - python -m pip install wheel - pip3 install wheel - python -m pip install --upgrade pip - apt-get update - apt-get install -y --no-install-recommends build-essential - apt-get install -y --no-install-recommends python3-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev - apt-get install -y --no-install-recommends dbus-tests - - name: Get branch name - run: | - INPUT=${{ github.event.inputs.IDF_branch }} - if [ -z "$INPUT" ]; then - echo "IDF_branch=main" >> $GITHUB_ENV - else - echo IDF_branch=${{ github.event.inputs.IDF_branch }} >> $GITHUB_ENV - fi - - name: Build wheels for IDF ${{ env.IDF_branch }} branch - env: - PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" - shell: pwsh - run: | - $env:PATH+=":/root/.cargo/bin" - rustc --version - if ( "${{ inputs.packages }}" ) { - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -NoReq -CompileWheels @(${{ env.packages }}) - } else { - # Cython==3.0.0 breaks the build of gevent. The build environment from https://github.com/gevent/gevent/blob/1.5.0/pyproject.toml - # must be patched with "cython<3" - # gevent is not possible to build in version 1.5.0 on Python 3.11 for ARM64 - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -BuildEnv @("setuptools", "wheel", "cython<3", "cffi") -CompileWheels @("gevent==1.5.0; python_version < '3.11'") - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -CompileWheels @("cryptography", "python-pkcs11") - } - - name: Copy cached wheels - run: cp -u `pip cache list --format=abspath` download - - name: Test wheels by installation - shell: pwsh - run: | - if ( "${{ inputs.packages }}" ) { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} -TestWheels @(${{ env.packages }}) - } else { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} - } - - name: Upload Release Asset To test s3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_BUCKET: ${{ secrets.DL_BUCKET }} - PREFIX: 'pypi' - shell: bash - run: | - chmod +x Upload-Wheels.sh - ./Upload-Wheels.sh $AWS_BUCKET - python3 create_index_pages.py $AWS_BUCKET - - name: Drop AWS cache - id: invalidate-index-cache - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - run: aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CACHE_INVALIDATION }} --paths "/pypi/*" diff --git a/.github/workflows/build-wheels-linux-armv7-self-hosted.yml b/.github/workflows/build-wheels-linux-armv7-self-hosted.yml deleted file mode 100644 index 1d40651..0000000 --- a/.github/workflows/build-wheels-linux-armv7-self-hosted.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: armv7-dispatch - -on: - # schedule: - # - cron: '0 0 * * 0,3' - workflow_dispatch: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - workflow_call: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - -jobs: - build-python-wheels: - name: Build Python Wheels for linux-armv7 - runs-on: linux-armv7-self-hosted - strategy: - fail-fast: false - matrix: - python-version: ['3.8.17', '3.9.17', '3.10.12', '3.11.4'] - container: - image: ghcr.io/espressif/github-esp-dockerfiles/pyenv_rust_powershell:v2 - options: --privileged - steps: - - name: Prepare package list - if: ${{ inputs.packages }} - run: | - echo "input packages: ${{ inputs.packages }}" - PACKAGES=$(echo "${{ inputs.packages }}" | - sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/\([^[:space:]]\{1,\}\)/"\1"/g;s/[[:space:]]\{1,\}/,/g') - echo "packages=$PACKAGES" >> $GITHUB_ENV - echo "output packages: $PACKAGES" - - name: Checkout repository - uses: actions/checkout@v2 - - name: Prepare download folder - run: mkdir download - - name: Add symlink for libffi.so.6 - run: ln -sfn /usr/lib/arm-linux-gnueabihf/libffi.so.7.1.0 /usr/lib/arm-linux-gnueabihf/libffi.so.6 - - name: Install build dependencies - run: | - apt update - apt-get install rustc -y - apt-get install libtiff5 libjpeg-dev libopenjp2-7 cmake libdbus-1-dev -y - pyenv global ${{ matrix.python-version }} - python -m pip install wheel - pip3 install wheel - apt-get install -y --no-install-recommends python3-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev - apt-get install -y --no-install-recommends dbus-tests - - name: Get branch name - run: | - INPUT=${{ github.event.inputs.IDF_branch }} - if [ -z "$INPUT" ]; then - echo "IDF_branch=main" >> $GITHUB_ENV - else - echo IDF_branch=${{ github.event.inputs.IDF_branch }} >> $GITHUB_ENV - fi - - name: Build wheels for IDF ${{ env.IDF_branch }} branch - env: - PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" - shell: pwsh - run: | - python -m pip install --upgrade pip - $env:PATH+=":/root/.cargo/bin" - rustc --version - if ( "${{ inputs.packages }}" ) { - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -NoReq -CompileWheels @(${{ env.packages }}) - } else { - # Cython==3.0.0 breaks the build of gevent. The build environment from https://github.com/gevent/gevent/blob/1.5.0/pyproject.toml - # must be patched with "cython<3" - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -BuildEnv @("setuptools", "wheel", "cython<3", "cffi") -CompileWheels @("gevent==1.5.0") - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -CompileWheels @("cryptography", "python-pkcs11") - } - - name: Copy cached wheels - run: cp -u `pip cache list --format=abspath` download - - name: Test wheels by installation - shell: pwsh - run: | - if ( "${{ inputs.packages }}" ) { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} -TestWheels @(${{ env.packages }}) - } else { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} - } - - name: Upload Release Asset To test s3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_BUCKET: ${{ secrets.DL_BUCKET }} - PREFIX: 'pypi' - shell: bash - run: | - chmod +x Upload-Wheels.sh - ./Upload-Wheels.sh $AWS_BUCKET - python3 create_index_pages.py $AWS_BUCKET - - name: Drop AWS cache - id: invalidate-index-cache - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - run: aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CACHE_INVALIDATION }} --paths "/pypi/*" diff --git a/.github/workflows/build-wheels-macos-M1-self-hosted.yml b/.github/workflows/build-wheels-macos-M1-self-hosted.yml deleted file mode 100644 index 4fef6f6..0000000 --- a/.github/workflows/build-wheels-macos-M1-self-hosted.yml +++ /dev/null @@ -1,168 +0,0 @@ -name: macos-M1-dispatch - -on: - # schedule: - # - cron: '0 0 * * 0,3' - workflow_dispatch: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - workflow_call: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - -jobs: - build-python-wheels: - name: Build Python Wheels for macos-M1 - runs-on: macos-m1-self-hosted - strategy: - fail-fast: false - matrix: - python-version: ['3.8.17', '3.9.17', '3.10.12', '3.11.4'] - steps: - - name: Prepare package list - if: ${{ inputs.packages }} - run: | - echo "input packages: ${{ inputs.packages }}" - PACKAGES=$(echo "${{ inputs.packages }}" | - sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/\([^[:space:]]\{1,\}\)/"\1"/g;s/[[:space:]]\{1,\}/,/g') - echo "packages=$PACKAGES" >> $GITHUB_ENV - echo "output packages: $PACKAGES" - - name: Checkout repository - uses: actions/checkout@v2 - - name: Cache Python - id: cache-python - uses: actions/cache@v2 - with: - path: ~/.pyenv - key: pyenv-${{ matrix.python-version }} - - name: Set up pyenv - if: steps.cache-python.outputs.cache-hit != 'true' - run: | - rm -rf ~/.pyenv - git clone https://github.com/pyenv/pyenv.git ~/.pyenv - export PYENV_ROOT=~/.pyenv - export PATH=$PYENV_ROOT/bin:$PATH - eval "$(pyenv init --path)" - pyenv --version - arch -arm64 pyenv install --skip-existing ${{ matrix.python-version }} - export CPPFLAGS=-I/Users/githubrunner/brew/opt/openssl/include - export LDFLAGS=-L/Users/githubrunner/brew/opt/openssl/lib - ~/.pyenv/versions/${{ matrix.python-version }}/bin/python -m pip install --upgrade pip - - name: Install stable Rust with clippy and rustfmt - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - components: rustfmt, clippy - - name: Rust version - run: | - rustc --version - rustup default - rustup update - - name: Prepare download folder - run: mkdir download - - name: Install build dependencies - run: ~/.pyenv/versions/${{ matrix.python-version }}/bin/python -m pip install wheel - - name: Get python3 version - run: ~/.pyenv/versions/${{ matrix.python-version }}/bin/python --version - - name: Get branch name - run: | - INPUT=${{ github.event.inputs.IDF_branch }} - if [ -z "$INPUT" ]; then - echo "IDF_branch=main" >> $GITHUB_ENV - else - echo IDF_branch=${{ github.event.inputs.IDF_branch }} >> $GITHUB_ENV - fi - - name: Add Homebrew to PATH and install dependencies - run: | - echo "Adding Homebrew to PATH" - export PATH="/opt/homebrew/bin:$PATH" - echo "Current PATH: $PATH" - - echo "Updating Homebrew and installing dependencies" - brew update - export HOMEBREW_NO_INSTALL_CLEANUP=TRUE - brew install libxcb - arch -arm64 brew install coreutils - - name: Build wheels for IDF ${{ env.IDF_branch }} branch - shell: pwsh - run: | - $env:CPPFLAGS="-I/Users/githubrunner/brew/opt/openssl/include" - $env:LDFLAGS="-L/Users/githubrunner/brew/opt/openssl/lib" - arch -arm64 pip3 install wheel - if ( "${{ inputs.packages }}" ) { - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -NoReq -CompileWheels @(${{ env.packages }}) - } else { - # Cython==3.0.0 breaks the build of gevent. The build environment from https://github.com/gevent/gevent/blob/1.5.0/pyproject.toml - # must be patched with "cython<3" - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -BuildEnv @("setuptools", "wheel", "cython<3", "cffi") -CompileWheels @("gevent==1.5.0") -Python ~/.pyenv/versions/${{ matrix.python-version }}/bin/python - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -CompileWheels @("cryptography", "python-pkcs11") -Python ~/.pyenv/versions/${{ matrix.python-version }}/bin/python - } - - name: Copy cached wheels - run: | - rsync -u `python3 -m pip cache list --format=abspath` download - - name: Test wheels by installation - shell: pwsh - run: | - $env:CPPFLAGS="-I/Users/githubrunner/brew/opt/openssl/include" - $env:LDFLAGS="-L/Users/githubrunner/brew/opt/openssl/lib" - if ( "${{ inputs.packages }}" ) { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} -Arch "-arm64" -Python ~/.pyenv/versions/${{ matrix.python-version }}/bin/python -TestWheels @(${{ env.packages }}) - } else { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} -Arch "-arm64" -Python ~/.pyenv/versions/${{ matrix.python-version }}/bin/python - } - - name: Upload Release Asset To test s3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_BUCKET: ${{ secrets.DL_BUCKET }} - PREFIX: 'pypi' - shell: bash - run: | - # AWS is not in PATH and has to be added, for new version maybe we should use GH action - # for example https://github.com/jakejarvis/s3-sync-action - export PATH=$PATH:/opt/homebrew/bin/ - chmod +x Upload-Wheels.sh - ./Upload-Wheels.sh $AWS_BUCKET - pip3 install boto3 - python3 create_index_pages.py $AWS_BUCKET - - name: Drop AWS cache - id: invalidate-index-cache - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - run: | - # AWS is not in PATH and has to be added, for new version maybe we should use GH action - # for example https://github.com/jakejarvis/s3-sync-action - export PATH=$PATH:/opt/homebrew/bin/ - aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CACHE_INVALIDATION }} --paths "/pypi/*" diff --git a/.github/workflows/build-wheels-macos-dispatch.yml b/.github/workflows/build-wheels-macos-dispatch.yml deleted file mode 100644 index d63c9d5..0000000 --- a/.github/workflows/build-wheels-macos-dispatch.yml +++ /dev/null @@ -1,130 +0,0 @@ -name: macos-x64-dispatch - -on: - # schedule: - # - cron: '0 0 * * 0,3' - workflow_dispatch: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - workflow_call: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - -jobs: - build-python-wheels: - name: Build Python Wheels for macos-latest - runs-on: macos-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.8.17', '3.9.17', '3.10.12', '3.11.4'] - steps: - - name: Prepare package list - if: ${{ inputs.packages }} - run: | - echo "input packages: ${{ inputs.packages }}" - PACKAGES=$(echo "${{ inputs.packages }}" | - sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/\([^[:space:]]\{1,\}\)/"\1"/g;s/[[:space:]]\{1,\}/,/g') - echo "packages=$PACKAGES" >> $GITHUB_ENV - echo "output packages: $PACKAGES" - - name: Identify - run: uname -a - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install stable Rust with clippy and rustfmt - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - components: rustfmt, clippy - - name: Rust version - run: rustc --version - - name: Prepare download folder - run: mkdir download - - name: Install build dependencies - run: python3 -m pip install wheel - - name: Get python3 version - run: python3 --version - - name: Get branch name - run: | - echo INPUT=${{ github.event.inputs.IDF_branch }} - if [ -z "$INPUT" ]; then - echo "IDF_branch=main" >> $GITHUB_ENV - else - echo IDF_branch=${{ github.event.inputs.IDF_branch }} >> $GITHUB_ENV - fi - - name: Build wheels for IDF ${{ env.IDF_branch }} branch - shell: pwsh - run: | - if ( "${{ inputs.packages }}" ) { - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -Arch "${{ matrix.ARCH }}" -NoReq -CompileWheels @(${{ env.packages }}) - } else { - # Cython==3.0.0 breaks the build of gevent. The build environment from https://github.com/gevent/gevent/blob/1.5.0/pyproject.toml - # must be patched with "cython<3" - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -BuildEnv @("setuptools", "wheel", "cython<3", "cffi") -CompileWheels @("gevent==1.5.0") - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -CompileWheels @("cryptography", "python-pkcs11") - } - - name: Copy cached wheels - run: | - brew install coreutils - rsync -u `python3 -m pip cache list --format=abspath` download - - name: Test wheels by installation - shell: pwsh - run: | - if ( "${{ inputs.packages }}" ) { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} -Arch "${{ matrix.ARCH }}" -TestWheels @(${{ env.packages }}) - } else { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} -Arch "${{ matrix.ARCH }}" - } - - name: Upload Release Asset To test s3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_BUCKET: ${{ secrets.DL_BUCKET }} - PREFIX: 'pypi' - shell: bash - run: | - chmod +x Upload-Wheels.sh - ./Upload-Wheels.sh $AWS_BUCKET - pip3 install boto3 - python3 create_index_pages.py $AWS_BUCKET - - name: Drop AWS cache - id: invalidate-index-cache - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - run: aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CACHE_INVALIDATION }} --paths "/pypi/*" diff --git a/.github/workflows/build-wheels-platforms.yml b/.github/workflows/build-wheels-platforms.yml index 2509964..e3ea042 100644 --- a/.github/workflows/build-wheels-platforms.yml +++ b/.github/workflows/build-wheels-platforms.yml @@ -125,40 +125,8 @@ jobs: uses: espressif/idf-python-wheels/.github/workflows/build-wheels-python-dependent.yml@main - # # TODO in following PR: use the separate workflow bellow, right branch to ref - # upload-python-wheels: - # needs: [build-wheels, build-python-version-dependent-wheels] - # name: Upload Python wheels - # uses: espressif/idf-python-wheels/.github/workflows/upload-python-wheels.yml@feat/python_version_specific_wheels upload-python-wheels: needs: [build-wheels, build-python-version-dependent-wheels] name: Upload Python wheels - runs-on: ubuntu-latest - strategy: - fail-fast: false - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_BUCKET: ${{ secrets.DL_BUCKET }} - PREFIX: 'pypi' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install dependencies - run: python -m pip install -r build_requirements.txt - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - path: ./downloaded_wheels - - - name: Upload release asset to S3 bucket - run: | - python upload_wheels.py $AWS_BUCKET - python create_index_pages.py $AWS_BUCKET - - - name: Drop AWS cache - id: invalidate-index-cache - run: aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CACHE_INVALIDATION }} --paths "/pypi/*" + uses: espressif/idf-python-wheels/.github/workflows/upload-python-wheels.yml@main + secrets: inherit diff --git a/.github/workflows/build-wheels-ubuntu-dispatch.yml b/.github/workflows/build-wheels-ubuntu-dispatch.yml deleted file mode 100644 index f24a2ad..0000000 --- a/.github/workflows/build-wheels-ubuntu-dispatch.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: ubuntu-dispatch - -on: - # schedule: - # - cron: '0 0 * * 0,3' - workflow_dispatch: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - workflow_call: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - -jobs: - build-python-wheels: - name: Build Python Wheels for ubuntu-latest - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.8.17', '3.9.17', '3.10.12', '3.11.4'] - steps: - - name: Prepare package list - if: ${{ inputs.packages }} - run: | - echo "input packages: ${{ inputs.packages }}" - PACKAGES=$(echo "${{ inputs.packages }}" | - sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/\([^[:space:]]\{1,\}\)/"\1"/g;s/[[:space:]]\{1,\}/,/g') - echo "packages=$PACKAGES" >> $GITHUB_ENV - echo "output packages: $PACKAGES" - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - check-latest: true - - name: Install stable Rust with clippy and rustfmt - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - components: rustfmt, clippy - - name: Rust version - run: rustc --version - - name: Prepare download folder - run: mkdir download - - name: Install build dependencies - run: | - sudo apt update - sudo apt-get install cmake libdbus-1-dev -y - python3 -m pip install wheel - sudo apt install git virtualenv build-essential python3-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev - - name: Get python3 version - run: python3 --version - - name: Get branch name - run: | - echo INPUT=${{ github.event.inputs.IDF_branch }} - if [ -z "$INPUT" ]; then - echo "IDF_branch=main" >> $GITHUB_ENV - else - echo IDF_branch=${{ github.event.inputs.IDF_branch }} >> $GITHUB_ENV - fi - - name: Build wheels for IDF ${{ env.IDF_branch }} branch - shell: pwsh - run: | - if ( "${{ inputs.packages }}" ) { - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -NoReq -CompileWheels @(${{ env.packages }}) - } else { - # Cython==3.0.0 breaks the build of gevent. The build environment from https://github.com/gevent/gevent/blob/1.5.0/pyproject.toml - # must be patched with "cython<3" - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -BuildEnv @("setuptools", "wheel", "cython<3", "cffi") -CompileWheels @("gevent==1.5.0") - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -CompileWheels @("cryptography", "python-pkcs11") - } - - name: Test wheels by installation - shell: pwsh - run: | - if ( "${{ inputs.packages }}" ) { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} -TestWheels @(${{ env.packages }}) - } else { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} - } - - name: Upload Release Asset To test s3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_BUCKET: ${{ secrets.DL_BUCKET }} - PREFIX: 'pypi' - shell: bash - run: | - chmod +x Upload-Wheels.sh - ./Upload-Wheels.sh $AWS_BUCKET - pip3 install boto3 - python3 create_index_pages.py $AWS_BUCKET - - name: Drop AWS cache - id: invalidate-index-cache - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - run: aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CACHE_INVALIDATION }} --paths "/pypi/*" diff --git a/.github/workflows/build-wheels-windows-dispatch.yml b/.github/workflows/build-wheels-windows-dispatch.yml deleted file mode 100644 index 5c6cd28..0000000 --- a/.github/workflows/build-wheels-windows-dispatch.yml +++ /dev/null @@ -1,134 +0,0 @@ -name: windows-dispatch - -on: - # schedule: - # - cron: '0 0 * * 0,3' - workflow_dispatch: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - workflow_call: - inputs: - IDF_branch: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - type: string - required: false - default: 'master' - packages: - description: > - Wheels will be built for this branch. This will - be ignored if specific packages are listed bellow. - Generate wheels for given packages separated by space. - Requirement specifiers can be used. - For example esptool~=4.5 esp-coredump~=1.2 - type: string - required: false - -jobs: - build-python-wheels: - name: Build Python Wheels for windows-latest - runs-on: windows-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.8.10', '3.9.13', '3.10.2', '3.11.4'] - steps: - - name: Prepare package list - shell: bash - if: ${{ inputs.packages }} - run: | - echo "input packages: ${{ inputs.packages }}" - PACKAGES=$(echo "${{ inputs.packages }}" | - sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/\([^[:space:]]\{1,\}\)/"\1"/g;s/[[:space:]]\{1,\}/,/g') - echo "packages=$PACKAGES" >> $GITHUB_ENV - echo "output packages: $PACKAGES" - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install stable Rust with clippy and rustfmt - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - components: rustfmt, clippy - - name: Rust version - run: rustc --version - - name: Prepare download folder - shell: pwsh - run: mkdir download - - name: Install build dependencies - shell: pwsh - run: python3 -m pip install wheel - - name: Get python3 version - shell: pwsh - run: python3 --version - - name: Get branch name - shell: bash - run: | - echo INPUT=${{ github.event.inputs.IDF_branch }} - if [[ $INPUT == "" ]]; then - echo "IDF_branch=main" >> $GITHUB_ENV - else - echo IDF_branch=${{ github.event.inputs.IDF_branch }} >> $GITHUB_ENV - fi - - name: Build wheels for IDF ${{ env.IDF_branch }} branch - env: - PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" - shell: pwsh - run: | - python -m pip install --upgrade pip - if ( "${{ inputs.packages }}" ) { - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -Arch "${{ matrix.ARCH }}" -NoReq -CompileWheels @(${{ env.packages }}) - } else { - # Cython==3.0.0 breaks the build of gevent. The build environment from https://github.com/gevent/gevent/blob/1.5.0/pyproject.toml - # must be patched with "cython<3" - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -BuildEnv @("setuptools", "wheel", "cython<3", "cffi") -CompileWheels @("gevent==1.5.0") - - # python-pkcs11 can't be build on Python 3.10 and above (3.10.2 works) https://github.com/danni/python-pkcs11/issues/159 - .\Build-Wheels.ps1 -Branch ${{ env.IDF_branch }} -CompileWheels @("cryptography", "windows-curses", "python-pkcs11;python_version<'3.11'") - } - - name: Test wheels by installation - shell: pwsh - run: | - if ( "${{ inputs.packages }}" ) { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} -Arch "${{ matrix.ARCH }}" -TestWheels @(${{ env.packages }}) - } else { - .\Test-Wheels.ps1 -Branch ${{ env.IDF_branch }} -Arch "${{ matrix.ARCH }}" - } - - name: Upload Release Asset To test s3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_BUCKET: ${{ secrets.DL_BUCKET }} - PREFIX: 'pypi' - shell: bash - run: | - chmod +x Upload-Wheels.sh - pip3 install boto3 - ./Upload-Wheels.sh $AWS_BUCKET - python3 create_index_pages.py $AWS_BUCKET - - name: Drop AWS cache - id: invalidate-index-cache - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - run: aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CACHE_INVALIDATION }} --paths "/pypi/*" diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml deleted file mode 100644 index 2c5ae51..0000000 --- a/.github/workflows/build-wheels.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: build-wheels - -on: - workflow_dispatch - -jobs: - create_release: - name: Create release - runs-on: ubuntu-latest - outputs: - upload_url: ${{ steps.create_release.outputs.upload_url }} - steps: - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - draft: false - prerelease: false - - build-python-wheels: - name: Build Python Wheels for ${{ matrix.os }} - needs: create_release - runs-on: ${{ matrix.os }} - strategy: - matrix: - #os: [macos-latest, self-hosted-macos-arm64, ubuntu-latest, windows-latest] - os: [macos-latest, ubuntu-latest, windows-latest, macos-m1-self-hosted, linux-armv7-self-hosted] - include: - - os: macos-latest - ARCH: '' - TARGET: macos-x64 - - os: macos-m1-self-hosted - ARCH: '-arm64' - TARGET: macos-arm64 - - os: linux-armv7-self-hosted - ARCH: '' - TARGET: linux-armv7 - - os: ubuntu-latest - ARCH: '' - TARGET: linux-amd64 - - os: windows-latest - ARCH: '' - TARGET: win64 - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set up Python - if: ${{ matrix.os == 'macos-latest' || matrix.os == 'ubuntu-latest' || matrix.os == 'windows-latest' }} - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Prepare download folder - shell: pwsh - run: mkdir download - - name: Install build dependencies - shell: pwsh - run: python3 -m pip install wheel - - name: Build wheels for IDF master - shell: pwsh - run: .\Build-Wheels.ps1 -Branch "master" -Arch "${{ matrix.ARCH }}" -CompileWheels @("greenlet==1.0.0", "gevent==1.5.0", "cryptography", "windows-curses", "python-pkcs11") - - name: Test wheels by installation - shell: pwsh - run: .\Test-Wheels.ps1 -Branch "master" -Arch "${{ matrix.ARCH }}" - - name: Build wheels for IDF v4.2.2 - shell: pwsh - run: .\Build-Wheels.ps1 -Branch "v4.2.2" -Arch "${{ matrix.ARCH }}" -CompileWheels @("greenlet==1.0.0", "gevent==1.5.0", "cryptography", "windows-curses") - - name: Test wheels by installation - shell: pwsh - run: .\Test-Wheels.ps1 -Branch "v4.2.2" -Arch "${{ matrix.ARCH }}" - - name: Build wheels for IDF v4.3 - shell: pwsh - run: .\Build-Wheels.ps1 -Branch "v4.3" -Arch "${{ matrix.ARCH }}" -CompileWheels @("windows-curses") - - name: Test wheels by installation - shell: pwsh - run: .\Test-Wheels.ps1 -Branch "v4.3" -Arch "${{ matrix.ARCH }}" - - name: Write version of package to file - shell: pwsh - run: python3 -c "print('3.8', file=open('download/version.txt', 'w'))" - - name: Archive artifact - shell: pwsh - run: Compress-Archive -Path "download\*" -DestinationPath "idf-python-wheels.zip" - - name: Upload Release Asset - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./idf-python-wheels.zip - asset_name: idf-python-wheels-3.8-${{ matrix.TARGET }}.zip - asset_content_type: application/zip diff --git a/Build-Wheels.ps1 b/Build-Wheels.ps1 deleted file mode 100644 index f0f1164..0000000 --- a/Build-Wheels.ps1 +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env pwsh - -param ( - [string]$Branch="main", - [string]$Python="python3", - [string]$Arch="", - [string[]]$BuildEnv=@(), - [string[]]$CompileWheels=@(), - [switch]$NoReq=$false -) - -"Using Python: $Python" -$env:IDF_PATH=(Get-Location).Path - -$BranchNum = ($Branch -replace "\D+[^0-9][^0-9]" , '') - -if ($BranchNum -eq "") { - $BranchNum = "5.1" # master -} - -if ($BranchNum -ge "5.0") { - $RequirementsUrl="https://dl.espressif.com/dl/esp-idf/espidf.constraints.v${BranchNum}.txt" -} else { - $RequirementsUrl="https://raw.githubusercontent.com/espressif/esp-idf/${Branch}/requirements.txt" -} - -$FileBranch = ${Branch}.Replace('/', '_') -$RequirementsTxt="requirements-${FileBranch}.txt" -$OnlyBinary = "" -"Processing: $RequirementsUrl" -Invoke-WebRequest $RequirementsUrl -OutFile $RequirementsTxt - -# dbus-python (https://github.com/posborne/dbus-python/tree/master) is deprecated -# we could migrate it to python-dbus-next (https://github.com/altdesktop/python-dbus-next) -# on Python 3.11 it is not possible for some platform to even install dbus-python -# and because of constraint file taken instead of requirements file there is condition: dbus-python<1.3; python_version > "3.10" -# that means dbus-python seems to be build only for Python 3.11 -# workaround for the dbus-python requirement for all platforms (install only for linux which should be working): -$Content = [System.IO.File]::ReadAllLines($RequirementsTxt) -$string = 'dbus-python<1.3; python_version > "3.10"' -$Content = $Content -replace $string,'dbus-python<1.3; python_version >"3.10" and sys_platform == "linux"' -$Content | Set-Content -Path $RequirementsTxt - -# If specific build environment is requested then build isolation must be disable in order to use the build environment. -# It might be necessary to clean or separate this build environment in the future. -if ($BuildEnv.count -ne 0) { - $ExtraPipArgs = "--no-build-isolation" -} - -# Install packages requested for the build environment -foreach ($build_req in $BuildEnv) { - if ("$Arch" -eq "") { - &$Python -m pip install $build_req - } else { - arch $Arch $Python -m pip install $build_req - } -} - -# Iterate over binaries which should be compiled. -# The build of next binary will receive list of previously build of binaries to avoid download -foreach ($wheelPrefix in $CompileWheels) { - $wheel=Get-Content $RequirementsTxt | % { if($_ -match "^$wheelPrefix") {$_}} - - # If wheel is not defined in requirements.txt use the prefix as name - if ("$wheel" -eq "") { - $wheel = $wheelPrefix - } - - if ($wheel.startswith('windows-curses')) { - $wheel = "windows-curses" - } - - "Processing: $wheel" - # Split by default splits on each space, so empty args are passed as - # requirements to pip-wheel, which complains with - # ERROR: Invalid requirement: '' - # " ".Split() vs. " ".split() | where {$_} - $OnlyBinarySplitted = $OnlyBinary.Split(' ') | where {$_} - if ("$Arch" -eq "") { - &$Python -m pip wheel --find-links download --wheel-dir download $ExtraPipArgs $OnlyBinarySplitted $wheel - } else { - arch $Arch $Python -m pip wheel --find-links download --wheel-dir download $ExtraPipArgs $OnlyBinarySplitted $wheel - } - if ($LASTEXITCODE -ne 0) { - exit 1 - } - #$cache=pip cache dir - #Get-ChildItem -Path $cache "{$wheel}*.whl" -Recurse | % {Copy-Item -Path $_.FullName -Destination download -Container } - #ls download - $OnlyBinary += " --only-binary $wheel" -} - -if ($NoReq) { - exit -} - -$OnlyBinarySplitted = $OnlyBinary.Split(' ') | where {$_} -if ("$Arch" -eq "") { - &$Python -m pip download $OnlyBinarySplitted --find-links download --dest download -r $RequirementsTxt -} else { - arch $Arch $Python -m pip download $OnlyBinarySplitted --find-links download --dest download -r $RequirementsTxt -} -if ($LASTEXITCODE -ne 0) { - exit 1 -} - -&$Python -m pip wheel --wheel-dir download --find-links download -r $RequirementsTxt -if ($LASTEXITCODE -ne 0) { - exit 1 -} diff --git a/README.md b/README.md index 75ebce5..3bac32d 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,13 @@ Supported architectures: - x86_64 - ARM64 +Supported Python versions: +* 3.8 +* 3.9 +* 3.10 +* 3.11 +* 3.12 + For each `release` branch of [ESP-IDF] starting from the version defined in GitHub variables and [ESP-IDF] `master` branch all the requirements and constraints files are automatically downloaded and wheels are built and uploaded. @@ -20,8 +27,10 @@ For each `release` branch of [ESP-IDF] starting from the version defined in GitH to change the [ESP-IDF] `release` branches to build wheels for. -## Usage of ad-hoc -Not supported yet. +## Usage of manual wheel(s) build - defined wheels workflow +If there is a need to manually build and upload wheel(s) the `defined-wheels` workflow can be used for this. The pip package needs to be specified with marker support (e.g. coredump~=1.2;sys_platform!='win32') and check the architecture(s) which should be wheel(s) build and upload for. Multiple wheels can be separated by space. + +Then the wheel(s) is built and uploaded for all supported Python versions. ## Requirements lists @@ -73,5 +82,8 @@ The syntax can be also converted into a sentence: "For assembled **main requirem ### build_requirements.txt File for the requirements needed for the build process and the build script. +### os_dependencies +When there is a need for additional OS dependencies to successfully build the wheel(s) on a specific platform and architecture, the `.sh` script in the `os_dependencies` directory can be adjusted. + [ESP-IDF]: https://github.com/espressif/esp-idf diff --git a/Test-Wheels.ps1 b/Test-Wheels.ps1 deleted file mode 100644 index db0bfa2..0000000 --- a/Test-Wheels.ps1 +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env pwsh - -param ( - [string]$Branch="main", - [string]$Python="python3", - [string]$Arch="", - [string[]]$TestWheels=@() -) - -$env:IDF_PATH=(Get-Location).Path -$FileBranch = ${Branch}.Replace('/', '_') -$RequirementsTxt="requirements-${FileBranch}.txt" - -if (@($TestWheels).length) { - if ("$Arch" -eq "") { - &$Python -m pip install --no-index --find-links download $TestWheels - } else { - arch $Arch $Python -m pip install --no-index --find-links download $TestWheels - } - exit -} - -if ("$Arch" -eq "") { - &$Python -m pip install --no-index --find-links download -r $RequirementsTxt -} else { - arch $Arch $Python -m pip install --no-index --find-links download -r $RequirementsTxt -} - diff --git a/Upload-Wheels.sh b/Upload-Wheels.sh deleted file mode 100755 index 0eea35a..0000000 --- a/Upload-Wheels.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -cd download || exit - -for wfile in *.whl; do - file_name=$(echo "${wfile%%-*}" | tr '[:upper:]' '[:lower:]' | tr '_' '-') - case "$(uname -sr)" in - CYGWIN*|MINGW*|MINGW32*|MSYS*) - if [[ "$file_name" == "esptool" ]]; then - continue - fi - ;; - *) - ;; - esac - aws s3 cp --acl=public-read --no-progress "$wfile" "s3://$1/pypi/$file_name/$wfile" -done diff --git a/tools/kconfig_new/esp-windows-curses/setup.py b/tools/kconfig_new/esp-windows-curses/setup.py deleted file mode 100644 index 0582872..0000000 --- a/tools/kconfig_new/esp-windows-curses/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2019 Espressif Systems (Shanghai) PTE LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from setuptools import setup - -setup(name='esp-windows-curses', - version='0.1', - description='Wrapper for the windows-curses package', - url='https://www.espressif.com', - author='Espressif Systems', - license='Apache License 2.0', - author_email='donotreply@espressif.com', - zip_safe=False, - # This wrapper exists only because of the following install_requires statement which ensures that the package - # dependency is not added for MSYS2 where it cannot be installed. There is no PEP 508 environment marker to - # detect MSYS2. - install_requires=('' if 'MSYSTEM' in os.environ else 'windows-curses; sys_platform == "win32"',) - ) diff --git a/upload-wheels/.gitignore b/upload-wheels/.gitignore deleted file mode 100644 index 869df07..0000000 --- a/upload-wheels/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target -Cargo.lock \ No newline at end of file diff --git a/upload-wheels/Cargo.toml b/upload-wheels/Cargo.toml deleted file mode 100644 index a280acd..0000000 --- a/upload-wheels/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "upload-wheels" -version = "0.1.0" -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -tokio = { version = "1.13.0", features = ["full"] } -#aws-config = "0.0.23-alpha" -#openssl-sys = "0.9" -#openssl = { version = "0.10", features = ["vendored"] } -#aws-sdk-rust = "0.1.40" -#rust-s3 = {version = "0.27", git = "https://github.com/JurajSadel/rust-s3.git", branch = "add_ap-east-1_region" } -rust-s3 = { version = "0.30.0" } -#rust-s3 = "0.27" -structopt = { version = "0.3", default-features = false } -regex = "1" -multimap = "0.8.3" -indicatif = "0.16.2" \ No newline at end of file diff --git a/upload-wheels/src/main.rs b/upload-wheels/src/main.rs deleted file mode 100644 index 03e8175..0000000 --- a/upload-wheels/src/main.rs +++ /dev/null @@ -1,127 +0,0 @@ -use s3::bucket::Bucket; -use s3::creds::Credentials; - -use multimap::MultiMap; -use std::env; -use regex::Regex; - -use indicatif::ProgressBar; - - -async fn upload_wheels(bucket: &Bucket, dl_prefix: &String) -> Result<(), Box> { - let pb = ProgressBar::new(25); - - let path = "./download/"; - let mut files_in_dir = tokio::fs::read_dir(&path).await?; - - while let Some(entry) = files_in_dir.next_entry().await? { - let name = entry.file_name().into_string().unwrap(); - if !name.contains(".whl") && !name.contains(".tar") { - continue; - } - - let re = Regex::new(r"^(.*?)-").unwrap(); //([a-zA-Z_\d]+)?\-.+ - let file = tokio::fs::read(entry.path()).await?; - let prefix = re.captures(&name).unwrap().get(1).unwrap().as_str().to_lowercase(); - bucket.put_object_with_content_type( dl_prefix.to_owned() + "/" + &prefix + "/" + &name, &file, "binary/octet-stream").await?; - pb.inc(1); - } - - pb.finish_with_message("Upload done"); - - Ok(()) -} - -async fn create_indexes(bucket: &Bucket, dl_prefix: &String) -> Result<(), Box> { - let header = r#" - - - - - Simple index - -"#; - - let footer = r#" - -"#; - - let result_list = bucket.list(dl_prefix.to_string(), None).await?; - let mut packages = MultiMap::new(); - let pb = ProgressBar::new(2 * result_list.len() as u64); - for file in result_list { - for item in file.contents { - if !item.key.contains(".whl") && !item.key.contains(".tar") { - continue; - } - let re = Regex::new(r"([a-zA-Z_\d]+)\-.+").unwrap(); - let m = re.captures(&item.key).unwrap(); - let name = m.get(1).unwrap().as_str().to_lowercase(); - let path = m.get(0).unwrap().as_str(); - packages.insert(name, path.to_string()); - pb.inc(1); - } - } - - let mut index = String::new(); - index.push_str(header); - - let pb = ProgressBar::new((packages.len()) as u64); - - for name in packages.keys() { - index.push_str(&format!("\n{item}/
", prefix=dl_prefix, item=name)); - pb.inc(1); - } - index.push_str(&footer); - - bucket.put_object_with_content_type( dl_prefix.to_owned() + "/" + "index.html", &index.into_bytes(), "text/html").await?; - - for (key, files) in packages { - - let mut index = String::new(); - index.push_str(header); - - for file in files { - index.push_str(&format!("\n{filename}", prefix=dl_prefix, item=key, filename=file)); - - } - index.push_str(&footer); - bucket.put_object_with_content_type( format!("{prefix}/{name}/index.html", prefix=dl_prefix, name=key), &index.into_bytes(), "text/html").await?; - pb.inc(1); - } - - pb.finish_with_message("Indexes created"); - - Ok(()) -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - - let dl_bucket = env::var("AWS_BUCKET").unwrap_or("".to_string()); - let prefix = env::var("PREFIX").unwrap_or("test".to_string()); - let aws_access_key = env::var("AWS_ACCESS_KEY_ID").unwrap_or("".to_string()); - let aws_secret_key = env::var("AWS_SECRET_ACCESS_KEY").unwrap_or("".to_string()); - let aws_region = env::var("AWS_DEFAULT_REGION").unwrap_or("eu-west-1".to_string()).parse()?; - - assert_ne!(dl_bucket, ""); - assert_ne!(aws_access_key, ""); - assert_ne!(aws_secret_key, ""); - //assert_ne!(AWS_DEFAULT_REGION, ""); - - let credentials = Credentials::new( - Some(&aws_access_key[..]), - Some(&aws_secret_key[..]), - None, None, None)?; - - let mut bucket = Bucket::new( - &dl_bucket[..], - aws_region, - credentials)?; - bucket.add_header("x-amz-acl", "public-read"); - - upload_wheels(&bucket, &prefix).await?; - create_indexes(&bucket, &prefix).await?; - - Ok(()) -} diff --git a/upload_wheels.py b/upload_wheels.py index 6e9038c..9a9b5ca 100644 --- a/upload_wheels.py +++ b/upload_wheels.py @@ -34,8 +34,8 @@ if match: wheel_name = match.group(1) - wheel_name = wheel_name.lower() - wheel_name = wheel_name.replace('_', '-') + wheel_name = wheel_name.lower() + wheel_name = wheel_name.replace('_', '-') - BUCKET.upload_file(f'{WHEELS_DIR}{os.sep}{subdir}{os.sep}{wheel}', f'pypi/{wheel_name}/{wheel}') - print(f'Uploaded {wheel}') + BUCKET.upload_file(f'{WHEELS_DIR}{os.sep}{subdir}{os.sep}{wheel}', f'pypi/{wheel_name}/{wheel}') + print(f'Uploaded {wheel}')