From 78ec4760c6ab73a22e328e2a1bc18d32149c472f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 24 Jul 2023 13:44:23 -0700 Subject: [PATCH] Switch to new wheel building pipeline (#3731) Moves the wheel build and test logic out of the workflow into the repo. This matches conda tests more closely and allows each repo to manage its own wheels more easily. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cugraph/pull/3731 --- .github/workflows/build.yaml | 27 ++++++--------------------- .github/workflows/pr.yaml | 29 ++++++++--------------------- .github/workflows/test.yaml | 13 ++++--------- ci/build_wheel.sh | 29 +++++++++++++++++++++++++++++ ci/build_wheel_cugraph.sh | 13 +++++++++++++ ci/build_wheel_pylibcugraph.sh | 8 ++++++++ ci/test_wheel.sh | 22 ++++++++++++++++++++++ ci/test_wheel_cugraph.sh | 22 ++++++++++++++++++++++ ci/test_wheel_pylibcugraph.sh | 6 ++++++ 9 files changed, 118 insertions(+), 51 deletions(-) create mode 100755 ci/build_wheel.sh create mode 100755 ci/build_wheel_cugraph.sh create mode 100755 ci/build_wheel_pylibcugraph.sh create mode 100755 ci/test_wheel.sh create mode 100755 ci/test_wheel_cugraph.sh create mode 100755 ci/test_wheel_pylibcugraph.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index de9caa0fabe..4354ca40327 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -68,27 +68,20 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: pylibcugraph - package-dir: python/pylibcugraph - - # Note that this approach to cloning repos obviates any modification to - # the CMake variables in get_cumlprims_mg.cmake since CMake will just use - # the clone as is. + script: ci/build_wheel_pylibcugraph.sh extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/" wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -98,28 +91,20 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: cugraph - package-dir: python/cugraph - - # Note that this approach to cloning repos obviates any modification to - # the CMake variables in get_cumlprims_mg.cmake since CMake will just use - # the clone as is. + script: ci/build_wheel_cugraph.sh extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse" - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/" wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4d52cd26de4..17d7ac48907 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -78,47 +78,34 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: pull-request - package-name: pylibcugraph - package-dir: python/pylibcugraph + script: ci/build_wheel_pylibcugraph.sh extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: pull-request - package-name: pylibcugraph - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests" - test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" + script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: pull-request - package-name: cugraph - package-dir: python/cugraph + script: ci/build_wheel_cugraph.sh extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse" - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: pull-request - package-name: cugraph - # Always want to test against latest dask/distributed. - test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - # Skip dataset downloads on arm to save CI time -- arm only runs smoke tests. - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests" - test-smoketest: "python ci/wheel_smoke_test_cugraph.py" + script: ci/test_wheel_cugraph.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d697b8f1649..1a579a87a7f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,24 +32,19 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - package-name: pylibcugraph - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests" + script: ci/test_wheel_pylibcugraph.sh wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - package-name: cugraph - # Always want to test against latest dask/distributed. - test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests" + script: ci/test_wheel_cugraph.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh new file mode 100755 index 00000000000..fbf1e6fce2d --- /dev/null +++ b/ci/build_wheel.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +package_name=$1 +package_dir=$2 + +source rapids-configure-sccache +source rapids-date-string + +# Use gha-tools rapids-pip-wheel-version to generate wheel version then +# update the necessary files +version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +bash ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" +echo "The package name and/or version was modified in the package source. The git diff is:" +git diff + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh new file mode 100755 index 00000000000..657220d372b --- /dev/null +++ b/ci/build_wheel_cugraph.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +RAPIDS_PY_WHEEL_NAME=pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX} rapids-download-wheels-from-s3 ./local-pylibcugraph +python -m pip install --no-deps ./local-pylibcugraph/pylibcugraph*.whl + +export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/" + +./ci/build_wheel.sh cugraph python/cugraph diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh new file mode 100755 index 00000000000..8d365bc250b --- /dev/null +++ b/ci/build_wheel_pylibcugraph.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/" + +./ci/build_wheel.sh pylibcugraph python/pylibcugraph diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100755 index 00000000000..146186ae2e7 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eoxu pipefail + +package_name=$1 +package_dir=$2 + +mkdir -p ./dist +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# echo to expand wildcard before adding `[extra]` requires for pip +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +python -m pip install $(echo ./dist/${package_name}*.whl)[test] + +# Run smoke tests for aarch64 pull requests +arch=$(uname -m) +if [[ "${arch}" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then + python ./ci/wheel_smoke_test_${package_name}.py +else + RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets python -m pytest ./python/${package_name}/${package_name}/tests +fi diff --git a/ci/test_wheel_cugraph.sh b/ci/test_wheel_cugraph.sh new file mode 100755 index 00000000000..90af5313619 --- /dev/null +++ b/ci/test_wheel_cugraph.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eoxu pipefail + +# Download the pylibcugraph built in the previous step +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep +python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl + +# Always install latest dask for testing +python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08 + +# Only download test data for x86 +arch=$(uname -m) +if [[ "${arch}" == "x86_64" ]]; then + pushd ./datasets + bash ./get_test_data.sh + popd +fi + +./ci/test_wheel.sh cugraph python/cugraph diff --git a/ci/test_wheel_pylibcugraph.sh b/ci/test_wheel_pylibcugraph.sh new file mode 100755 index 00000000000..d04cb358d21 --- /dev/null +++ b/ci/test_wheel_pylibcugraph.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eoxu pipefail + +./ci/test_wheel.sh pylibcugraph python/pylibcugraph