From 7bd0be5f1de60fd5a8013c1bdcb1ff7176ae4200 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 28 Aug 2024 12:54:22 +0900 Subject: [PATCH] feat(dev/release): add Rust release process (#2107) Fixes #2104. --- ci/scripts/rust_build.sh | 26 +++++++ ci/scripts/rust_test.sh | 30 ++++++++ dev/release/post-08-rust.sh | 51 +++++++++++++ ...cts.sh => post-09-remove-old-artifacts.sh} | 0 ...p-versions.sh => post-10-bump-versions.sh} | 0 ...{post-10-website.sh => post-11-website.sh} | 0 dev/release/verify-release-candidate.sh | 71 +++++++++++++++++-- docs/source/development/releasing.rst | 28 ++++++-- rust/Cargo.lock | 4 +- 9 files changed, 198 insertions(+), 12 deletions(-) create mode 100755 ci/scripts/rust_build.sh create mode 100755 ci/scripts/rust_test.sh create mode 100644 dev/release/post-08-rust.sh rename dev/release/{post-08-remove-old-artifacts.sh => post-09-remove-old-artifacts.sh} (100%) rename dev/release/{post-09-bump-versions.sh => post-10-bump-versions.sh} (100%) rename dev/release/{post-10-website.sh => post-11-website.sh} (100%) diff --git a/ci/scripts/rust_build.sh b/ci/scripts/rust_build.sh new file mode 100755 index 0000000000..42630136c0 --- /dev/null +++ b/ci/scripts/rust_build.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euxo pipefail + +source_dir="${1}/rust" + +pushd "${source_dir}" +cargo build --all-features --all-targets --workspace +popd diff --git a/ci/scripts/rust_test.sh b/ci/scripts/rust_test.sh new file mode 100755 index 0000000000..439ad5e33c --- /dev/null +++ b/ci/scripts/rust_test.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euxo pipefail + +source_dir="${1}/rust" +cpp_libs_dir="${2}" + +export LD_LIBRARY_PATH="${cpp_libs_dir}/lib:${LD_LIBRARY_PATH:-}" +export DYLD_LIBRARY_PATH="${cpp_libs_dir}/lib:${DYLD_LIBRARY_PATH:-}" + +pushd "${source_dir}" +cargo test --all-features --workspace +popd diff --git a/dev/release/post-08-rust.sh b/dev/release/post-08-rust.sh new file mode 100644 index 0000000000..e31427d5b7 --- /dev/null +++ b/dev/release/post-08-rust.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# -*- indent-tabs-mode: nil; sh-indentation: 2; sh-basic-offset: 2 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e +set -u +set -o pipefail + +SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SOURCE_DIR}/utils-common.sh" +source "${SOURCE_DIR}/utils-prepare.sh" + +main() { + if [ "$#" -ne 0 ]; then + echo "Usage: $0" + exit + fi + + local -r tag="apache-arrow-adbc-${RELEASE}" + # Ensure we are being run from the tag + if [[ $(git describe --exact-match --tags) != "${tag}" ]]; then + echo "This script must be run from the tag ${tag}" + exit 1 + fi + + pushd "${SOURCE_TOP_DIR}/rust" + cargo publish --all-features -p adbc_core + popd + + echo "Success! The released Cargo crate is available here:" + echo " https://crates.io/crates/adbc_core" +} + +main "$@" diff --git a/dev/release/post-08-remove-old-artifacts.sh b/dev/release/post-09-remove-old-artifacts.sh similarity index 100% rename from dev/release/post-08-remove-old-artifacts.sh rename to dev/release/post-09-remove-old-artifacts.sh diff --git a/dev/release/post-09-bump-versions.sh b/dev/release/post-10-bump-versions.sh similarity index 100% rename from dev/release/post-09-bump-versions.sh rename to dev/release/post-10-bump-versions.sh diff --git a/dev/release/post-10-website.sh b/dev/release/post-11-website.sh similarity index 100% rename from dev/release/post-10-website.sh rename to dev/release/post-11-website.sh diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 49b7b2dc9c..bc69948d49 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -340,6 +340,45 @@ install_go() { GO_ALREADY_INSTALLED=1 } +install_rust() { + if [ "${RUST_ALREADY_INSTALLED:-0}" -gt 0 ]; then + show_info "Rust already installed at $(command -v cargo)" + show_info "$(cargo --version)" + return 0 + fi + + if [[ -f ${ARROW_TMPDIR}/cargo/env ]]; then + source ${ARROW_TMPDIR}/cargo/env + rustup default stable + show_info "$(cargo version) installed at $(command -v cargo)" + RUST_ALREADY_INSTALLED=1 + return 0 + fi + + if command -v cargo > /dev/null; then + show_info "Found $(cargo version) at $(command -v cargo)" + RUST_ALREADY_INSTALLED=1 + return 0 + fi + + show_info "Installing Rust..." + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs |\ + env \ + RUSTUP_HOME=${ARROW_TMPDIR}/rustup \ + CARGO_HOME=${ARROW_TMPDIR}/cargo \ + sh -s -- \ + --default-toolchain stable \ + --no-modify-path \ + -y + + source ${ARROW_TMPDIR}/cargo/env + rustup default stable + + show_info "$(cargo version) installed at $(command -v cargo)" + + RUST_ALREADY_INSTALLED=1 +} + install_conda() { # Setup short-lived miniconda for Python and integration tests show_info "Ensuring that Conda is installed..." @@ -462,6 +501,13 @@ maybe_setup_go() { fi } +maybe_setup_rust() { + show_info "Ensuring that Rust is installed..." + if [ "${USE_CONDA}" -eq 0 ]; then + install_rust + fi +} + test_cpp() { show_header "Build, install and test C++ libraries" @@ -475,9 +521,9 @@ test_cpp() { if [ "${USE_CONDA}" -gt 0 ]; then export CMAKE_PREFIX_PATH="${CONDA_BACKUP_CMAKE_PREFIX_PATH}:${CMAKE_PREFIX_PATH}" # The CMake setup forces RPATH to be the Conda prefix - local -r install_prefix="${CONDA_PREFIX}" + export CPP_INSTALL_PREFIX="${CONDA_PREFIX}" else - local -r install_prefix="${ARROW_TMPDIR}/local" + export CPP_INSTALL_PREFIX="${ARROW_TMPDIR}/local" fi export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:-${NPROC}} @@ -486,14 +532,14 @@ test_cpp() { export ADBC_CMAKE_ARGS="-DADBC_INSTALL_NAME_RPATH=OFF" export ADBC_USE_ASAN=OFF export ADBC_USE_UBSAN=OFF - "${ADBC_DIR}/ci/scripts/cpp_build.sh" "${ADBC_SOURCE_DIR}" "${ARROW_TMPDIR}/cpp-build" "${install_prefix}" + "${ADBC_DIR}/ci/scripts/cpp_build.sh" "${ADBC_SOURCE_DIR}" "${ARROW_TMPDIR}/cpp-build" "${CPP_INSTALL_PREFIX}" # FlightSQL driver requires running database for testing export BUILD_DRIVER_FLIGHTSQL=0 # PostgreSQL driver requires running database for testing export BUILD_DRIVER_POSTGRESQL=0 # Snowflake driver requires snowflake creds for testing export BUILD_DRIVER_SNOWFLAKE=0 - "${ADBC_DIR}/ci/scripts/cpp_test.sh" "${ARROW_TMPDIR}/cpp-build" "${install_prefix}" + "${ADBC_DIR}/ci/scripts/cpp_test.sh" "${ARROW_TMPDIR}/cpp-build" "${CPP_INSTALL_PREFIX}" export BUILD_DRIVER_FLIGHTSQL=1 export BUILD_DRIVER_POSTGRESQL=1 export BUILD_DRIVER_SNOWFLAKE=1 @@ -637,6 +683,17 @@ test_go() { "${ADBC_DIR}/ci/scripts/go_test.sh" "${ADBC_SOURCE_DIR}" "${ARROW_TMPDIR}/go-build" "${install_prefix}" } +test_rust() { + show_header "Build and test Rust libraries" + + maybe_setup_rust || exit 1 + maybe_setup_conda rust || exit 1 + + # We expect the C++ libraries to exist. + "${ADBC_DIR}/ci/scripts/rust_build.sh" "${ADBC_SOURCE_DIR}" + "${ADBC_DIR}/ci/scripts/rust_test.sh" "${ADBC_SOURCE_DIR}" "${CPP_INSTALL_PREFIX}" +} + ensure_source_directory() { show_header "Ensuring source directory" @@ -722,6 +779,9 @@ test_source_distribution() { if [ ${TEST_R} -gt 0 ]; then test_r fi + if [ ${TEST_RUST} -gt 0 ]; then + test_rust + fi popd } @@ -874,9 +934,10 @@ test_jars() { : ${TEST_JS:=${TEST_SOURCE}} : ${TEST_GO:=${TEST_SOURCE}} : ${TEST_R:=${TEST_SOURCE}} +: ${TEST_RUST:=${TEST_SOURCE}} # Automatically test if its activated by a dependent -TEST_CPP=$((${TEST_CPP} + ${TEST_GO} + ${TEST_GLIB} + ${TEST_PYTHON})) +TEST_CPP=$((${TEST_CPP} + ${TEST_GO} + ${TEST_GLIB} + ${TEST_PYTHON} + ${TEST_RUST})) # Execute tests in a conda enviroment : ${USE_CONDA:=0} diff --git a/docs/source/development/releasing.rst b/docs/source/development/releasing.rst index 460534b5c2..b6d2de271b 100644 --- a/docs/source/development/releasing.rst +++ b/docs/source/development/releasing.rst @@ -386,6 +386,24 @@ Be sure to go through on the following checklist: # dev/release/post-07-csharp.sh dev/release/post-07-csharp.sh +.. dropdown:: Upload Rust crates to crates.io + :class-title: sd-fs-5 + :class-container: sd-shadow-md + + You must be one of owners of the package. If you aren't an owner yet, an + existing owner can add you at https://crates.io. + + You will need to [create an API token](https://crates.io/settings/tokens). + + An owner can upload: + + .. code-block:: bash + + cargo login + + # dev/release/post-08-rust.sh + dev/release/post-08-rust.sh + .. dropdown:: Update conda-forge packages :class-title: sd-fs-5 :class-container: sd-shadow-md @@ -411,7 +429,7 @@ Be sure to go through on the following checklist: .. code-block:: Bash - dev/release/post-08-remove-old-artifacts.sh + dev/release/post-09-remove-old-artifacts.sh .. dropdown:: Bump versions :class-title: sd-fs-5 @@ -421,8 +439,8 @@ Be sure to go through on the following checklist: .. code-block:: Bash - # dev/release/post-09-bump-versions.sh ../arrow - dev/release/post-09-bump-versions.sh + # dev/release/post-10-bump-versions.sh ../arrow + dev/release/post-10-bump-versions.sh .. dropdown:: Publish release blog post :class-title: sd-fs-5 @@ -434,7 +452,7 @@ Be sure to go through on the following checklist: .. code-block:: Bash - # dev/release/post-10-website.sh ../arrow-site - dev/release/post-10-website.sh + # dev/release/post-11-website.sh ../arrow-site + dev/release/post-11-website.sh .. _nightly-website.yml: https://github.com/apache/arrow-adbc/actions/workflows/nightly-website.yml diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 883ccf72f6..ea15345040 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -4,7 +4,7 @@ version = 3 [[package]] name = "adbc_core" -version = "0.13.0" +version = "0.14.0" dependencies = [ "arrow", "libloading", @@ -13,7 +13,7 @@ dependencies = [ [[package]] name = "adbc_dummy" -version = "0.13.0" +version = "0.14.0" dependencies = [ "adbc_core", "arrow",