From 9b3bf08f78f690605394ad3815177ece931f06d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 3 Aug 2023 23:04:15 +0200 Subject: [PATCH] GH-36752: [Python] Remove AWS SDK bundling when building wheels (#36925) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change In https://github.com/apache/arrow/pull/12227 we decided to use a bundled version of the AWS SDK when compiling Python wheels, in order to downgrade the AWS SDK version. Now that we have fixed S3 finalization issues (https://github.com/apache/arrow/pull/36442), it should be ok to rely on the vcpkg-installed version of the AWS SDK again. ### What changes are included in this PR? Remove use of bundled AWS SDK and use S3 vcpkg feature for requirements. ### Are these changes tested? On CI and via crossbow ### Are there any user-facing changes? No * Closes: #36752 Authored-by: Raúl Cumplido Signed-off-by: Sutou Kouhei --- .env | 2 +- ci/docker/python-wheel-manylinux.dockerfile | 3 ++- ci/docker/python-wheel-windows-vs2017.dockerfile | 3 ++- ci/scripts/java_jni_macos_build.sh | 1 - ci/scripts/python_wheel_manylinux_build.sh | 4 ---- ci/scripts/python_wheel_windows_build.bat | 4 ---- 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.env b/.env index 25b2743f6542f..c9cd6c8094ed8 100644 --- a/.env +++ b/.env @@ -98,7 +98,7 @@ VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1" # 2023.04.15 Release # ci/docker/python-wheel-windows-vs2017.dockerfile. # This is a workaround for our CI problem that "archery docker build" doesn't # use pulled built images in dev/tasks/python-wheels/github.windows.yml. -PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12 +PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2023-08-02 # Use conanio/${CONAN} for "docker-compose run --rm conan". See # https://github.com/conan-io/conan-docker-tools#readme for available diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index ed329ecdef6de..0f7779c878505 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -84,7 +84,8 @@ RUN vcpkg install \ --x-feature=flight \ --x-feature=gcs \ --x-feature=json \ - --x-feature=parquet + --x-feature=parquet \ + --x-feature=s3 # Configure Python for applications running in the bash shell of this Dockerfile ARG python=3.8 diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 01152dae232d8..531c4e678fafe 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -66,7 +66,8 @@ RUN vcpkg install \ --x-feature=flight \ --x-feature=gcs \ --x-feature=json \ - --x-feature=parquet + --x-feature=parquet \ + --x-feature=s3 # Remove previous installations of python from the base image # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7 diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 4a6f9444ec25f..d66c39a37c5bd 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -81,7 +81,6 @@ cmake \ -DARROW_PARQUET=${ARROW_PARQUET} \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ - -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh index cb5c2fbb7cc62..58e42fea88088 100755 --- a/ci/scripts/python_wheel_manylinux_build.sh +++ b/ci/scripts/python_wheel_manylinux_build.sh @@ -85,9 +85,6 @@ fi mkdir /tmp/arrow-build pushd /tmp/arrow-build -# ARROW-17501: We can remove -DAWSSDK_SOURCE=BUNDLED once -# https://github.com/aws/aws-sdk-cpp/issues/1809 is fixed and vcpkg -# ships the fix. cmake \ -DARROW_ACERO=${ARROW_ACERO} \ -DARROW_BUILD_SHARED=ON \ @@ -120,7 +117,6 @@ cmake \ -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \ -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \ -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \ - -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \ diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index ee879c7050dad..ffb43b3481e55 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -49,9 +49,6 @@ set ARROW_WITH_LZ4=ON set ARROW_WITH_SNAPPY=ON set ARROW_WITH_ZLIB=ON set ARROW_WITH_ZSTD=ON -@rem Workaround for https://github.com/aws/aws-sdk-cpp/issues/1809 . -@rem Use (old) bundled AWS SDK C++ instead of (newer) AWS SDK C++. -set AWSSDK_SOURCE=BUNDLED set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 set VCPKG_ROOT=C:\vcpkg @@ -90,7 +87,6 @@ cmake ^ -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^ -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^ -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^ - -DAWSSDK_SOURCE=%AWSSDK_SOURCE% ^ -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ -DCMAKE_CXX_COMPILER=clcache ^ -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^