diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml deleted file mode 100644 index c698baba2c816..0000000000000 --- a/.github/workflows/archery.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Archery & Crossbow - -on: - push: - paths: - - '.github/workflows/archery.yml' - - 'dev/archery/**' - - 'dev/tasks/**' - - 'docker-compose.yml' - pull_request: - paths: - - '.github/workflows/archery.yml' - - 'dev/archery/**' - - 'dev/tasks/**' - - 'docker-compose.yml' - -env: - ARCHERY_DEBUG: 1 - ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - ARCHERY_USE_DOCKER_CLI: 1 - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - - test: - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - name: Archery Unittests and Crossbow Check Config - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Git Fixup - shell: bash - run: git branch $ARCHERY_DEFAULT_BRANCH origin/$ARCHERY_DEFAULT_BRANCH || true - - name: Setup Python - uses: actions/setup-python@v5.1.0 - with: - python-version: '3.9' - - name: Install pygit2 binary wheel - run: pip install pygit2 --only-binary pygit2 - - name: Install Archery, Crossbow- and Test Dependencies - run: | - pip install -e dev/archery[all] - pip install -r dev/archery/requirements-test.txt - - name: Archery Unittests - working-directory: dev/archery - run: pytest -v archery - - name: Archery Docker Validation - run: archery docker check-config - - name: Crossbow Check Config - working-directory: dev/tasks - run: archery crossbow check-config diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml deleted file mode 100644 index a34856d2dc81a..0000000000000 --- a/.github/workflows/comment_bot.yml +++ /dev/null @@ -1,194 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Comment Bot - -on: - # TODO(kszucs): support pull_request_review_comment - issue_comment: - types: - - created - - edited - -permissions: - contents: read - pull-requests: write - -jobs: - crossbow: - name: Listen! - if: startsWith(github.event.comment.body, '@github-actions crossbow') - runs-on: ubuntu-latest - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - path: arrow - # fetch the tags for version number generation - fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.12 - - name: Install Archery and Crossbow dependencies - run: pip install -e arrow/dev/archery[bot] - - name: Handle GitHub comment event - env: - ARROW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN }} - run: | - archery --debug trigger-bot \ - --event-name ${{ github.event_name }} \ - --event-payload ${{ github.event_path }} - - autotune: - name: "Fix all the things" - if: startsWith(github.event.comment.body, '@github-actions autotune') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - - uses: r-lib/actions/pr-fetch@11a22a908006c25fe054c4ef0ac0436b1de3edbe # v2.6.4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: See what is different - run: | - set -ex - DEFAULT_BRANCH=${{ github.event.repository.default_branch }} - git remote add upstream https://github.com/apache/arrow - git fetch upstream - - changed() { - git diff --name-only upstream/$DEFAULT_BRANCH... | grep -e "$1" >/dev/null 2>&1 - } - if changed '^r/.*\.R$'; then - echo "R_DOCS=true" >> $GITHUB_ENV - echo "R_CODE=true" >> $GITHUB_ENV - fi - if changed 'cmake' || changed 'CMake'; then - echo "CMAKE_FORMAT=true" >> $GITHUB_ENV - fi - if changed '^cpp/src'; then - echo "CLANG_FORMAT_CPP=true" >> $GITHUB_ENV - fi - if changed '^r/src'; then - echo "CLANG_FORMAT_R=true" >> $GITHUB_ENV - fi - - name: Ensure clang-format has the appropriate version - if: env.CMAKE_FORMAT == 'true' || - env.CLANG_FORMAT_CPP == 'true' || - env.CLANG_FORMAT_R == 'true' || - endsWith(github.event.comment.body, 'everything') - run: | - set -e - . .env # To get the clang version we use - sudo apt update - sudo apt install -y clang-format-${CLANG_TOOLS} - - name: Run cmake_format - if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - set -ex - export PATH=/home/runner/.local/bin:$PATH - python3 -m pip install --upgrade pip setuptools wheel - python3 -m pip install -e dev/archery[lint] - archery lint --cmake-format --fix - - name: Run clang-format on cpp - if: env.CLANG_FORMAT_CPP == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - . .env # To get the clang version we use - cpp/build-support/run_clang_format.py \ - --clang_format_binary=clang-format-${CLANG_TOOLS} \ - --exclude_glob=cpp/build-support/lint_exclusions.txt \ - --source_dir=cpp/src --quiet --fix - - name: Run clang-format on r - if: env.CLANG_FORMAT_R == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - . .env # To get the clang version we use - cpp/build-support/run_clang_format.py \ - --clang_format_binary=clang-format-${CLANG_TOOLS} \ - --exclude_glob=cpp/build-support/lint_exclusions.txt \ - --source_dir=r/src --quiet --fix - - uses: r-lib/actions/setup-r@11a22a908006c25fe054c4ef0ac0436b1de3edbe # v2.6.4 - if: env.R_DOCS == 'true' || env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything') - - name: Update R docs - if: env.R_DOCS == 'true' || endsWith(github.event.comment.body, 'everything') - shell: Rscript {0} - run: | - source("ci/etc/rprofile") - install.packages(c("remotes", "roxygen2")) - remotes::install_deps("r") - roxygen2::roxygenize("r") - - name: Style R code - if: env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything') - shell: Rscript {0} - run: | - changed_files <- system("git diff --name-only upstream/${{ github.event.repository.default_branch }}... 2>&1", intern = TRUE) - # only grab the .R files under r/ - changed_files <- grep('^r/.*\\.R$', changed_files, value = TRUE) - # remove codegen.R and other possible exclusions - changed_files <- changed_files[!changed_files %in% file.path("r", source("r/.styler_excludes.R")$value)] - source("ci/etc/rprofile") - install.packages(c("remotes", "styler")) - remotes::install_deps("r") - styler::style_file(changed_files) - - name: Commit results - run: | - git config user.name "$(git log -1 --pretty=format:%an)" - git config user.email "$(git log -1 --pretty=format:%ae)" - git commit -a -m 'Autoformat/render all the things [automated commit]' || echo "No changes to commit" - - uses: r-lib/actions/pr-push@11a22a908006c25fe054c4ef0ac0436b1de3edbe # v2.6.4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - rebase: - name: "Rebase" - if: startsWith(github.event.comment.body, '@github-actions rebase') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - - uses: r-lib/actions/pr-fetch@11a22a908006c25fe054c4ef0ac0436b1de3edbe # v2.6.4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Rebase on ${{ github.repository }} default branch - run: | - set -ex - git config user.name "$(git log -1 --pretty=format:%an)" - git config user.email "$(git log -1 --pretty=format:%ae)" - git remote add upstream https://github.com/${{ github.repository }} - git fetch --unshallow upstream ${{ github.event.repository.default_branch }} - git rebase upstream/${{ github.event.repository.default_branch }} - - uses: r-lib/actions/pr-push@11a22a908006c25fe054c4ef0ac0436b1de3edbe # v2.6.4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - args: "--force" - - issue_assign: - name: "Assign issue" - permissions: - issues: write - if: github.event.comment.body == 'take' - runs-on: ubuntu-latest - steps: - - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - github.rest.issues.addAssignees({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.issue.number, - assignees: context.payload.comment.user.login - }); diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml deleted file mode 100644 index dd5abbe1b4b1b..0000000000000 --- a/.github/workflows/cpp.yml +++ /dev/null @@ -1,480 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C++ - -on: - push: - paths: - - '.github/workflows/cpp.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_*' - - 'ci/scripts/install_azurite.sh' - - 'ci/scripts/install_gcs_testbench.sh' - - 'ci/scripts/install_minio.sh' - - 'ci/scripts/msys2_*' - - 'ci/scripts/util_*' - - 'cpp/**' - - 'docker-compose.yml' - - 'format/Flight.proto' - - 'testing' - pull_request: - paths: - - '.github/workflows/cpp.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_*' - - 'ci/scripts/install_azurite.sh' - - 'ci/scripts/install_gcs_testbench.sh' - - 'ci/scripts/install_minio.sh' - - 'ci/scripts/msys2_*' - - 'ci/scripts/util_*' - - 'cpp/**' - - 'docker-compose.yml' - - 'format/Flight.proto' - - 'testing' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - docker-targets: - name: Docker targets - runs-on: ubuntu-latest - outputs: - targets: ${{ steps.detect-targets.outputs.targets }} - steps: - - name: Detect targets - id: detect-targets - run: | - echo "targets<> "$GITHUB_OUTPUT" - echo "[" >> "$GITHUB_OUTPUT" - cat <> "$GITHUB_OUTPUT" - { - "arch": "amd64", - "clang-tools": "14", - "image": "conda-cpp", - "llvm": "14", - "runs-on": "ubuntu-latest", - "simd-level": "AVX2", - "title": "AMD64 Conda C++ AVX2", - "ubuntu": "22.04" - }, - { - "arch": "amd64", - "clang-tools": "14", - "image": "ubuntu-cpp-sanitizer", - "llvm": "14", - "runs-on": "ubuntu-latest", - "title": "AMD64 Ubuntu 22.04 C++ ASAN UBSAN", - "ubuntu": "22.04" - } - JSON - if [ "$GITHUB_REPOSITORY_OWNER" = "apache" ]; then - echo "," >> "$GITHUB_OUTPUT" - cat <> "$GITHUB_OUTPUT" - { - "arch": "arm64v8", - "archery-use-docker-cli": "0", - "clang-tools": "10", - "image": "ubuntu-cpp", - "llvm": "10", - "runs-on": ["self-hosted", "arm", "linux"], - "title": "ARM64 Ubuntu 20.04 C++", - "ubuntu": "20.04" - } - JSON - fi - echo "]" >> "$GITHUB_OUTPUT" - echo "JSON" >> "$GITHUB_OUTPUT" - - docker: - name: ${{ matrix.title }} - needs: docker-targets - runs-on: ${{ matrix.runs-on }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.docker-targets.outputs.targets) }} - env: - ARCH: ${{ matrix.arch }} - # By default, use Docker CLI because docker-compose v1 is obsolete, - # except where the Docker client version is too old. - ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }} - ARROW_SIMD_LEVEL: ${{ matrix.simd-level }} - CLANG_TOOLS: ${{ matrix.clang-tools }} - LLVM: ${{ matrix.llvm }} - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }} - restore-keys: ${{ matrix.image }}- - - name: Setup Python - run: | - sudo apt update - sudo apt install -y --no-install-recommends python3 python3-dev python3-pip - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes - sudo sysctl -w vm.mmap_rnd_bits=28 - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ${{ matrix.image }} - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - build-example: - name: C++ Minimal Build Example - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 45 - steps: - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Check CMake presets - run: | - cd cpp - cmake --list-presets - - name: Run minimal example - run: | - cd cpp/examples/minimal_build - docker-compose run --rm minimal - - macos: - name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} C++ - runs-on: macos-${{ matrix.macos-version }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - include: - - architecture: AMD64 - macos-version: "12" - - architecture: ARM64 - macos-version: "14" - env: - ARROW_AZURE: ON - ARROW_BUILD_TESTS: ON - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_GCS: ON - ARROW_HDFS: ON - ARROW_HOME: /tmp/local - ARROW_JEMALLOC: ON - ARROW_ORC: ON - ARROW_PARQUET: ON - ARROW_S3: ON - ARROW_SUBSTRAIT: ON - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - # GH-36013 disabling opentelemetry here because we can't - # get the patched version from conda - # ARROW_WITH_OPENTELEMETRY: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - GTest_SOURCE: BUNDLED - steps: - - name: CPU Info - run: | - sysctl -a | grep cpu - sysctl -a | grep "hw.optional" - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Install Dependencies - run: | - brew bundle --file=cpp/Brewfile - - name: Install MinIO - run: | - $(brew --prefix bash)/bin/bash \ - ci/scripts/install_minio.sh latest ${ARROW_HOME} - - name: Set up Python - uses: actions/setup-python@v5.1.0 - with: - python-version: 3.12 - - name: Install Google Cloud Storage Testbench - run: ci/scripts/install_gcs_testbench.sh default - - name: Install Azurite Storage Emulator - run: ci/scripts/install_azurite.sh - - name: Setup ccache - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: cpp-ccache-macos-${{ matrix.macos-version }}-${{ hashFiles('cpp/**') }} - restore-keys: cpp-ccache-macos-${{ matrix.macos-version }}- - - name: Build - run: | - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: | - sudo sysctl -w kern.coredump=1 - sudo sysctl -w kern.corefile=core.%N.%P - ulimit -c unlimited # must enable within the same shell - ci/scripts/cpp_test.sh $(pwd) $(pwd)/build - - windows: - name: ${{ matrix.title }} - runs-on: ${{ matrix.os }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - os: - - windows-2019 - include: - - os: windows-2019 - simd-level: AVX2 - title: AMD64 Windows 2019 C++17 AVX2 - env: - ARROW_BOOST_USE_SHARED: OFF - ARROW_BUILD_BENCHMARKS: ON - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_DATASET: ON - ARROW_FLIGHT: OFF - ARROW_HDFS: ON - ARROW_HOME: /usr - ARROW_JEMALLOC: OFF - ARROW_MIMALLOC: ON - ARROW_ORC: ON - ARROW_PARQUET: ON - ARROW_SIMD_LEVEL: ${{ matrix.simd-level }} - ARROW_SUBSTRAIT: ON - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: OFF - ARROW_WITH_BZ2: OFF - ARROW_WITH_LZ4: OFF - ARROW_WITH_OPENTELEMETRY: OFF - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - BOOST_SOURCE: BUNDLED - CMAKE_CXX_STANDARD: "17" - CMAKE_GENERATOR: Ninja - CMAKE_INSTALL_LIBDIR: bin - CMAKE_INSTALL_PREFIX: /usr - CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Download Timezone Database - shell: bash - run: ci/scripts/download_tz_database.sh - - name: Install ccache - shell: bash - run: | - ci/scripts/install_ccache.sh 4.6.3 /usr - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: cpp-ccache-windows-${{ env.CACHE_VERSION }}-${{ hashFiles('cpp/**') }} - restore-keys: cpp-ccache-windows-${{ env.CACHE_VERSION }}- - env: - # We can invalidate the current cache by updating this. - CACHE_VERSION: "2022-09-13" - - name: Build - shell: cmd - run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build" - - name: Test - shell: bash - run: | - # For ORC - export TZDIR=/c/msys64/usr/share/zoneinfo - ci/scripts/cpp_test.sh $(pwd) $(pwd)/build - - windows-mingw: - name: AMD64 Windows MinGW ${{ matrix.msystem_upper }} C++ - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - # Build may take 1h+ without cache. - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - include: - - msystem_lower: mingw64 - msystem_upper: MINGW64 - - msystem_lower: clang64 - msystem_upper: CLANG64 - env: - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_BUILD_TYPE: release - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_FLIGHT_SQL: ON - ARROW_GANDIVA: ON - ARROW_GCS: ON - ARROW_HDFS: OFF - ARROW_HOME: /${{ matrix.msystem_lower}} - ARROW_JEMALLOC: OFF - ARROW_PARQUET: ON - ARROW_S3: ON - ARROW_SUBSTRAIT: ON - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_OPENTELEMETRY: OFF - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - # Don't use preinstalled Boost by empty BOOST_ROOT and - # -DBoost_NO_BOOST_CMAKE=ON - BOOST_ROOT: "" - ARROW_CMAKE_ARGS: >- - -DARROW_PACKAGE_PREFIX=/${{ matrix.msystem_lower}} - -DBoost_NO_BOOST_CMAKE=ON - -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON - # We can't use unity build because we don't have enough memory on - # GitHub Actions. - # CMAKE_UNITY_BUILD: ON - GTest_SOURCE: BUNDLED - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - uses: msys2/setup-msys2@v2 - with: - msystem: ${{ matrix.msystem_upper }} - update: true - - name: Setup MSYS2 - shell: msys2 {0} - run: ci/scripts/msys2_setup.sh cpp - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ccache - key: cpp-ccache-${{ matrix.msystem_lower}}-${{ hashFiles('cpp/**') }} - restore-keys: cpp-ccache-${{ matrix.msystem_lower}}- - - name: Build - shell: msys2 {0} - run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS - ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" - - name: Download Timezone Database - shell: bash - run: ci/scripts/download_tz_database.sh - - name: Download MinIO - shell: msys2 {0} - run: | - mkdir -p /usr/local/bin - wget \ - --output-document /usr/local/bin/minio.exe \ - https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z - chmod +x /usr/local/bin/minio.exe - - name: Set up Python - uses: actions/setup-python@v5.1.0 - with: - python-version: 3.9 - - name: Install Google Cloud Storage Testbench - shell: bash - run: | - ci/scripts/install_gcs_testbench.sh default - echo "PYTHON_BIN_DIR=$(cygpath --windows $(dirname $(which python3.exe)))" >> $GITHUB_ENV - - name: Test - shell: msys2 {0} - run: | - PATH="$(cygpath --unix ${PYTHON_BIN_DIR}):${PATH}" - ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build" diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml deleted file mode 100644 index 7ae3606a44812..0000000000000 --- a/.github/workflows/csharp.yml +++ /dev/null @@ -1,126 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C# - -on: - push: - paths: - - '.github/workflows/csharp.yml' - - 'ci/scripts/csharp_*' - - 'csharp/**' - pull_request: - paths: - - '.github/workflows/csharp.yml' - - 'ci/scripts/csharp_*' - - 'csharp/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - - ubuntu: - name: AMD64 Ubuntu 18.04 C# ${{ matrix.dotnet }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - dotnet: ['8.0.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v4 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install Source Link - shell: bash - run: dotnet tool install --global sourcelink - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) - - windows: - name: AMD64 Windows 2019 18.04 C# ${{ matrix.dotnet }} - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - dotnet: ['8.0.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v4 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install Source Link - run: dotnet tool install --global sourcelink - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) - - macos: - name: ARM64 macOS 14 C# ${{ matrix.dotnet }} - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - dotnet: ['8.0.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v4 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.12 - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install Source Link - shell: bash - run: dotnet tool install --global sourcelink - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml deleted file mode 100644 index 6111d1d2e5fe3..0000000000000 --- a/.github/workflows/dev.yml +++ /dev/null @@ -1,132 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Dev - -on: - # always trigger - push: - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - -jobs: - - lint: - name: Lint C++, Python, R, Docker, RAT - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.12 - - name: Install pre-commit - run: | - python -m pip install pre-commit - pre-commit run --show-diff-on-failure --color=always - - name: Cache pre-commit - uses: actions/cache@v4 - with: - path: ~/.cache/pre-commit - key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} - - name: Run pre-commit - run: | - pre-commit run --all-files --color=always --show-diff-on-failure - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run -e GITHUB_ACTIONS=true ubuntu-lint - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ubuntu-lint - - release: - name: Source Release and Merge Script on ${{ matrix.runs-on }} - runs-on: ${{ matrix.runs-on }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - runs-on: - - macos-latest - - ubuntu-latest - env: - GIT_AUTHOR_NAME: "github-actions[bot]" - GIT_AUTHOR_EMAIL: "github-actions[bot]@users.noreply.github.com" - GIT_COMMITTER_NAME: "github-actions[bot]" - GIT_COMMITTER_EMAIL: "github-actions[bot]@users.noreply.github.com" - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - - name: Install Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: '3.12' - - name: Install Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ruby - - name: Install .NET - uses: actions/setup-dotnet@4d6c8fcf3c8f7a60068d26b594648e99df24cee3 # v4.0.0 - with: - dotnet-version: '8.0.x' - - name: Install Dependencies - shell: bash - run: | - gem install test-unit - pip install "cython>=0.29.31" setuptools six pytest jira - - name: Run Release Test - env: - ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - ci/scripts/release_test.sh $(pwd) - - name: Run Merge Script Test - shell: bash - run: pytest -v dev/test_merge_arrow_pr.py diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml deleted file mode 100644 index 10b33c96d2129..0000000000000 --- a/.github/workflows/dev_pr.yml +++ /dev/null @@ -1,106 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Dev PR - -on: - # TODO: Enable this when eps1lon/actions-label-merge-conflict is available. - # push: - # branches: - # - master - # - main - pull_request_target: - types: - - opened - - edited - - synchronize - -concurrency: - group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event.number }} - cancel-in-progress: true - -permissions: - contents: read - pull-requests: write - issues: write - -jobs: - process: - name: Process - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - repository: apache/arrow - ref: main - persist-credentials: false - - - name: Comment JIRA link - if: | - (github.event.action == 'opened' || - github.event.action == 'edited') - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr/link.js`); - script({github, context}); - - - name: Check title - if: | - (github.event.action == 'opened' || - github.event.action == 'edited') - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr/title_check.js`); - script({github, context}); - - - name: Check Issue - if: | - (github.event.action == 'opened' || - github.event.action == 'edited') - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - with: - debug: true - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr/issue_check.js`); - script({github, context}); - - - name: Assign GitHub labels - if: | - (github.event.action == 'opened' || - github.event.action == 'synchronize') - uses: actions/labeler@ac9175f8a1f3625fd0d4fb234536d26811351594 # v4.3.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - configuration-path: .github/workflows/dev_pr/labeler.yml - sync-labels: true - - # TODO: Enable this when eps1lon/actions-label-merge-conflict is available. - # - name: Checks if PR needs rebase - # if: | - # github.event_name == 'push' || - # (github.event_name == 'pull_request_target' && - # (github.event.action == 'opened' || - # github.event.action == 'synchronize')) - # uses: eps1lon/actions-label-merge-conflict@releases/2.x - # with: - # dirtyLabel: "needs-rebase" - # repoToken: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 36a0dc014db8d..0000000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Docs - -on: - push: - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - complete: - name: AMD64 Debian 12 Complete Documentation - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 150 - env: - JDK: 17 - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: debian-docs-${{ hashFiles('cpp/**') }} - restore-keys: debian-docs- - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - JDK: 17 - run: archery docker run debian-docs - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push debian-docs diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml deleted file mode 100644 index 947e2ac21b83c..0000000000000 --- a/.github/workflows/docs_light.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Docs - -on: - pull_request: - paths: - - 'docs/**' - - '.github/workflows/docs_light.yml' - - 'ci/docker/conda.dockerfile' - - 'ci/docker/conda-cpp.dockerfile' - - 'ci/docker/conda-python.dockerfile' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/python_build.sh' - - 'docker-compose.yml' - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - light: - name: AMD64 Conda Python 3.9 Sphinx Documentation - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - env: - PYTHON: "3.9" - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: conda-docs-${{ hashFiles('cpp/**') }} - restore-keys: conda-docs- - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run conda-python-docs diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 11dc29dcae54e..0000000000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,485 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Go - -on: - push: - paths: - - '.github/workflows/go.yml' - - 'ci/docker/*_go.dockerfile' - - 'ci/scripts/go_*' - - 'docker-compose.yml' - - 'go/**' - pull_request: - paths: - - '.github/workflows/go.yml' - - 'ci/docker/*_go.dockerfile' - - 'ci/docker/**' - - 'ci/scripts/go_*' - - 'docker-compose.yml' - - 'go/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - -jobs: - - docker-targets: - name: Docker targets - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - outputs: - targets: ${{ steps.detect-targets.outputs.targets }} - steps: - - name: Detect targets - id: detect-targets - run: | - echo "targets<> "$GITHUB_OUTPUT" - echo "[" >> "$GITHUB_OUTPUT" - cat <> "$GITHUB_OUTPUT" - { - "arch-label": "AMD64", - "arch": "amd64", - "go": "1.21", - "runs-on": "ubuntu-latest" - }, - { - "arch-label": "AMD64", - "arch": "amd64", - "go": "1.22", - "runs-on": "ubuntu-latest" - } - JSON - if [ "$GITHUB_REPOSITORY_OWNER" = "apache" ]; then - echo "," >> "$GITHUB_OUTPUT" - cat <> "$GITHUB_OUTPUT" - { - "arch-label": "ARM64", - "arch": "arm64v8", - "archery-use-docker-cli": "0", - "go": "1.21", - "runs-on": ["self-hosted", "arm", "linux"] - }, - { - "arch-label": "ARM64", - "arch": "arm64v8", - "archery-use-docker-cli": "0", - "go": "1.22", - "runs-on": ["self-hosted", "arm", "linux"] - } - JSON - fi - echo "]" >> "$GITHUB_OUTPUT" - echo "JSON" >> "$GITHUB_OUTPUT" - - docker: - name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }} - needs: docker-targets - runs-on: ${{ matrix.runs-on }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.docker-targets.outputs.targets) }} - env: - ARCH: ${{ matrix.arch }} - # By default, use Docker CLI because docker-compose v1 is obsolete, - # except where the Docker client version is too old. - ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }} - GO: ${{ matrix.go }} - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Setup Python - run: | - sudo apt update - sudo apt install -y --no-install-recommends python3 python3-dev python3-pip - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run debian-go - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push debian-go - - name: Install Go ${{ matrix.go }} for Benchmarks - if: >- - success() && - matrix.arch == 'amd64' && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 - with: - go-version: ${{ matrix.go }} - cache: true - cache-dependency-path: go/go.sum - - name: Run Benchmarks - if: >- - success() && - matrix.arch == 'amd64' && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - CONBENCH_URL: https://conbench.ursa.dev - CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }} - CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }} - CONBENCH_REF: ${{ github.ref_name }} - CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12 - run: | - python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python - python3 ci/scripts/go_bench_adapt.py - - build386: - name: Go Cross-build for 386 - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 20 - steps: - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Get required Go version - run: | - (. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version: "${{ env.GO_VERSION }}" - cache: true - cache-dependency-path: go/go.sum - - name: Run build - run: | - cd go - GOARCH=386 go build ./... - - docker_cgo: - name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - go: ['1.21', '1.22'] - env: - GO: ${{ matrix.go }} - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run debian-go-cgo - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push debian-go-cgo - - - docker_cgo_python: - name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: ['1.21', '1.22'] - env: - GO: ${{ matrix.go }} - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run debian-go-cgo-python - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push debian-go-cgo-python - - windows: - name: AMD64 Windows 2019 Go ${{ matrix.go }} - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 25 - strategy: - fail-fast: false - matrix: - go: ['1.21', '1.22'] - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Install go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 - with: - go-version: ${{ matrix.go }} - cache: true - cache-dependency-path: go/go.sum - - name: Install staticcheck - shell: bash - run: | - . .env - go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK} - - name: Build - shell: bash - run: ci/scripts/go_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/go_test.sh $(pwd) - - macos: - name: AMD64 macOS 11 Go ${{ matrix.go }} - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - go: ['1.21', '1.22'] - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Install go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 - with: - go-version: ${{ matrix.go }} - cache: true - cache-dependency-path: go/go.sum - - name: Install staticcheck - run: | - . .env - go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK} - - name: Build - shell: bash - run: ci/scripts/go_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/go_test.sh $(pwd) - - name: Setup Python - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: '3.10' - - name: Run Benchmarks - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - shell: bash - env: - CONBENCH_URL: 'https://conbench.ursa.dev' - CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }} - CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }} - CONBENCH_REF: ${{ github.ref_name }} - CONBENCH_MACHINE_INFO_NAME: amd64-macos-11 - run: | - pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python - python ci/scripts/go_bench_adapt.py - - - macos-cgo: - name: AMD64 macOS 11 Go ${{ matrix.go }} - CGO - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - go: ['1.21', '1.22'] - env: - ARROW_GO_TESTCGO: "1" - steps: - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Install go - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go }} - cache: true - cache-dependency-path: go/go.sum - - name: Brew Install Arrow and pkg-config - shell: bash - run: brew install apache-arrow pkg-config - - name: Install staticcheck - run: | - . .env - go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK} - - name: Add To pkg config path - shell: bash - run: | - echo "PKG_CONFIG_PATH=$(brew --prefix openssl@3)/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV - - name: Build - shell: bash - run: ci/scripts/go_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/go_test.sh $(pwd) - - windows-mingw: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} CGO - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - mingw-n-bits: - #- 32 runtime handling for CGO needs 64-bit currently - - 64 - env: - ARROW_GO_TESTCGO: "1" - MINGW_LINT: "1" - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW${{ matrix.mingw-n-bits }} - update: true - - name: Setup MSYS2 - shell: msys2 {0} - run: | - ci/scripts/msys2_setup.sh cgo - - name: Get required Go version - run: | - (. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV - - name: Update CGO Env vars - shell: msys2 {0} - run: | - echo "CGO_CPPFLAGS=-I$(cygpath --windows ${MINGW_PREFIX}/include)" >> $GITHUB_ENV - echo "CGO_LDFLAGS=-g -O2 -L$(cygpath --windows ${MINGW_PREFIX}/lib) -L$(cygpath --windows ${MINGW_PREFIX}/bin)" >> $GITHUB_ENV - echo "MINGW_PREFIX=$(cygpath --windows ${MINGW_PREFIX})" >> $GITHUB_ENV - - name: Install go - uses: actions/setup-go@v5 - with: - go-version: "${{ env.GO_VERSION }}" - cache: true - cache-dependency-path: go/go.sum - - name: Install staticcheck - shell: bash - run: | - . .env - go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK} - - name: Build - shell: bash - run: ci/scripts/go_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/go_test.sh $(pwd) - - tinygo: - name: TinyGo - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - env: - TINYGO_VERSION: 0.27.0 - timeout-minutes: 60 - steps: - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Build and Run Example - run: | - docker run --rm -v $(pwd)/go:/src -v $(pwd)/ci/scripts:/ci-scripts "tinygo/tinygo:$TINYGO_VERSION" /ci-scripts/go_tinygo_example.sh diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index f53f4aeb505d2..0000000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,118 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Integration - -on: - push: - paths: - - '.github/workflows/integration.yml' - - 'ci/**' - - 'dev/archery/**' - - 'docker-compose.yml' - - 'go/**' - - 'integration/**' - - 'js/**' - - 'cpp/**' - - 'java/**' - - 'csharp/**' - - 'format/**' - pull_request: - paths: - - '.github/workflows/integration.yml' - - 'ci/**' - - 'dev/archery/**' - - 'docker-compose.yml' - - 'go/**' - - 'integration/**' - - 'js/**' - - 'cpp/**' - - 'csharp/**' - - 'java/**' - - 'format/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - docker: - name: AMD64 Conda Integration Test - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Checkout Arrow Rust - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - repository: apache/arrow-rs - path: rust - - name: Checkout Arrow nanoarrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - repository: apache/arrow-nanoarrow - path: nanoarrow - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: conda-${{ hashFiles('cpp/**') }} - restore-keys: conda- - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: > - archery docker run \ - -e ARCHERY_DEFAULT_BRANCH=${{ github.event.repository.default_branch }} \ - -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ - -e ARCHERY_INTEGRATION_WITH_RUST=1 \ - conda-integration - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push conda-integration diff --git a/.github/workflows/issue_bot.yml b/.github/workflows/issue_bot.yml deleted file mode 100644 index ec614ca1e7c56..0000000000000 --- a/.github/workflows/issue_bot.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Issue Bot - -on: - issues: - types: - - opened - - edited - -permissions: - contents: read - issues: write - -jobs: - label_components: - name: Label Components - if: github.event.issue.pull_request == null - runs-on: ubuntu-latest - steps: - - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - with: - script: | - let split_body = context.payload.issue.body.split('### Component(s)'); - if (split_body.length != 2) throw new Error('No components found!'); - - let current_labels = await github.rest.issues.listLabelsOnIssue({ - "owner": context.repo.owner, - "repo": context.repo.repo, - "per_page": 100, - "issue_number": context.payload.issue.number, - }); - - let current_label_names = current_labels.data.map(label => label.name); - - // keep non-component labels - let non_component_labels = current_label_names.filter( - label => !label.startsWith("Component: ") - ); - - let component_labels = split_body[1] - .split(',') - .map(component => component.trim()) - .map(component => "Component: " + component); - - let repo_labels = await github.rest.issues.listLabelsForRepo({ - "owner": context.repo.owner, - "repo": context.repo.repo, - "per_page": 100, - }); - - // this removes nonexistent labels - component_labels = component_labels.filter( - label => repo_labels.data.some(repo_label => repo_label.name === label) - ); - - if (component_labels.length == 0) throw new Error('No components found!'); - - await github.rest.issues.setLabels({ - "owner": context.repo.owner, - "repo": context.repo.repo, - "issue_number": context.payload.issue.number, - "labels": component_labels.concat(non_component_labels), - }); diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml deleted file mode 100644 index e92d3f4fc5877..0000000000000 --- a/.github/workflows/java.yml +++ /dev/null @@ -1,167 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Java - -on: - push: - paths: - - '.github/workflows/java.yml' - - 'ci/docker/*java*' - - 'ci/scripts/java*.sh' - - 'ci/scripts/util_*.sh' - - 'docker-compose.yml' - - 'format/Flight.proto' - - 'java/**' - pull_request: - paths: - - '.github/workflows/java.yml' - - 'ci/docker/*java*' - - 'ci/scripts/java*.sh' - - 'ci/scripts/util_*.sh' - - 'docker-compose.yml' - - 'format/Flight.proto' - - 'java/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - ubuntu: - name: AMD64 Ubuntu 22.04 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [8, 11, 17, 21, 22] - maven: [3.9.6] - image: [java] - env: - JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: maven-${{ hashFiles('java/**') }} - restore-keys: maven- - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: | - archery docker run \ - -e CI=true \ - -e "GRADLE_ENTERPRISE_ACCESS_KEY=$GRADLE_ENTERPRISE_ACCESS_KEY" \ - ${{ matrix.image }} - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - macos: - name: AMD64 macOS 11 Java JDK ${{ matrix.jdk }} - runs-on: macos-latest - if: github.event_name == 'push' - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [11] - steps: - - name: Set up Java - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: ${{ matrix.jdk }} - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Build - shell: bash - env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build - - windows: - name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [11] - steps: - - name: Set up Java - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Build - shell: bash - env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml deleted file mode 100644 index 958216ac7669d..0000000000000 --- a/.github/workflows/java_jni.yml +++ /dev/null @@ -1,139 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Java JNI - -on: - push: - paths: - - '.github/workflows/java_jni.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/java_*' - - 'cpp/**' - - 'docker-compose.yml' - - 'java/**' - pull_request: - paths: - - '.github/workflows/java_jni.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/java_*' - - 'cpp/**' - - 'docker-compose.yml' - - 'java/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - docker: - name: AMD64 manylinux2014 Java JNI - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: java-jni-manylinux-2014-${{ hashFiles('cpp/**', 'java/**') }} - restore-keys: java-jni-manylinux-2014- - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run java-jni-manylinux-2014 - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push java-jni-manylinux-2014 - - docker_integration_python: - name: AMD64 Conda Java C Data Interface Integration - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: maven-${{ hashFiles('java/**') }} - restore-keys: maven- - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: | - archery docker run \ - -e CI=true \ - -e "GRADLE_ENTERPRISE_ACCESS_KEY=$GRADLE_ENTERPRISE_ACCESS_KEY" \ - conda-python-java-integration - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push conda-python-java-integration diff --git a/.github/workflows/java_nightly.yml b/.github/workflows/java_nightly.yml deleted file mode 100644 index f40d4ce5b42d6..0000000000000 --- a/.github/workflows/java_nightly.yml +++ /dev/null @@ -1,139 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Upload Java Nightly builds - -on: - workflow_dispatch: - inputs: - prefix: - description: Job prefix to use. - required: false - default: '' - keep: - description: Number of versions to keep. - required: false - default: 14 - schedule: - - cron: '0 14 * * *' - -permissions: - contents: read - -jobs: - upload: - if: github.repository == 'apache/arrow' - env: - PREFIX: ${{ github.event.inputs.prefix || ''}} - CROSSBOW_GITHUB_TOKEN: ${{ github.token }} - runs-on: ubuntu-latest - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 1 - path: arrow - repository: apache/arrow - ref: main - submodules: recursive - - name: Checkout Crossbow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - path: crossbow - repository: ursacomputing/crossbow - ref: main - - name: Set up Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - cache: 'pip' - python-version: 3.12 - - name: Install Archery - shell: bash - run: pip install -e arrow/dev/archery[all] - - run: mkdir -p binaries - - name: Download Artifacts - run: | - if [ -z $PREFIX ]; then - PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0 - fi - echo $PREFIX - archery crossbow download-artifacts -f java-jars -t binaries $PREFIX - - name: Sync from Remote - uses: ./arrow/.github/actions/sync-nightlies - with: - switches: -avzh --update --delete --progress - local_path: repo - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} - - shell: bash - name: Show local repo sync from remote - run: | - for i in `ls -t repo/org/apache/arrow`; do - echo "- $i: $(find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d \ - | wc -l \ - | xargs) versions available" - done - - shell: bash - name: Build Repository - run: | - DATE=$(date +%Y-%m-%d) - if [ -z $PREFIX ]; then - PREFIX=nightly-packaging-${DATE}-0 - fi - PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9]+.[0-9]+.[0-9]+-SNAPSHOT)' - mkdir -p repo/org/apache/arrow/ - for LIBRARY in $(ls binaries/$PREFIX/java-jars | grep -E '.jar|.json|.pom|.xml' | grep SNAPSHOT); do - [[ $LIBRARY =~ $PATTERN_TO_GET_LIB_AND_VERSION ]] - mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} - mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE} - # Copy twice to maintain a latest snapshot and some earlier versions - cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} - touch repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} - cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE} - echo "Artifacts $LIBRARY configured" - done - - name: Prune Repository - shell: bash - env: - KEEP: ${{ github.event.inputs.keep || 14 }} - run: | - for i in `ls -t repo/org/apache/arrow`; do - find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d -print0 \ - | xargs -0 ls -t -d \ - | tail -n +$((KEEP + 1)) \ - | xargs rm -rf - done - - name: Show repo contents - run: tree repo - - name: Sync to Remote - if: ${{ github.repository == 'apache/arrow' }} - uses: ./arrow/.github/actions/sync-nightlies - with: - upload: true - switches: -avzh --update --delete --progress - local_path: repo - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml deleted file mode 100644 index c9b7d7b742d88..0000000000000 --- a/.github/workflows/js.yml +++ /dev/null @@ -1,142 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: NodeJS - -on: - push: - paths: - - '.github/workflows/js.yml' - - 'ci/docker/*js.dockerfile' - - 'ci/scripts/js_*' - - 'js/**' - pull_request: - paths: - - '.github/workflows/js.yml' - - 'ci/docker/*js.dockerfile' - - 'ci/scripts/js_*' - - 'js/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - -jobs: - - docker: - name: AMD64 Debian 12 NodeJS 18 - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run debian-js - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push debian-js - - macos: - name: AMD64 macOS 11 NodeJS ${{ matrix.node }} - runs-on: macos-latest - if: github.event_name == 'push' - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - node: [18] - steps: - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Jest Cache - uses: actions/cache@v4 - with: - path: js/.jest-cache - key: js-jest-cache-${{ runner.os }}-${{ hashFiles('js/src/**/*.ts', 'js/test/**/*.ts', 'js/yarn.lock') }} - restore-keys: js-jest-cache-${{ runner.os }}- - - name: Install NodeJS - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node }} - - name: Build - shell: bash - run: ci/scripts/js_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/js_test.sh $(pwd) - - windows: - name: AMD64 Windows NodeJS ${{ matrix.node }} - runs-on: windows-latest - if: github.event_name == 'push' - strategy: - fail-fast: false - matrix: - node: [18] - steps: - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Jest Cache - uses: actions/cache@v4 - with: - path: js/.jest-cache - key: js-jest-cache-${{ runner.os }}-${{ hashFiles('js/src/**/*.ts', 'js/test/**/*.ts', 'js/yarn.lock') }} - restore-keys: js-jest-cache-${{ runner.os }}- - - name: Install NodeJS - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node }} - - name: Build - shell: bash - run: ci/scripts/js_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/js_test.sh $(pwd) diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml deleted file mode 100644 index ca8280927f4a5..0000000000000 --- a/.github/workflows/matlab.yml +++ /dev/null @@ -1,194 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: MATLAB - -on: - push: - paths: - - '.github/workflows/matlab.yml' - - 'ci/scripts/matlab*.sh' - - 'matlab/**' - - 'cpp/src/arrow/**' - pull_request: - paths: - - '.github/workflows/matlab.yml' - - 'ci/scripts/matlab*.sh' - - 'matlab/**' - - 'cpp/src/arrow/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - - ubuntu: - name: AMD64 Ubuntu 20.04 MATLAB - # Explicitly pin the Ubuntu version to 20.04 for the time being because: - # - # 1. The version of GLIBCXX shipped with Ubuntu 22.04 is not binary compatible - # with the GLIBCXX bundled with MATLAB R2023a. This is a relatively common - # issue. - # - # For example, see: - # - # https://www.mathworks.com/matlabcentral/answers/1907290-how-to-manually-select-the-libstdc-library-to-use-to-resolve-a-version-glibcxx_-not-found - # - # 2. The version of GLIBCXX shipped with Ubuntu 22.04 is not binary compatible with - # the version of GLIBCXX shipped with Debian 11. Several of the Arrow community - # members who work on the MATLAB bindings use Debian 11 locally for qualification. - # Using Ubuntu 20.04 eases development workflows for these community members. - # - # In the future, we can investigate adding support for building against more Linux (e.g. `ubuntu-22.04`) and MATLAB versions (e.g. R2023b). - runs-on: ubuntu-20.04 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install ninja-build - run: sudo apt-get install ninja-build - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v2 - with: - release: R2024a - - name: Install ccache - run: sudo apt-get install ccache - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: matlab-ccache-ubuntu-${{ hashFiles('cpp/**', 'matlab/**') }} - restore-keys: matlab-ccache-ubuntu- - - name: Build MATLAB Interface - run: ci/scripts/matlab_build.sh $(pwd) - - name: Run MATLAB Tests - env: - # Add the installation directory to the MATLAB Search Path by - # setting the MATLABPATH environment variable. - MATLABPATH: matlab/install/arrow_matlab - uses: matlab-actions/run-tests@v2 - with: - select-by-folder: matlab/test - strict: true - macos: - name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} MATLAB - runs-on: macos-${{ matrix.macos-version }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - matrix: - include: - - architecture: AMD64 - macos-version: "12" - - architecture: ARM64 - macos-version: "14" - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install ninja-build - run: brew install ninja - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v2 - with: - release: R2024a - - name: Install ccache - run: brew install ccache - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: matlab-ccache-macos-${{ hashFiles('cpp/**', 'matlab/**') }} - restore-keys: matlab-ccache-macos- - - name: Build MATLAB Interface - run: ci/scripts/matlab_build.sh $(pwd) - - name: Run MATLAB Tests - env: - # Add the installation directory to the MATLAB Search Path by - # setting the MATLABPATH environment variable. - MATLABPATH: matlab/install/arrow_matlab - uses: matlab-actions/run-tests@v2 - with: - select-by-folder: matlab/test - strict: true - windows: - name: AMD64 Windows 2022 MATLAB - runs-on: windows-2022 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v2 - with: - release: R2024a - - name: Download Timezone Database - shell: bash - run: ci/scripts/download_tz_database.sh - - name: Install ccache - shell: bash - run: ci/scripts/install_ccache.sh 4.6.3 /usr - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v4 - with: - path: | - ${{ steps.ccache-info.outputs.cache-dir }} - key: matlab-ccache-windows-${{ hashFiles('cpp/**', 'matlab/**') }} - restore-keys: matlab-ccache-windows- - - name: Build MATLAB Interface - shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/matlab_build.sh $(pwd)" - - name: Run MATLAB Tests - env: - # Add the installation directory to the MATLAB Search Path by - # setting the MATLABPATH environment variable. - MATLABPATH: matlab/install/arrow_matlab - uses: matlab-actions/run-tests@v2 - with: - select-by-folder: matlab/test - strict: true diff --git a/.github/workflows/pr_bot.yml b/.github/workflows/pr_bot.yml deleted file mode 100644 index e589610f536b3..0000000000000 --- a/.github/workflows/pr_bot.yml +++ /dev/null @@ -1,105 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: "Workflow label bot" -on: - pull_request_target: - types: - - opened - - converted_to_draft - - ready_for_review - - synchronize - workflow_run: - workflows: ["Label when reviewed"] - types: ['completed'] - -permissions: - contents: read - pull-requests: write - issues: write - -jobs: - pr-workflow-bot-job: - name: "PR Workflow bot" - runs-on: ubuntu-latest - steps: - - name: 'Download PR review payload' - id: 'download' - if: github.event_name == 'workflow_run' - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - with: - script: | - const run_id = "${{ github.event.workflow_run.id }}"; - let artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: run_id, - }); - let pr_review_artifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "pr_review_payload" - })[0]; - let pr_review_download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: pr_review_artifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/pr_review.zip', Buffer.from(pr_review_download.data)); - - name: Extract artifact - id: extract - if: github.event_name == 'workflow_run' - run: | - unzip pr_review.zip - echo "pr_review_path=$(pwd)/event.json" >> $GITHUB_OUTPUT - - name: Download committers file - id: committers - run: | - url="https://raw.githubusercontent.com/apache/arrow-site/main/_data/committers.yml" - curl -sL -o committers.yml $url - echo "committers_path=$(pwd)/committers.yml" >> $GITHUB_OUTPUT - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - path: arrow - repository: apache/arrow - ref: main - persist-credentials: false - # fetch the tags for version number generation - fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.12 - - name: Install Archery and Crossbow dependencies - run: pip install -e arrow/dev/archery[bot] - - name: Handle PR workflow event - env: - ARROW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - if [ "${GITHUB_EVENT_NAME}" = "workflow_run" ]; then - # workflow_run is executed on PR review. Update to original event. - archery trigger-bot \ - --event-name "pull_request_review" \ - --event-payload "${{ steps.extract.outputs.pr_review_path }}" \ - --committers-file "${{ steps.committers.outputs.committers_path }}" - else - archery trigger-bot \ - --event-name "${{ github.event_name }}" \ - --event-payload "${{ github.event_path }}" \ - --committers-file "${{ steps.committers.outputs.committers_path }}" - fi diff --git a/.github/workflows/pr_review_trigger.yml b/.github/workflows/pr_review_trigger.yml deleted file mode 100644 index 0cd89b3206715..0000000000000 --- a/.github/workflows/pr_review_trigger.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: "Label when reviewed" -on: pull_request_review - -permissions: - contents: read - -jobs: - # due to GitHub Actions permissions we can't change labels on the pull_request_review - # workflow. We trigger a new workflow run which will have permissions to add labels. - label-when-reviewed: - name: "Label PRs when reviewed" - runs-on: ubuntu-latest - steps: - - name: "Upload PR review Payload" - uses: actions/upload-artifact@v3 - with: - path: "${{ github.event_path }}" - name: "pr_review_payload" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index a568f8346e7fc..0000000000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,206 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Python - -on: - push: - paths: - - '.github/workflows/python.yml' - - 'ci/**' - - 'cpp/**' - - 'docker-compose.yml' - - 'python/**' - pull_request: - paths: - - '.github/workflows/python.yml' - - 'ci/**' - - 'cpp/**' - - 'docker-compose.yml' - - 'python/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - docker: - name: ${{ matrix.title }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - name: - - conda-python-docs - - conda-python-3.9-nopandas - - conda-python-3.8-pandas-1.0 - - conda-python-3.10-pandas-latest - include: - - name: conda-python-docs - cache: conda-python-3.9 - image: conda-python-docs - title: AMD64 Conda Python 3.9 Sphinx & Numpydoc - python: 3.9 - - name: conda-python-3.9-nopandas - cache: conda-python-3.9 - image: conda-python - title: AMD64 Conda Python 3.9 Without Pandas - python: 3.9 - - name: conda-python-3.8-pandas-1.0 - cache: conda-python-3.8 - image: conda-python-pandas - title: AMD64 Conda Python 3.8 Pandas 1.0 - python: 3.8 - pandas: "1.0" - numpy: 1.16 - - name: conda-python-3.10-pandas-latest - cache: conda-python-3.10 - image: conda-python-pandas - title: AMD64 Conda Python 3.10 Pandas latest - python: "3.10" - pandas: latest - env: - PYTHON: ${{ matrix.python || 3.8 }} - UBUNTU: ${{ matrix.ubuntu || 20.04 }} - PANDAS: ${{ matrix.pandas || 'latest' }} - NUMPY: ${{ matrix.numpy || 'latest' }} - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }} - restore-keys: ${{ matrix.cache }}- - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ${{ matrix.image }} - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - macos: - name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} Python 3 - runs-on: macos-${{ matrix.macos-version }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - architecture: AMD64 - macos-version: "12" - - architecture: ARM64 - macos-version: "14" - env: - ARROW_HOME: /tmp/local - ARROW_AZURE: ON - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_GCS: OFF - ARROW_HDFS: ON - ARROW_JEMALLOC: ON - ARROW_ORC: ON - ARROW_PARQUET: ON - PARQUET_REQUIRE_ENCRYPTION: ON - ARROW_PYTHON: ON - ARROW_S3: ON - ARROW_SUBSTRAIT: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_ZSTD: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_BROTLI: ON - ARROW_BUILD_TESTS: OFF - PYARROW_TEST_LARGE_MEMORY: ON - # Current oldest supported version according to https://endoflife.date/macos - MACOSX_DEPLOYMENT_TARGET: 10.15 - steps: - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Setup Python - uses: actions/setup-python@v5.1.0 - with: - python-version: '3.11' - - name: Install Dependencies - shell: bash - run: | - brew bundle --file=cpp/Brewfile - python -m pip install \ - -r python/requirements-build.txt \ - -r python/requirements-test.txt - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: python-ccache-macos-${{ matrix.macos-version }}-${{ hashFiles('cpp/**', 'python/**') }} - restore-keys: python-ccache-macos-${{ matrix.macos-version }}- - - name: Build - shell: bash - run: | - python -m pip install wheel - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - ci/scripts/python_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: ci/scripts/python_test.sh $(pwd) $(pwd)/build diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml deleted file mode 100644 index aba77347659cd..0000000000000 --- a/.github/workflows/r.yml +++ /dev/null @@ -1,407 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: R - -on: - push: - paths: - - ".github/workflows/r.yml" - - "ci/docker/**" - - "ci/etc/rprofile" - - "ci/scripts/PKGBUILD" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/install_minio.sh" - - "ci/scripts/r_*.sh" - - "cpp/**" - - "docker-compose.yml" - - "r/**" - pull_request: - paths: - - ".github/workflows/r.yml" - - "ci/docker/**" - - "ci/etc/rprofile" - - "ci/scripts/PKGBUILD" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/install_minio.sh" - - "ci/scripts/r_*.sh" - - "cpp/**" - - "docker-compose.yml" - - "r/**" - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - ubuntu-minimum-cpp-version: - name: Check minimum supported Arrow C++ Version (${{ matrix.cpp_version }}) - runs-on: ubuntu-latest - strategy: - matrix: - include: - - cpp_version: "13.0.0" - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - path: src - submodules: recursive - - - name: Install Arrow C++ (${{ matrix.cpp_version }}) - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - # We have to list all packages to avoid version conflicts. - sudo apt install -y -V libarrow-dev=${{ matrix.cpp_version }}-1 \ - libarrow-acero-dev=${{ matrix.cpp_version }}-1 \ - libparquet-dev=${{ matrix.cpp_version }}-1 \ - libarrow-dataset-dev=${{ matrix.cpp_version }}-1 - - - name: Install checkbashisms - run: | - sudo apt-get install devscripts - - - uses: r-lib/actions/setup-r@v2 - with: - use-public-rspm: true - install-r: false - - - uses: r-lib/actions/setup-r-dependencies@v2 - with: - extra-packages: any::rcmdcheck - needs: check - working-directory: src/r - - - uses: r-lib/actions/check-r-package@v2 - with: - working-directory: src/r - env: - LIBARROW_BINARY: "false" - LIBARROW_BUILD: "false" - ARROW_R_VERBOSE_TEST: "true" - ARROW_R_ALLOW_CPP_VERSION_MISMATCH: "true" - - - name: Show install output - if: always() - run: find src/r/check -name '00install.out*' -exec cat '{}' \; || true - shell: bash - - - ubuntu: - name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }} Force-Tests ${{ matrix.force-tests }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - r: ["4.4"] - ubuntu: [20.04] - force-tests: ["true"] - env: - R: ${{ matrix.r }} - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - # As this key is identical on both matrix builds only one will be able to successfully cache, - # this is fine as there are no differences in the build - key: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} - restore-keys: | - ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- - ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}- - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - # Setting a non-default and non-probable Marquesas French Polynesia time - # it has both with a .45 offset and very very few people who live there. - archery docker run -e TZ=MART -e ARROW_R_FORCE_TESTS=${{ matrix.force-tests }} ubuntu-r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 - with: - name: test-output - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ubuntu-r - - bundled: - name: "${{ matrix.config.org }}/${{ matrix.config.image }}:${{ matrix.config.tag }}" - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - config: - - { org: "rhub", image: "ubuntu-gcc12", tag: "latest" } - env: - R_ORG: ${{ matrix.config.org }} - R_IMAGE: ${{ matrix.config.image }} - R_TAG: ${{ matrix.config.tag }} - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - # Don't set a TZ here to test that case. These builds will have the following warning in them: - # System has not been booted with systemd as init system (PID 1). Can't operate. - # Failed to connect to bus: Host is down - archery docker run -e TZ="" r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 - with: - name: test-output - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push r - - windows-cpp: - name: AMD64 Windows C++ RTools ${{ matrix.config.rtools }} ${{ matrix.config.arch }} - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - config: - - { rtools: 40, arch: 'ucrt64' } - steps: - - run: git config --global core.autocrlf false - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ccache - key: r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} - restore-keys: | - r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- - r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- - - uses: r-lib/actions/setup-r@v2 - with: - # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ - # static library. The R is not used here but R 4.1 was the last R to use - # Rtools40. - r-version: "4.1" - rtools-version: 40 - Ncpus: 2 - - name: Build Arrow C++ - shell: bash - env: - MINGW_ARCH: ${{ matrix.config.arch }} - run: ci/scripts/r_windows_build.sh - - name: Rename libarrow.zip - # So that they're unique when multiple are downloaded in the next step - shell: bash - run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - - uses: actions/upload-artifact@v3 - with: - name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - - windows-r: - needs: [windows-cpp] - name: AMD64 Windows R ${{ matrix.config.rversion }} - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - config: - - { rversion: "release" } - - env: - ARROW_R_CXXFLAGS: "-Werror" - _R_CHECK_TESTS_NLINES_: 0 - steps: - - run: git config --global core.autocrlf false - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - # This must be done before r-lib/actions/setup-r because curl in - # Rtools doesn't work on non Rtools' MSYS2 environment. If we - # use "shell: bash" after r-lib/actions/setup-r, bash in Rtools - # is used on non Rtools' MSYS2 environment. - - name: Install MinIO - shell: bash - run: | - mkdir -p "$HOME/.local/bin" - ci/scripts/install_minio.sh latest "$HOME/.local" - echo "$HOME/.local/bin" >> $GITHUB_PATH - - run: mkdir r/windows - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - name: libarrow-rtools40-ucrt64.zip - path: r/windows - - name: Unzip and rezip libarrows - shell: bash - run: | - cd r/windows - ls *.zip | xargs -n 1 unzip -uo - rm -rf *.zip - - uses: r-lib/actions/setup-r@v2 - with: - r-version: ${{ matrix.config.rversion }} - Ncpus: 2 - - uses: r-lib/actions/setup-r-dependencies@v2 - env: - GITHUB_PAT: "${{ github.token }}" - with: - # For some arcane reason caching does not work on the windows runners - # most likely due to https://github.com/actions/cache/issues/815 - cache: false - working-directory: 'r' - extra-packages: | - any::rcmdcheck - # TODO(ARROW-17149): figure out why the GCS tests are hanging on Windows - # - name: Install Google Cloud Storage Testbench - # shell: bash - # run: ci/scripts/install_gcs_testbench.sh default - - name: Check - shell: Rscript {0} - run: | - # Because we do R CMD build and r/windows is in .Rbuildignore, - # assemble the libarrow.zip file and pass it as an env var - setwd("r/windows") - zip("libarrow.zip", ".") - setwd("..") - - Sys.setenv( - RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), - MAKEFLAGS = paste0("-j", parallel::detectCores()), - ARROW_R_DEV = TRUE, - "_R_CHECK_FORCE_SUGGESTS_" = FALSE, - "_R_CHECK_STOP_ON_INVALID_NUMERIC_VERSION_INPUTS_" = TRUE - ) - rcmdcheck::rcmdcheck(".", - build_args = '--no-build-vignettes', - args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'), - error_on = 'warning', - check_dir = 'check', - timeout = 3600 - ) - - name: Run lintr - if: ${{ matrix.config.rversion == 'release' }} - env: - NOT_CRAN: "true" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: Rscript {0} - working-directory: r - run: | - Sys.setenv( - RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), - MAKEFLAGS = paste0("-j", parallel::detectCores()), - ARROW_R_DEV = TRUE, - "_R_CHECK_FORCE_SUGGESTS_" = FALSE - ) - # we use pak for package installation since it is faster, safer and more convenient - pak::local_install() - pak::pak("lintr") - lintr::expect_lint_free() - - name: Dump install logs - shell: cmd - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - shell: bash - run: find r/check -name 'testthat.Rout*' -exec cat '{}' \; || true - if: always() diff --git a/.github/workflows/r_nightly.yml b/.github/workflows/r_nightly.yml deleted file mode 100644 index af5382f90834c..0000000000000 --- a/.github/workflows/r_nightly.yml +++ /dev/null @@ -1,192 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Upload R Nightly builds -# This workflow downloads the (nightly) binaries created in crossbow and uploads them -# to nightlies.apache.org. Due to authorization requirements, this upload can't be done -# from the crossbow repository. - -on: - workflow_dispatch: - inputs: - prefix: - description: Job prefix to use. - required: false - default: '' - keep: - description: Number of versions to keep. - required: false - default: 14 - - schedule: - #Crossbow packaging runs at 0 8 * * * - - cron: '0 14 * * *' - -permissions: - contents: read - -jobs: - upload: - if: github.repository == 'apache/arrow' - runs-on: ubuntu-latest - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 1 - path: arrow - repository: apache/arrow - ref: main - submodules: recursive - - name: Checkout Crossbow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - path: crossbow - repository: ursacomputing/crossbow - ref: main - - name: Set up Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - cache: 'pip' - python-version: 3.12 - - name: Install Archery - shell: bash - run: pip install -e arrow/dev/archery[all] - - run: mkdir -p binaries - - name: Download Artifacts - env: - PREFIX: ${{ github.event.inputs.prefix || ''}} - run: | - if [ -z $PREFIX ]; then - PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0 - fi - echo $PREFIX - - archery crossbow download-artifacts -f r-binary-packages -t binaries $PREFIX - - if [ -n "$(ls -A binaries/*/*/)" ]; then - echo "Found files!" - else - echo "No files found. Stopping upload." - exit 1 - fi - - name: Cache Repo - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: repo - key: r-nightly-${{ github.run_id }} - restore-keys: r-nightly- - - name: Sync from Remote - uses: ./arrow/.github/actions/sync-nightlies - with: - switches: -avzh --update --delete --progress - local_path: repo - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/r - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} - - run: tree repo - - name: Build Repository - shell: Rscript {0} - run: | - # folder that we sync to nightlies.apache.org - repo_root <- "repo" - # The binaries are in a nested dir - # so we need to find the correct path. - art_path <- list.files("binaries", - recursive = TRUE, - include.dirs = TRUE, - pattern = "r-binary-packages$", - full.names = TRUE - ) - - current_path <- list.files(art_path, full.names = TRUE, recursive = TRUE) - files <- sub("r-(pkg|lib)", repo_root, current_path) - - # decode contrib.url from artifact name: - # bin__windows__contrib__4.1 -> bin/windows/contrib/4.1 - new_paths <- gsub("__", "/", files) - # strip superfluous nested dirs - new_paths <- sub(art_path, ".", new_paths) - dirs <- dirname(new_paths) - sapply(dirs, dir.create, recursive = TRUE, showWarnings = FALSE) - - # overwrite allows us to "force push" a new version with the same name - copy_result <- file.copy(current_path, new_paths, overwrite = TRUE) - - if (!all(copy_result)) { - stop("There was an issue while copying the files!") - } - - name: Prune Repository - shell: bash - env: - KEEP: ${{ github.event.inputs.keep || 14 }} - run: | - prune() { - # list files | retain $KEEP newest files | delete everything else - ls -t $1/arrow* | tail -n +$((KEEP + 1)) | xargs --no-run-if-empty rm - } - - # find leaf sub dirs - repo_dirs=$(find repo -type d -links 2) - - # We want to retain $keep (14) versions of each pkg/lib so we call - # prune on each leaf dir and not on repo/. - for dir in ${repo_dirs[@]}; do - prune $dir - done - - name: Update Repository Index - shell: Rscript {0} - run: | - # folder that we sync to nightlies.apache.org - repo_root <- "repo" - tools::write_PACKAGES(file.path(repo_root, "src/contrib"), - type = "source", - verbose = TRUE, - latestOnly = FALSE - ) - - repo_dirs <- list.dirs(repo_root) - # find dirs with binary R packages: e.g. */contrib/4.1 - pkg_dirs <- grep(".+contrib\\/\\d.+", repo_dirs, value = TRUE) - - - for (dir in pkg_dirs) { - on_win <- grepl("windows", dir) - tools::write_PACKAGES(dir, - type = ifelse(on_win, "win.binary", "mac.binary"), - verbose = TRUE, - latestOnly = FALSE - ) - } - - name: Show repo contents - run: tree repo - - name: Sync to Remote - uses: ./arrow/.github/actions/sync-nightlies - with: - upload: true - switches: -avzh --update --delete --progress - local_path: repo - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/r - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml deleted file mode 100644 index 04f944f56c665..0000000000000 --- a/.github/workflows/ruby.yml +++ /dev/null @@ -1,399 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C GLib & Ruby - -on: - push: - paths: - - '.github/workflows/ruby.yml' - - 'ci/docker/**' - - 'ci/scripts/c_glib_*' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/ruby_*' - - 'ci/scripts/util_*' - - 'c_glib/**' - - 'cpp/**' - - 'docker-compose.yml' - - 'ruby/**' - pull_request: - paths: - - '.github/workflows/ruby.yml' - - 'ci/docker/**' - - 'ci/scripts/c_glib_*' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/ruby_*' - - 'ci/scripts/util_*' - - 'c_glib/**' - - 'cpp/**' - - 'docker-compose.yml' - - 'ruby/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - ubuntu: - name: AMD64 Ubuntu ${{ matrix.ubuntu }} GLib & Ruby - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - ubuntu: - - 22.04 - env: - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: ubuntu-${{ matrix.ubuntu }}-ruby-${{ hashFiles('cpp/**') }} - restore-keys: ubuntu-${{ matrix.ubuntu }}-ruby- - - name: Setup Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run \ - -e ARROW_FLIGHT=ON \ - -e ARROW_FLIGHT_SQL=ON \ - -e ARROW_GCS=ON \ - -e Protobuf_SOURCE=BUNDLED \ - -e gRPC_SOURCE=BUNDLED \ - ubuntu-ruby - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - shell: bash - run: archery docker push ubuntu-ruby - - macos: - name: AMD64 macOS 14 GLib & Ruby - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - env: - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: OFF - ARROW_BUILD_UTILITIES: OFF - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_FLIGHT_SQL: ON - ARROW_GANDIVA: ON - ARROW_GCS: ON - ARROW_GLIB_GTK_DOC: true - ARROW_GLIB_WERROR: true - ARROW_HOME: /tmp/local - ARROW_JEMALLOC: OFF - ARROW_ORC: OFF - ARROW_PARQUET: ON - ARROW_WITH_BROTLI: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - steps: - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Install Homebrew Dependencies - shell: bash - run: | - brew bundle --file=cpp/Brewfile - brew bundle --file=c_glib/Brewfile - - name: Install Ruby Dependencies - run: | - export MAKEFLAGS="-j$(sysctl -n hw.ncpu)" - bundle install --gemfile c_glib/Gemfile - bundle install --gemfile ruby/Gemfile - for ruby_package_gemfile in ruby/*/Gemfile; do \ - bundle install --gemfile ${ruby_package_gemfile} - done - - name: Setup ccache - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: ruby-ccache-macos-${{ hashFiles('cpp/**') }} - restore-keys: ruby-ccache-macos- - - name: Build C++ - run: | - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Build GLib - run: | - ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build - - name: Test GLib - shell: bash - run: ci/scripts/c_glib_test.sh $(pwd) $(pwd)/build - - name: Test Ruby - shell: bash - run: ci/scripts/ruby_test.sh $(pwd) $(pwd)/build - - windows-mingw: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} GLib & Ruby - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - mingw-n-bits: - - 64 - ruby-version: - # TODO: Use the latest Ruby again when we fix GH-39130. - # - ruby - - "3.1" - env: - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: OFF - ARROW_BUILD_UTILITIES: OFF - ARROW_BUILD_TYPE: release - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_FLIGHT_SQL: ON - ARROW_GANDIVA: ON - ARROW_GCS: ON - ARROW_HDFS: OFF - ARROW_HOME: /ucrt${{ matrix.mingw-n-bits }} - ARROW_JEMALLOC: OFF - ARROW_PARQUET: ON - ARROW_PYTHON: OFF - ARROW_S3: ON - ARROW_USE_GLOG: OFF - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - # Don't use preinstalled Boost by empty BOOST_ROOT and - # -DBoost_NO_BOOST_CMAKE=ON - BOOST_ROOT: "" - ARROW_CMAKE_ARGS: >- - -DARROW_PACKAGE_PREFIX=/ucrt${{ matrix.mingw-n-bits }} - -DBoost_NO_BOOST_CMAKE=ON - -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON - CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby-version }} - - name: Setup MSYS2 - run: | - ridk exec bash ci\scripts\msys2_setup.sh ruby - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ccache - key: ruby-ccache-ucrt${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }} - restore-keys: ruby-ccache-ucrt${{ matrix.mingw-n-bits }}- - - name: Build C++ - run: | - $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\cpp_build.sh "${source_dir}" "${build_dir}" - - name: Build GLib - run: | - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\c_glib_build.sh "${source_dir}" "${build_dir}" - - name: RubyGems info - id: rubygems-info - run: | - Write-Output "gem-dir=$(ridk exec gem env gemdir)" | ` - Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - - name: Cache RubyGems - uses: actions/cache@v4 - with: - path: ${{ steps.rubygems-info.outputs.gem-dir }} - key: ruby-rubygems-ucrt${{ matrix.mingw-n-bits }}-${{ hashFiles('**/Gemfile', 'ruby/*/*.gemspec') }} - restore-keys: ruby-rubygems-ucrt${{ matrix.mingw-n-bits }}- - - name: Install test dependencies - run: | - bundle install --gemfile c_glib\Gemfile - bundle install --gemfile ruby\Gemfile - Get-ChildItem ruby\*\Gemfile | ` - ForEach-Object {bundle install --gemfile $_} - - name: Test GLib - run: | - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\c_glib_test.sh "${source_dir}" "${build_dir}" - - name: Test Ruby - run: | - $Env:PKG_CONFIG_PATH = ` - "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/pkgconfig")" - $Env:GI_TYPELIB_PATH = ` - "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/girepository-1.0")" - $Env:RUBYOPTS = "-rdevkit" - $Env:MAKE = "ridk exec make" - $ErrorActionPreference = "Continue" - rake -f ruby\Rakefile - - windows-msvc: - name: AMD64 Windows MSVC GLib - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - strategy: - fail-fast: false - env: - ARROW_BOOST_USE_SHARED: OFF - ARROW_BUILD_BENCHMARKS: OFF - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: OFF - ARROW_ACERO: ON - ARROW_DATASET: ON - ARROW_FLIGHT: OFF - ARROW_FLIGHT_SQL: OFF - ARROW_GANDIVA: OFF - ARROW_HDFS: OFF - ARROW_HOME: "${{ github.workspace }}/dist" - ARROW_JEMALLOC: OFF - ARROW_MIMALLOC: ON - ARROW_ORC: OFF - ARROW_PARQUET: ON - ARROW_SUBSTRAIT: OFF - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: OFF - ARROW_WITH_BZ2: OFF - ARROW_WITH_LZ4: OFF - ARROW_WITH_OPENTELEMETRY: OFF - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: OFF - ARROW_WITH_ZSTD: ON - BOOST_SOURCE: BUNDLED - CMAKE_CXX_STANDARD: "17" - CMAKE_GENERATOR: Ninja - CMAKE_INSTALL_PREFIX: "${{ github.workspace }}/dist" - CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Install vcpkg - shell: bash - run: | - ci/scripts/install_vcpkg.sh ./vcpkg - - name: Install meson - run: | - python -m pip install meson - - name: Install ccache - shell: bash - run: | - ci/scripts/install_ccache.sh 4.6.3 /usr - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: glib-ccache-msvc-${{ env.CACHE_VERSION }}-${{ hashFiles('cpp/**') }} - restore-keys: glib-ccache-msvc-${{ env.CACHE_VERSION }}- - env: - # We can invalidate the current cache by updating this. - CACHE_VERSION: "2024-05-09" - - name: Build C++ - shell: cmd - run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build" - - name: Build GLib - shell: cmd - run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "VCPKG_ROOT=\"$(pwd)/vcpkg\" ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build" diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml deleted file mode 100644 index 3f039315b505a..0000000000000 --- a/.github/workflows/swift.yml +++ /dev/null @@ -1,80 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Swift - -on: - push: - paths: - - '.github/workflows/swift.yml' - - 'ci/docker/*swift*' - - 'ci/scripts/swift_*' - - 'docker-compose.yml' - - 'swift/**' - pull_request: - paths: - - '.github/workflows/swift.yml' - - 'ci/docker/*swift*' - - 'ci/scripts/swift_*' - - 'docker-compose.yml' - - 'swift/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - docker: - name: AMD 64 Ubuntu Swift 5.7 - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ubuntu-swift - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ubuntu-swift